Skip to content

Commit dc5ec76

Browse files
committed
fix(frame): verify raw payload digests and bounded zstd frames
1 parent cb201ac commit dc5ec76

4 files changed

Lines changed: 277 additions & 34 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "mst2-codec"
3-
version = "0.3.1"
3+
version = "0.4.0"
44
edition = "2021"
55
description = "MST/2 wire codec: ServingDescriptor (MSD2), MTP2/1 metadata pages, TreeFrame v2, Chunk Map (MCM2/MCL2). Pure byte-level parsing and encoding per Mega_ScorpioFS_MST2 specs 03/05/06/07."
66
license = "Apache-2.0"

‎src/treeframe.rs‎

Lines changed: 154 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ pub const OBJECT_MAX_LEN: u32 = 262_144;
2929
pub const OBJECT_MAX_RAW: usize = 1_048_576;
3030
pub const CHUNK_MAX_LEN: u32 = 1_048_576;
3131
pub const ERROR_MAX_BYTES: usize = 4096;
32+
pub const DATA_MAX_WIRE: usize = 2_097_152;
3233

3334
/// A parsed frame header (payload digest included).
3435
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -127,21 +128,29 @@ pub fn parse_frame(buf: &[u8]) -> CodecResult<(Frame, usize)> {
127128
));
128129
}
129130

130-
let total = HEADER_LEN + wire_len as usize;
131+
let raw_limit = match kind {
132+
KIND_META => META_MAX_RAW,
133+
KIND_OBJECT => OBJECT_MAX_RAW,
134+
KIND_CHUNK => 76 + CHUNK_MAX_LEN as usize,
135+
KIND_END => 48,
136+
KIND_ERROR => ERROR_MAX_BYTES,
137+
_ => return Err(CodecError::BadConstant("frame kind")),
138+
};
139+
if raw_len as usize > raw_limit || wire_len as usize > DATA_MAX_WIRE {
140+
return Err(CodecError::BadLength("frame payload exceeds kind limit"));
141+
}
142+
let total = HEADER_LEN
143+
.checked_add(wire_len as usize)
144+
.ok_or(CodecError::BadLength("frame length overflow"))?;
131145
if buf.len() < total {
132146
return Err(CodecError::Truncated("frame payload"));
133147
}
134148
let wire_payload = &buf[HEADER_LEN..total];
135149

136-
// Header digest always covers the on-wire (compressed) payload.
137-
let wire_digest = sha256(&[wire_payload]);
138-
if wire_digest != payload_sha256 {
139-
return Err(CodecError::DigestMismatch("frame payload"));
140-
}
141-
142150
// Recover the canonical raw payload, enforcing the single-frame
143151
// zstd contract when compressed (window ≤ 8 MiB, exact raw_len, no
144152
// concatenated/skippable/trailing data).
153+
#[cfg(feature = "zstd")]
145154
let owned_payload: Vec<u8>;
146155
let payload: &[u8] = if flags & FLAG_ZSTD == 0 {
147156
wire_payload
@@ -158,6 +167,9 @@ pub fn parse_frame(buf: &[u8]) -> CodecResult<(Frame, usize)> {
158167
));
159168
}
160169
};
170+
if sha256(&[payload]) != payload_sha256 {
171+
return Err(CodecError::DigestMismatch("frame raw payload"));
172+
}
161173

162174
let frame = match kind {
163175
KIND_META => Frame::Meta(decode_meta(payload)?),
@@ -576,7 +588,7 @@ fn frame_bytes(kind: u8, payload: &[u8], stream_id: u32, sequence: u64) -> Vec<u
576588
}
577589

578590
/// Compress `raw_payload` into one zstd frame (flags `FLAG_ZSTD`).
579-
/// The header digest covers the compressed wire payload, per spec 06.
591+
/// The header digest covers the raw payload, per spec 06.
580592
/// Not for END/ERROR frames — callers reject those.
581593
#[cfg(feature = "zstd")]
582594
fn frame_bytes_zstd(
@@ -586,7 +598,7 @@ fn frame_bytes_zstd(
586598
sequence: u64,
587599
) -> CodecResult<Vec<u8>> {
588600
let wire = crate::zstd1::compress(raw_payload, 3)?;
589-
let digest = sha256(&[&wire]);
601+
let digest = sha256(&[raw_payload]);
590602
let mut out = header(
591603
kind,
592604
FLAG_ZSTD,
@@ -719,6 +731,9 @@ impl ChunkPayload {
719731

720732
#[cfg(feature = "zstd")]
721733
pub fn encode_zstd(&self, stream_id: u32, sequence: u64) -> CodecResult<Vec<u8>> {
734+
if self.chunk_bytes.len() as u64 > CHUNK_MAX_LEN as u64 {
735+
return Err(CodecError::BadLength("chunk over 1MiB"));
736+
}
722737
let payload = self.raw_payload();
723738
frame_bytes_zstd(KIND_CHUNK, &payload, stream_id, sequence)
724739
}
@@ -934,6 +949,36 @@ mod tests {
934949
assert!(parse_stream(&bad).is_err());
935950
}
936951

952+
#[test]
953+
fn oversized_lengths_and_unknown_kinds_fail_before_reading_payload() {
954+
for (kind, raw_limit) in [
955+
(KIND_META, META_MAX_RAW),
956+
(KIND_OBJECT, OBJECT_MAX_RAW),
957+
(KIND_CHUNK, 76 + CHUNK_MAX_LEN as usize),
958+
(KIND_END, 48),
959+
(KIND_ERROR, ERROR_MAX_BYTES),
960+
] {
961+
let raw_len = (raw_limit + 1) as u32;
962+
let bytes = header(kind, 0, raw_len, raw_len, 1, 0, [0; 32]);
963+
assert!(matches!(parse_frame(&bytes), Err(CodecError::BadLength(_))));
964+
}
965+
let bytes = header(
966+
KIND_OBJECT,
967+
FLAG_ZSTD,
968+
DATA_MAX_WIRE as u32 + 1,
969+
100,
970+
1,
971+
0,
972+
[0; 32],
973+
);
974+
assert!(matches!(parse_frame(&bytes), Err(CodecError::BadLength(_))));
975+
let bytes = header(99, FLAG_ZSTD, 1, 1, 1, 0, [0; 32]);
976+
assert!(matches!(
977+
parse_frame(&bytes),
978+
Err(CodecError::BadConstant("frame kind"))
979+
));
980+
}
981+
937982
#[test]
938983
fn header_rules() {
939984
let p = EndPayload {
@@ -1013,6 +1058,86 @@ mod zstd_tests {
10131058
assert_eq!(f, Frame::Object(raw));
10141059
}
10151060

1061+
#[test]
1062+
fn compression_representations_share_the_raw_payload_digest() {
1063+
use crate::metapage::{page_id, Page};
1064+
let page = Page::Leaf { entries: vec![] }.encode().unwrap();
1065+
let meta = MetaPayload {
1066+
pages: vec![(page_id(&page), page)],
1067+
};
1068+
let object = obj(0x21, 100_000);
1069+
let chunk = ChunkPayload {
1070+
map_id: [1; 32],
1071+
file_content_id: [2; 32],
1072+
chunk_index: 3,
1073+
chunk_bytes: vec![0x42; CHUNK_MAX_LEN as usize],
1074+
};
1075+
for (identity, compressed) in [
1076+
(meta.encode(7, 0).unwrap(), meta.encode_zstd(7, 0).unwrap()),
1077+
(
1078+
object.encode(7, 0).unwrap(),
1079+
object.encode_zstd(7, 0).unwrap(),
1080+
),
1081+
(
1082+
chunk.encode(7, 0).unwrap(),
1083+
chunk.encode_zstd(7, 0).unwrap(),
1084+
),
1085+
] {
1086+
let raw = &identity[HEADER_LEN..];
1087+
assert_eq!(&compressed[32..64], &sha256(&[raw]));
1088+
assert_eq!(&identity[32..64], &compressed[32..64]);
1089+
let expected = parse_frame(&identity).unwrap().0;
1090+
assert_eq!(parse_frame(&compressed).unwrap().0, expected);
1091+
for level in [1, 19] {
1092+
let wire = zstd::bulk::compress(raw, level).unwrap();
1093+
let mut independent = header(
1094+
identity[6],
1095+
FLAG_ZSTD,
1096+
wire.len() as u32,
1097+
raw.len() as u32,
1098+
7,
1099+
0,
1100+
sha256(&[raw]),
1101+
);
1102+
independent.extend_from_slice(&wire);
1103+
assert_eq!(parse_frame(&independent).unwrap().0, expected);
1104+
}
1105+
let mut legacy = compressed;
1106+
let wire_digest = sha256(&[&legacy[HEADER_LEN..]]);
1107+
assert_ne!(&legacy[32..64], &wire_digest);
1108+
legacy[32..64].copy_from_slice(&wire_digest);
1109+
assert!(matches!(
1110+
parse_frame(&legacy),
1111+
Err(CodecError::DigestMismatch(_))
1112+
));
1113+
}
1114+
}
1115+
1116+
#[test]
1117+
fn raw_digest_is_checked_even_when_object_content_is_valid() {
1118+
let mut encoded = obj(0x45, 20_000).encode_zstd(1, 0).unwrap();
1119+
encoded[32] ^= 1;
1120+
assert!(matches!(
1121+
parse_frame(&encoded),
1122+
Err(CodecError::DigestMismatch(_))
1123+
));
1124+
}
1125+
1126+
#[test]
1127+
fn both_encodings_refuse_oversized_chunks_before_encoding() {
1128+
let chunk = ChunkPayload {
1129+
map_id: [1; 32],
1130+
file_content_id: [2; 32],
1131+
chunk_index: 0,
1132+
chunk_bytes: vec![0x61; CHUNK_MAX_LEN as usize + 1],
1133+
};
1134+
assert!(matches!(chunk.encode(1, 0), Err(CodecError::BadLength(_))));
1135+
assert!(matches!(
1136+
chunk.encode_zstd(1, 0),
1137+
Err(CodecError::BadLength(_))
1138+
));
1139+
}
1140+
10161141
#[test]
10171142
fn end_and_error_frames_are_never_compressed() {
10181143
// No encode_zstd exists for End/Error; even hand-built zstd headers
@@ -1034,7 +1159,9 @@ mod zstd_tests {
10341159

10351160
#[test]
10361161
fn concatenated_zstd_frames_are_rejected() {
1037-
let p = obj(0x22, 50_000).encode_zstd(1, 0).unwrap();
1162+
let object = obj(0x22, 50_000);
1163+
let identity = object.encode(1, 0).unwrap();
1164+
let p = object.encode_zstd(1, 0).unwrap();
10381165
let wire1 = &p[HEADER_LEN..];
10391166
// Two independent zstd frames concatenated in one payload region.
10401167
let mut doubled = Vec::new();
@@ -1044,50 +1171,58 @@ mod zstd_tests {
10441171
KIND_OBJECT,
10451172
FLAG_ZSTD,
10461173
doubled.len() as u32,
1047-
50_000,
1174+
(identity.len() - HEADER_LEN) as u32,
10481175
1,
10491176
0,
1050-
sha256(&[&doubled]),
1177+
sha256(&[&identity[HEADER_LEN..]]),
10511178
);
10521179
out.extend_from_slice(&doubled);
10531180
assert!(parse_frame(&out).is_err(), "two frames must not decode");
10541181
}
10551182

10561183
#[test]
10571184
fn trailing_bytes_after_frame_are_rejected() {
1058-
let p = obj(0x33, 50_000).encode_zstd(1, 0).unwrap();
1185+
let object = obj(0x33, 50_000);
1186+
let identity = object.encode(1, 0).unwrap();
1187+
let p = object.encode_zstd(1, 0).unwrap();
10591188
let wire = &p[HEADER_LEN..];
10601189
let mut padded = wire.to_vec();
10611190
padded.push(0x00);
10621191
let mut out = header(
10631192
KIND_OBJECT,
10641193
FLAG_ZSTD,
10651194
padded.len() as u32,
1066-
50_000,
1195+
(identity.len() - HEADER_LEN) as u32,
10671196
1,
10681197
0,
1069-
sha256(&[&padded]),
1198+
sha256(&[&identity[HEADER_LEN..]]),
10701199
);
10711200
out.extend_from_slice(&padded);
10721201
assert!(parse_frame(&out).is_err(), "trailing byte must fail");
10731202
}
10741203

10751204
#[test]
10761205
fn wrong_raw_len_is_rejected_short_and_long() {
1077-
let p = obj(0x44, 50_000).encode_zstd(1, 0).unwrap();
1206+
let object = obj(0x44, 50_000);
1207+
let identity = object.encode(1, 0).unwrap();
1208+
let p = object.encode_zstd(1, 0).unwrap();
10781209
let wire = &p[HEADER_LEN..];
1079-
for advertised in [10u32, 50_001] {
1210+
let raw = &identity[HEADER_LEN..];
1211+
for advertised in [raw.len() as u32 - 1, raw.len() as u32 + 1] {
10801212
let mut out = header(
10811213
KIND_OBJECT,
10821214
FLAG_ZSTD,
10831215
wire.len() as u32,
10841216
advertised,
10851217
1,
10861218
0,
1087-
sha256(&[wire]),
1219+
sha256(&[raw]),
10881220
);
10891221
out.extend_from_slice(wire);
1090-
assert!(parse_frame(&out).is_err(), "raw_len={advertised} must fail");
1222+
assert!(
1223+
matches!(parse_frame(&out), Err(CodecError::BadLength(_))),
1224+
"raw_len={advertised} must fail"
1225+
);
10911226
}
10921227
}
10931228

0 commit comments

Comments
 (0)