fix: address parameter set parsing review

Author: alexliyu7352

libflv/source/hevc-annexbtomp4.c

@@ -29,7 +29,7 @@ struct h265_annexbtomp4_handle_t
 	size_t capacity;
 };
 
-int mpeg4_h264_read_ue(const uint8_t* data, size_t bytes, size_t* offset, uint8_t* value);
+int mpeg4_h264_read_ue(const uint8_t* data, size_t bytes, size_t* offset, uint32_t* value);
 
 static size_t hevc_rbsp_decode(const uint8_t* nalu, size_t bytes, uint8_t* sodb, size_t len)
 {
@@ -127,13 +127,17 @@ static uint8_t hevc_vps_id(const uint8_t* rbsp, size_t bytes, struct mpeg4_hevc_
 
 static uint8_t hevc_sps_id(const uint8_t* rbsp, size_t bytes, struct mpeg4_hevc_t* hevc, uint8_t* ptr, size_t len, uint8_t* vps)
 {
+	int ptl_bytes;
 	size_t n;
 	size_t sodb;
-	uint8_t sps;
+	uint32_t sps;
+	uint32_t chroma_format;
+	uint32_t bit_depth_luma_minus8;
+	uint32_t bit_depth_chroma_minus8;
 	uint8_t sps_max_sub_layers_minus1;
 	uint8_t sps_temporal_id_nesting_flag;
 	uint8_t conformance_window_flag;
-	uint8_t value;
+	uint32_t value;
 
 	sodb = hevc_rbsp_decode(rbsp, bytes, ptr, len);
 	if (sodb < 12+3)
@@ -142,15 +146,17 @@ static uint8_t hevc_sps_id(const uint8_t* rbsp, size_t bytes, struct mpeg4_hevc_
 	*vps = ptr[2] >> 4;  // 2-nalu type
 	sps_max_sub_layers_minus1 = (ptr[2] >> 1) & 0x07;
 	sps_temporal_id_nesting_flag = ptr[2] & 0x01;
-	n = hevc_profile_tier_level(ptr + 3, sodb - 3, sps_max_sub_layers_minus1, hevc);
-	if (n <= 0)
+	ptl_bytes = hevc_profile_tier_level(ptr + 3, sodb - 3, sps_max_sub_layers_minus1, hevc);
+	if (ptl_bytes <= 0)
 		return 0xFF;
 
-	n = (n + 3) * 8;
+	n = ((size_t)ptl_bytes + 3) * 8;
 	if (mpeg4_h264_read_ue(ptr, sodb, &n, &sps) < 0
-		|| mpeg4_h264_read_ue(ptr, sodb, &n, &hevc->chromaFormat) < 0)
+		|| mpeg4_h264_read_ue(ptr, sodb, &n, &chroma_format) < 0
+		|| sps >= 16 || chroma_format > 3)
 		return 0xFF;
-	if (3 == hevc->chromaFormat)
+	hevc->chromaFormat = (uint8_t)chroma_format;
+	if (3 == chroma_format)
 	{
 		if (n / 8 >= sodb)
 			return 0xFF;
@@ -170,28 +176,47 @@ static uint8_t hevc_sps_id(const uint8_t* rbsp, size_t bytes, struct mpeg4_hevc_
 			|| mpeg4_h264_read_ue(ptr, sodb, &n, &value) < 0) // conf_win_bottom_offset
 			return 0xFF;
 	}
-	if (mpeg4_h264_read_ue(ptr, sodb, &n, &hevc->bitDepthLumaMinus8) < 0
-		|| mpeg4_h264_read_ue(ptr, sodb, &n, &hevc->bitDepthChromaMinus8) < 0)
+	if (mpeg4_h264_read_ue(ptr, sodb, &n, &bit_depth_luma_minus8) < 0
+		|| mpeg4_h264_read_ue(ptr, sodb, &n, &bit_depth_chroma_minus8) < 0
+		|| bit_depth_luma_minus8 > 7 || bit_depth_chroma_minus8 > 7)
 		return 0xFF;
+	hevc->bitDepthLumaMinus8 = (uint8_t)bit_depth_luma_minus8;
+	hevc->bitDepthChromaMinus8 = (uint8_t)bit_depth_chroma_minus8;
 
 	// TODO: vui_parameters
 	//mp4->hevc->min_spatial_segmentation_idc; // min_spatial_segmentation_idc
-	return sps;
+	return (uint8_t)sps;
+}
+
+static void hevc_sps_apply(struct mpeg4_hevc_t* hevc, const struct mpeg4_hevc_t* parsed)
+{
+	hevc->general_profile_space = parsed->general_profile_space;
+	hevc->general_tier_flag = parsed->general_tier_flag;
+	hevc->general_profile_idc = parsed->general_profile_idc;
+	hevc->general_profile_compatibility_flags = parsed->general_profile_compatibility_flags;
+	hevc->general_constraint_indicator_flags = parsed->general_constraint_indicator_flags;
+	hevc->general_level_idc = parsed->general_level_idc;
+	hevc->chromaFormat = parsed->chromaFormat;
+	hevc->bitDepthLumaMinus8 = parsed->bitDepthLumaMinus8;
+	hevc->bitDepthChromaMinus8 = parsed->bitDepthChromaMinus8;
 }
 
 static uint8_t hevc_pps_id(const uint8_t* rbsp, size_t bytes, struct mpeg4_hevc_t* hevc, uint8_t* ptr, size_t len, uint8_t* sps)
 {
-	uint8_t pps;
+	uint32_t pps;
+	uint32_t spsid;
 	size_t sodb;
 	size_t offset = 2 * 8;  // 2-nalu type
 	sodb = hevc_rbsp_decode(rbsp, bytes, ptr, len);
 	if (sodb < 3)
 		return 0xFF;
 	(void)hevc;
 	if (mpeg4_h264_read_ue(ptr, sodb, &offset, &pps) < 0
-		|| mpeg4_h264_read_ue(ptr, sodb, &offset, sps) < 0)
+		|| mpeg4_h264_read_ue(ptr, sodb, &offset, &spsid) < 0
+		|| pps >= 64 || spsid >= 16)
 		return 0xFF;
-	return pps;
+	*sps = (uint8_t)spsid;
+	return (uint8_t)pps;
 }
 
 static void mpeg4_hevc_remove(struct mpeg4_hevc_t* hevc, uint8_t* ptr, size_t bytes, const uint8_t* end)
@@ -273,25 +298,40 @@ static int h265_vps_copy(struct mpeg4_hevc_t* hevc, const uint8_t* nalu, size_t
 static int h265_sps_copy(struct mpeg4_hevc_t* hevc, const uint8_t* nalu, size_t bytes)
 {
 	int i;
+	int r;
 	uint8_t spsid;
+	uint8_t spsid2;
 	uint8_t vpsid, vpsid2;
+	struct mpeg4_hevc_t parsed;
+	struct mpeg4_hevc_t parsed2;
 
 	if (bytes < 13 + 2)
 	{
 		assert(0);
 		return -1; // invalid length
 	}
 
-	spsid = hevc_sps_id(nalu, bytes, hevc, hevc->data + hevc->off, sizeof(hevc->data) - hevc->off, &vpsid);
+	spsid = hevc_sps_id(nalu, bytes, &parsed, hevc->data + hevc->off, sizeof(hevc->data) - hevc->off, &vpsid);
 	if (0xFF == spsid)
 		return -1;
 	for (i = 0; i < hevc->numOfArrays; i++)
 	{
-		if (H265_NAL_SPS == hevc->nalu[i].type && spsid == hevc_sps_id(hevc->nalu[i].data, hevc->nalu[i].bytes, hevc, hevc->data + hevc->off, sizeof(hevc->data) - hevc->off, &vpsid2) && vpsid == vpsid2)
-			return mpeg4_hevc_update2(hevc, i, nalu, bytes);
+		if (H265_NAL_SPS != hevc->nalu[i].type)
+			continue;
+		spsid2 = hevc_sps_id(hevc->nalu[i].data, hevc->nalu[i].bytes, &parsed2, hevc->data + hevc->off, sizeof(hevc->data) - hevc->off, &vpsid2);
+		if (spsid == spsid2 && vpsid == vpsid2)
+		{
+			r = mpeg4_hevc_update2(hevc, i, nalu, bytes);
+			if (r >= 0)
+				hevc_sps_apply(hevc, &parsed);
+			return r;
+		}
 	}
 
-	return mpeg4_hevc_add(hevc, H265_NAL_SPS, nalu, bytes);
+	r = mpeg4_hevc_add(hevc, H265_NAL_SPS, nalu, bytes);
+	if (r >= 0)
+		hevc_sps_apply(hevc, &parsed);
+	return r;
 }
 
 static int h265_pps_copy(struct mpeg4_hevc_t* hevc, const uint8_t* nalu, size_t bytes)
@@ -476,12 +516,15 @@ void hevc_annexbtomp4_test(void)
 	const uint8_t vps[] = { 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x9d, 0xc0, 0x90 };
 	const uint8_t sps[] = { 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0xa0, 0x03, 0xc0, 0x80, 0x32, 0x16, 0x59, 0xde, 0x49, 0x1b, 0x6b, 0x80, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x17, 0x70, 0x02 };
 	const uint8_t pps[] = { 0x44, 0x01, 0xc1, 0x73, 0xd1, 0x89 };
+	const uint8_t truncated_ptl_sps[] = { 0x42, 0x01, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+	const uint8_t truncated_sps[] = { 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78 };
 	const uint8_t incomplete_pps[] = { 0x44, 0x01, 0x01 };
 	const uint8_t annexb[] = { 0x00, 0x00, 0x00, 0x01, 0x4e, 0x01, 0x06, 0x01, 0xd0, 0x80, 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x9d, 0xc0, 0x90, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0xa0, 0x03, 0xc0, 0x80, 0x32, 0x16, 0x59, 0xde, 0x49, 0x1b, 0x6b, 0x80, 0x40, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x17, 0x70, 0x02, 0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc1, 0x73, 0xd1, 0x89 };
 	uint8_t output[512];
 	int vcl, update;
 
 	struct mpeg4_hevc_t hevc;
+	struct mpeg4_hevc_t before;
 	memset(&hevc, 0, sizeof(hevc));
 	assert(h265_annexbtomp4(&hevc, annexb, sizeof(annexb), output, sizeof(output), &vcl, &update) > 0);
 	assert(3 == hevc.numOfArrays && vcl == 0 && update == 1);
@@ -492,5 +535,21 @@ void hevc_annexbtomp4_test(void)
 	memset(&hevc, 0, sizeof(hevc));
 	assert(mpeg4_hevc_update(&hevc, incomplete_pps, sizeof(incomplete_pps)) < 0);
 	assert(0 == hevc.numOfArrays && 0 == hevc.off);
+
+	memset(&hevc, 0x5a, offsetof(struct mpeg4_hevc_t, numOfArrays));
+	hevc.numOfArrays = 0;
+	hevc.off = 0;
+	before = hevc;
+	assert(mpeg4_hevc_update(&hevc, truncated_ptl_sps, sizeof(truncated_ptl_sps)) < 0);
+	assert(0 == memcmp(&before, &hevc, offsetof(struct mpeg4_hevc_t, numOfArrays)));
+	assert(0 == hevc.numOfArrays && 0 == hevc.off);
+
+	memset(&hevc, 0x5a, offsetof(struct mpeg4_hevc_t, numOfArrays));
+	hevc.numOfArrays = 0;
+	hevc.off = 0;
+	before = hevc;
+	assert(mpeg4_hevc_update(&hevc, truncated_sps, sizeof(truncated_sps)) < 0);
+	assert(0 == memcmp(&before, &hevc, offsetof(struct mpeg4_hevc_t, numOfArrays)));
+	assert(0 == hevc.numOfArrays && 0 == hevc.off);
 }
 #endif

libflv/source/mpeg4-annexbtomp4.c

@@ -161,7 +161,7 @@ int mpeg4_h264_annexb_nalu(const void* h264, size_t bytes, void (*handler)(void*
 	return 0;
 }
 
-int mpeg4_h264_read_ue(const uint8_t* data, size_t bytes, size_t* offset, uint8_t* value)
+int mpeg4_h264_read_ue(const uint8_t* data, size_t bytes, size_t* offset, uint32_t* value)
 {
 	int bit, i;
 	int leadingZeroBits = 0;
@@ -188,7 +188,7 @@ int mpeg4_h264_read_ue(const uint8_t* data, size_t bytes, size_t* offset, uint8_
 		++*offset;
 	}
 
-	*value = (uint8_t)((1U << leadingZeroBits) - 1 + suffix);
+	*value = (1U << leadingZeroBits) - 1 + suffix;
 	return 0;
 }
 
@@ -215,8 +215,8 @@ static int h264_sps_copy(struct mpeg4_avc_t* avc, const uint8_t* nalu, size_t by
 {
 	size_t i;
 	size_t offset;
-	uint8_t spsid;
-	uint8_t spsid2;
+	uint32_t spsid;
+	uint32_t spsid2;
 
 	if (bytes < 4 + 1)
 	{
@@ -225,7 +225,7 @@ static int h264_sps_copy(struct mpeg4_avc_t* avc, const uint8_t* nalu, size_t by
 	}
 
 	offset = 4 * 8; // 1-NALU + 3-profile+flags+level
-	if (mpeg4_h264_read_ue(nalu, bytes, &offset, &spsid) < 0)
+	if (mpeg4_h264_read_ue(nalu, bytes, &offset, &spsid) < 0 || spsid >= 32)
 		return -1;
 
 	for (i = 0; i < avc->nb_sps; i++)
@@ -276,10 +276,10 @@ static int h264_pps_copy(struct mpeg4_avc_t* avc, const uint8_t* nalu, size_t by
 {
 	size_t i;
 	size_t offset;
-	uint8_t spsid;
-	uint8_t spsid2;
-	uint8_t ppsid;
-	uint8_t ppsid2;
+	uint32_t spsid;
+	uint32_t spsid2;
+	uint32_t ppsid;
+	uint32_t ppsid2;
 
 	if (bytes < 1 + 1)
 	{
@@ -289,7 +289,8 @@ static int h264_pps_copy(struct mpeg4_avc_t* avc, const uint8_t* nalu, size_t by
 
 	offset = 1 * 8; // 1-NALU
 	if (mpeg4_h264_read_ue(nalu, bytes, &offset, &ppsid) < 0
-		|| mpeg4_h264_read_ue(nalu, bytes, &offset, &spsid) < 0)
+		|| mpeg4_h264_read_ue(nalu, bytes, &offset, &spsid) < 0
+		|| ppsid >= 256 || spsid >= 32)
 		return -1;
 
 	for (i = 0; i < avc->nb_pps; i++)
@@ -359,7 +360,7 @@ int mpeg4_avc_update(struct mpeg4_avc_t* avc, const uint8_t* nalu, size_t bytes)
 	{
 	case H264_NAL_SPS:
 		r = h264_sps_copy(avc, nalu, bytes);
-		if (1 == r || 1 == avc->nb_sps)
+		if (r >= 0 && (1 == r || 1 == avc->nb_sps))
 		{
 			// update profile/level
 			avc->profile = nalu[1];
@@ -520,9 +521,13 @@ void mpeg4_annexbtomp4_test(void)
 {
 	const uint8_t sps[] = { 0x67,0x42,0xe0,0x1e,0xab };
 	const uint8_t pps[] = { 0x28,0xce,0x3c,0x80 };
+	const uint8_t oversized_sps_id[] = { 0x67,0x64,0x00,0x28,0x00,0x80,0x80 };
+	const uint8_t truncated_sps[] = { 0x67,0x64,0x00,0x28,0x00 };
 	const uint8_t truncated_pps[] = { 0x68,0x01 };
 	const uint8_t annexb[] = { 0x00,0x00,0x00,0x01,0x67,0x42,0xe0,0x1e,0xab, 0x00,0x00,0x00,0x01,0x28,0xce,0x3c,0x80,0x00,0x00,0x00,0x01,0x65,0x11 };
 	uint8_t output[256];
+	uint8_t profile, compatibility, level;
+	size_t off;
 	int vcl, update;
 
 	struct mpeg4_avc_t avc;
@@ -536,6 +541,20 @@ void mpeg4_annexbtomp4_test(void)
 	assert(mpeg4_avc_update(&avc, truncated_pps, sizeof(truncated_pps)) < 0);
 	assert(0 == avc.nb_pps && 0 == avc.off);
 
+	memset(&avc, 0, sizeof(avc));
+	assert(mpeg4_avc_update(&avc, oversized_sps_id, sizeof(oversized_sps_id)) < 0);
+	assert(0 == avc.nb_sps && 0 == avc.off);
+
+	memset(&avc, 0, sizeof(avc));
+	assert(mpeg4_avc_update(&avc, sps, sizeof(sps)) > 0);
+	profile = avc.profile;
+	compatibility = avc.compatibility;
+	level = avc.level;
+	off = avc.off;
+	assert(mpeg4_avc_update(&avc, truncated_sps, sizeof(truncated_sps)) < 0);
+	assert(profile == avc.profile && compatibility == avc.compatibility && level == avc.level);
+	assert(1 == avc.nb_sps && off == avc.off && 0 == memcmp(avc.sps[0].data, sps, sizeof(sps)));
+
 	mpeg4_annexbtomp4_test2();
 	mpeg4_h264_bitstream_format_test();
 }

libflv/source/mpeg4-vvc.c

@@ -426,5 +426,6 @@ void mpeg4_vvc_test(void)
 	assert(2 == vvc.numOfArrays && H266_SPS == vvc.nalu[0].type && 0x2a == vvc.nalu[0].bytes && H266_PPS == vvc.nalu[1].type && 0x0c == vvc.nalu[1].bytes);
 	assert(sizeof(data) == mpeg4_vvc_decoder_configuration_record_save(&vvc, buffer, sizeof(buffer)) && 0 == memcmp(buffer, data, sizeof(data)));
 	mpeg4_vvc_codecs_test(&vvc);
+	vvc_annexbtomp4_test();
 }
 #endif