Skip to content

fix: Compatible with flv video stream decoding without audio #253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
UmbraCi wants to merge 3 commits into
base: master
Choose a base branch
from
Open

fix: Compatible with flv video stream decoding without audio #253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d0fd213
fix: Compatible with flv video stream decoding without audio
UmbraCi May 29, 2025
787727d
feat: Enhance FLV demuxer with H.264 slice header parsing and MacOS d…
UmbraCi Feb 6, 2026
79b85dd
fix: Improve error handling in slice header parsing and enhance loggi…
UmbraCi Feb 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 154 additions & 2 deletions src/demux/flv-demuxer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import H265Parser from './h265-parser.js';
import buffersAreEqual from '../utils/typedarray-equality.ts';
import AV1OBUParser from './av1-parser.ts';
import ExpGolomb from './exp-golomb.js';
import {FrameType, SliceHeaderParser} from './slice-header-parser.js';

function Swap16(src) {
return (((src >>> 8) & 0xFF) |
Expand All @@ -46,12 +47,20 @@ function ReadBig32(array, index) {
(array[index + 3]));
}

function isMacOS() {
if (navigator.userAgentData?.platform === "macOS") return true;

return false;
}

class FLVDemuxer {

constructor(probeData, config) {
this.TAG = 'FLVDemuxer';

this._isMacOS = isMacOS();
Log.d(this.TAG, 'isMacOS: ' + this._isMacOS);

this._config = config;

this._onError = null;
Expand Down Expand Up @@ -81,6 +90,16 @@ class FLVDemuxer {
this._audioMetadata = null;
this._videoMetadata = null;

this._h264SpsInfo = null;
this._h264MaxFrameNum = -1;
this._h264HasBFrame = false;
this._h264DroppingFrame = false;
this._h264LastVideoFrame = -1;
this._h264LastVideoFrameDts = -1;
this._h264LastVideoFramePts = -1;
this._h264LastIFrameDts = -1;
this._h264MinGopDuration = -1;

this._naluLengthSize = 4;
this._timestampBase = 0; // int32, in milliseconds
this._timescale = 1000;
Expand Down Expand Up @@ -363,6 +382,15 @@ class FLVDemuxer {
offset += 11 + dataSize + 4; // tagBody + dataSize + prevTagSize
}

if (this._hasAudio && this._hasVideo && !this._audioInitialMetadataDispatched) {
// both audio & video, but audio initial meta data still not dispatched
let samples = this._videoTrack.samples;
if (samples.length > 0 && samples[samples.length - 1].dts > samples[0].dts + 3000) {
Log.d(this.TAG, 'we need regard it as video only, last sample: ' + samples[samples.length - 1].dts + ', first sample: ' + samples[0].dts);
this._hasAudio = false;
}
}

// dispatch parsed frames to consumer (typically, the remuxer)
if (this._isInitialMetadataDispatched()) {
if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) {
Expand Down Expand Up @@ -1321,6 +1349,11 @@ class FLVDemuxer {
continue;
}

if (Object.keys(config).length === 0) continue;

this._h264SpsInfo = config;
this._h264MaxFrameNum = Math.pow(2, config.log2_max_frame_num_minus4 + 4);

meta.codecWidth = config.codec_size.width;
meta.codecHeight = config.codec_size.height;
meta.presentWidth = config.present_size.width;
Expand Down Expand Up @@ -1686,11 +1719,130 @@ class FLVDemuxer {
cts: cts,
pts: (dts + cts)
};

if (keyframe) {
avcSample.fileposition = tagPosition;
if (this._h264LastIFrameDts !== -1) {
let gopDuration = dts - this._h264LastIFrameDts;
if (gopDuration > 0 && gopDuration < 50000) {
// valid GOP duration 0~50s
if (this._h264MinGopDuration === -1 || gopDuration < this._h264MinGopDuration) {
this._h264MinGopDuration = gopDuration;
Log.v(this.TAG, 'GOP minimum duration: ' + this._h264MinGopDuration);
}
}
}

this._h264LastIFrameDts = dts;
}

let dropThisFrame = false;
let slice = units[0].data.slice(lengthSize, units[0].data.length - lengthSize);
let result = SliceHeaderParser.parseSliceHeader(slice, this._h264SpsInfo);
if (result.success === true) {
Log.d(this.TAG, 'video sample, dts: ' + dts + ', size: ' + length + ', frame_type: ' + result.data.frame_type +
', frame_num: ' + result.data.frame_num);

if (result.data.frame_type === FrameType.FrameType_B) {
this._h264HasBFrame = true;
}

// drop frame only for no B Frame case
if (this._h264HasBFrame === false) {
if (result.data.frame_type === FrameType.FrameType_I) {
// I frame
if (this._h264DroppingFrame) {
// after dropping frames, we need modify the timestamp for I frame
// since if non I frame has big duration, MSE will hang
let meta = this._videoMetadata;
if (meta) {
Log.w(this.TAG, 'dropping meet I frame, need stop dropping, last video frame dts: ' +
this._h264LastVideoFrameDts + ', I frame dts: ' + dts + ', ref duration: ' + meta.refSampleDuration);
avcSample.dts = this._h264LastVideoFrameDts + meta.refSampleDuration;
avcSample.pts = this._h264LastVideoFramePts + meta.refSampleDuration;
} else {
Log.w(this.TAG, 'dropping meet I frame, need stop dropping, last video frame dts: ' +
this._h264LastVideoFrameDts + ', I frame dts: ' + dts + ', ref duration: nil');
avcSample.dts = this._h264LastVideoFrameDts + 40;
avcSample.pts = this._h264LastVideoFramePts + 40;
}
}
this._h264DroppingFrame = false;
this._h264LastVideoFrame = result.data.frame_num;
this._h264LastVideoFrameDts = dts;
this._h264LastVideoFramePts = (dts + cts);
} else {
// not I frame
if (this._h264DroppingFrame) {
dropThisFrame = true;
} else {
// if not dropping frame, we need judge if need start dropping
// 1, first frame;
// 2, normal case;
// 3, normal frame_num overflow case;
if ((this._h264LastVideoFrame === -1) ||
(this._h264LastVideoFrame + 1 === result.data.frame_num) ||
((this._h264LastVideoFrame + 1) === this._h264MaxFrameNum && result.data.frame_num === 0)) {
this._h264LastVideoFrame = result.data.frame_num;
this._h264LastVideoFrameDts = dts;
this._h264LastVideoFramePts = (dts + cts);
} else if (this._h264LastVideoFrame === result.data.frame_num) {
// if same frame_num, we need judge timestamp
if (this._h264MinGopDuration !== -1 && dts > this._h264LastVideoFrameDts + this._h264MinGopDuration / 2) {
// maybe cross GOP
Log.w(this.TAG, 'frame_num not continuous(cross GOP): ' + this._h264LastVideoFrame + '(' + this._h264LastVideoFrameDts + '), ' +
result.data.frame_num + '(' + dts + '), need dropping...');
this._h264DroppingFrame = true;
dropThisFrame = true;
} else {
// in same GOP, it is normal case since non-reference frame will use the same frame_num as previous reference frame
this._h264LastVideoFrame = result.data.frame_num;
this._h264LastVideoFrameDts = dts;
this._h264LastVideoFramePts = (dts + cts);
}
} else {
if (this._isMacOS) {
Log.w(this.TAG, 'frame_num not continuous: ' + this._h264LastVideoFrame + '(' + this._h264LastVideoFrameDts + '), ' +
result.data.frame_num + '(' + dts + '), need dropping...');
this._h264DroppingFrame = true;
dropThisFrame = true;
} else {
if (this._h264LastIFrameDts !== -1 && this._h264MinGopDuration !== -1 &&
dts >= this._h264LastIFrameDts + this._h264MinGopDuration) {
// only I frame dropped, we need drop frames to next I frame
// it seems MSE can handle P frames dropped case
Log.w(this.TAG, 'frame_num not continuous: ' + this._h264LastVideoFrame + '(' + this._h264LastVideoFrameDts + '), ' +
result.data.frame_num + '(' + dts + '), and I frame seems dropped, last I frame ' + this._h264LastIFrameDts +
', gop duration: ' + this._h264MinGopDuration + ', need dropping...');
this._h264DroppingFrame = true;
dropThisFrame = true;
} else {
Log.w(this.TAG, 'frame_num not continuous: ' + this._h264LastVideoFrame + '(' + this._h264LastVideoFrameDts + '), ' +
result.data.frame_num + '(' + dts + '), but I frame not dropped, last I frame ' + this._h264LastIFrameDts +
', gop duration: ' + this._h264MinGopDuration + ', just warning.');
this._h264LastVideoFrame = result.data.frame_num;
this._h264LastVideoFrameDts = dts;
this._h264LastVideoFramePts = (dts + cts);
}
}
}
}
}
} else {
// B frame detected, disable drop frame mechanism
this._h264DroppingFrame = false;
this._h264LastVideoFrame = -1;
}
} else {
Log.w(this.TAG, 'parse slice fail, video sample, dts: ' + dts + ', size: ' + length + ', keyframe: ' + keyframe);
dropThisFrame = true;
}

if (!dropThisFrame) {
// Log.d(this.TAG, 'video sample, dts: ' + dts + ', size: ' + length + ', keyframe: ' + keyframe);
track.samples.push(avcSample);
track.length += length;
}
track.samples.push(avcSample);
track.length += length;
}
}

Expand Down
90 changes: 90 additions & 0 deletions src/demux/slice-header-parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright (C) 2016 Bilibili. All Rights Reserved.
*
* @author East Zhou <zrdong@ulucu.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import Log from '../utils/logger.js';
import ExpGolomb from './exp-golomb.js';

export const FrameType = {
FrameType_U: 0, // unknown
FrameType_I: 1, // I frame
FrameType_P: 2, // P frame
FrameType_B: 3 // B frame
};

export class SliceHeaderParser {
static parseSliceHeader(uint8array, sps_info) {
do {
if (!sps_info) {
Log.e('SliceHeaderParser', 'missing sps or pps!');
break;
}

try {
let gb = new ExpGolomb(uint8array);

let nalu_header = gb.readByte();
if (nalu_header & 0x80) // forbidden_zero_bit：在H.264规范中规定了这一位必须为0。
break;

if (!(nalu_header & 0x60)) // 取00~11，似乎指示这个NALU的重要性, 如00的NALU解码器可以丢弃它而不影响图像的回放。
break;

let unitType = nalu_header & 0x1F;
gb.readUEG(); // first_mb_in_slice
let slice_type = gb.readUEG(); // slice_type
let slice_type_i = (slice_type % 5 === 2);
let slice_type_p = (slice_type % 5 === 0);
let slice_type_b = (slice_type % 5 === 1);
let slice_type_si = (slice_type % 5 === 4);
let slice_type_sp = (slice_type % 5 === 3);

let frame_type = FrameType.FrameType_I;
if (slice_type_p || slice_type_sp) {
frame_type = FrameType.FrameType_P;
} else if (slice_type_b) {
frame_type = FrameType.FrameType_B;
}

gb.readUEG(); // pic_parameter_set_id

if (sps_info.separate_colour_plane_flag) {
gb.readBits(16); // colour_plane_id
}

let frame_num = gb.readBits(sps_info.log2_max_frame_num_minus4 + 4);

return {
success: true,
data: {
frame_type: frame_type,
frame_num: frame_num
}
};
} catch (error) {
Log.e('SliceHeaderParser', error.message);
break;
}
} while (0);

return {
success: false,
data: {
}
};
}
}
33 changes: 26 additions & 7 deletions src/demux/sps-parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
* limitations under the License.
*/

import Log from '../utils/logger.js';
import ExpGolomb from './exp-golomb.js';

class SPSParser {
Expand Down Expand Up @@ -56,13 +57,14 @@ class SPSParser {

gb.readByte();
let profile_idc = gb.readByte(); // profile_idc
gb.readByte(); // constraint_set_flags[5] + reserved_zero[3]
gb.readByte(); // constraint_set_flags[6] + reserved_zero[2]
let level_idc = gb.readByte(); // level_idc
gb.readUEG(); // seq_parameter_set_id

let profile_string = SPSParser.getProfileString(profile_idc);
let level_string = SPSParser.getLevelString(level_idc);
let chroma_format_idc = 1;
let separate_colour_plane_flag = 0;
let chroma_format = 420;
let chroma_format_table = [0, 420, 422, 444];
let bit_depth_luma = 8;
Expand All @@ -72,10 +74,15 @@ class SPSParser {
profile_idc === 244 || profile_idc === 44 || profile_idc === 83 ||
profile_idc === 86 || profile_idc === 118 || profile_idc === 128 ||
profile_idc === 138 || profile_idc === 144) {

chroma_format_idc = gb.readUEG();
if (chroma_format_idc > 3) {
Log.e('SPSParser', 'illegal chroma format idc: ' + chroma_format_idc);
return {
};
}

if (chroma_format_idc === 3) {
gb.readBits(1); // separate_colour_plane_flag
separate_colour_plane_flag = gb.readBits(1); // separate_colour_plane_flag
}
if (chroma_format_idc <= 3) {
chroma_format = chroma_format_table[chroma_format_idc];
Expand All @@ -97,12 +104,14 @@ class SPSParser {
}
}
}
gb.readUEG(); // log2_max_frame_num_minus4
let pic_order_cnt_type = gb.readUEG();
let log2_max_frame_num_minus4 = gb.readUEG(); // log2_max_frame_num_minus4
let pic_order_cnt_type = gb.readUEG(); // pic_order_cnt_type
let log2_max_pic_order_cnt_lsb_minus_4 = 0;
let delta_pic_order_always_zero_flag = 0;
if (pic_order_cnt_type === 0) {
gb.readUEG(); // log2_max_pic_order_cnt_lsb_minus_4
log2_max_pic_order_cnt_lsb_minus_4 = gb.readUEG(); // log2_max_pic_order_cnt_lsb_minus_4
} else if (pic_order_cnt_type === 1) {
gb.readBits(1); // delta_pic_order_always_zero_flag
delta_pic_order_always_zero_flag = gb.readBits(1); // delta_pic_order_always_zero_flag
gb.readSEG(); // offset_for_non_ref_pic
gb.readSEG(); // offset_for_top_to_bottom_field
let num_ref_frames_in_pic_order_cnt_cycle = gb.readUEG();
Expand Down Expand Up @@ -212,9 +221,14 @@ class SPSParser {
profile_string, // baseline, high, high10, ...
level_string, // 3, 3.1, 4, 4.1, 5, 5.1, ...
chroma_format_idc,
separate_colour_plane_flag,
bit_depth: bit_depth_luma, // 8bit, 10bit, ...
bit_depth_luma,
bit_depth_chroma,
log2_max_frame_num_minus4,
pic_order_cnt_type,
log2_max_pic_order_cnt_lsb_minus_4,
delta_pic_order_always_zero_flag,
ref_frames,
chroma_format, // 4:2:0, 4:2:2, ...
chroma_format_string: SPSParser.getChromaFormatString(chroma_format),
Expand Down Expand Up @@ -293,6 +307,11 @@ class SPSParser {
}
}

static getChromaFormat(chroma_format_idc) {
const chroma_format_table = [0, 420, 422, 444];
return chroma_format_table[chroma_format_idc];
}

}

export default SPSParser;
Loading