/* sync.c Copyright (c) 2003-2018 HandBrake Team This file is part of the HandBrake source code Homepage: . It may be used under the terms of the GNU General Public License v2. For full terms see the file COPYING file or visit http://www.gnu.org/licenses/gpl-2.0.html */ #include "hb.h" #include "hbffmpeg.h" #include #include "samplerate.h" #define SYNC_MAX_VIDEO_QUEUE_LEN 40 #define SYNC_MIN_VIDEO_QUEUE_LEN 20 // Audio is small, buffer a lot. It helps to ensure that we see // the initial PTS from all input streams before setting the 'zero' point. #define SYNC_MAX_AUDIO_QUEUE_LEN 200 #define SYNC_MIN_AUDIO_QUEUE_LEN 30 // We do not place a limit on the number of subtitle frames // that are buffered (max_len == INT_MAX) because there are // cases where we will receive all the subtitles for a file // all at once (SSA subs). // // If we did not buffer these subs here, the following deadlock // condition would occur: // 1. Subtitle decoder blocks trying to generate more subtitle // lines than will fit in sync input buffers. // 2. This blocks the reader. Reader doesn't read any more // audio or video, so sync won't receive buffers it needs // to unblock subtitles. #define SYNC_MAX_SUBTITLE_QUEUE_LEN INT_MAX #define SYNC_MIN_SUBTITLE_QUEUE_LEN 0 typedef enum { SYNC_TYPE_VIDEO, SYNC_TYPE_AUDIO, SYNC_TYPE_SUBTITLE, } sync_type_t; typedef struct { int64_t pts; int64_t delta; } sync_delta_t; typedef struct { int link; int merge; hb_buffer_list_t list_current; } subtitle_sanitizer_t; typedef struct sync_common_s sync_common_t; #define SCR_HASH_SZ (2 << 3) #define SCR_HASH_MASK (SCR_HASH_SZ - 1) typedef struct { int scr_sequence; int64_t scr_offset; } scr_t; typedef struct { sync_common_t * common; // Stream I/O control int done; int flush; hb_list_t * in_queue; hb_list_t * scr_delay_queue; int max_len; int min_len; hb_fifo_t * fifo_in; hb_fifo_t * fifo_out; // PTS synchronization hb_list_t * delta_list; int64_t pts_slip; double next_pts; double last_pts; // SCR recovery int last_scr_sequence; double last_scr_pts; double last_duration; // frame statistics int64_t first_pts; int64_t min_frame_duration; int64_t max_frame_duration; int64_t current_duration; int frame_count; // Error reporting stats int64_t drop_duration; int drop; int64_t drop_pts; int64_t gap_duration; int64_t gap_pts; int first_frame; // stream type specific context sync_type_t type; union { struct { int id; int cadence[12]; int new_chap; } video; // Audio stream context struct { hb_audio_t * audio; // Audio filter settings // Samplerate conversion struct { SRC_STATE * ctx; SRC_DATA pkt; } src; double gain_factor; } audio; // Subtitle stream context struct { hb_subtitle_t * subtitle; subtitle_sanitizer_t sanitizer; } subtitle; }; } sync_stream_t; struct sync_common_s { // Audio/Video sync thread synchronization hb_job_t * job; hb_lock_t * mutex; int stream_count; sync_stream_t * streams; int found_first_pts; // SCR adjustments scr_t scr[SCR_HASH_SZ]; int first_scr; // point-to-point support int start_found; int64_t pts_to_start; int64_t start_pts; int64_t stop_pts; int wait_for_frame; int wait_for_pts; // sync audio work objects hb_list_t * list_work; // UpdateState Statistics int est_frame_count; uint64_t st_counts[4]; uint64_t st_dates[4]; uint64_t st_first; int chapter; }; struct hb_work_private_s { sync_common_t * common; sync_stream_t * stream; }; /*********************************************************************** * Local prototypes **********************************************************************/ static void UpdateState( sync_common_t * common, int frame_count ); static void UpdateSearchState( sync_common_t * common, int64_t start, int frame_count ); static int UpdateSCR( sync_stream_t * stream, hb_buffer_t * buf ); static hb_buffer_t * FilterAudioFrame( sync_stream_t * stream, hb_buffer_t *buf ); static void SortedQueueBuffer( sync_stream_t * stream, hb_buffer_t * buf ); static hb_buffer_t * sanitizeSubtitle(sync_stream_t * stream, hb_buffer_t * sub); static void saveChap( sync_stream_t * stream, hb_buffer_t * buf ) { if (stream->type != SYNC_TYPE_VIDEO || buf == NULL) { return; } if (buf->s.new_chap > 0) { stream->video.new_chap = buf->s.new_chap; } } static void restoreChap( sync_stream_t * stream, hb_buffer_t * buf ) { if (stream->type != SYNC_TYPE_VIDEO || buf == NULL) { return; } if (stream->video.new_chap > 0 && buf->s.new_chap <= 0) { buf->s.new_chap = stream->video.new_chap; stream->video.new_chap = 0; } } static int fillQueues( sync_common_t * common ) { int ii, wait = 0, abort = 0; for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t *stream = &common->streams[ii]; // Don't let the queues grow indefinitely // abort when too large if (hb_list_count(stream->in_queue) > stream->max_len) { abort = 1; } if (hb_list_count(stream->in_queue) <= stream->min_len) { wait = 1; } } return !wait || abort; } static void scrSlip( sync_common_t * common, int64_t delta ) { int ii; for (ii = 0; ii < SCR_HASH_SZ; ii++) { common->scr[ii].scr_offset += delta; } } static void shiftTS( sync_common_t * common, int64_t delta ) { int ii, jj; scrSlip(common, delta); for (ii = 0; ii < common->stream_count; ii++) { hb_buffer_t * buf = NULL; sync_stream_t * stream = &common->streams[ii]; int count = hb_list_count(stream->in_queue); for (jj = 0; jj < count; jj++) { buf = hb_list_item(stream->in_queue, jj); if (buf->s.start != AV_NOPTS_VALUE) { buf->s.start -= delta; } if (buf->s.stop != AV_NOPTS_VALUE) { buf->s.stop -= delta; } } if (buf != NULL && buf->s.start != AV_NOPTS_VALUE) { stream->last_scr_pts = buf->s.start + buf->s.duration; } else { stream->last_scr_pts = (int64_t)AV_NOPTS_VALUE; } } } static hb_buffer_t * CreateSilenceBuf( sync_stream_t * stream, int64_t dur, int64_t pts ) { double frame_dur, next_pts, duration; int size; hb_buffer_list_t list; hb_buffer_t * buf; if (stream->audio.audio->config.out.codec & HB_ACODEC_PASS_FLAG) { return NULL; } duration = dur; frame_dur = (90000. * stream->audio.audio->config.in.samples_per_frame) / stream->audio.audio->config.in.samplerate; size = sizeof(float) * stream->audio.audio->config.in.samples_per_frame * av_get_channel_layout_nb_channels( stream->audio.audio->config.in.channel_layout ); hb_buffer_list_clear(&list); next_pts = pts; while (duration >= frame_dur) { buf = hb_buffer_init(size); memset(buf->data, 0, buf->size); buf->s.start = next_pts; next_pts += frame_dur; buf->s.stop = next_pts; buf->s.duration = frame_dur; duration -= frame_dur; hb_buffer_list_append(&list, buf); } if (duration > 0) { // Make certain size is even multiple of sample size * num channels size = (int)(duration * stream->audio.audio->config.in.samplerate / 90000) * sizeof(float) * av_get_channel_layout_nb_channels( stream->audio.audio->config.in.channel_layout); if (size > 0) { buf = hb_buffer_init(size); memset(buf->data, 0, buf->size); buf->s.start = next_pts; next_pts += duration; buf->s.stop = next_pts; buf->s.duration = duration; hb_buffer_list_append(&list, buf); } } return hb_buffer_list_clear(&list); } static hb_buffer_t * CreateBlackBuf( sync_stream_t * stream, int64_t dur, int64_t pts ) { // I would like to just allocate one black frame here and give // it the full duration. But encoders that use B-Frames compute // the dts delay based on the pts of the 2nd or 3rd frame which // can be a very large value in this case. This large dts delay // causes problems with computing the dts of the frames (which is // extrapolated from the pts and the computed dts delay). And the // dts problems lead to problems with frame duration. double frame_dur, next_pts, duration; hb_buffer_list_t list; hb_buffer_t * buf = NULL; hb_buffer_list_clear(&list); duration = dur; next_pts = pts; frame_dur = 90000. * stream->common->job->title->vrate.den / stream->common->job->title->vrate.num; // Only create black buffers of frame_dur or longer while (duration >= frame_dur) { if (buf == NULL) { buf = hb_frame_buffer_init(AV_PIX_FMT_YUV420P, stream->common->job->title->geometry.width, stream->common->job->title->geometry.height); memset(buf->plane[0].data, 0x00, buf->plane[0].size); memset(buf->plane[1].data, 0x80, buf->plane[1].size); memset(buf->plane[2].data, 0x80, buf->plane[2].size); } else { buf = hb_buffer_dup(buf); } buf->s.start = next_pts; next_pts += frame_dur; buf->s.stop = next_pts; buf->s.duration = frame_dur; duration -= frame_dur; hb_buffer_list_append(&list, buf); } if (buf != NULL) { if (buf->s.stop < pts + dur) { // Extend the duration of the last black buffer to fill // the remaining gap. buf->s.duration += pts + dur - buf->s.stop; buf->s.stop = pts + dur; } } return hb_buffer_list_clear(&list); } static void setNextPts( sync_common_t * common ) { int ii; for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; hb_buffer_t * buf = hb_list_item(stream->in_queue, 0); if (buf != NULL) { stream->next_pts = buf->s.start; } else { stream->next_pts = (int64_t)AV_NOPTS_VALUE; } } } static void alignStream( sync_common_t * common, sync_stream_t * stream, int64_t pts ) { if (hb_list_count(stream->in_queue) <= 0 || stream->type == SYNC_TYPE_SUBTITLE) { return; } hb_buffer_t * buf = hb_list_item(stream->in_queue, 0); int64_t gap = buf->s.start - pts; if (gap == 0) { return; } if (gap < 0) { int ii; // Drop frames from other streams for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * other_stream = &common->streams[ii]; if (stream == other_stream) { continue; } while (hb_list_count(other_stream->in_queue) > 0) { buf = hb_list_item(other_stream->in_queue, 0); if (buf->s.start < pts) { hb_list_rem(other_stream->in_queue, buf); hb_buffer_close(&buf); } else { // Fill the partial frame gap left after dropping frames alignStream(common, other_stream, pts); break; } } } } else { hb_buffer_t * blank_buf = NULL; // Insert a blank frame to fill the gap if (stream->type == SYNC_TYPE_AUDIO) { // Can't add silence padding to passthru streams if (!(stream->audio.audio->config.out.codec & HB_ACODEC_PASS_FLAG)) { blank_buf = CreateSilenceBuf(stream, gap, pts); } } else if (stream->type == SYNC_TYPE_VIDEO) { blank_buf = CreateBlackBuf(stream, gap, pts); } int64_t last_stop = pts; hb_buffer_t * next; int pos; for (pos = 0; blank_buf != NULL; blank_buf = next, pos++) { last_stop = blank_buf->s.stop; next = blank_buf->next; blank_buf->next = NULL; hb_list_insert(stream->in_queue, pos, blank_buf); } if (stream->type == SYNC_TYPE_VIDEO && last_stop < buf->s.start) { // Extend the duration of the first frame to fill the remaining gap. buf->s.duration += buf->s.start - last_stop; buf->s.start = last_stop; } } } static void alignStreams( sync_common_t * common, int64_t pts ) { int ii; hb_buffer_t * buf; if (common->job->align_av_start) { int64_t first_pts = AV_NOPTS_VALUE; int audio_passthru = 0; for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; buf = hb_list_item(stream->in_queue, 0); // P-to-P encoding will pass the start point in pts. // Drop any buffers that are before the start point. while (buf != NULL && buf->s.start < pts) { hb_list_rem(stream->in_queue, buf); hb_buffer_close(&buf); buf = hb_list_item(stream->in_queue, 0); } if (buf == NULL) { continue; } if (stream->type == SYNC_TYPE_AUDIO && stream->audio.audio->config.out.codec & HB_ACODEC_PASS_FLAG) { // Find the largest initial pts of all passthru audio streams. // We can not add silence to passthru audio streams. // To align with a passthru audio stream, we must drop // buffers from all other streams that are before // the first buffer in the passthru audio stream. audio_passthru = 1; if (first_pts < buf->s.start) { first_pts = buf->s.start; } } else if (!audio_passthru) { // Find the smallest initial pts of all streams when // there is *no* passthru audio. // When there is no passthru audio stream, we insert // silence or black buffers to fill any gaps between // the start of any stream and the start of the stream // with the smallest pts. if (first_pts == AV_NOPTS_VALUE || first_pts > buf->s.start) { first_pts = buf->s.start; } } } if (first_pts != AV_NOPTS_VALUE) { for (ii = 0; ii < common->stream_count; ii++) { // Fill the gap (or drop frames for passthru audio) alignStream(common, &common->streams[ii], first_pts); } } } } static void computeInitialTS( sync_common_t * common, sync_stream_t * first_stream ) { int ii; hb_buffer_t * prev, * buf; // Process first_stream first since it has the initial PTS prev = NULL; for (ii = 0; ii < hb_list_count(first_stream->in_queue);) { buf = hb_list_item(first_stream->in_queue, ii); if (!UpdateSCR(first_stream, buf)) { hb_list_rem(first_stream->in_queue, buf); } else { ii++; if (first_stream->type == SYNC_TYPE_VIDEO && prev != NULL) { double duration = buf->s.start - prev->s.start; if (duration > 0) { prev->s.duration = duration; prev->s.stop = buf->s.start; } } prev = buf; } } for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; if (stream == first_stream) { // skip first_stream, already done continue; } int jj; prev = NULL; for (jj = 0; jj < hb_list_count(stream->in_queue);) { buf = hb_list_item(stream->in_queue, jj); if (!UpdateSCR(stream, buf)) { // Subtitle put into delay queue, remove it from in_queue hb_list_rem(stream->in_queue, buf); } else { jj++; if (stream->type == SYNC_TYPE_VIDEO && prev != NULL) { double duration = buf->s.start - prev->s.start; if (duration > 0) { prev->s.duration = duration; prev->s.stop = buf->s.start; } } prev = buf; } } } alignStreams(common, AV_NOPTS_VALUE); } static void checkFirstPts( sync_common_t * common ) { int ii; int64_t first_pts = INT64_MAX; sync_stream_t * first_stream = NULL; for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t *stream = &common->streams[ii]; if (stream->type == SYNC_TYPE_SUBTITLE) { // only wait for audio and video continue; } // If buffers are queued, find the lowest initial PTS while (hb_list_count(stream->in_queue) > 0) { hb_buffer_t * buf = hb_list_item(stream->in_queue, 0); if (buf->s.start != AV_NOPTS_VALUE) { // We require an initial pts for every stream if (buf->s.start < first_pts) { first_pts = buf->s.start; first_stream = stream; } break; } else { hb_list_rem(stream->in_queue, buf); hb_buffer_close(&buf); } } } // We should *always* find a first pts because we let the queues // fill before performing this test. if (first_pts != INT64_MAX) { common->found_first_pts = 1; // We may have buffers that have no timestamps (i.e. AV_NOPTS_VALUE). // Compute these timestamps based on previous buffer's timestamp // and duration. computeInitialTS(common, first_stream); // After this initialization, all AV_NOPTS_VALUE timestamps // will be filled in by UpdateSCR() } else { // This should never happen hb_error("checkFirstPts: No initial PTS found!\n"); } } static void addDelta( sync_common_t * common, int64_t start, int64_t delta) { int ii; for (ii = 0; ii < common->stream_count; ii++) { sync_delta_t * delta_item = malloc(sizeof(sync_delta_t)); delta_item->pts = start; delta_item->delta = delta; hb_list_add(common->streams[ii].delta_list, delta_item); } } static void applyDeltas( sync_common_t * common ) { int ii; // Apply delta to any applicable buffers in the queue for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; // Make adjustments for deltas found in other streams sync_delta_t * delta = hb_list_item(stream->delta_list, 0); if (delta != NULL) { int jj, index = -1; int64_t prev_start, max = 0; hb_buffer_t * buf; prev_start = stream->next_pts; for (jj = 0; jj < hb_list_count(stream->in_queue); jj++) { buf = hb_list_item(stream->in_queue, jj); if (stream->type == SYNC_TYPE_SUBTITLE) { if (buf->s.start > delta->pts) { // absorb gaps in subtitles as soon as possible index = jj; break; } } else if (buf->s.start > delta->pts) { // absorb gap in largest gap found in this stream. if (buf->s.start - prev_start > max) { max = buf->s.start - prev_start; index = jj; } if (stream->type == SYNC_TYPE_AUDIO && max >= delta->delta) { // absorb gaps in audio as soon as possible // when there is a gap that will absorb it. break; } } prev_start = buf->s.start; } if (index >= 0) { for (jj = index; jj < hb_list_count(stream->in_queue); jj++) { buf = hb_list_item(stream->in_queue, jj); buf->s.start -= delta->delta; if (buf->s.stop != AV_NOPTS_VALUE) { buf->s.stop -= delta->delta; } } // Correct the duration of the video buffer before // the affected timestamp correction. if (stream->type == SYNC_TYPE_VIDEO && index > 0) { buf = hb_list_item(stream->in_queue, index - 1); if (buf->s.duration > delta->delta) { buf->s.duration -= delta->delta; buf->s.stop -= delta->delta; } else { buf->s.duration = 0; buf->s.stop = buf->s.start; } } stream->pts_slip += delta->delta; hb_list_rem(stream->delta_list, delta); free(delta); } } } } static void removeVideoJitter( sync_stream_t * stream, int stop ) { int ii; hb_buffer_t * buf; double frame_duration, next_pts; frame_duration = 90000. * stream->common->job->title->vrate.den / stream->common->job->title->vrate.num; buf = hb_list_item(stream->in_queue, 0); buf->s.start = stream->next_pts; next_pts = stream->next_pts + frame_duration; for (ii = 1; ii <= stop; ii++) { buf->s.duration = frame_duration; buf->s.stop = next_pts; buf = hb_list_item(stream->in_queue, ii); buf->s.start = next_pts; next_pts += frame_duration; } } // Look for a sequence of packets whose duration as measure by // vrate closely matches the duration as measured by timestamp // differences. When close matches are found, smooth the timestamps. // // Most often, video dejitter is applied when there is jitter due to // soft telecine. I also have a couple sample files that have very // bad video jitter that this corrects. static void dejitterVideo( sync_stream_t * stream ) { int ii, count, jitter_stop; double frame_duration, duration; hb_buffer_t * buf; count = hb_list_count(stream->in_queue); if (count < 2) { return; } frame_duration = 90000. * stream->common->job->title->vrate.den / stream->common->job->title->vrate.num; // Look for start of jittered sequence buf = hb_list_item(stream->in_queue, 1); duration = buf->s.start - stream->next_pts; if (ABS(duration - frame_duration) < 1.1) { // Ignore small jitter buf->s.start = stream->next_pts + frame_duration; buf = hb_list_item(stream->in_queue, 0); buf->s.start = stream->next_pts; buf->s.duration = frame_duration; buf->s.stop = stream->next_pts + frame_duration; return; } // Look for end of jittered sequence jitter_stop = 0; for (ii = 1; ii < count; ii++) { buf = hb_list_item(stream->in_queue, ii); duration = buf->s.start - stream->next_pts; // Only dejitter video that aligns periodically // with the frame durations. if (ABS(duration - ii * frame_duration) < frame_duration / 3) { jitter_stop = ii; } } if (jitter_stop > 0) { removeVideoJitter(stream, jitter_stop); } } // Fix video overlaps that could not be corrected by dejitter static void fixVideoOverlap( sync_stream_t * stream ) { int drop = 0; int64_t overlap; hb_buffer_t * buf; // If time goes backwards drop the frame. // Check if subsequent buffers also overlap. while ((buf = hb_list_item(stream->in_queue, 0)) != NULL) { // For video, an overlap is where the entire frame is // in the past. overlap = stream->next_pts - buf->s.stop; if (overlap >= 0) { if (stream->drop == 0) { stream->drop_pts = buf->s.start; } hb_list_rem(stream->in_queue, buf); // Video frame durations are assumed to be variable and are // adjusted based on the start time of the next frame before // we get to this point. // // Estimate duration dropped based on average framerate stream->drop_duration += 90000. * stream->common->job->title->vrate.den / stream->common->job->title->vrate.num; stream->drop++; drop++; saveChap(stream, buf); hb_buffer_close(&buf); } else { break; } } if (drop <= 0 && stream->drop > 0) { hb_log("sync: video time went backwards %d ms, dropped %d frames. " "PTS %"PRId64"", (int)stream->drop_duration / 90, stream->drop, stream->drop_pts); stream->drop_duration = 0; stream->drop = 0; } } static void removeAudioJitter(sync_stream_t * stream, int stop) { int ii; hb_buffer_t * buf; double next_pts; // If duration of sum of packet durations is close to duration // as measured by timestamps, align timestamps to packet durations. // The packet durations are computed based on samplerate and // number of samples and are therefore a reliable measure // of the actual duration of an audio frame. buf = hb_list_item(stream->in_queue, 0); buf->s.start = stream->next_pts; next_pts = stream->next_pts + buf->s.duration; for (ii = 1; ii <= stop; ii++) { // Duration can be fractional, so track fractional PTS buf->s.stop = next_pts; buf = hb_list_item(stream->in_queue, ii); buf->s.start = next_pts; next_pts += buf->s.duration; } } // Look for a sequence of packets whose duration as measure by packet // durations closely matches the duration as measured by timestamp // differences. When close matches are found, smooth the timestamps. // // This fixes issues where there are false overlaps and gaps in audio // timestamps. libav creates this type of problem sometimes with it's // timestamp guessing code. static void dejitterAudio( sync_stream_t * stream ) { int ii, count, jitter_stop; double duration; hb_buffer_t * buf, * buf0, * buf1; count = hb_list_count(stream->in_queue); if (count < 4) { return; } // Look for start of jitter sequence jitter_stop = 0; buf0 = hb_list_item(stream->in_queue, 0); buf1 = hb_list_item(stream->in_queue, 1); if (ABS(buf0->s.duration - (buf1->s.start - stream->next_pts)) < 1.1) { // Ignore very small jitter return; } buf = hb_list_item(stream->in_queue, 0); duration = buf->s.duration; // Look for end of jitter sequence for (ii = 1; ii < count; ii++) { buf = hb_list_item(stream->in_queue, ii); if (ABS(duration - (buf->s.start - stream->next_pts)) < (90 * 40)) { // Finds the largest span that has low jitter jitter_stop = ii; } duration += buf->s.duration; } if (jitter_stop >= 4) { removeAudioJitter(stream, jitter_stop); } } // Fix audio gaps that could not be corrected with dejitter static void fixAudioGap( sync_stream_t * stream ) { int64_t gap; hb_buffer_t * buf; if (hb_list_count(stream->in_queue) < 1 || !stream->first_frame) { // Can't find gaps with < 1 buffers return; } buf = hb_list_item(stream->in_queue, 0); gap = buf->s.start - stream->next_pts; // If there's a gap of more than a minute between the last // frame and this, assume we got a corrupted timestamp. if (gap > 90 * 20 && gap < 90000LL * 60) { if (stream->gap_duration <= 0) { stream->gap_pts = buf->s.start; } buf = CreateSilenceBuf(stream, gap, stream->next_pts); if (buf != NULL) { hb_buffer_t * next; int pos; for (pos = 0; buf != NULL; buf = next, pos++) { next = buf->next; buf->next = NULL; hb_list_insert(stream->in_queue, pos, buf); } } else { addDelta(stream->common, stream->next_pts, gap); applyDeltas(stream->common); } stream->gap_duration += gap; } else { // If not fixing the gap, carry to the next frame // so they do not accumulate. Do not carry negative gaps. // Those are overlaps and are handled by fixAudioOverlap. if (gap >= 90000LL * 60) { // Fix "corrupted" timestamp buf->s.start = stream->next_pts; } if (stream->gap_duration > 0) { hb_deep_log(3, "sync: audio 0x%x time gap %d ms. PTS %"PRId64"", stream->audio.audio->id, (int)stream->gap_duration / 90, stream->gap_pts); stream->gap_duration = 0; } } } // Fix audio overlaps that could not be corrected with dejitter static void fixAudioOverlap( sync_stream_t * stream ) { int drop = 0; int64_t overlap; hb_buffer_t * buf; if (!stream->first_frame) { // There are no overlaps if we haven't seen the first frame yet. return; } // If time goes backwards drop the frame. // Check if subsequent buffers also overlap. while ((buf = hb_list_item(stream->in_queue, 0)) != NULL) { overlap = stream->next_pts - buf->s.start; if (overlap > 90 * 20) { if (stream->drop == 0) { stream->drop_pts = buf->s.start; } // This is likely to generate a gap in audio timestamps // This will be subsequently handled by the call to // fix AudioGap in Synchronize(). Small gaps will be handled // by just shifting the timestamps and carrying the gap // along. hb_list_rem(stream->in_queue, buf); stream->drop_duration += buf->s.duration; stream->drop++; drop++; hb_buffer_close(&buf); } else { break; } } if (drop <= 0 && stream->drop > 0) { hb_log("sync: audio 0x%x time went backwards %d ms, dropped %d frames. " "PTS %"PRId64"", stream->audio.audio->id, (int)stream->drop_duration / 90, stream->drop, stream->drop_pts); stream->drop_duration = 0; stream->drop = 0; } } static void fixSubtitleOverlap( sync_stream_t * stream ) { hb_buffer_t * buf; buf = hb_list_item(stream->in_queue, 0); if (buf == NULL || (buf->s.flags & HB_BUF_FLAG_EOS) || (buf->s.flags & HB_BUF_FLAG_EOF)) { // marker to indicate the end of a subtitle return; } // Only SSA subs can overlap if (stream->subtitle.subtitle->source != SSASUB && stream->subtitle.subtitle->config.dest == PASSTHRUSUB && buf->s.start <= stream->last_pts) { int64_t overlap; overlap = stream->last_pts - buf->s.start; hb_log("sync: subtitle 0x%x time went backwards %d ms, PTS %"PRId64"", stream->subtitle.subtitle->id, (int)overlap / 90, buf->s.start); hb_list_rem(stream->in_queue, buf); hb_buffer_close(&buf); } } static void fixStreamTimestamps( sync_stream_t * stream ) { // Fix gaps and overlaps in queue if (stream->type == SYNC_TYPE_AUDIO) { dejitterAudio(stream); fixAudioOverlap(stream); fixAudioGap(stream); } else if (stream->type == SYNC_TYPE_VIDEO) { dejitterVideo(stream); fixVideoOverlap(stream); } else if (stream->type == SYNC_TYPE_SUBTITLE) { fixSubtitleOverlap(stream); } } static void fifo_push( hb_fifo_t * fifo, hb_buffer_t * buf ) { if (fifo != NULL) { hb_fifo_push(fifo, buf); } else { hb_buffer_close(&buf); } } static void streamFlush( sync_stream_t * stream ) { while (hb_list_count(stream->in_queue) > 0) { hb_buffer_t * buf; if (!stream->common->found_first_pts) { checkFirstPts(stream->common); } if (stream->next_pts == (int64_t)AV_NOPTS_VALUE) { // Initialize next_pts, it is used to make timestamp corrections buf = hb_list_item(stream->in_queue, 0); if (buf != NULL) { stream->next_pts = buf->s.start; } } fixStreamTimestamps(stream); buf = hb_list_item(stream->in_queue, 0); if (buf != NULL) { hb_list_rem(stream->in_queue, buf); if (!stream->first_frame) { if (buf->s.start >= 0) { switch (stream->type) { case SYNC_TYPE_VIDEO: hb_log("sync: first pts video is %"PRId64, buf->s.start); break; case SYNC_TYPE_AUDIO: hb_log("sync: first pts audio 0x%x is %"PRId64, stream->audio.audio->id, buf->s.start); break; case SYNC_TYPE_SUBTITLE: hb_log("sync: first pts subtitle 0x%x is %"PRId64, stream->subtitle.subtitle->id, buf->s.start); break; default: break; } stream->first_frame = 1; stream->first_pts = buf->s.start; stream->min_frame_duration = buf->s.duration; } stream->next_pts = buf->s.start; } if (stream->type == SYNC_TYPE_AUDIO) { buf = FilterAudioFrame(stream, buf); if (buf == NULL) { // FilterAudioFrame can consume the buffer with no output continue; } } int64_t subtitle_last_pts = AV_NOPTS_VALUE; if (stream->type == SYNC_TYPE_SUBTITLE) { buf = sanitizeSubtitle(stream, buf); if (buf == NULL) { // sanitizeSubtitle can consume the buffer with no output continue; } // sanitizeSubtitle can return a list of subtitles. // Find the pts of the last subtitle in the list hb_buffer_t * sub = buf; while (sub != NULL) { subtitle_last_pts = sub->s.start; sub = sub->next; } } if (stream->type == SYNC_TYPE_AUDIO || stream->type == SYNC_TYPE_VIDEO) { buf->s.start = stream->next_pts; buf->s.stop = stream->next_pts + buf->s.duration; stream->last_pts = stream->next_pts; stream->next_pts += buf->s.duration; } else { stream->next_pts = stream->last_pts = subtitle_last_pts; } if (buf->s.stop > 0) { stream->current_duration = buf->s.stop - stream->first_pts; } stream->frame_count++; if (buf->s.duration > 0 && stream->min_frame_duration > buf->s.duration) { stream->min_frame_duration = buf->s.duration; } if (stream->max_frame_duration < buf->s.duration) { stream->max_frame_duration = buf->s.duration; } if ((buf->s.start < 0) || (stream->type == SYNC_TYPE_VIDEO && buf->s.duration < 256)) { // The pipeline can't handle negative timestamps // and it is sometimes not possible to avoid one // at the start of the video. There can be a // significant delay before we see the first buffers // from all streams. We can't buffer indefinitely // until we have seen the first PTS for all streams // so sometimes we may start before we have seen // the earliest PTS // // Also, encx264.c can't handle timestamps that are spaced // less than 256 ticks apart. saveChap(stream, buf); hb_buffer_close(&buf); } restoreChap(stream, buf); fifo_push(stream->fifo_out, buf); } } fifo_push(stream->fifo_out, hb_buffer_eof_init()); } static void flushStreams( sync_common_t * common ) { int ii; // Make sure all streams are complete for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; if (!stream->done && !stream->flush) { return; } } // Flush all streams for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; if (stream->done) { continue; } streamFlush(stream); } } static void flushStreamsLock( sync_common_t * common ) { hb_lock(common->mutex); flushStreams(common); hb_unlock(common->mutex); } static void log_chapter( sync_common_t *common, int chap_num, int nframes, int64_t pts ) { hb_chapter_t *c; if ( !common->job ) return; c = hb_list_item( common->job->list_chapter, chap_num - 1 ); if ( c && c->title ) { hb_log("sync: \"%s\" (%d) at frame %d time %"PRId64, c->title, chap_num, nframes, pts); } else { hb_log("sync: Chapter %d at frame %d time %"PRId64, chap_num, nframes, pts ); } } #define TOP_FIRST PIC_FLAG_TOP_FIELD_FIRST #define PROGRESSIVE PIC_FLAG_PROGRESSIVE_FRAME #define REPEAT_FIRST PIC_FLAG_REPEAT_FIRST_FIELD #define TB 8 #define BT 16 #define BT_PROG 32 #define BTB_PROG 64 #define TB_PROG 128 #define TBT_PROG 256 static void checkCadence( int * cadence, hb_buffer_t * buf ) { /* Rotate the cadence tracking. */ int i = 0; for (i = 11; i > 0; i--) { cadence[i] = cadence[i-1]; } if (!(buf->s.flags & PROGRESSIVE) && !(buf->s.flags & TOP_FIRST)) { /* Not progressive, not top first... That means it's probably bottom first, 2 fields displayed. */ //hb_log("MPEG2 Flag: Bottom field first, 2 fields displayed."); cadence[0] = BT; } else if (!(buf->s.flags & PROGRESSIVE) && (buf->s.flags & TOP_FIRST)) { /* Not progressive, top is first, Two fields displayed. */ //hb_log("MPEG2 Flag: Top field first, 2 fields displayed."); cadence[0] = TB; } else if ((buf->s.flags & PROGRESSIVE) && !(buf->s.flags & TOP_FIRST) && !(buf->s.flags & REPEAT_FIRST)) { /* Progressive, but noting else. That means Bottom first, 2 fields displayed. */ //hb_log("MPEG2 Flag: Progressive. Bottom field first, 2 fields displayed."); cadence[0] = BT_PROG; } else if ((buf->s.flags & PROGRESSIVE) && !(buf->s.flags & TOP_FIRST) && (buf->s.flags & REPEAT_FIRST)) { /* Progressive, and repeat. . That means Bottom first, 3 fields displayed. */ //hb_log("MPEG2 Flag: Progressive repeat. Bottom field first, 3 fields displayed."); cadence[0] = BTB_PROG; } else if ((buf->s.flags & PROGRESSIVE) && (buf->s.flags & TOP_FIRST) && !(buf->s.flags & REPEAT_FIRST)) { /* Progressive, top first. That means top first, 2 fields displayed. */ //hb_log("MPEG2 Flag: Progressive. Top field first, 2 fields displayed."); cadence[0] = TB_PROG; } else if ((buf->s.flags & PROGRESSIVE) && (buf->s.flags & TOP_FIRST) && (buf->s.flags & REPEAT_FIRST)) { /* Progressive, top, repeat. That means top first, 3 fields displayed. */ //hb_log("MPEG2 Flag: Progressive repeat. Top field first, 3 fields displayed."); cadence[0] = TBT_PROG; } if ((cadence[2] <= TB) && (cadence[1] <= TB) && (cadence[0] > TB) && (cadence[11])) { hb_log("%fs: Video -> Film", (float)buf->s.start / 90000); } if ((cadence[2] > TB) && (cadence[1] <= TB) && (cadence[0] <= TB) && (cadence[11])) { hb_log("%fs: Film -> Video", (float)buf->s.start / 90000); } } // When doing point-to-point encoding, subtitles can cause a long // delay in finishing the job when the stop point is reached. This // is due to the sparse nature of subtitles. We may not even see // any additional subtitle till we reach the end of the file. // // So when all audio and video streams have reached the end point, // force the termination of all subtitle streams by pushing an // end-of-stream buffer into each subtitles input fifo. // This will cause each subtitle sync worker to wake and return // HB_WORK_DONE. static void terminateSubtitleStreams( sync_common_t * common ) { int ii; // Make sure all streams are complete for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; if (stream->type == SYNC_TYPE_SUBTITLE) { continue; } if (!stream->done) { return; } } // Terminate all subtitle streams for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; if (stream->done || stream->type != SYNC_TYPE_SUBTITLE) { continue; } fifo_push(stream->fifo_out, hb_buffer_eof_init()); fifo_push(stream->fifo_in, hb_buffer_eof_init()); stream->done = 1; } } // OutputBuffer pulls buffers from the internal sync buffer queues in // lowest PTS first order. It then processes the queue the buffer is // pulled from for frame overlaps and gaps. // // If a queue reaches MAX depth, it is possible another queue is too low // to achieve both goals of pulling lowest PTS first *and* perform // timestamp correction. In this scenario, we forego lowest PTS and pull // the next lowest PTS that has enough buffers in the queue to perform // timestamp correction. static void OutputBuffer( sync_common_t * common ) { int ii, full; int64_t pts; sync_stream_t * out_stream; hb_buffer_t * buf; do { full = 0; out_stream = NULL; pts = INT64_MAX; // Find lowest PTS and output that buffer for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; int min = stream->min_len; // Ignore minimum buffer requirements for video if we have // not yet found the PtoP start frame. if (!common->start_found && common->wait_for_frame && stream->type == SYNC_TYPE_VIDEO) { min = 0; } // We need at least 2 buffers in the queue in order to fix // frame overlaps and inter-frame gaps. So if a queue is // low, do not do normal PTS interleaving with this queue. // Except for subtitles which are not processed for gaps // and overlaps. if (hb_list_count(stream->in_queue) > min) { buf = hb_list_item(stream->in_queue, 0); if (buf->s.start < pts) { pts = buf->s.start; out_stream = stream; } } // But continue output of buffers as long as one of the queues // is above the maximum queue level. if(hb_list_count(stream->in_queue) > stream->max_len) { full = 1; } } if (out_stream == NULL) { // This should only happen if all queues are below the // minimum queue level break; } if (out_stream->done) { buf = hb_list_item(out_stream->in_queue, 0); hb_list_rem(out_stream->in_queue, buf); hb_buffer_close(&buf); continue; } if (out_stream->next_pts == (int64_t)AV_NOPTS_VALUE) { // Initialize next_pts, it is used to make timestamp corrections // If doing p-to-p encoding, it will get reinitialized when // we find the start point. buf = hb_list_item(out_stream->in_queue, 0); out_stream->next_pts = buf->s.start; } // Make timestamp adjustments to eliminate jitter, gaps, and overlaps fixStreamTimestamps(out_stream); buf = hb_list_item(out_stream->in_queue, 0); if (buf == NULL) { // In case some timestamp sanitization causes the one and // only buffer in the queue to be deleted... // This really shouldn't happen. continue; } if (!common->start_found) { // pts_to_start or frame_to_start were specified. // Wait for the appropriate start point. if (common->wait_for_frame) { if (out_stream->type != SYNC_TYPE_VIDEO) { // We haven't found the PtoP start frame yet and // this buffer is either before the start frame or // the video queue was empty. out_stream->next_pts = buf->s.start + buf->s.duration; hb_list_rem(out_stream->in_queue, buf); hb_buffer_close(&buf); continue; } common->start_pts = buf->s.start + 1; if (out_stream->frame_count >= common->job->frame_to_start) { common->start_found = 1; out_stream->frame_count = 0; } } else if (common->wait_for_pts && out_stream->type != SYNC_TYPE_SUBTITLE) { if (buf->s.start >= common->pts_to_start) { common->start_found = 1; common->streams[0].frame_count = 0; } } if (!common->start_found) { if (out_stream->type == SYNC_TYPE_VIDEO) { UpdateSearchState(common, buf->s.start, out_stream->frame_count); out_stream->frame_count++; } if (buf->s.start < common->start_pts) { out_stream->next_pts = buf->s.start + buf->s.duration; hb_list_rem(out_stream->in_queue, buf); hb_buffer_close(&buf); } continue; } // reset frame count to track number of frames after // the start position till the end of encode. out_stream->frame_count = 0; shiftTS(common, buf->s.start); alignStreams(common, buf->s.start); setNextPts(common); buf = hb_list_item(out_stream->in_queue, 0); if (buf == NULL) { // In case aligning timestamps causes all buffers in // out_stream to be deleted... continue; } } // If pts_to_stop or frame_to_stop were specified, stop output if (common->stop_pts && buf->s.start >= common->stop_pts ) { switch (out_stream->type) { case SYNC_TYPE_VIDEO: hb_log("sync: reached video pts %"PRId64", exiting early", buf->s.start); break; case SYNC_TYPE_AUDIO: hb_log("sync: reached audio 0x%x pts %"PRId64 ", exiting early", out_stream->audio.audio->id, buf->s.start); break; case SYNC_TYPE_SUBTITLE: hb_log("sync: reached subtitle 0x%x pts %"PRId64 ", exiting early", out_stream->subtitle.subtitle->id, buf->s.start); break; default: break; } out_stream->done = 1; fifo_push(out_stream->fifo_out, hb_buffer_eof_init()); terminateSubtitleStreams(common); flushStreams(common); continue; } if (out_stream->type == SYNC_TYPE_VIDEO && common->job->frame_to_stop && out_stream->frame_count >= common->job->frame_to_stop) { hb_log("sync: reached video frame %d, exiting early", out_stream->frame_count); common->stop_pts = buf->s.start; out_stream->done = 1; fifo_push(out_stream->fifo_out, hb_buffer_eof_init()); terminateSubtitleStreams(common); flushStreams(common); continue; } if (out_stream->type == SYNC_TYPE_VIDEO) { checkCadence(out_stream->video.cadence, buf); } // Out the buffer goes... hb_list_rem(out_stream->in_queue, buf); if (out_stream->type == SYNC_TYPE_VIDEO) { UpdateState(common, out_stream->frame_count); } if (!out_stream->first_frame) { if (buf->s.start >= 0) { switch (out_stream->type) { case SYNC_TYPE_VIDEO: hb_log("sync: first pts video is %"PRId64, buf->s.start); break; case SYNC_TYPE_AUDIO: hb_log("sync: first pts audio 0x%x is %"PRId64, out_stream->audio.audio->id, buf->s.start); break; case SYNC_TYPE_SUBTITLE: hb_log("sync: first pts subtitle 0x%x is %"PRId64, out_stream->subtitle.subtitle->id, buf->s.start); break; default: break; } out_stream->first_frame = 1; out_stream->first_pts = buf->s.start; out_stream->min_frame_duration = buf->s.duration; } out_stream->next_pts = buf->s.start; } if (out_stream->type == SYNC_TYPE_AUDIO) { buf = FilterAudioFrame(out_stream, buf); if (buf == NULL) { // FilterAudioFrame can consume the buffer with no output continue; } } int64_t subtitle_last_pts = AV_NOPTS_VALUE; if (out_stream->type == SYNC_TYPE_SUBTITLE) { buf = sanitizeSubtitle(out_stream, buf); if (buf == NULL) { // sanitizeSubtitle can consume the buffer with no output continue; } // sanitizeSubtitle can return a list of subtitles. // Find the pts of the last subtitle in the list hb_buffer_t * sub = buf; while (sub != NULL) { subtitle_last_pts = sub->s.start; sub = sub->next; } } if (out_stream->type == SYNC_TYPE_AUDIO || out_stream->type == SYNC_TYPE_VIDEO) { buf->s.start = out_stream->next_pts; buf->s.stop = out_stream->next_pts + buf->s.duration; out_stream->last_pts = out_stream->next_pts; out_stream->next_pts += buf->s.duration; } else { out_stream->next_pts = out_stream->last_pts = subtitle_last_pts; } if (buf->s.stop > 0) { out_stream->current_duration = buf->s.stop - out_stream->first_pts; } out_stream->frame_count++; if (buf->s.duration > 0 && out_stream->min_frame_duration > buf->s.duration) { out_stream->min_frame_duration = buf->s.duration; } if (out_stream->max_frame_duration < buf->s.duration) { out_stream->max_frame_duration = buf->s.duration; } if (out_stream->type == SYNC_TYPE_VIDEO && buf->s.new_chap > common->chapter) { common->chapter = buf->s.new_chap; log_chapter(common, buf->s.new_chap, out_stream->frame_count, buf->s.start); } if ((buf->s.start < 0) || (out_stream->type == SYNC_TYPE_VIDEO && buf->s.duration < 256)) { // The pipeline can't handle negative timestamps // and it is sometimes not possible to avoid one // at the start of the video. There can be a // significant delay before we see the first buffers // from all streams. We can't buffer indefinitely // until we have seen the first PTS for all streams // so sometimes we may start before we have seen // the earliest PTS // // Also, encx264.c can't handle timestamps that are spaced // less than 256 ticks apart. saveChap(out_stream, buf); hb_buffer_close(&buf); } restoreChap(out_stream, buf); fifo_push(out_stream->fifo_out, buf); } while (full); } static void FlushBuffer( sync_common_t * common ) { hb_lock(common->mutex); if (!common->found_first_pts) { checkFirstPts(common); } OutputBuffer(common); hb_unlock(common->mutex); } static void Synchronize( sync_stream_t * stream ) { sync_common_t * common = stream->common; // Sync deposits output directly into fifos, so work_loop is not // blocking when output fifos become full. Wait here before // performing any output when the output fifo for the input stream // is full if (stream->fifo_out != NULL && common->start_found) { while (!common->job->done && !*common->job->die) { if (hb_fifo_full_wait(stream->fifo_out)) { break; } } } hb_lock(common->mutex); if (!fillQueues(common)) { hb_unlock(common->mutex); return; } if (!common->found_first_pts) { checkFirstPts(common); } OutputBuffer(common); hb_unlock(common->mutex); } static void updateDuration( sync_stream_t * stream ) { // The video decoder sets a nominal duration for frames. But the // actual duration needs to be computed from timestamps. if (stream->type == SYNC_TYPE_VIDEO) { int count = hb_list_count(stream->in_queue); if (count >= 2) { hb_buffer_t * buf1 = hb_list_item(stream->in_queue, count - 1); hb_buffer_t * buf2 = hb_list_item(stream->in_queue, count - 2); double duration = buf1->s.start - buf2->s.start; if (duration > 0) { buf2->s.duration = duration; buf2->s.stop = buf1->s.start; } else { buf2->s.duration = 0.; buf2->s.start = buf2->s.stop = buf1->s.start; } } } } static void ProcessSCRDelayQueue( sync_common_t * common ) { int ii, jj; for (ii = 0; ii < common->stream_count; ii++) { sync_stream_t * stream = &common->streams[ii]; for (jj = 0; jj < hb_list_count(stream->scr_delay_queue);) { hb_buffer_t * buf = hb_list_item(stream->scr_delay_queue, jj); int hash = buf->s.scr_sequence & SCR_HASH_MASK; if (buf->s.scr_sequence < 0) { // Unset scr_sequence inidicates an external stream // (e.g. SRT subtitle) that is not on the same timebase // as the source tracks. Do not adjust timestamps for // scr_offset in this case. hb_list_rem(stream->scr_delay_queue, buf); SortedQueueBuffer(stream, buf); } else if (buf->s.scr_sequence == common->scr[hash].scr_sequence) { if (buf->s.start != AV_NOPTS_VALUE) { buf->s.start -= common->scr[hash].scr_offset; buf->s.start -= stream->pts_slip; } if (buf->s.stop != AV_NOPTS_VALUE) { buf->s.stop -= common->scr[hash].scr_offset; buf->s.stop -= stream->pts_slip; } hb_list_rem(stream->scr_delay_queue, buf); SortedQueueBuffer(stream, buf); } else { jj++; } } } } static const char * getStreamType( sync_stream_t * stream ) { switch (stream->type) { case SYNC_TYPE_VIDEO: return "Video"; case SYNC_TYPE_AUDIO: return "Audio"; case SYNC_TYPE_SUBTITLE: return "Subtitle"; default: return "Unknown"; } } static int getStreamId( sync_stream_t * stream ) { switch (stream->type) { case SYNC_TYPE_VIDEO: return stream->video.id; case SYNC_TYPE_AUDIO: return stream->audio.audio->id; case SYNC_TYPE_SUBTITLE: return stream->subtitle.subtitle->id; default: return -1; } } static int UpdateSCR( sync_stream_t * stream, hb_buffer_t * buf ) { int hash = buf->s.scr_sequence & SCR_HASH_MASK; sync_common_t * common = stream->common; double last_scr_pts, last_duration; int64_t scr_offset = 0; if (buf->s.scr_sequence < stream->last_scr_sequence) { // In decoder error conditions, the decoder can send us out of // order frames. Often the stream error will also trigger a // discontinuity detection. An out of order frame will // cause an incorrect SCR offset, so drop such frames. hb_deep_log(3, "SCR sequence went backwards %d -> %d", stream->last_scr_sequence, buf->s.scr_sequence); hb_buffer_close(&buf); return 0; } if (buf->s.scr_sequence >= 0) { if (buf->s.scr_sequence != common->scr[hash].scr_sequence) { if (stream->type == SYNC_TYPE_SUBTITLE || (stream->last_scr_pts == (int64_t)AV_NOPTS_VALUE && common->first_scr)) { // We got a new scr, but we have no last_scr_pts to base it // off of. Delay till we can compute the scr offset from a // different stream. hb_list_add(stream->scr_delay_queue, buf); return 0; } if (buf->s.start != AV_NOPTS_VALUE) { last_scr_pts = stream->last_scr_pts; last_duration = stream->last_duration; if (last_scr_pts == (int64_t)AV_NOPTS_VALUE) { last_scr_pts = 0.; last_duration = 0.; common->first_scr = 1; } // New SCR. Compute SCR offset common->scr[hash].scr_sequence = buf->s.scr_sequence; common->scr[hash].scr_offset = buf->s.start - (last_scr_pts + last_duration); hb_deep_log(4, "New SCR: type %8s id %x scr seq %d scr offset %"PRId64" " "start %"PRId64" last %f dur %f", getStreamType(stream), getStreamId(stream), buf->s.scr_sequence, common->scr[hash].scr_offset, buf->s.start, last_scr_pts, last_duration); ProcessSCRDelayQueue(common); } } scr_offset = common->scr[hash].scr_offset; } // Adjust buffer timestamps for SCR offset if (buf->s.start != AV_NOPTS_VALUE) { buf->s.start -= scr_offset; last_scr_pts = buf->s.start; } else if (stream->last_scr_pts != (int64_t)AV_NOPTS_VALUE) { last_scr_pts = stream->last_scr_pts + stream->last_duration; buf->s.start = last_scr_pts; } else { // This should happen extremely rarely if ever. // But if we get here, this is the first buffer received for // this stream. Normally, some buffers will be queued for // every stream before we ever call UpdateSCR. // We don't really know what it's timestamp should be. // So drop the buffer. hb_buffer_close(&buf); return 0; } if (buf->s.stop != AV_NOPTS_VALUE) { buf->s.stop -= scr_offset; } if (last_scr_pts > stream->last_scr_pts) { stream->last_scr_pts = last_scr_pts; } if (buf->s.scr_sequence > stream->last_scr_sequence) { stream->last_scr_sequence = buf->s.scr_sequence; } stream->last_duration = buf->s.duration; return 1; } // Handle broken timestamps that are out of order // These are usually due to a broken decoder (e.g. QSV and libav AVI packed // b-frame support). But sometimes can come from a severely broken or // corrupted source file. // // We can pretty reliably fix out of order timestamps *if* every frame // has a timestamp. But some container formats allow frames with no // timestamp (e.g. TS and PS). When there is no timestamp, we will // compute one based on the last frames timestamp. If the missing // timestamp is out of order and really belonged on an earlier frame (A), // this will result in the frame before (A) being long and the frame // after the current will overlap current. // // The condition above of one long frame and one overlap will most likely // get fixed by dejitterVideo. dejitterVideo finds sequences where the // sum of the durations of frames 1..N == (1/fps) * N. When it finds such // a sequence, it adjusts the frame durations to all be 1/fps. Since the // vast majority of video is constant framerate, this will fix the above // problem most of the time. static void SortedQueueBuffer( sync_stream_t * stream, hb_buffer_t * buf ) { int64_t start; int ii, count; start = buf->s.start; hb_list_add(stream->in_queue, buf); // Search for the first earlier timestamp that is < this one. // Under normal circumstances where the timestamps are not broken, // this will only check the next to last buffer in the queue // before aborting. count = hb_list_count(stream->in_queue); for (ii = count - 2; ii >= 0; ii--) { buf = hb_list_item(stream->in_queue, ii); if (buf->s.start < start || start == AV_NOPTS_VALUE) { break; } } if (ii < count - 2) { hb_buffer_t * prev = NULL; int jj; // The timestamp was out of order. // The timestamp belongs at position ii + 1 // Every timestamp from ii + 2 to count - 1 needs to be shifted up. if (ii >= 0) { prev = hb_list_item(stream->in_queue, ii); } for (jj = ii + 1; jj < count; jj++) { int64_t tmp_start; buf = hb_list_item(stream->in_queue, jj); tmp_start = buf->s.start; buf->s.start = start; start = tmp_start; if (stream->type == SYNC_TYPE_VIDEO && prev != NULL) { // recompute video buffer duration prev->s.duration = buf->s.start - prev->s.start; prev->s.stop = buf->s.start; } prev = buf; } } } static void QueueBuffer( sync_stream_t * stream, hb_buffer_t * buf ) { hb_lock(stream->common->mutex); while (hb_list_count(stream->in_queue) > stream->max_len && !stream->done && !stream->common->job->done && !*stream->common->job->die) { // If the in_queue is full, we have to force some output to // unblock it. Blocking here would back up the pipeline and // stall out reader eventually. hb_unlock(stream->common->mutex); Synchronize(stream); hb_lock(stream->common->mutex); } // Reader can change job->reader_pts_offset after initialization // and before we receive the first buffer here. Calculate // common->pts_to_start here since this is the first opportunity where // all the necessary information exists. if (stream->common->pts_to_start == AV_NOPTS_VALUE) { hb_job_t * job = stream->common->job; if (job->pts_to_start > 0) { stream->common->pts_to_start = MAX(0, job->pts_to_start - job->reader_pts_offset); } } // Render offset is only useful for decoders, which are all // upstream of sync. Squash it. buf->s.renderOffset = AV_NOPTS_VALUE; hb_deep_log(11, "type %8s id %x scr seq %d start %"PRId64" stop %"PRId64" dur %f", getStreamType(stream), getStreamId(stream), buf->s.scr_sequence, buf->s.start, buf->s.stop, buf->s.duration); if (stream->common->found_first_pts) { if (UpdateSCR(stream, buf) > 0) { // Apply any stream slips. // Stream slips will only temporarily differ between // the streams. The slips get updated in applyDeltas. When // all the deltas are absorbed, the stream slips will all // be equal. buf->s.start -= stream->pts_slip; if (buf->s.stop != AV_NOPTS_VALUE) { buf->s.stop -= stream->pts_slip; } SortedQueueBuffer(stream, buf); updateDuration(stream); } } else { if (buf->s.start == AV_NOPTS_VALUE && hb_list_count(stream->in_queue) == 0) { // We require an initial pts to start synchronization saveChap(stream, buf); hb_buffer_close(&buf); hb_unlock(stream->common->mutex); return; } SortedQueueBuffer(stream, buf); } // Make adjustments for gaps found in other streams applyDeltas(stream->common); hb_unlock(stream->common->mutex); } static int InitAudio( sync_common_t * common, int index ) { hb_work_object_t * w = NULL; hb_work_private_t * pv; hb_audio_t * audio; audio = hb_list_item(common->job->list_audio, index); if (audio->priv.fifo_raw == NULL) { // No input fifo, not configured return 0; } pv = calloc(1, sizeof(hb_work_private_t)); if (pv == NULL) goto fail; w = hb_get_work(common->job->h, WORK_SYNC_AUDIO); w->private_data = pv; w->audio = audio; w->fifo_in = audio->priv.fifo_raw; if (audio->config.out.codec & HB_ACODEC_PASS_FLAG) { w->fifo_out = audio->priv.fifo_out; } else { w->fifo_out = audio->priv.fifo_sync; } pv->common = common; pv->stream = &common->streams[1 + index]; pv->stream->common = common; pv->stream->in_queue = hb_list_init(); pv->stream->scr_delay_queue = hb_list_init(); pv->stream->max_len = SYNC_MAX_AUDIO_QUEUE_LEN; pv->stream->min_len = SYNC_MIN_AUDIO_QUEUE_LEN; if (pv->stream->in_queue == NULL) goto fail; pv->stream->delta_list = hb_list_init(); if (pv->stream->delta_list == NULL) goto fail; pv->stream->type = SYNC_TYPE_AUDIO; pv->stream->first_pts = AV_NOPTS_VALUE; pv->stream->next_pts = (int64_t)AV_NOPTS_VALUE; pv->stream->last_pts = (int64_t)AV_NOPTS_VALUE; pv->stream->last_scr_pts = (int64_t)AV_NOPTS_VALUE; pv->stream->last_scr_sequence = -1; pv->stream->last_duration = (int64_t)AV_NOPTS_VALUE; pv->stream->audio.audio = audio; pv->stream->fifo_out = w->fifo_out; if (!(audio->config.out.codec & HB_ACODEC_PASS_FLAG) && audio->config.in.samplerate != audio->config.out.samplerate) { /* Initialize libsamplerate */ int error; pv->stream->audio.src.ctx = src_new(SRC_SINC_MEDIUM_QUALITY, hb_mixdown_get_discrete_channel_count(audio->config.out.mixdown), &error); if (pv->stream->audio.src.ctx == NULL) goto fail; pv->stream->audio.src.pkt.end_of_input = 0; } pv->stream->audio.gain_factor = pow(10, audio->config.out.gain / 20); hb_list_add(common->list_work, w); return 0; fail: if (pv != NULL) { if (pv->stream != NULL) { if (pv->stream->audio.src.ctx) { src_delete(pv->stream->audio.src.ctx); } hb_list_close(&pv->stream->delta_list); hb_list_close(&pv->stream->in_queue); } } free(pv); free(w); return 1; } static int InitSubtitle( sync_common_t * common, int index ) { hb_work_object_t * w = NULL; hb_work_private_t * pv; hb_subtitle_t * subtitle; subtitle = hb_list_item(common->job->list_subtitle, index); if (subtitle->fifo_raw == NULL) { // No input fifo, not configured return 0; } pv = calloc(1, sizeof(hb_work_private_t)); if (pv == NULL) goto fail; pv->common = common; pv->stream = &common->streams[1 + hb_list_count(common->job->list_audio) + index]; pv->stream->common = common; pv->stream->in_queue = hb_list_init(); pv->stream->scr_delay_queue = hb_list_init(); pv->stream->max_len = SYNC_MAX_SUBTITLE_QUEUE_LEN; pv->stream->min_len = SYNC_MIN_SUBTITLE_QUEUE_LEN; if (pv->stream->in_queue == NULL) goto fail; pv->stream->delta_list = hb_list_init(); if (pv->stream->delta_list == NULL) goto fail; pv->stream->type = SYNC_TYPE_SUBTITLE; pv->stream->first_pts = AV_NOPTS_VALUE; pv->stream->next_pts = (int64_t)AV_NOPTS_VALUE; pv->stream->last_pts = (int64_t)AV_NOPTS_VALUE; pv->stream->last_scr_pts = (int64_t)AV_NOPTS_VALUE; pv->stream->last_scr_sequence = -1; pv->stream->last_duration = (int64_t)AV_NOPTS_VALUE; pv->stream->subtitle.subtitle = subtitle; pv->stream->fifo_out = subtitle->fifo_out; pv->stream->fifo_in = subtitle->fifo_in; w = hb_get_work(common->job->h, WORK_SYNC_SUBTITLE); w->private_data = pv; w->subtitle = subtitle; w->fifo_in = subtitle->fifo_raw; w->fifo_out = subtitle->fifo_out; memset(&pv->stream->subtitle.sanitizer, 0, sizeof(pv->stream->subtitle.sanitizer)); if (subtitle->format == TEXTSUB && subtitle->config.dest == PASSTHRUSUB && (common->job->mux & HB_MUX_MASK_MP4)) { // Merge overlapping subtitles since mpv tx3g does not support them pv->stream->subtitle.sanitizer.merge = 1; } // PGS subtitles don't need to be linked because there are explicit // "clear" subtitle packets that indicate the end time of the // previous subtitle if (subtitle->config.dest == PASSTHRUSUB && subtitle->source != PGSSUB) { // Fill in stop time when it is missing pv->stream->subtitle.sanitizer.link = 1; } hb_buffer_list_clear(&pv->stream->subtitle.sanitizer.list_current); hb_list_add(common->list_work, w); return 0; fail: if (pv != NULL) { if (pv->stream != NULL) { hb_list_close(&pv->stream->delta_list); hb_list_close(&pv->stream->in_queue); } } free(pv); free(w); return 1; } /*********************************************************************** * Initialize the work object **********************************************************************/ static int syncVideoInit( hb_work_object_t * w, hb_job_t * job) { hb_work_private_t * pv; int ii; pv = calloc(1, sizeof(hb_work_private_t)); if (pv == NULL) goto fail; w->private_data = pv; pv->common = calloc(1, sizeof(sync_common_t)); if (pv->common == NULL) goto fail; pv->common->job = job; // count number of streams we need pv->common->stream_count = 1; pv->common->stream_count += hb_list_count(job->list_audio); pv->common->stream_count += hb_list_count(job->list_subtitle); pv->common->streams = calloc(pv->common->stream_count, sizeof(sync_stream_t)); // Allocate streams if (pv->common->streams == NULL) goto fail; // create audio and subtitle work list pv->common->list_work = hb_list_init(); if (pv->common->list_work == NULL) goto fail; // mutex to mediate access to pv->common pv->common->mutex = hb_lock_init(); if (pv->common->mutex == NULL) goto fail; // Set up video sync work object pv->stream = &pv->common->streams[0]; pv->stream->common = pv->common; pv->stream->in_queue = hb_list_init(); pv->stream->scr_delay_queue = hb_list_init(); pv->stream->max_len = SYNC_MAX_VIDEO_QUEUE_LEN; pv->stream->min_len = SYNC_MIN_VIDEO_QUEUE_LEN; if (pv->stream->in_queue == NULL) goto fail; pv->stream->delta_list = hb_list_init(); if (pv->stream->delta_list == NULL) goto fail; pv->stream->type = SYNC_TYPE_VIDEO; pv->stream->first_pts = AV_NOPTS_VALUE; pv->stream->next_pts = (int64_t)AV_NOPTS_VALUE; pv->stream->last_pts = (int64_t)AV_NOPTS_VALUE; pv->stream->last_scr_pts = (int64_t)AV_NOPTS_VALUE; pv->stream->last_scr_sequence = -1; pv->stream->last_duration = (int64_t)AV_NOPTS_VALUE; pv->stream->fifo_out = job->fifo_sync; pv->stream->video.id = job->title->video_id; w->fifo_in = job->fifo_raw; w->fifo_out = job->fifo_sync; if (job->pass_id == HB_PASS_ENCODE_2ND) { /* We already have an accurate frame count from pass 1 */ hb_interjob_t * interjob = hb_interjob_get(job->h); pv->common->est_frame_count = interjob->frame_count; } else { /* Calculate how many video frames we are expecting */ if (job->frame_to_stop) { pv->common->est_frame_count = job->frame_to_stop; } else { int64_t duration, total_duration, extra_duration = 0; if (job->pts_to_stop) { duration = job->pts_to_stop + 90000; } else { total_duration = 0; for (ii = 0; ii <= hb_list_count(job->list_chapter); ii++) { hb_chapter_t * chapter; chapter = hb_list_item(job->list_chapter, ii - 1); if (chapter != NULL) { total_duration += chapter->duration; } } // Some titles are longer than the sum duration of their // chapters. Account for this extra duration. if (job->title->duration > total_duration) { extra_duration = job->title->duration - total_duration; } duration = 0; for (ii = job->chapter_start; ii <= job->chapter_end; ii++) { hb_chapter_t * chapter; chapter = hb_list_item(job->list_chapter, ii - 1); if (chapter != NULL) { duration += chapter->duration; } } if (job->chapter_end == hb_list_count(job->list_chapter)) { duration += extra_duration; } } pv->common->est_frame_count = duration * job->title->vrate.num / job->title->vrate.den / 90000; } } hb_log("sync: expecting %d video frames", pv->common->est_frame_count); // Initialize audio sync work objects for (ii = 0; ii < hb_list_count(job->list_audio); ii++ ) { if (InitAudio(pv->common, ii)) goto fail; } // Initialize subtitle sync work objects for (ii = 0; ii < hb_list_count(job->list_subtitle); ii++ ) { if (InitSubtitle(pv->common, ii)) goto fail; } /* Launch work processing threads */ for (ii = 0; ii < hb_list_count(pv->common->list_work); ii++) { hb_work_object_t * work; work = hb_list_item(pv->common->list_work, ii); work->done = w->done; work->thread = hb_thread_init(work->name, hb_work_loop, work, HB_LOW_PRIORITY); } if (job->frame_to_start || job->pts_to_start) { pv->common->start_found = 0; pv->common->start_pts = job->pts_to_start; pv->common->wait_for_frame = !!job->frame_to_start; pv->common->wait_for_pts = !!job->pts_to_start; if (job->pts_to_start) { pv->common->pts_to_start = AV_NOPTS_VALUE; } } else { pv->common->start_found = 1; } if (job->pts_to_stop) { pv->common->stop_pts = job->pts_to_stop; } return 0; fail: if (pv != NULL) { if (pv->common != NULL) { for (ii = 0; ii < hb_list_count(pv->common->list_work); ii++) { hb_work_object_t * work; work = hb_list_item(pv->common->list_work, ii); if (work->close) work->close(work); } hb_list_close(&pv->common->list_work); hb_lock_close(&pv->common->mutex); if (pv->stream != NULL) { hb_list_close(&pv->stream->delta_list); hb_list_close(&pv->stream->in_queue); } free(pv->common->streams); free(pv->common); } } free(pv); w->private_data = NULL; return 1; } /*********************************************************************** * Close Video *********************************************************************** * **********************************************************************/ static void syncVideoClose( hb_work_object_t * w ) { hb_work_private_t * pv = w->private_data; hb_job_t * job; if (pv == NULL) { return; } job = pv->common->job; hb_log("sync: got %d frames, %d expected", pv->stream->frame_count, pv->common->est_frame_count ); if (pv->stream->min_frame_duration > 0 && pv->stream->max_frame_duration > 0 && pv->stream->current_duration > 0) { hb_log("sync: framerate min %.3f fps, max %.3f fps, avg %.3f fps", 90000. / pv->stream->max_frame_duration, 90000. / pv->stream->min_frame_duration, (pv->stream->frame_count * 90000.) / pv->stream->current_duration); } /* save data for second pass */ if( job->pass_id == HB_PASS_ENCODE_1ST ) { /* Preserve frame count for better accuracy in pass 2 */ hb_interjob_t * interjob = hb_interjob_get( job->h ); interjob->frame_count = pv->stream->frame_count; } sync_delta_t * delta; while ((delta = hb_list_item(pv->stream->delta_list, 0)) != NULL) { hb_list_rem(pv->stream->delta_list, delta); free(delta); } hb_list_close(&pv->stream->delta_list); hb_list_empty(&pv->stream->in_queue); hb_list_empty(&pv->stream->scr_delay_queue); // Close work threads hb_work_object_t * work; while ((work = hb_list_item(pv->common->list_work, 0))) { hb_list_rem(pv->common->list_work, work); if (work->thread != NULL) { hb_thread_close(&work->thread); } if (work->close) work->close(work); free(work); } hb_list_close(&pv->common->list_work); hb_lock_close(&pv->common->mutex); free(pv->common->streams); free(pv->common); free(pv); w->private_data = NULL; } static hb_buffer_t * merge_ssa(hb_buffer_t *a, hb_buffer_t *b) { int len, ii; char *text; hb_buffer_t *buf; if (a == NULL && b == NULL) { return NULL; } if (a == NULL) { return hb_buffer_dup(b); } if (b == NULL) { return hb_buffer_dup(a); } buf = hb_buffer_init(a->size + b->size); buf->s = a->s; // Find the text in the second SSA sub text = (char*)b->data; for (ii = 0; ii < 8; ii++) { text = strchr(text, ','); if (text == NULL) break; text++; } if (text != NULL) { len = sprintf((char*)buf->data, "%s\n%s", a->data, text); if (len >= 0) buf->size = len + 1; } else { memcpy(buf->data, a->data, a->size); buf->size = a->size; } return buf; } static hb_buffer_t * setSubDuration(sync_stream_t * stream, hb_buffer_t * sub) { if (sub->s.flags & HB_BUF_FLAG_EOS) { return sub; } if (sub->s.stop != AV_NOPTS_VALUE) { sub->s.duration = sub->s.stop - sub->s.start; if (sub->s.duration <= 0) { hb_log("sync: subtitle 0x%x duration <= 0, PTS %"PRId64"", stream->subtitle.subtitle->id, sub->s.start); hb_buffer_close(&sub); } } else if (stream->subtitle.sanitizer.link) { hb_log("sync: subtitle 0x%x duration not set, PTS %"PRId64"", stream->subtitle.subtitle->id, sub->s.start); sub->s.duration = (int64_t)AV_NOPTS_VALUE; } return sub; } // Create a list of buffers that overlap the given start time. // Returns a list of buffers and the smallest stop time of those // buffers. static hb_buffer_t * findOverlap(subtitle_sanitizer_t *sanitizer, int64_t start, int64_t *stop_out) { hb_buffer_list_t list; hb_buffer_t * buf; int64_t stop; stop = INT64_MAX; hb_buffer_list_clear(&list); buf = hb_buffer_list_head(&sanitizer->list_current); while (buf != NULL) { if (buf->s.flags & HB_BUF_FLAG_EOF) { break; } if (buf->s.start > start) { if (stop > buf->s.start) { *stop_out = buf->s.start; } break; } if (buf->s.start <= start && start < buf->s.stop) { hb_buffer_t * tmp = hb_buffer_dup(buf); tmp->s.start = start; hb_buffer_list_append(&list, tmp); if (stop > buf->s.stop) { stop = buf->s.stop; *stop_out = stop; } } buf = buf->next; } return hb_buffer_list_clear(&list); } // Find all subtitles in the list that start "now" and overlap for // some period of time. Create a new subtitle buffer that is the // merged results of the overlapping parts and update start times // of non-overlapping parts. static int mergeSubtitleOverlaps(subtitle_sanitizer_t *sanitizer) { hb_buffer_t * merged_buf = NULL; hb_buffer_t * a, * b; a = hb_buffer_list_head(&sanitizer->list_current); if (a != NULL && (a->s.flags & HB_BUF_FLAG_EOF)) { // EOF return 0; } if (a == NULL || a->s.start == AV_NOPTS_VALUE || a->s.stop == AV_NOPTS_VALUE) { // Not enough information to resolve an overlap return -1; } b = a->next; if (b != NULL && a->s.stop <= b->s.start) { // No overlap return 0; } // Check that we have 2 non-overlapping buffers in the list // and that all timestamps are valid up to the non-overlap. // This ensures that multiple overlapping subtitles have been // completely merged. while (b != NULL && b->s.start < a->s.stop && !(b->s.flags & HB_BUF_FLAG_EOF)) { if (b->s.start == AV_NOPTS_VALUE || b->s.stop == AV_NOPTS_VALUE) { // Not enough information to resolve an overlap return -1; } b = b->next; } if (b == NULL) { // Not enough information to resolve an overlap return -1; } hb_buffer_list_t merged_list; int64_t start, stop, last; if (b->s.flags & HB_BUF_FLAG_EOF) { last = INT64_MAX; } else { last = b->s.start; } hb_buffer_list_clear(&merged_list); a = hb_buffer_list_head(&sanitizer->list_current); stop = start = a->s.start; while (start < last) { hb_buffer_t * merge = findOverlap(sanitizer, start, &stop); if (merge == NULL) { break; } a = merge; merged_buf = NULL; while (a != NULL) { hb_buffer_t * tmp; tmp = merge_ssa(merged_buf, a); hb_buffer_close(&merged_buf); merged_buf = tmp; a = a->next; } merged_buf->s.stop = stop; hb_buffer_close(&merge); hb_buffer_list_append(&merged_list, merged_buf); start = stop; // Remove merged buffers a = hb_buffer_list_head(&sanitizer->list_current); while (a != NULL && a->s.start < stop && !(a->s.flags & HB_BUF_FLAG_EOF)) { hb_buffer_t * next = a->next; if (a->s.stop <= stop) { // Buffer consumed hb_buffer_list_rem(&sanitizer->list_current, a); hb_buffer_close(&a); } else { a->s.start = stop; } a = next; } } merged_buf = hb_buffer_list_clear(&merged_list); hb_buffer_list_prepend(&sanitizer->list_current, merged_buf); return 0; } static hb_buffer_t * mergeSubtitles(sync_stream_t * stream) { hb_buffer_t * buf; hb_buffer_list_t list; subtitle_sanitizer_t * sanitizer = &stream->subtitle.sanitizer; hb_buffer_list_clear(&list); if (!sanitizer->merge) { // Handle all but the last buffer // The last buffer may not have been "linked" yet while (hb_buffer_list_count(&sanitizer->list_current) > 0) { buf = hb_buffer_list_head(&sanitizer->list_current); if (!(buf->s.flags & HB_BUF_FLAG_EOF) && buf->s.stop != AV_NOPTS_VALUE) { buf = hb_buffer_list_rem_head(&sanitizer->list_current); buf = setSubDuration(stream, buf); hb_buffer_list_append(&list, buf); } else { break; } } return hb_buffer_list_clear(&list); } // We only reach here if we are merging subtitles while (hb_buffer_list_count(&sanitizer->list_current) > 0) { buf = hb_buffer_list_head(&sanitizer->list_current); if (buf->s.flags & HB_BUF_FLAG_EOF) { // remove EOF from list, add to output buf = hb_buffer_list_rem_head(&sanitizer->list_current); hb_buffer_list_append(&list, buf); break; } int result = mergeSubtitleOverlaps(sanitizer); if (result < 0) { // not enough information available yet to resolve the overlap break; } // Overlap resolved, output a buffer buf = hb_buffer_list_rem_head(&sanitizer->list_current); if (buf != NULL && !(buf->s.flags & HB_BUF_FLAG_EOF)) { buf = setSubDuration(stream, buf); hb_buffer_list_append(&list, buf); } } return hb_buffer_list_clear(&list); } static hb_buffer_t * sanitizeSubtitle( sync_stream_t * stream, hb_buffer_t * sub) { hb_buffer_list_t list; subtitle_sanitizer_t * sanitizer = &stream->subtitle.sanitizer; if (sub == NULL) { return NULL; } hb_buffer_list_set(&list, sub); if (!sanitizer->link && !sanitizer->merge) { hb_buffer_list_t out_list; hb_buffer_list_clear(&out_list); sub = hb_buffer_list_rem_head(&list); while (sub != NULL && !(sub->s.flags & HB_BUF_FLAG_EOF)) { sub = setSubDuration(stream, sub); hb_buffer_list_append(&out_list, sub); sub = hb_buffer_list_rem_head(&list); } return hb_buffer_list_clear(&out_list); } sub = hb_buffer_list_rem_head(&list); while (sub != NULL) { if (sub->s.flags & HB_BUF_FLAG_EOF) { hb_buffer_list_append(&sanitizer->list_current, sub); break; } hb_buffer_t *last = hb_buffer_list_tail(&sanitizer->list_current); if (last != NULL && last->s.stop == AV_NOPTS_VALUE) { last->s.stop = sub->s.start; if (last->s.stop <= last->s.start) { // Subtitle duration <= 0. Drop it. hb_log("sync: subtitle 0x%x has no duration, PTS %"PRId64"", stream->subtitle.subtitle->id, sub->s.start); hb_buffer_list_rem_tail(&sanitizer->list_current); hb_buffer_close(&last); } } if (sub->s.flags & HB_BUF_FLAG_EOS) { // Used to indicate "clear" subtitles when the duration // of subtitles is not encoded in the stream hb_buffer_close(&sub); } hb_buffer_list_append(&sanitizer->list_current, sub); sub = hb_buffer_list_rem_head(&list); } return mergeSubtitles(stream); } /*********************************************************************** * syncVideoWork *********************************************************************** * **********************************************************************/ static int syncVideoWork( hb_work_object_t * w, hb_buffer_t ** buf_in, hb_buffer_t ** buf_out ) { hb_work_private_t * pv = w->private_data; hb_buffer_t * in = *buf_in; if (pv->stream->done) { FlushBuffer(pv->stream->common); return HB_WORK_DONE; } if (in->s.flags & HB_BUF_FLAG_EOF) { pv->stream->flush = 1; flushStreamsLock(pv->common); // Ideally, we would only do this subtitle scan check in // syncSubtitleWork, but someone might try to do a subtitle // scan on a source that has no subtitles :-( if (pv->common->job->indepth_scan) { // When doing subtitle indepth scan, the pipeline ends at sync. // Terminate job when EOF reached. *w->done = 1; } return HB_WORK_DONE; } *buf_in = NULL; QueueBuffer(pv->stream, in); Synchronize(pv->stream); if (pv->stream->done) { FlushBuffer(pv->stream->common); return HB_WORK_DONE; } return HB_WORK_OK; } hb_work_object_t hb_sync_video = { WORK_SYNC_VIDEO, "Video Synchronization", syncVideoInit, syncVideoWork, syncVideoClose }; /*********************************************************************** * Close Audio *********************************************************************** * **********************************************************************/ static void syncAudioClose( hb_work_object_t * w ) { hb_work_private_t * pv = w->private_data; if (pv == NULL) { return; } // Free samplerate conversion context if (pv->stream->audio.src.ctx) { src_delete(pv->stream->audio.src.ctx); } sync_delta_t * delta; while ((delta = hb_list_item(pv->stream->delta_list, 0)) != NULL) { hb_list_rem(pv->stream->delta_list, delta); free(delta); } hb_list_close(&pv->stream->delta_list); hb_list_empty(&pv->stream->in_queue); hb_list_empty(&pv->stream->scr_delay_queue); free(pv); w->private_data = NULL; } static int syncAudioInit( hb_work_object_t * w, hb_job_t * job) { return 0; } /*********************************************************************** * SyncAudio *********************************************************************** * **********************************************************************/ static int syncAudioWork( hb_work_object_t * w, hb_buffer_t ** buf_in, hb_buffer_t ** buf_out ) { hb_work_private_t * pv = w->private_data; hb_buffer_t * in = *buf_in; if (pv->stream->done) { FlushBuffer(pv->stream->common); return HB_WORK_DONE; } if (in->s.flags & HB_BUF_FLAG_EOF) { pv->stream->flush = 1; flushStreamsLock(pv->common); return HB_WORK_DONE; } *buf_in = NULL; QueueBuffer(pv->stream, in); Synchronize(pv->stream); if (pv->stream->done) { FlushBuffer(pv->stream->common); return HB_WORK_DONE; } return HB_WORK_OK; } hb_work_object_t hb_sync_audio = { WORK_SYNC_AUDIO, "Audio Synchronization", syncAudioInit, syncAudioWork, syncAudioClose }; // FilterAudioFrame is called after audio timestamp discontinuities // have all been corrected. So we expect smooth continuous audio // here. static hb_buffer_t * FilterAudioFrame( sync_stream_t * stream, hb_buffer_t *buf ) { hb_audio_t * audio = stream->audio.audio; // Can't count of buf->s.stop - buf->s.start for accurate duration // due to integer rounding, so use buf->s.duration when it is set // (which should be always if I didn't miss anything) if (buf->s.duration <= 0) { buf->s.duration = buf->s.stop - buf->s.start; } if (!(audio->config.out.codec & HB_ACODEC_PASS_FLAG)) { // TODO: this should all be replaced by an audio filter chain. // Audio is not passthru. Check if we need to modify the audio // in any way. if (stream->audio.src.ctx != NULL) { /* do sample rate conversion */ int count_in, count_out; hb_buffer_t * buf_raw = buf; int sample_size = hb_mixdown_get_discrete_channel_count( audio->config.out.mixdown ) * sizeof( float ); count_in = buf_raw->size / sample_size; /* * When using stupid rates like 44.1 there will always be some * truncation error. E.g., a 1536 sample AC3 frame will turn into a * 1536*44.1/48.0 = 1411.2 sample frame. If we just truncate the .2 * the error will build up over time and eventually the audio will * substantially lag the video. libsamplerate will keep track of the * fractional sample & give it to us when appropriate if we give it * an extra sample of space in the output buffer. */ count_out = (buf->s.duration * audio->config.out.samplerate) / 90000 + 1; stream->audio.src.pkt.input_frames = count_in; stream->audio.src.pkt.output_frames = count_out; stream->audio.src.pkt.src_ratio = (double)audio->config.out.samplerate / audio->config.in.samplerate; buf = hb_buffer_init( count_out * sample_size ); buf->s = buf_raw->s; stream->audio.src.pkt.data_in = (float *) buf_raw->data; stream->audio.src.pkt.data_out = (float *) buf->data; if (src_process(stream->audio.src.ctx, &stream->audio.src.pkt)) { /* XXX If this happens, we're screwed */ hb_error("sync: audio 0x%x src_process failed", audio->id); } hb_buffer_close(&buf_raw); if (stream->audio.src.pkt.output_frames_gen <= 0) { hb_buffer_close(&buf); return NULL; } buf->size = stream->audio.src.pkt.output_frames_gen * sample_size; buf->s.duration = 90000. * stream->audio.src.pkt.output_frames_gen / audio->config.out.samplerate; buf->s.start = stream->next_pts; buf->s.stop = stream->next_pts + buf->s.duration; } if (audio->config.out.gain > 0.0) { int count, ii; count = buf->size / sizeof(float); for ( ii = 0; ii < count; ii++ ) { double sample; sample = (double)*(((float*)buf->data)+ii); sample *= stream->audio.gain_factor; if (sample > 0) sample = MIN(sample, 1.0); else sample = MAX(sample, -1.0); *(((float*)buf->data)+ii) = sample; } } else if( audio->config.out.gain < 0.0 ) { int count, ii; count = buf->size / sizeof(float); for ( ii = 0; ii < count; ii++ ) { double sample; sample = (double)*(((float*)buf->data)+ii); sample *= stream->audio.gain_factor; *(((float*)buf->data)+ii) = sample; } } } buf->s.type = AUDIO_BUF; buf->s.frametype = HB_FRAME_AUDIO; return buf; } static void UpdateState( sync_common_t * common, int frame_count ) { hb_job_t * job = common->job; hb_state_t state; if (job->indepth_scan) { // Progress for indept scan is handled by reader // frame_count is used during indepth_scan // to find start & end points. return; } if (frame_count == 0) { common->st_first = hb_get_date(); job->st_pause_date = -1; job->st_paused = 0; } if (hb_get_date() > common->st_dates[3] + 1000) { memmove( &common->st_dates[0], &common->st_dates[1], 3 * sizeof( uint64_t ) ); memmove( &common->st_counts[0], &common->st_counts[1], 3 * sizeof( uint64_t ) ); common->st_dates[3] = hb_get_date(); common->st_counts[3] = frame_count; } hb_get_state2(job->h, &state); state.state = HB_STATE_WORKING; #define p state.param.working p.progress = (float)frame_count / common->est_frame_count; if (p.progress > 1.0) { p.progress = 1.0; } p.rate_cur = 1000.0 * (common->st_counts[3] - common->st_counts[0]) / (common->st_dates[3] - common->st_dates[0]); if (hb_get_date() > common->st_first + 4000) { int eta; p.rate_avg = 1000.0 * common->st_counts[3] / (common->st_dates[3] - common->st_first - job->st_paused); eta = (common->est_frame_count - common->st_counts[3]) / p.rate_avg; p.hours = eta / 3600; p.minutes = (eta % 3600) / 60; p.seconds = eta % 60; } else { p.rate_avg = 0.0; p.hours = -1; p.minutes = -1; p.seconds = -1; } #undef p hb_set_state(job->h, &state); } static void UpdateSearchState( sync_common_t * common, int64_t start, int frame_count ) { hb_job_t * job = common->job; hb_state_t state; uint64_t now; double avg; if (job->indepth_scan) { // Progress for indept scan is handled by reader // frame_count is used during indepth_scan // to find start & end points. return; } now = hb_get_date(); if (frame_count == 0) { common->st_first = now; job->st_pause_date = -1; job->st_paused = 0; } hb_get_state2(job->h, &state); state.state = HB_STATE_SEARCHING; #define p state.param.working if (common->wait_for_frame) p.progress = (float)frame_count / job->frame_to_start; else if (common->wait_for_pts) p.progress = (float) start / common->pts_to_start; else p.progress = 0; if (p.progress > 1.0) { p.progress = 1.0; } if (now > common->st_first) { int eta = 0; if (common->wait_for_frame) { avg = 1000.0 * frame_count / (now - common->st_first); eta = (job->frame_to_start - frame_count ) / avg; } else if (common->wait_for_pts) { avg = 1000.0 * start / (now - common->st_first); eta = (common->pts_to_start - start) / avg; } p.hours = eta / 3600; p.minutes = (eta % 3600) / 60; p.seconds = eta % 60; } else { p.rate_avg = 0.0; p.hours = -1; p.minutes = -1; p.seconds = -1; } #undef p hb_set_state(job->h, &state); } static int syncSubtitleInit( hb_work_object_t * w, hb_job_t * job ) { return 0; } static void syncSubtitleClose( hb_work_object_t * w ) { hb_work_private_t * pv = w->private_data; if (pv == NULL) { return; } sync_delta_t * delta; while ((delta = hb_list_item(pv->stream->delta_list, 0)) != NULL) { hb_list_rem(pv->stream->delta_list, delta); free(delta); } hb_list_close(&pv->stream->delta_list); hb_list_empty(&pv->stream->in_queue); hb_list_empty(&pv->stream->scr_delay_queue); hb_buffer_list_close(&pv->stream->subtitle.sanitizer.list_current); free(pv); w->private_data = NULL; } static int syncSubtitleWork( hb_work_object_t * w, hb_buffer_t ** buf_in, hb_buffer_t ** buf_out ) { hb_work_private_t * pv = w->private_data; hb_buffer_t * in = *buf_in; if (pv->stream->done) { FlushBuffer(pv->stream->common); return HB_WORK_DONE; } if (in->s.flags & HB_BUF_FLAG_EOF) { pv->stream->flush = 1; // sanitizeSubtitle requires EOF buffer to recognize that // it needs to flush all subtitles. hb_list_add(pv->stream->in_queue, hb_buffer_eof_init()); flushStreamsLock(pv->common); if (pv->common->job->indepth_scan) { // When doing subtitle indepth scan, the pipeline ends at sync. // Terminate job when EOF reached. *w->done = 1; } return HB_WORK_DONE; } *buf_in = NULL; QueueBuffer(pv->stream, in); Synchronize(pv->stream); if (pv->stream->done) { FlushBuffer(pv->stream->common); return HB_WORK_DONE; } return HB_WORK_OK; } hb_work_object_t hb_sync_subtitle = { WORK_SYNC_SUBTITLE, "Subitle Synchronization", syncSubtitleInit, syncSubtitleWork, syncSubtitleClose };