/* sync.c
Copyright (c) 2003-2019 HandBrake Team
This file is part of the HandBrake source code
Homepage: .
It may be used under the terms of the GNU General Public License v2.
For full terms see the file COPYING file or visit http://www.gnu.org/licenses/gpl-2.0.html
*/
#include "hb.h"
#include "hbffmpeg.h"
#include
#include "audio_resample.h"
#define SYNC_MAX_VIDEO_QUEUE_LEN 40
#define SYNC_MIN_VIDEO_QUEUE_LEN 20
// Audio is small, buffer a lot. It helps to ensure that we see
// the initial PTS from all input streams before setting the 'zero' point.
#define SYNC_MAX_AUDIO_QUEUE_LEN 200
#define SYNC_MIN_AUDIO_QUEUE_LEN 30
// We do not place a limit on the number of subtitle frames
// that are buffered (max_len == INT_MAX) because there are
// cases where we will receive all the subtitles for a file
// all at once (SSA subs).
//
// If we did not buffer these subs here, the following deadlock
// condition would occur:
// 1. Subtitle decoder blocks trying to generate more subtitle
// lines than will fit in sync input buffers.
// 2. This blocks the reader. Reader doesn't read any more
// audio or video, so sync won't receive buffers it needs
// to unblock subtitles.
#define SYNC_MAX_SUBTITLE_QUEUE_LEN INT_MAX
#define SYNC_MIN_SUBTITLE_QUEUE_LEN 0
typedef enum
{
SYNC_TYPE_VIDEO,
SYNC_TYPE_AUDIO,
SYNC_TYPE_SUBTITLE,
} sync_type_t;
typedef struct
{
int64_t pts;
int64_t delta;
} sync_delta_t;
typedef struct
{
int link;
int merge;
hb_buffer_list_t list_current;
} subtitle_sanitizer_t;
typedef struct sync_common_s sync_common_t;
#define SCR_HASH_SZ (2 << 3)
#define SCR_HASH_MASK (SCR_HASH_SZ - 1)
typedef struct
{
int scr_sequence;
int64_t scr_offset;
} scr_t;
typedef struct
{
sync_common_t * common;
// Stream I/O control
int done;
int flush;
hb_list_t * in_queue;
hb_list_t * scr_delay_queue;
int max_len;
int min_len;
hb_fifo_t * fifo_in;
hb_fifo_t * fifo_out;
// PTS synchronization
hb_list_t * delta_list;
int64_t pts_slip;
double next_pts;
double last_pts;
// SCR recovery
int last_scr_sequence;
double last_scr_pts;
double last_duration;
// frame statistics
int64_t first_pts;
double min_frame_duration;
double max_frame_duration;
int64_t current_duration;
int frame_count;
// Error reporting stats
int64_t drop_duration;
int drop;
int64_t drop_pts;
int64_t gap_duration;
int64_t gap_pts;
int first_frame;
// stream type specific context
sync_type_t type;
union
{
struct
{
int id;
int cadence[12];
int new_chap;
} video;
// Audio stream context
struct
{
hb_audio_t * audio;
// Audio filter settings
// Samplerate conversion
hb_audio_resample_t * resample;
double gain_factor;
} audio;
// Subtitle stream context
struct
{
hb_subtitle_t * subtitle;
subtitle_sanitizer_t sanitizer;
} subtitle;
};
} sync_stream_t;
struct sync_common_s
{
// Audio/Video sync thread synchronization
hb_job_t * job;
hb_lock_t * mutex;
int stream_count;
sync_stream_t * streams;
int found_first_pts;
// SCR adjustments
scr_t scr[SCR_HASH_SZ];
int first_scr;
// point-to-point support
int start_found;
int64_t pts_to_start;
int64_t start_pts;
int64_t stop_pts;
int wait_for_frame;
int wait_for_pts;
// sync audio work objects
hb_list_t * list_work;
// UpdateState Statistics
int est_frame_count;
uint64_t st_counts[4];
uint64_t st_dates[4];
uint64_t st_first;
int chapter;
};
struct hb_work_private_s
{
sync_common_t * common;
sync_stream_t * stream;
};
/***********************************************************************
* Local prototypes
**********************************************************************/
static void UpdateState( sync_common_t * common, int frame_count );
static void UpdateSearchState( sync_common_t * common, int64_t start,
int frame_count );
static int UpdateSCR( sync_stream_t * stream, hb_buffer_t * buf );
static hb_buffer_t * FilterAudioFrame( sync_stream_t * stream,
hb_buffer_t *buf );
static void SortedQueueBuffer( sync_stream_t * stream, hb_buffer_t * buf );
static hb_buffer_t * sanitizeSubtitle(sync_stream_t * stream,
hb_buffer_t * sub);
static void saveChap( sync_stream_t * stream, hb_buffer_t * buf )
{
if (stream->type != SYNC_TYPE_VIDEO || buf == NULL)
{
return;
}
if (buf->s.new_chap > 0)
{
stream->video.new_chap = buf->s.new_chap;
}
}
static void restoreChap( sync_stream_t * stream, hb_buffer_t * buf )
{
if (stream->type != SYNC_TYPE_VIDEO || buf == NULL)
{
return;
}
if (stream->video.new_chap > 0 && buf->s.new_chap <= 0)
{
buf->s.new_chap = stream->video.new_chap;
stream->video.new_chap = 0;
}
}
static int fillQueues( sync_common_t * common )
{
int ii, wait = 0, abort = 0;
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t *stream = &common->streams[ii];
// Don't let the queues grow indefinitely
// abort when too large
if (hb_list_count(stream->in_queue) > stream->max_len)
{
abort = 1;
}
if (hb_list_count(stream->in_queue) <= stream->min_len)
{
wait = 1;
}
}
return !wait || abort;
}
static void scrSlip( sync_common_t * common, int64_t delta )
{
int ii;
for (ii = 0; ii < SCR_HASH_SZ; ii++)
{
common->scr[ii].scr_offset += delta;
}
}
static void shiftTS( sync_common_t * common, int64_t delta )
{
int ii, jj;
scrSlip(common, delta);
for (ii = 0; ii < common->stream_count; ii++)
{
hb_buffer_t * buf = NULL;
sync_stream_t * stream = &common->streams[ii];
int count = hb_list_count(stream->in_queue);
for (jj = 0; jj < count; jj++)
{
buf = hb_list_item(stream->in_queue, jj);
if (buf->s.start != AV_NOPTS_VALUE)
{
buf->s.start -= delta;
}
if (buf->s.stop != AV_NOPTS_VALUE)
{
buf->s.stop -= delta;
}
}
if (buf != NULL && buf->s.start != AV_NOPTS_VALUE)
{
stream->last_scr_pts = buf->s.start + buf->s.duration;
}
else
{
stream->last_scr_pts = (int64_t)AV_NOPTS_VALUE;
}
}
}
static hb_buffer_t * CreateSilenceBuf( sync_stream_t * stream,
int64_t dur, int64_t pts )
{
double frame_dur, next_pts, duration;
int size;
hb_buffer_list_t list;
hb_buffer_t * buf;
if (stream->audio.audio->config.out.codec & HB_ACODEC_PASS_FLAG)
{
return NULL;
}
duration = dur;
// Although frame size isn't technically important any more, we
// keep audio buffer durations <= input audio buffer durations.
frame_dur = (90000. * stream->audio.audio->config.in.samples_per_frame) /
stream->audio.audio->config.in.samplerate;
// Audio mixdown occurs in decoders before sync.
// So number of channels here is output channel count.
// But audio samplerate conversion happens in later here in sync.c
// FilterAudioFrame, so samples_per_frame is still the input sample count.
size = sizeof(float) * stream->audio.audio->config.in.samples_per_frame *
hb_mixdown_get_discrete_channel_count(
stream->audio.audio->config.out.mixdown);
hb_buffer_list_clear(&list);
next_pts = pts;
while (duration >= frame_dur)
{
buf = hb_buffer_init(size);
memset(buf->data, 0, buf->size);
buf->s.start = next_pts;
next_pts += frame_dur;
buf->s.stop = next_pts;
buf->s.duration = frame_dur;
duration -= frame_dur;
hb_buffer_list_append(&list, buf);
}
if (duration > 0)
{
// Make certain size is even multiple of sample size * num channels
size = (int)(duration * stream->audio.audio->config.in.samplerate /
90000) * sizeof(float) *
hb_mixdown_get_discrete_channel_count(
stream->audio.audio->config.out.mixdown);
if (size > 0)
{
buf = hb_buffer_init(size);
memset(buf->data, 0, buf->size);
buf->s.start = next_pts;
next_pts += duration;
buf->s.stop = next_pts;
buf->s.duration = duration;
hb_buffer_list_append(&list, buf);
}
}
return hb_buffer_list_clear(&list);
}
static hb_buffer_t * CreateBlackBuf( sync_stream_t * stream,
int64_t dur, int64_t pts )
{
// I would like to just allocate one black frame here and give
// it the full duration. But encoders that use B-Frames compute
// the dts delay based on the pts of the 2nd or 3rd frame which
// can be a very large value in this case. This large dts delay
// causes problems with computing the dts of the frames (which is
// extrapolated from the pts and the computed dts delay). And the
// dts problems lead to problems with frame duration.
double frame_dur, next_pts, duration;
hb_buffer_list_t list;
hb_buffer_t * buf = NULL;
hb_buffer_list_clear(&list);
duration = dur;
next_pts = pts;
frame_dur = 90000. * stream->common->job->title->vrate.den /
stream->common->job->title->vrate.num;
// Only create black buffers of frame_dur or longer
while (duration >= frame_dur)
{
if (buf == NULL)
{
buf = hb_frame_buffer_init(AV_PIX_FMT_YUV420P,
stream->common->job->title->geometry.width,
stream->common->job->title->geometry.height);
memset(buf->plane[0].data, 0x00, buf->plane[0].size);
memset(buf->plane[1].data, 0x80, buf->plane[1].size);
memset(buf->plane[2].data, 0x80, buf->plane[2].size);
}
else
{
buf = hb_buffer_dup(buf);
}
buf->s.start = next_pts;
next_pts += frame_dur;
buf->s.stop = next_pts;
buf->s.duration = frame_dur;
duration -= frame_dur;
hb_buffer_list_append(&list, buf);
}
if (buf != NULL)
{
if (buf->s.stop < pts + dur)
{
// Extend the duration of the last black buffer to fill
// the remaining gap.
buf->s.duration += pts + dur - buf->s.stop;
buf->s.stop = pts + dur;
}
}
return hb_buffer_list_clear(&list);
}
static void setNextPts( sync_common_t * common )
{
int ii;
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
hb_buffer_t * buf = hb_list_item(stream->in_queue, 0);
if (buf != NULL)
{
stream->next_pts = buf->s.start;
}
else
{
stream->next_pts = (int64_t)AV_NOPTS_VALUE;
}
}
}
static void alignStream( sync_common_t * common, sync_stream_t * stream,
int64_t pts )
{
if (hb_list_count(stream->in_queue) <= 0 ||
stream->type == SYNC_TYPE_SUBTITLE)
{
return;
}
hb_buffer_t * buf = hb_list_item(stream->in_queue, 0);
int64_t gap = buf->s.start - pts;
if (gap == 0)
{
return;
}
if (gap < 0)
{
int ii;
// Drop frames from other streams
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * other_stream = &common->streams[ii];
if (stream == other_stream)
{
continue;
}
while (hb_list_count(other_stream->in_queue) > 0)
{
buf = hb_list_item(other_stream->in_queue, 0);
if (buf->s.start < pts)
{
hb_list_rem(other_stream->in_queue, buf);
hb_buffer_close(&buf);
}
else
{
// Fill the partial frame gap left after dropping frames
alignStream(common, other_stream, pts);
break;
}
}
}
}
else
{
hb_buffer_t * blank_buf = NULL;
// Insert a blank frame to fill the gap
if (stream->type == SYNC_TYPE_AUDIO)
{
// Can't add silence padding to passthru streams
if (!(stream->audio.audio->config.out.codec & HB_ACODEC_PASS_FLAG))
{
blank_buf = CreateSilenceBuf(stream, gap, pts);
}
}
else if (stream->type == SYNC_TYPE_VIDEO)
{
blank_buf = CreateBlackBuf(stream, gap, pts);
}
int64_t last_stop = pts;
hb_buffer_t * next;
int pos;
for (pos = 0; blank_buf != NULL; blank_buf = next, pos++)
{
last_stop = blank_buf->s.stop;
next = blank_buf->next;
blank_buf->next = NULL;
hb_list_insert(stream->in_queue, pos, blank_buf);
}
if (stream->type == SYNC_TYPE_VIDEO && last_stop < buf->s.start)
{
// Extend the duration of the first frame to fill the remaining gap.
buf->s.duration += buf->s.start - last_stop;
buf->s.start = last_stop;
}
}
}
static void alignStreams( sync_common_t * common, int64_t pts )
{
int ii;
hb_buffer_t * buf;
if (common->job->align_av_start)
{
int64_t first_pts = AV_NOPTS_VALUE;
int audio_passthru = 0;
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
buf = hb_list_item(stream->in_queue, 0);
// P-to-P encoding will pass the start point in pts.
// Drop any buffers that are before the start point.
while (buf != NULL && buf->s.start < pts)
{
hb_list_rem(stream->in_queue, buf);
hb_buffer_close(&buf);
buf = hb_list_item(stream->in_queue, 0);
}
if (buf == NULL)
{
continue;
}
if (stream->type == SYNC_TYPE_AUDIO &&
stream->audio.audio->config.out.codec & HB_ACODEC_PASS_FLAG)
{
// Find the largest initial pts of all passthru audio streams.
// We can not add silence to passthru audio streams.
// To align with a passthru audio stream, we must drop
// buffers from all other streams that are before
// the first buffer in the passthru audio stream.
audio_passthru = 1;
if (first_pts < buf->s.start)
{
first_pts = buf->s.start;
}
}
else if (!audio_passthru)
{
// Find the smallest initial pts of all streams when
// there is *no* passthru audio.
// When there is no passthru audio stream, we insert
// silence or black buffers to fill any gaps between
// the start of any stream and the start of the stream
// with the smallest pts.
if (first_pts == AV_NOPTS_VALUE || first_pts > buf->s.start)
{
first_pts = buf->s.start;
}
}
}
if (first_pts != AV_NOPTS_VALUE)
{
for (ii = 0; ii < common->stream_count; ii++)
{
// Fill the gap (or drop frames for passthru audio)
alignStream(common, &common->streams[ii], first_pts);
}
}
}
}
static void computeInitialTS( sync_common_t * common,
sync_stream_t * first_stream )
{
int ii;
hb_buffer_t * prev, * buf;
// Process first_stream first since it has the initial PTS
prev = NULL;
for (ii = 0; ii < hb_list_count(first_stream->in_queue);)
{
buf = hb_list_item(first_stream->in_queue, ii);
if (!UpdateSCR(first_stream, buf))
{
hb_list_rem(first_stream->in_queue, buf);
}
else
{
ii++;
if (first_stream->type == SYNC_TYPE_VIDEO && prev != NULL)
{
double duration = buf->s.start - prev->s.start;
if (duration > 0)
{
prev->s.duration = duration;
prev->s.stop = buf->s.start;
}
}
prev = buf;
}
}
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
if (stream == first_stream)
{
// skip first_stream, already done
continue;
}
int jj;
prev = NULL;
for (jj = 0; jj < hb_list_count(stream->in_queue);)
{
buf = hb_list_item(stream->in_queue, jj);
if (!UpdateSCR(stream, buf))
{
// Subtitle put into delay queue, remove it from in_queue
hb_list_rem(stream->in_queue, buf);
}
else
{
jj++;
if (stream->type == SYNC_TYPE_VIDEO && prev != NULL)
{
double duration = buf->s.start - prev->s.start;
if (duration > 0)
{
prev->s.duration = duration;
prev->s.stop = buf->s.start;
}
}
prev = buf;
}
}
}
alignStreams(common, AV_NOPTS_VALUE);
}
static void checkFirstPts( sync_common_t * common )
{
int ii;
int64_t first_pts = INT64_MAX;
sync_stream_t * first_stream = NULL;
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t *stream = &common->streams[ii];
if (stream->type == SYNC_TYPE_SUBTITLE)
{
// only wait for audio and video
continue;
}
// If buffers are queued, find the lowest initial PTS
while (hb_list_count(stream->in_queue) > 0)
{
hb_buffer_t * buf = hb_list_item(stream->in_queue, 0);
if (buf->s.start != AV_NOPTS_VALUE)
{
// We require an initial pts for every stream
if (buf->s.start < first_pts)
{
first_pts = buf->s.start;
first_stream = stream;
}
break;
}
else
{
hb_list_rem(stream->in_queue, buf);
hb_buffer_close(&buf);
}
}
}
// We should *always* find a first pts because we let the queues
// fill before performing this test.
if (first_pts != INT64_MAX)
{
common->found_first_pts = 1;
// We may have buffers that have no timestamps (i.e. AV_NOPTS_VALUE).
// Compute these timestamps based on previous buffer's timestamp
// and duration.
computeInitialTS(common, first_stream);
// After this initialization, all AV_NOPTS_VALUE timestamps
// will be filled in by UpdateSCR()
}
else
{
// This should never happen
hb_error("checkFirstPts: No initial PTS found!\n");
}
}
static void addDelta( sync_common_t * common, int64_t start, int64_t delta)
{
int ii;
for (ii = 0; ii < common->stream_count; ii++)
{
sync_delta_t * delta_item = malloc(sizeof(sync_delta_t));
delta_item->pts = start;
delta_item->delta = delta;
hb_list_add(common->streams[ii].delta_list, delta_item);
}
}
static void applyDeltas( sync_common_t * common )
{
int ii;
// Apply delta to any applicable buffers in the queue
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
// Make adjustments for deltas found in other streams
sync_delta_t * delta = hb_list_item(stream->delta_list, 0);
if (delta != NULL)
{
int jj, index = -1;
int64_t prev_start, max = 0;
hb_buffer_t * buf;
prev_start = stream->next_pts;
for (jj = 0; jj < hb_list_count(stream->in_queue); jj++)
{
buf = hb_list_item(stream->in_queue, jj);
if (stream->type == SYNC_TYPE_SUBTITLE)
{
if (buf->s.start > delta->pts)
{
// absorb gaps in subtitles as soon as possible
index = jj;
break;
}
}
else if (buf->s.start > delta->pts)
{
// absorb gap in largest gap found in this stream.
if (buf->s.start - prev_start > max)
{
max = buf->s.start - prev_start;
index = jj;
}
if (stream->type == SYNC_TYPE_AUDIO && max >= delta->delta)
{
// absorb gaps in audio as soon as possible
// when there is a gap that will absorb it.
break;
}
}
prev_start = buf->s.start;
}
if (index >= 0)
{
for (jj = index; jj < hb_list_count(stream->in_queue); jj++)
{
buf = hb_list_item(stream->in_queue, jj);
buf->s.start -= delta->delta;
if (buf->s.stop != AV_NOPTS_VALUE)
{
buf->s.stop -= delta->delta;
}
}
// Correct the duration of the video buffer before
// the affected timestamp correction.
if (stream->type == SYNC_TYPE_VIDEO && index > 0)
{
buf = hb_list_item(stream->in_queue, index - 1);
if (buf->s.duration > delta->delta)
{
buf->s.duration -= delta->delta;
buf->s.stop -= delta->delta;
}
else
{
buf->s.duration = 0;
buf->s.stop = buf->s.start;
}
}
stream->pts_slip += delta->delta;
hb_list_rem(stream->delta_list, delta);
free(delta);
}
}
}
}
static void removeVideoJitter( sync_stream_t * stream, int stop )
{
int ii;
hb_buffer_t * buf;
double frame_duration, next_pts;
frame_duration = 90000. * stream->common->job->title->vrate.den /
stream->common->job->title->vrate.num;
buf = hb_list_item(stream->in_queue, 0);
buf->s.start = stream->next_pts;
next_pts = stream->next_pts + frame_duration;
for (ii = 1; ii <= stop; ii++)
{
buf->s.duration = frame_duration;
buf->s.stop = next_pts;
buf = hb_list_item(stream->in_queue, ii);
buf->s.start = next_pts;
next_pts += frame_duration;
}
}
// Look for a sequence of packets whose duration as measure by
// vrate closely matches the duration as measured by timestamp
// differences. When close matches are found, smooth the timestamps.
//
// Most often, video dejitter is applied when there is jitter due to
// soft telecine. I also have a couple sample files that have very
// bad video jitter that this corrects.
static void dejitterVideo( sync_stream_t * stream )
{
int ii, count, jitter_stop;
double frame_duration, duration;
hb_buffer_t * buf;
count = hb_list_count(stream->in_queue);
if (count < 2)
{
return;
}
frame_duration = 90000. * stream->common->job->title->vrate.den /
stream->common->job->title->vrate.num;
// Look for start of jittered sequence
buf = hb_list_item(stream->in_queue, 1);
duration = buf->s.start - stream->next_pts;
if (ABS(duration - frame_duration) < 1.1)
{
// Ignore small jitter
buf->s.start = stream->next_pts + frame_duration;
buf = hb_list_item(stream->in_queue, 0);
buf->s.start = stream->next_pts;
buf->s.duration = frame_duration;
buf->s.stop = stream->next_pts + frame_duration;
return;
}
// Look for end of jittered sequence
jitter_stop = 0;
for (ii = 1; ii < count; ii++)
{
buf = hb_list_item(stream->in_queue, ii);
duration = buf->s.start - stream->next_pts;
// Only dejitter video that aligns periodically
// with the frame durations.
if (ABS(duration - ii * frame_duration) < frame_duration / 3)
{
jitter_stop = ii;
}
}
if (jitter_stop > 0)
{
removeVideoJitter(stream, jitter_stop);
}
}
// Fix video overlaps that could not be corrected by dejitter
static void fixVideoOverlap( sync_stream_t * stream )
{
int drop = 0;
int64_t overlap;
hb_buffer_t * buf;
// If time goes backwards drop the frame.
// Check if subsequent buffers also overlap.
while ((buf = hb_list_item(stream->in_queue, 0)) != NULL)
{
// For video, an overlap is where the entire frame is
// in the past.
overlap = stream->next_pts - buf->s.stop;
if (overlap >= 0)
{
if (stream->drop == 0)
{
stream->drop_pts = buf->s.start;
}
hb_list_rem(stream->in_queue, buf);
// Video frame durations are assumed to be variable and are
// adjusted based on the start time of the next frame before
// we get to this point.
//
// Estimate duration dropped based on average framerate
stream->drop_duration +=
90000. * stream->common->job->title->vrate.den /
stream->common->job->title->vrate.num;
stream->drop++;
drop++;
saveChap(stream, buf);
hb_buffer_close(&buf);
}
else
{
break;
}
}
if (drop <= 0 && stream->drop > 0)
{
hb_log("sync: video time went backwards %d ms, dropped %d frames. "
"PTS %"PRId64"",
(int)stream->drop_duration / 90, stream->drop, stream->drop_pts);
stream->drop_duration = 0;
stream->drop = 0;
}
}
static void removeAudioJitter(sync_stream_t * stream, int stop)
{
int ii;
hb_buffer_t * buf;
double next_pts;
// If duration of sum of packet durations is close to duration
// as measured by timestamps, align timestamps to packet durations.
// The packet durations are computed based on samplerate and
// number of samples and are therefore a reliable measure
// of the actual duration of an audio frame.
buf = hb_list_item(stream->in_queue, 0);
buf->s.start = stream->next_pts;
next_pts = stream->next_pts + buf->s.duration;
for (ii = 1; ii <= stop; ii++)
{
// Duration can be fractional, so track fractional PTS
buf->s.stop = next_pts;
buf = hb_list_item(stream->in_queue, ii);
buf->s.start = next_pts;
next_pts += buf->s.duration;
}
}
// Look for a sequence of packets whose duration as measure by packet
// durations closely matches the duration as measured by timestamp
// differences. When close matches are found, smooth the timestamps.
//
// This fixes issues where there are false overlaps and gaps in audio
// timestamps. libav creates this type of problem sometimes with it's
// timestamp guessing code.
static void dejitterAudio( sync_stream_t * stream )
{
int ii, count, jitter_stop;
double duration;
hb_buffer_t * buf, * buf0, * buf1;
count = hb_list_count(stream->in_queue);
if (count < 4)
{
return;
}
// Look for start of jitter sequence
jitter_stop = 0;
buf0 = hb_list_item(stream->in_queue, 0);
buf1 = hb_list_item(stream->in_queue, 1);
if (ABS(buf0->s.duration - (buf1->s.start - stream->next_pts)) < 1.1)
{
// Ignore very small jitter
return;
}
buf = hb_list_item(stream->in_queue, 0);
duration = buf->s.duration;
// Look for end of jitter sequence
for (ii = 1; ii < count; ii++)
{
buf = hb_list_item(stream->in_queue, ii);
if (ABS(duration - (buf->s.start - stream->next_pts)) < (90 * 40))
{
// Finds the largest span that has low jitter
jitter_stop = ii;
}
duration += buf->s.duration;
}
if (jitter_stop >= 4)
{
removeAudioJitter(stream, jitter_stop);
}
}
// Fix audio gaps that could not be corrected with dejitter
static void fixAudioGap( sync_stream_t * stream )
{
int64_t gap;
hb_buffer_t * buf;
if (hb_list_count(stream->in_queue) < 1 || !stream->first_frame)
{
// Can't find gaps with < 1 buffers
return;
}
buf = hb_list_item(stream->in_queue, 0);
gap = buf->s.start - stream->next_pts;
// If there's a gap of more than a minute between the last
// frame and this, assume we got a corrupted timestamp.
if (gap > 90 * 20 && gap < 90000LL * 60)
{
if (stream->gap_duration <= 0)
{
stream->gap_pts = buf->s.start;
}
buf = CreateSilenceBuf(stream, gap, stream->next_pts);
if (buf != NULL)
{
hb_buffer_t * next;
int pos;
for (pos = 0; buf != NULL; buf = next, pos++)
{
next = buf->next;
buf->next = NULL;
hb_list_insert(stream->in_queue, pos, buf);
}
}
else
{
addDelta(stream->common, stream->next_pts, gap);
applyDeltas(stream->common);
}
stream->gap_duration += gap;
}
else
{
// If not fixing the gap, carry to the next frame
// so they do not accumulate. Do not carry negative gaps.
// Those are overlaps and are handled by fixAudioOverlap.
if (gap >= 90000LL * 60)
{
// Fix "corrupted" timestamp
buf->s.start = stream->next_pts;
}
if (stream->gap_duration > 0)
{
hb_deep_log(3, "sync: audio 0x%x time gap %d ms. PTS %"PRId64"",
stream->audio.audio->id, (int)stream->gap_duration / 90,
stream->gap_pts);
stream->gap_duration = 0;
}
}
}
// Fix audio overlaps that could not be corrected with dejitter
static void fixAudioOverlap( sync_stream_t * stream )
{
int drop = 0;
int64_t overlap;
hb_buffer_t * buf;
if (!stream->first_frame)
{
// There are no overlaps if we haven't seen the first frame yet.
return;
}
// If time goes backwards drop the frame.
// Check if subsequent buffers also overlap.
while ((buf = hb_list_item(stream->in_queue, 0)) != NULL)
{
overlap = stream->next_pts - buf->s.start;
if (overlap > 90 * 20)
{
if (stream->drop == 0)
{
stream->drop_pts = buf->s.start;
}
// This is likely to generate a gap in audio timestamps
// This will be subsequently handled by the call to
// fix AudioGap in Synchronize(). Small gaps will be handled
// by just shifting the timestamps and carrying the gap
// along.
hb_list_rem(stream->in_queue, buf);
stream->drop_duration += buf->s.duration;
stream->drop++;
drop++;
hb_buffer_close(&buf);
}
else
{
break;
}
}
if (drop <= 0 && stream->drop > 0)
{
hb_log("sync: audio 0x%x time went backwards %d ms, dropped %d frames. "
"PTS %"PRId64"",
stream->audio.audio->id, (int)stream->drop_duration / 90,
stream->drop, stream->drop_pts);
stream->drop_duration = 0;
stream->drop = 0;
}
}
static void fixSubtitleOverlap( sync_stream_t * stream )
{
hb_buffer_t * buf;
buf = hb_list_item(stream->in_queue, 0);
if (buf == NULL || (buf->s.flags & HB_BUF_FLAG_EOS) ||
(buf->s.flags & HB_BUF_FLAG_EOF))
{
// marker to indicate the end of a subtitle
return;
}
// Only SSA subs can overlap
if (stream->subtitle.subtitle->source != SSASUB &&
stream->subtitle.subtitle->config.dest == PASSTHRUSUB &&
buf->s.start <= stream->last_pts)
{
int64_t overlap;
overlap = stream->last_pts - buf->s.start;
hb_log("sync: subtitle 0x%x time went backwards %d ms, PTS %"PRId64"",
stream->subtitle.subtitle->id, (int)overlap / 90,
buf->s.start);
hb_list_rem(stream->in_queue, buf);
hb_buffer_close(&buf);
}
}
static void fixStreamTimestamps( sync_stream_t * stream )
{
// Fix gaps and overlaps in queue
if (stream->type == SYNC_TYPE_AUDIO)
{
dejitterAudio(stream);
fixAudioOverlap(stream);
fixAudioGap(stream);
}
else if (stream->type == SYNC_TYPE_VIDEO)
{
dejitterVideo(stream);
fixVideoOverlap(stream);
}
else if (stream->type == SYNC_TYPE_SUBTITLE)
{
fixSubtitleOverlap(stream);
}
}
static void fifo_push( hb_fifo_t * fifo, hb_buffer_t * buf )
{
if (fifo != NULL)
{
hb_fifo_push(fifo, buf);
}
else
{
hb_buffer_close(&buf);
}
}
static void streamFlush( sync_stream_t * stream )
{
while (hb_list_count(stream->in_queue) > 0)
{
hb_buffer_t * buf;
if (!stream->common->found_first_pts)
{
checkFirstPts(stream->common);
}
if (stream->next_pts == (int64_t)AV_NOPTS_VALUE)
{
// Initialize next_pts, it is used to make timestamp corrections
buf = hb_list_item(stream->in_queue, 0);
if (buf != NULL)
{
stream->next_pts = buf->s.start;
}
}
fixStreamTimestamps(stream);
buf = hb_list_item(stream->in_queue, 0);
if (buf != NULL)
{
hb_list_rem(stream->in_queue, buf);
if (!stream->first_frame)
{
if (buf->s.start >= 0)
{
switch (stream->type)
{
case SYNC_TYPE_VIDEO:
hb_log("sync: first pts video is %"PRId64,
buf->s.start);
break;
case SYNC_TYPE_AUDIO:
hb_log("sync: first pts audio 0x%x is %"PRId64,
stream->audio.audio->id, buf->s.start);
break;
case SYNC_TYPE_SUBTITLE:
hb_log("sync: first pts subtitle 0x%x is %"PRId64,
stream->subtitle.subtitle->id, buf->s.start);
break;
default:
break;
}
stream->first_frame = 1;
stream->first_pts = buf->s.start;
stream->min_frame_duration = buf->s.duration;
}
stream->next_pts = buf->s.start;
}
if (stream->type == SYNC_TYPE_AUDIO)
{
buf = FilterAudioFrame(stream, buf);
if (buf == NULL)
{
// FilterAudioFrame can consume the buffer with no output
continue;
}
}
int64_t subtitle_last_pts = AV_NOPTS_VALUE;
if (stream->type == SYNC_TYPE_SUBTITLE)
{
buf = sanitizeSubtitle(stream, buf);
if (buf == NULL)
{
// sanitizeSubtitle can consume the buffer with no output
continue;
}
// sanitizeSubtitle can return a list of subtitles.
// Find the pts of the last subtitle in the list
hb_buffer_t * sub = buf;
while (sub != NULL)
{
subtitle_last_pts = sub->s.start;
sub = sub->next;
}
}
if (stream->type == SYNC_TYPE_AUDIO ||
stream->type == SYNC_TYPE_VIDEO)
{
buf->s.start = stream->next_pts;
buf->s.stop = stream->next_pts + buf->s.duration;
stream->last_pts = stream->next_pts;
stream->next_pts += buf->s.duration;
}
else
{
stream->next_pts =
stream->last_pts = subtitle_last_pts;
}
if (buf->s.stop > 0)
{
stream->current_duration = buf->s.stop - stream->first_pts;
}
stream->frame_count++;
if (buf->s.duration > 0 &&
stream->min_frame_duration > buf->s.duration)
{
stream->min_frame_duration = buf->s.duration;
}
if (stream->max_frame_duration < buf->s.duration)
{
stream->max_frame_duration = buf->s.duration;
}
if (buf->s.start < 0)
{
// The pipeline can't handle negative timestamps
// and it is sometimes not possible to avoid one
// at the start of the video. There can be a
// significant delay before we see the first buffers
// from all streams. We can't buffer indefinitely
// until we have seen the first PTS for all streams
// so sometimes we may start before we have seen
// the earliest PTS
saveChap(stream, buf);
hb_buffer_close(&buf);
}
restoreChap(stream, buf);
fifo_push(stream->fifo_out, buf);
}
}
fifo_push(stream->fifo_out, hb_buffer_eof_init());
}
static void flushStreams( sync_common_t * common )
{
int ii;
// Make sure all streams are complete
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
if (!stream->done && !stream->flush)
{
return;
}
}
// Flush all streams
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
if (stream->done)
{
continue;
}
streamFlush(stream);
}
}
static void flushStreamsLock( sync_common_t * common )
{
hb_lock(common->mutex);
flushStreams(common);
hb_unlock(common->mutex);
}
static void log_chapter( sync_common_t *common, int chap_num,
int nframes, int64_t pts )
{
hb_chapter_t *c;
if ( !common->job )
return;
c = hb_list_item( common->job->list_chapter, chap_num - 1 );
if ( c && c->title )
{
hb_log("sync: \"%s\" (%d) at frame %d time %"PRId64,
c->title, chap_num, nframes, pts);
}
else
{
hb_log("sync: Chapter %d at frame %d time %"PRId64,
chap_num, nframes, pts );
}
}
#define TOP_FIRST PIC_FLAG_TOP_FIELD_FIRST
#define PROGRESSIVE PIC_FLAG_PROGRESSIVE_FRAME
#define REPEAT_FIRST PIC_FLAG_REPEAT_FIRST_FIELD
#define TB 8
#define BT 16
#define BT_PROG 32
#define BTB_PROG 64
#define TB_PROG 128
#define TBT_PROG 256
static void checkCadence( int * cadence, hb_buffer_t * buf )
{
/* Rotate the cadence tracking. */
int i = 0;
for (i = 11; i > 0; i--)
{
cadence[i] = cadence[i-1];
}
if (!(buf->s.flags & PROGRESSIVE) && !(buf->s.flags & TOP_FIRST))
{
/* Not progressive, not top first...
That means it's probably bottom
first, 2 fields displayed.
*/
//hb_log("MPEG2 Flag: Bottom field first, 2 fields displayed.");
cadence[0] = BT;
}
else if (!(buf->s.flags & PROGRESSIVE) && (buf->s.flags & TOP_FIRST))
{
/* Not progressive, top is first,
Two fields displayed.
*/
//hb_log("MPEG2 Flag: Top field first, 2 fields displayed.");
cadence[0] = TB;
}
else if ((buf->s.flags & PROGRESSIVE) &&
!(buf->s.flags & TOP_FIRST) && !(buf->s.flags & REPEAT_FIRST))
{
/* Progressive, but noting else.
That means Bottom first,
2 fields displayed.
*/
//hb_log("MPEG2 Flag: Progressive. Bottom field first, 2 fields displayed.");
cadence[0] = BT_PROG;
}
else if ((buf->s.flags & PROGRESSIVE) &&
!(buf->s.flags & TOP_FIRST) && (buf->s.flags & REPEAT_FIRST))
{
/* Progressive, and repeat. .
That means Bottom first,
3 fields displayed.
*/
//hb_log("MPEG2 Flag: Progressive repeat. Bottom field first, 3 fields displayed.");
cadence[0] = BTB_PROG;
}
else if ((buf->s.flags & PROGRESSIVE) &&
(buf->s.flags & TOP_FIRST) && !(buf->s.flags & REPEAT_FIRST))
{
/* Progressive, top first.
That means top first,
2 fields displayed.
*/
//hb_log("MPEG2 Flag: Progressive. Top field first, 2 fields displayed.");
cadence[0] = TB_PROG;
}
else if ((buf->s.flags & PROGRESSIVE) &&
(buf->s.flags & TOP_FIRST) && (buf->s.flags & REPEAT_FIRST))
{
/* Progressive, top, repeat.
That means top first,
3 fields displayed.
*/
//hb_log("MPEG2 Flag: Progressive repeat. Top field first, 3 fields displayed.");
cadence[0] = TBT_PROG;
}
if ((cadence[2] <= TB) && (cadence[1] <= TB) &&
(cadence[0] > TB) && (cadence[11]))
{
hb_log("%fs: Video -> Film", (float)buf->s.start / 90000);
}
if ((cadence[2] > TB) && (cadence[1] <= TB) &&
(cadence[0] <= TB) && (cadence[11]))
{
hb_log("%fs: Film -> Video", (float)buf->s.start / 90000);
}
}
// When doing point-to-point encoding, subtitles can cause a long
// delay in finishing the job when the stop point is reached. This
// is due to the sparse nature of subtitles. We may not even see
// any additional subtitle till we reach the end of the file.
//
// So when all audio and video streams have reached the end point,
// force the termination of all subtitle streams by pushing an
// end-of-stream buffer into each subtitles input fifo.
// This will cause each subtitle sync worker to wake and return
// HB_WORK_DONE.
static void terminateSubtitleStreams( sync_common_t * common )
{
int ii;
// Make sure all streams are complete
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
if (stream->type == SYNC_TYPE_SUBTITLE)
{
continue;
}
if (!stream->done)
{
return;
}
}
// Terminate all subtitle streams
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
if (stream->done || stream->type != SYNC_TYPE_SUBTITLE)
{
continue;
}
fifo_push(stream->fifo_out, hb_buffer_eof_init());
fifo_push(stream->fifo_in, hb_buffer_eof_init());
stream->done = 1;
}
}
// OutputBuffer pulls buffers from the internal sync buffer queues in
// lowest PTS first order. It then processes the queue the buffer is
// pulled from for frame overlaps and gaps.
//
// If a queue reaches MAX depth, it is possible another queue is too low
// to achieve both goals of pulling lowest PTS first *and* perform
// timestamp correction. In this scenario, we forego lowest PTS and pull
// the next lowest PTS that has enough buffers in the queue to perform
// timestamp correction.
static void OutputBuffer( sync_common_t * common )
{
int ii, full;
int64_t pts;
sync_stream_t * out_stream;
hb_buffer_t * buf;
do
{
full = 0;
out_stream = NULL;
pts = INT64_MAX;
// Find lowest PTS and output that buffer
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
int min = stream->min_len;
// Ignore minimum buffer requirements for video if we have
// not yet found the PtoP start frame.
if (!common->start_found && common->wait_for_frame &&
stream->type == SYNC_TYPE_VIDEO)
{
min = 0;
}
// We need at least 2 buffers in the queue in order to fix
// frame overlaps and inter-frame gaps. So if a queue is
// low, do not do normal PTS interleaving with this queue.
// Except for subtitles which are not processed for gaps
// and overlaps.
if (hb_list_count(stream->in_queue) > min)
{
buf = hb_list_item(stream->in_queue, 0);
if (buf->s.start < pts)
{
pts = buf->s.start;
out_stream = stream;
}
}
// But continue output of buffers as long as one of the queues
// is above the maximum queue level.
if(hb_list_count(stream->in_queue) > stream->max_len)
{
full = 1;
}
}
if (out_stream == NULL)
{
// This should only happen if all queues are below the
// minimum queue level
break;
}
if (out_stream->done)
{
buf = hb_list_item(out_stream->in_queue, 0);
hb_list_rem(out_stream->in_queue, buf);
hb_buffer_close(&buf);
continue;
}
if (out_stream->next_pts == (int64_t)AV_NOPTS_VALUE)
{
// Initialize next_pts, it is used to make timestamp corrections
// If doing p-to-p encoding, it will get reinitialized when
// we find the start point.
buf = hb_list_item(out_stream->in_queue, 0);
out_stream->next_pts = buf->s.start;
}
// Make timestamp adjustments to eliminate jitter, gaps, and overlaps
fixStreamTimestamps(out_stream);
buf = hb_list_item(out_stream->in_queue, 0);
if (buf == NULL)
{
// In case some timestamp sanitization causes the one and
// only buffer in the queue to be deleted...
// This really shouldn't happen.
continue;
}
if (!common->start_found)
{
// pts_to_start or frame_to_start were specified.
// Wait for the appropriate start point.
if (common->wait_for_frame)
{
if (out_stream->type != SYNC_TYPE_VIDEO)
{
// We haven't found the PtoP start frame yet and
// this buffer is either before the start frame or
// the video queue was empty.
out_stream->next_pts = buf->s.start + buf->s.duration;
hb_list_rem(out_stream->in_queue, buf);
hb_buffer_close(&buf);
continue;
}
common->start_pts = buf->s.start + 1;
if (out_stream->frame_count >= common->job->frame_to_start)
{
common->start_found = 1;
out_stream->frame_count = 0;
}
}
else if (common->wait_for_pts &&
out_stream->type != SYNC_TYPE_SUBTITLE)
{
if (buf->s.start >= common->pts_to_start)
{
common->start_found = 1;
common->streams[0].frame_count = 0;
}
}
if (!common->start_found)
{
if (out_stream->type == SYNC_TYPE_VIDEO)
{
UpdateSearchState(common, buf->s.start,
out_stream->frame_count);
out_stream->frame_count++;
}
if (out_stream->type == SYNC_TYPE_SUBTITLE &&
buf->s.stop > common->start_pts)
{
// Subtitle ends after start time, keep sub and
// adjust it's start time
buf->s.start = common->start_pts;
}
else if (buf->s.start < common->start_pts)
{
out_stream->next_pts = buf->s.start + buf->s.duration;
hb_list_rem(out_stream->in_queue, buf);
hb_buffer_close(&buf);
}
continue;
}
// reset frame count to track number of frames after
// the start position till the end of encode.
out_stream->frame_count = 0;
shiftTS(common, buf->s.start);
alignStreams(common, buf->s.start);
setNextPts(common);
buf = hb_list_item(out_stream->in_queue, 0);
if (buf == NULL)
{
// In case aligning timestamps causes all buffers in
// out_stream to be deleted...
continue;
}
}
// If pts_to_stop or frame_to_stop were specified, stop output
if (common->stop_pts &&
buf->s.start >= common->stop_pts )
{
switch (out_stream->type)
{
case SYNC_TYPE_VIDEO:
hb_log("sync: reached video pts %"PRId64", exiting early",
buf->s.start);
break;
case SYNC_TYPE_AUDIO:
hb_log("sync: reached audio 0x%x pts %"PRId64
", exiting early",
out_stream->audio.audio->id, buf->s.start);
break;
case SYNC_TYPE_SUBTITLE:
hb_log("sync: reached subtitle 0x%x pts %"PRId64
", exiting early",
out_stream->subtitle.subtitle->id, buf->s.start);
break;
default:
break;
}
out_stream->done = 1;
fifo_push(out_stream->fifo_out, hb_buffer_eof_init());
terminateSubtitleStreams(common);
flushStreams(common);
continue;
}
if (out_stream->type == SYNC_TYPE_VIDEO &&
common->job->frame_to_stop &&
out_stream->frame_count >= common->job->frame_to_stop)
{
hb_log("sync: reached video frame %d, exiting early",
out_stream->frame_count);
common->stop_pts = buf->s.start;
out_stream->done = 1;
fifo_push(out_stream->fifo_out, hb_buffer_eof_init());
terminateSubtitleStreams(common);
flushStreams(common);
continue;
}
if (out_stream->type == SYNC_TYPE_VIDEO)
{
checkCadence(out_stream->video.cadence, buf);
}
// Out the buffer goes...
hb_list_rem(out_stream->in_queue, buf);
if (out_stream->type == SYNC_TYPE_VIDEO)
{
UpdateState(common, out_stream->frame_count);
}
if (!out_stream->first_frame)
{
if (buf->s.start >= 0)
{
switch (out_stream->type)
{
case SYNC_TYPE_VIDEO:
hb_log("sync: first pts video is %"PRId64,
buf->s.start);
break;
case SYNC_TYPE_AUDIO:
hb_log("sync: first pts audio 0x%x is %"PRId64,
out_stream->audio.audio->id, buf->s.start);
break;
case SYNC_TYPE_SUBTITLE:
hb_log("sync: first pts subtitle 0x%x is %"PRId64,
out_stream->subtitle.subtitle->id, buf->s.start);
break;
default:
break;
}
out_stream->first_frame = 1;
out_stream->first_pts = buf->s.start;
out_stream->min_frame_duration = buf->s.duration;
}
out_stream->next_pts = buf->s.start;
}
if (out_stream->type == SYNC_TYPE_AUDIO)
{
buf = FilterAudioFrame(out_stream, buf);
if (buf == NULL)
{
// FilterAudioFrame can consume the buffer with no output
continue;
}
}
int64_t subtitle_last_pts = AV_NOPTS_VALUE;
if (out_stream->type == SYNC_TYPE_SUBTITLE)
{
buf = sanitizeSubtitle(out_stream, buf);
if (buf == NULL)
{
// sanitizeSubtitle can consume the buffer with no output
continue;
}
// sanitizeSubtitle can return a list of subtitles.
// Find the pts of the last subtitle in the list
hb_buffer_t * sub = buf;
while (sub != NULL)
{
subtitle_last_pts = sub->s.start;
sub = sub->next;
}
}
if (out_stream->type == SYNC_TYPE_AUDIO ||
out_stream->type == SYNC_TYPE_VIDEO)
{
buf->s.start = out_stream->next_pts;
buf->s.stop = out_stream->next_pts + buf->s.duration;
out_stream->last_pts = out_stream->next_pts;
out_stream->next_pts += buf->s.duration;
}
else
{
out_stream->next_pts =
out_stream->last_pts = subtitle_last_pts;
}
if (buf->s.stop > 0)
{
out_stream->current_duration = buf->s.stop -
out_stream->first_pts;
}
out_stream->frame_count++;
if (buf->s.duration > 0 &&
out_stream->min_frame_duration > buf->s.duration)
{
out_stream->min_frame_duration = buf->s.duration;
}
if (out_stream->max_frame_duration < buf->s.duration)
{
out_stream->max_frame_duration = buf->s.duration;
}
if (out_stream->type == SYNC_TYPE_VIDEO &&
buf->s.new_chap > common->chapter)
{
common->chapter = buf->s.new_chap;
log_chapter(common, buf->s.new_chap, out_stream->frame_count,
buf->s.start);
}
if (buf->s.start < 0)
{
// The pipeline can't handle negative timestamps
// and it is sometimes not possible to avoid one
// at the start of the video. There can be a
// significant delay before we see the first buffers
// from all streams. We can't buffer indefinitely
// until we have seen the first PTS for all streams
// so sometimes we may start before we have seen
// the earliest PTS
saveChap(out_stream, buf);
hb_buffer_close(&buf);
}
restoreChap(out_stream, buf);
fifo_push(out_stream->fifo_out, buf);
} while (full);
}
static void FlushBuffer( sync_common_t * common )
{
hb_lock(common->mutex);
if (!common->found_first_pts)
{
checkFirstPts(common);
}
OutputBuffer(common);
hb_unlock(common->mutex);
}
static void Synchronize( sync_stream_t * stream )
{
sync_common_t * common = stream->common;
// Sync deposits output directly into fifos, so work_loop is not
// blocking when output fifos become full. Wait here before
// performing any output when the output fifo for the input stream
// is full
if (stream->fifo_out != NULL && common->start_found)
{
while (!common->job->done && !*common->job->die)
{
if (hb_fifo_full_wait(stream->fifo_out))
{
break;
}
}
}
hb_lock(common->mutex);
if (!fillQueues(common))
{
hb_unlock(common->mutex);
return;
}
if (!common->found_first_pts)
{
checkFirstPts(common);
}
OutputBuffer(common);
hb_unlock(common->mutex);
}
static void updateDuration( sync_stream_t * stream )
{
// The video decoder sets a nominal duration for frames. But the
// actual duration needs to be computed from timestamps.
if (stream->type == SYNC_TYPE_VIDEO)
{
int count = hb_list_count(stream->in_queue);
if (count >= 2)
{
hb_buffer_t * buf1 = hb_list_item(stream->in_queue, count - 1);
hb_buffer_t * buf2 = hb_list_item(stream->in_queue, count - 2);
double duration = buf1->s.start - buf2->s.start;
if (duration > 0)
{
buf2->s.duration = duration;
buf2->s.stop = buf1->s.start;
}
else
{
buf2->s.duration = 0.;
buf2->s.start = buf2->s.stop = buf1->s.start;
}
}
}
}
static void ProcessSCRDelayQueue( sync_common_t * common )
{
int ii, jj;
for (ii = 0; ii < common->stream_count; ii++)
{
sync_stream_t * stream = &common->streams[ii];
for (jj = 0; jj < hb_list_count(stream->scr_delay_queue);)
{
hb_buffer_t * buf = hb_list_item(stream->scr_delay_queue, jj);
int hash = buf->s.scr_sequence & SCR_HASH_MASK;
if (buf->s.scr_sequence < 0)
{
// Unset scr_sequence indicates an external stream
// (e.g. SRT subtitle) that is not on the same timebase
// as the source tracks. Do not adjust timestamps for
// scr_offset in this case.
hb_list_rem(stream->scr_delay_queue, buf);
SortedQueueBuffer(stream, buf);
}
else if (buf->s.scr_sequence == common->scr[hash].scr_sequence)
{
if (buf->s.start != AV_NOPTS_VALUE)
{
buf->s.start -= common->scr[hash].scr_offset;
buf->s.start -= stream->pts_slip;
}
if (buf->s.stop != AV_NOPTS_VALUE)
{
buf->s.stop -= common->scr[hash].scr_offset;
buf->s.stop -= stream->pts_slip;
}
hb_list_rem(stream->scr_delay_queue, buf);
SortedQueueBuffer(stream, buf);
}
else
{
jj++;
}
}
}
}
static const char * getStreamType( sync_stream_t * stream )
{
switch (stream->type)
{
case SYNC_TYPE_VIDEO:
return "Video";
case SYNC_TYPE_AUDIO:
return "Audio";
case SYNC_TYPE_SUBTITLE:
return "Subtitle";
default:
return "Unknown";
}
}
static int getStreamId( sync_stream_t * stream )
{
switch (stream->type)
{
case SYNC_TYPE_VIDEO:
return stream->video.id;
case SYNC_TYPE_AUDIO:
return stream->audio.audio->id;
case SYNC_TYPE_SUBTITLE:
return stream->subtitle.subtitle->id;
default:
return -1;
}
}
static int UpdateSCR( sync_stream_t * stream, hb_buffer_t * buf )
{
int hash = buf->s.scr_sequence & SCR_HASH_MASK;
sync_common_t * common = stream->common;
double last_scr_pts, last_duration;
int64_t scr_offset = 0;
if (buf->s.scr_sequence < stream->last_scr_sequence)
{
// In decoder error conditions, the decoder can send us out of
// order frames. Often the stream error will also trigger a
// discontinuity detection. An out of order frame will
// cause an incorrect SCR offset, so drop such frames.
hb_deep_log(3, "SCR sequence went backwards %d -> %d",
stream->last_scr_sequence, buf->s.scr_sequence);
hb_buffer_close(&buf);
return 0;
}
if (buf->s.scr_sequence >= 0)
{
if (buf->s.scr_sequence != common->scr[hash].scr_sequence)
{
if (stream->type == SYNC_TYPE_SUBTITLE ||
(stream->last_scr_pts == (int64_t)AV_NOPTS_VALUE &&
common->first_scr))
{
// We got a new scr, but we have no last_scr_pts to base it
// off of. Delay till we can compute the scr offset from a
// different stream.
hb_list_add(stream->scr_delay_queue, buf);
return 0;
}
if (buf->s.start != AV_NOPTS_VALUE)
{
last_scr_pts = stream->last_scr_pts;
last_duration = stream->last_duration;
if (last_scr_pts == (int64_t)AV_NOPTS_VALUE)
{
last_scr_pts = 0.;
last_duration = 0.;
common->first_scr = 1;
}
// New SCR. Compute SCR offset
common->scr[hash].scr_sequence = buf->s.scr_sequence;
common->scr[hash].scr_offset = buf->s.start -
(last_scr_pts + last_duration);
hb_deep_log(4,
"New SCR: type %8s id %x scr seq %d scr offset %"PRId64" "
"start %"PRId64" last %f dur %f",
getStreamType(stream), getStreamId(stream),
buf->s.scr_sequence, common->scr[hash].scr_offset,
buf->s.start, last_scr_pts, last_duration);
ProcessSCRDelayQueue(common);
}
}
scr_offset = common->scr[hash].scr_offset;
}
// Adjust buffer timestamps for SCR offset
if (buf->s.start != AV_NOPTS_VALUE)
{
buf->s.start -= scr_offset;
last_scr_pts = buf->s.start;
}
else if (stream->last_scr_pts != (int64_t)AV_NOPTS_VALUE)
{
last_scr_pts = stream->last_scr_pts + stream->last_duration;
buf->s.start = last_scr_pts;
}
else
{
// This should happen extremely rarely if ever.
// But if we get here, this is the first buffer received for
// this stream. Normally, some buffers will be queued for
// every stream before we ever call UpdateSCR.
// We don't really know what it's timestamp should be.
// So drop the buffer.
hb_buffer_close(&buf);
return 0;
}
if (buf->s.stop != AV_NOPTS_VALUE)
{
buf->s.stop -= scr_offset;
}
if (last_scr_pts > stream->last_scr_pts)
{
stream->last_scr_pts = last_scr_pts;
}
if (buf->s.scr_sequence > stream->last_scr_sequence)
{
stream->last_scr_sequence = buf->s.scr_sequence;
}
stream->last_duration = buf->s.duration;
return 1;
}
// Handle broken timestamps that are out of order
// These are usually due to a broken decoder (e.g. QSV and libav AVI packed
// b-frame support). But sometimes can come from a severely broken or
// corrupted source file.
//
// We can pretty reliably fix out of order timestamps *if* every frame
// has a timestamp. But some container formats allow frames with no
// timestamp (e.g. TS and PS). When there is no timestamp, we will
// compute one based on the last frames timestamp. If the missing
// timestamp is out of order and really belonged on an earlier frame (A),
// this will result in the frame before (A) being long and the frame
// after the current will overlap current.
//
// The condition above of one long frame and one overlap will most likely
// get fixed by dejitterVideo. dejitterVideo finds sequences where the
// sum of the durations of frames 1..N == (1/fps) * N. When it finds such
// a sequence, it adjusts the frame durations to all be 1/fps. Since the
// vast majority of video is constant framerate, this will fix the above
// problem most of the time.
static void SortedQueueBuffer( sync_stream_t * stream, hb_buffer_t * buf )
{
int64_t start;
int ii, count;
start = buf->s.start;
hb_list_add(stream->in_queue, buf);
// Search for the first earlier timestamp that is < this one.
// Under normal circumstances where the timestamps are not broken,
// this will only check the next to last buffer in the queue
// before aborting.
count = hb_list_count(stream->in_queue);
for (ii = count - 2; ii >= 0; ii--)
{
buf = hb_list_item(stream->in_queue, ii);
if (buf->s.start < start || start == AV_NOPTS_VALUE)
{
break;
}
}
if (ii < count - 2)
{
hb_buffer_t * prev = NULL;
int jj;
// The timestamp was out of order.
// The timestamp belongs at position ii + 1
// Every timestamp from ii + 2 to count - 1 needs to be shifted up.
if (ii >= 0)
{
prev = hb_list_item(stream->in_queue, ii);
}
for (jj = ii + 1; jj < count; jj++)
{
int64_t tmp_start;
buf = hb_list_item(stream->in_queue, jj);
tmp_start = buf->s.start;
buf->s.start = start;
start = tmp_start;
if (stream->type == SYNC_TYPE_VIDEO && prev != NULL)
{
// recompute video buffer duration
prev->s.duration = buf->s.start - prev->s.start;
prev->s.stop = buf->s.start;
}
prev = buf;
}
}
}
static void QueueBuffer( sync_stream_t * stream, hb_buffer_t * buf )
{
hb_lock(stream->common->mutex);
while (hb_list_count(stream->in_queue) > stream->max_len &&
!stream->done && !stream->common->job->done &&
!*stream->common->job->die)
{
// If the in_queue is full, we have to force some output to
// unblock it. Blocking here would back up the pipeline and
// stall out reader eventually.
hb_unlock(stream->common->mutex);
Synchronize(stream);
hb_lock(stream->common->mutex);
}
// Reader can change job->reader_pts_offset after initialization
// and before we receive the first buffer here. Calculate
// common->pts_to_start here since this is the first opportunity where
// all the necessary information exists.
if (stream->common->pts_to_start == AV_NOPTS_VALUE)
{
hb_job_t * job = stream->common->job;
if (job->pts_to_start > 0)
{
stream->common->start_pts =
stream->common->pts_to_start =
MAX(0, job->pts_to_start - job->reader_pts_offset);
}
}
// Render offset is only useful for decoders, which are all
// upstream of sync. Squash it.
buf->s.renderOffset = AV_NOPTS_VALUE;
hb_deep_log(11,
"type %8s id %x scr seq %d start %"PRId64" stop %"PRId64" dur %f",
getStreamType(stream), getStreamId(stream), buf->s.scr_sequence,
buf->s.start, buf->s.stop, buf->s.duration);
if (stream->common->found_first_pts)
{
if (UpdateSCR(stream, buf) > 0)
{
// Apply any stream slips.
// Stream slips will only temporarily differ between
// the streams. The slips get updated in applyDeltas. When
// all the deltas are absorbed, the stream slips will all
// be equal.
buf->s.start -= stream->pts_slip;
if (buf->s.stop != AV_NOPTS_VALUE)
{
buf->s.stop -= stream->pts_slip;
}
SortedQueueBuffer(stream, buf);
updateDuration(stream);
}
}
else
{
if (buf->s.start == AV_NOPTS_VALUE &&
hb_list_count(stream->in_queue) == 0)
{
// We require an initial pts to start synchronization
saveChap(stream, buf);
hb_buffer_close(&buf);
hb_unlock(stream->common->mutex);
return;
}
SortedQueueBuffer(stream, buf);
}
// Make adjustments for gaps found in other streams
applyDeltas(stream->common);
hb_unlock(stream->common->mutex);
}
static int InitAudio( sync_common_t * common, int index )
{
hb_work_object_t * w = NULL;
hb_work_private_t * pv;
hb_audio_t * audio;
audio = hb_list_item(common->job->list_audio, index);
if (audio->priv.fifo_raw == NULL)
{
// No input fifo, not configured
return 0;
}
pv = calloc(1, sizeof(hb_work_private_t));
if (pv == NULL) goto fail;
w = hb_get_work(common->job->h, WORK_SYNC_AUDIO);
w->private_data = pv;
w->audio = audio;
w->fifo_in = audio->priv.fifo_raw;
if (audio->config.out.codec & HB_ACODEC_PASS_FLAG)
{
w->fifo_out = audio->priv.fifo_out;
}
else
{
w->fifo_out = audio->priv.fifo_sync;
}
pv->common = common;
pv->stream = &common->streams[1 + index];
pv->stream->common = common;
pv->stream->in_queue = hb_list_init();
pv->stream->scr_delay_queue = hb_list_init();
pv->stream->max_len = SYNC_MAX_AUDIO_QUEUE_LEN;
pv->stream->min_len = SYNC_MIN_AUDIO_QUEUE_LEN;
if (pv->stream->in_queue == NULL) goto fail;
pv->stream->delta_list = hb_list_init();
if (pv->stream->delta_list == NULL) goto fail;
pv->stream->type = SYNC_TYPE_AUDIO;
pv->stream->first_pts = AV_NOPTS_VALUE;
pv->stream->next_pts = (int64_t)AV_NOPTS_VALUE;
pv->stream->last_pts = (int64_t)AV_NOPTS_VALUE;
pv->stream->last_scr_pts = (int64_t)AV_NOPTS_VALUE;
pv->stream->last_scr_sequence = -1;
pv->stream->last_duration = (int64_t)AV_NOPTS_VALUE;
pv->stream->audio.audio = audio;
pv->stream->fifo_out = w->fifo_out;
if (!(audio->config.out.codec & HB_ACODEC_PASS_FLAG) &&
audio->config.in.samplerate != audio->config.out.samplerate)
{
/* Initialize samplerate conversion */
pv->stream->audio.resample =
hb_audio_resample_init(AV_SAMPLE_FMT_FLT,
audio->config.out.samplerate,
audio->config.out.mixdown,
audio->config.out.normalize_mix_level);
if (pv->stream->audio.resample == NULL)
{
hb_error("sync: audio 0x%x resample init failed", audio->id);
goto fail;
}
hb_audio_resample_set_sample_rate(pv->stream->audio.resample,
audio->config.in.samplerate);
if (hb_audio_resample_update(pv->stream->audio.resample))
{
hb_error("sync: audio 0x%x resample update failed", audio->id);
goto fail;
}
}
pv->stream->audio.gain_factor = pow(10, audio->config.out.gain / 20);
hb_list_add(common->list_work, w);
return 0;
fail:
if (pv != NULL)
{
if (pv->stream != NULL)
{
if (pv->stream->audio.resample)
{
hb_audio_resample_free(pv->stream->audio.resample);
}
hb_list_close(&pv->stream->delta_list);
hb_list_close(&pv->stream->in_queue);
}
}
free(pv);
free(w);
return 1;
}
static int InitSubtitle( sync_common_t * common, int index )
{
hb_work_object_t * w = NULL;
hb_work_private_t * pv;
hb_subtitle_t * subtitle;
subtitle = hb_list_item(common->job->list_subtitle, index);
if (subtitle->fifo_raw == NULL)
{
// No input fifo, not configured
return 0;
}
pv = calloc(1, sizeof(hb_work_private_t));
if (pv == NULL) goto fail;
pv->common = common;
pv->stream =
&common->streams[1 + hb_list_count(common->job->list_audio) + index];
pv->stream->common = common;
pv->stream->in_queue = hb_list_init();
pv->stream->scr_delay_queue = hb_list_init();
pv->stream->max_len = SYNC_MAX_SUBTITLE_QUEUE_LEN;
pv->stream->min_len = SYNC_MIN_SUBTITLE_QUEUE_LEN;
if (pv->stream->in_queue == NULL) goto fail;
pv->stream->delta_list = hb_list_init();
if (pv->stream->delta_list == NULL) goto fail;
pv->stream->type = SYNC_TYPE_SUBTITLE;
pv->stream->first_pts = AV_NOPTS_VALUE;
pv->stream->next_pts = (int64_t)AV_NOPTS_VALUE;
pv->stream->last_pts = (int64_t)AV_NOPTS_VALUE;
pv->stream->last_scr_pts = (int64_t)AV_NOPTS_VALUE;
pv->stream->last_scr_sequence = -1;
pv->stream->last_duration = (int64_t)AV_NOPTS_VALUE;
pv->stream->subtitle.subtitle = subtitle;
pv->stream->fifo_out = subtitle->fifo_out;
pv->stream->fifo_in = subtitle->fifo_in;
w = hb_get_work(common->job->h, WORK_SYNC_SUBTITLE);
w->private_data = pv;
w->subtitle = subtitle;
w->fifo_in = subtitle->fifo_raw;
w->fifo_out = subtitle->fifo_out;
memset(&pv->stream->subtitle.sanitizer, 0,
sizeof(pv->stream->subtitle.sanitizer));
if (subtitle->format == TEXTSUB && subtitle->config.dest == PASSTHRUSUB &&
(common->job->mux & HB_MUX_MASK_MP4))
{
// Merge overlapping subtitles since mpv tx3g does not support them
pv->stream->subtitle.sanitizer.merge = 1;
}
// PGS subtitles don't need to be linked because there are explicit
// "clear" subtitle packets that indicate the end time of the
// previous subtitle
if (subtitle->config.dest == PASSTHRUSUB &&
subtitle->source != PGSSUB)
{
// Fill in stop time when it is missing
pv->stream->subtitle.sanitizer.link = 1;
}
hb_buffer_list_clear(&pv->stream->subtitle.sanitizer.list_current);
hb_list_add(common->list_work, w);
return 0;
fail:
if (pv != NULL)
{
if (pv->stream != NULL)
{
hb_list_close(&pv->stream->delta_list);
hb_list_close(&pv->stream->in_queue);
}
}
free(pv);
free(w);
return 1;
}
/***********************************************************************
* Initialize the work object
**********************************************************************/
static int syncVideoInit( hb_work_object_t * w, hb_job_t * job)
{
hb_work_private_t * pv;
int ii;
pv = calloc(1, sizeof(hb_work_private_t));
if (pv == NULL) goto fail;
w->private_data = pv;
pv->common = calloc(1, sizeof(sync_common_t));
if (pv->common == NULL) goto fail;
pv->common->job = job;
// count number of streams we need
pv->common->stream_count = 1;
pv->common->stream_count += hb_list_count(job->list_audio);
pv->common->stream_count += hb_list_count(job->list_subtitle);
pv->common->streams = calloc(pv->common->stream_count,
sizeof(sync_stream_t));
// Allocate streams
if (pv->common->streams == NULL) goto fail;
// create audio and subtitle work list
pv->common->list_work = hb_list_init();
if (pv->common->list_work == NULL) goto fail;
// mutex to mediate access to pv->common
pv->common->mutex = hb_lock_init();
if (pv->common->mutex == NULL) goto fail;
// Set up video sync work object
pv->stream = &pv->common->streams[0];
pv->stream->common = pv->common;
pv->stream->in_queue = hb_list_init();
pv->stream->scr_delay_queue = hb_list_init();
pv->stream->max_len = SYNC_MAX_VIDEO_QUEUE_LEN;
pv->stream->min_len = SYNC_MIN_VIDEO_QUEUE_LEN;
if (pv->stream->in_queue == NULL) goto fail;
pv->stream->delta_list = hb_list_init();
if (pv->stream->delta_list == NULL) goto fail;
pv->stream->type = SYNC_TYPE_VIDEO;
pv->stream->first_pts = AV_NOPTS_VALUE;
pv->stream->next_pts = (int64_t)AV_NOPTS_VALUE;
pv->stream->last_pts = (int64_t)AV_NOPTS_VALUE;
pv->stream->last_scr_pts = (int64_t)AV_NOPTS_VALUE;
pv->stream->last_scr_sequence = -1;
pv->stream->last_duration = (int64_t)AV_NOPTS_VALUE;
pv->stream->fifo_out = job->fifo_sync;
pv->stream->video.id = job->title->video_id;
w->fifo_in = job->fifo_raw;
w->fifo_out = job->fifo_sync;
if (job->pass_id == HB_PASS_ENCODE_2ND)
{
/* We already have an accurate frame count from pass 1 */
hb_interjob_t * interjob = hb_interjob_get(job->h);
pv->common->est_frame_count = interjob->frame_count;
}
else
{
/* Calculate how many video frames we are expecting */
if (job->frame_to_stop)
{
pv->common->est_frame_count = job->frame_to_stop;
}
else
{
int64_t duration, total_duration, extra_duration = 0;
if (job->pts_to_stop)
{
duration = job->pts_to_stop + 90000;
}
else
{
total_duration = 0;
for (ii = 0; ii <= hb_list_count(job->list_chapter); ii++)
{
hb_chapter_t * chapter;
chapter = hb_list_item(job->list_chapter, ii - 1);
if (chapter != NULL)
{
total_duration += chapter->duration;
}
}
// Some titles are longer than the sum duration of their
// chapters. Account for this extra duration.
if (job->title->duration > total_duration)
{
extra_duration = job->title->duration - total_duration;
}
duration = 0;
for (ii = job->chapter_start; ii <= job->chapter_end; ii++)
{
hb_chapter_t * chapter;
chapter = hb_list_item(job->list_chapter, ii - 1);
if (chapter != NULL)
{
duration += chapter->duration;
}
}
if (job->chapter_end == hb_list_count(job->list_chapter))
{
duration += extra_duration;
}
}
pv->common->est_frame_count = duration * job->title->vrate.num /
job->title->vrate.den / 90000;
}
}
hb_log("sync: expecting %d video frames", pv->common->est_frame_count);
// Initialize audio sync work objects
for (ii = 0; ii < hb_list_count(job->list_audio); ii++ )
{
if (InitAudio(pv->common, ii)) goto fail;
}
// Initialize subtitle sync work objects
for (ii = 0; ii < hb_list_count(job->list_subtitle); ii++ )
{
if (InitSubtitle(pv->common, ii)) goto fail;
}
/* Launch work processing threads */
for (ii = 0; ii < hb_list_count(pv->common->list_work); ii++)
{
hb_work_object_t * work;
work = hb_list_item(pv->common->list_work, ii);
work->done = w->done;
work->thread = hb_thread_init(work->name, hb_work_loop,
work, HB_LOW_PRIORITY);
}
if (job->frame_to_start || job->pts_to_start)
{
pv->common->start_found = 0;
pv->common->start_pts = job->pts_to_start;
pv->common->wait_for_frame = !!job->frame_to_start;
pv->common->wait_for_pts = !!job->pts_to_start;
if (job->pts_to_start)
{
pv->common->pts_to_start = AV_NOPTS_VALUE;
}
}
else
{
pv->common->start_found = 1;
}
if (job->pts_to_stop)
{
pv->common->stop_pts = job->pts_to_stop;
}
return 0;
fail:
if (pv != NULL)
{
if (pv->common != NULL)
{
for (ii = 0; ii < hb_list_count(pv->common->list_work); ii++)
{
hb_work_object_t * work;
work = hb_list_item(pv->common->list_work, ii);
if (work->close) work->close(work);
}
hb_list_close(&pv->common->list_work);
hb_lock_close(&pv->common->mutex);
if (pv->stream != NULL)
{
hb_list_close(&pv->stream->delta_list);
hb_list_close(&pv->stream->in_queue);
}
free(pv->common->streams);
free(pv->common);
}
}
free(pv);
w->private_data = NULL;
return 1;
}
/***********************************************************************
* Close Video
***********************************************************************
*
**********************************************************************/
static void syncVideoClose( hb_work_object_t * w )
{
hb_work_private_t * pv = w->private_data;
hb_job_t * job;
if (pv == NULL)
{
return;
}
job = pv->common->job;
hb_log("sync: got %d frames, %d expected",
pv->stream->frame_count, pv->common->est_frame_count );
if (pv->stream->min_frame_duration > 0 &&
pv->stream->max_frame_duration > 0 &&
pv->stream->current_duration > 0)
{
hb_log("sync: framerate min %.3f fps, max %.3f fps, avg %.3f fps",
90000. / pv->stream->max_frame_duration,
90000. / pv->stream->min_frame_duration,
(pv->stream->frame_count * 90000.) /
pv->stream->current_duration);
}
/* save data for second pass */
if( job->pass_id == HB_PASS_ENCODE_1ST )
{
/* Preserve frame count for better accuracy in pass 2 */
hb_interjob_t * interjob = hb_interjob_get( job->h );
interjob->frame_count = pv->stream->frame_count;
}
sync_delta_t * delta;
while ((delta = hb_list_item(pv->stream->delta_list, 0)) != NULL)
{
hb_list_rem(pv->stream->delta_list, delta);
free(delta);
}
hb_list_close(&pv->stream->delta_list);
hb_list_empty(&pv->stream->in_queue);
hb_list_empty(&pv->stream->scr_delay_queue);
// Close work threads
hb_work_object_t * work;
while ((work = hb_list_item(pv->common->list_work, 0)))
{
hb_list_rem(pv->common->list_work, work);
if (work->thread != NULL)
{
hb_thread_close(&work->thread);
}
if (work->close) work->close(work);
free(work);
}
hb_list_close(&pv->common->list_work);
hb_lock_close(&pv->common->mutex);
free(pv->common->streams);
free(pv->common);
free(pv);
w->private_data = NULL;
}
static hb_buffer_t * merge_ssa(hb_buffer_t *a, hb_buffer_t *b)
{
int len, ii;
char *text;
hb_buffer_t *buf;
if (a == NULL && b == NULL)
{
return NULL;
}
if (a == NULL)
{
return hb_buffer_dup(b);
}
if (b == NULL)
{
return hb_buffer_dup(a);
}
buf = hb_buffer_init(a->size + b->size);
buf->s = a->s;
// Find the text in the second SSA sub
text = (char*)b->data;
for (ii = 0; ii < 8; ii++)
{
text = strchr(text, ',');
if (text == NULL)
break;
text++;
}
if (text != NULL)
{
// Strip trailing CR and/or LF
len = strlen((char*)a->data);
if (len > 0 && a->data[len - 1] == '\n')
{
a->data[len - 1] = 0;
len--;
if (len > 0 && a->data[len - 1] == '\r')
{
a->data[len - 1] = 0;
}
}
// Text subtitles are SSA internally. Use SSA newline code
// and force style reset at beginning of new line.
len = sprintf((char*)buf->data, "%s\\N{\\r}%s", a->data, text);
if (len >= 0)
buf->size = len + 1;
}
else
{
memcpy(buf->data, a->data, a->size);
buf->size = a->size;
}
return buf;
}
static hb_buffer_t * setSubDuration(sync_stream_t * stream, hb_buffer_t * sub)
{
if (sub->s.flags & HB_BUF_FLAG_EOS)
{
return sub;
}
if (sub->s.stop != AV_NOPTS_VALUE)
{
sub->s.duration = sub->s.stop - sub->s.start;
if (sub->s.duration <= 0)
{
hb_log("sync: subtitle 0x%x duration <= 0, PTS %"PRId64"",
stream->subtitle.subtitle->id, sub->s.start);
hb_buffer_close(&sub);
}
}
else if (stream->subtitle.sanitizer.link)
{
hb_log("sync: subtitle 0x%x duration not set, PTS %"PRId64"",
stream->subtitle.subtitle->id, sub->s.start);
sub->s.duration = (int64_t)AV_NOPTS_VALUE;
}
return sub;
}
// Create a list of buffers that overlap the given start time.
// Returns a list of buffers and the smallest stop time of those
// buffers.
static hb_buffer_t * findOverlap(subtitle_sanitizer_t *sanitizer,
int64_t start, int64_t *stop_out)
{
hb_buffer_list_t list;
hb_buffer_t * buf;
int64_t stop;
stop = INT64_MAX;
hb_buffer_list_clear(&list);
buf = hb_buffer_list_head(&sanitizer->list_current);
while (buf != NULL)
{
if (buf->s.flags & HB_BUF_FLAG_EOF)
{
break;
}
if (buf->s.start > start)
{
if (stop > buf->s.start)
{
*stop_out = buf->s.start;
}
break;
}
if (buf->s.start <= start && start < buf->s.stop)
{
hb_buffer_t * tmp = hb_buffer_dup(buf);
tmp->s.start = start;
hb_buffer_list_append(&list, tmp);
if (stop > buf->s.stop)
{
stop = buf->s.stop;
*stop_out = stop;
}
}
buf = buf->next;
}
return hb_buffer_list_clear(&list);
}
// Find all subtitles in the list that start "now" and overlap for
// some period of time. Create a new subtitle buffer that is the
// merged results of the overlapping parts and update start times
// of non-overlapping parts.
static int mergeSubtitleOverlaps(subtitle_sanitizer_t *sanitizer)
{
hb_buffer_t * merged_buf = NULL;
hb_buffer_t * a, * b;
a = hb_buffer_list_head(&sanitizer->list_current);
if (a != NULL && (a->s.flags & HB_BUF_FLAG_EOF))
{
// EOF
return 0;
}
if (a == NULL ||
a->s.start == AV_NOPTS_VALUE || a->s.stop == AV_NOPTS_VALUE)
{
// Not enough information to resolve an overlap
return -1;
}
b = a->next;
if (b != NULL && a->s.stop <= b->s.start)
{
// No overlap
return 0;
}
// Check that we have 2 non-overlapping buffers in the list
// and that all timestamps are valid up to the non-overlap.
// This ensures that multiple overlapping subtitles have been
// completely merged.
while (b != NULL &&
b->s.start < a->s.stop && !(b->s.flags & HB_BUF_FLAG_EOF))
{
if (b->s.start == AV_NOPTS_VALUE || b->s.stop == AV_NOPTS_VALUE)
{
// Not enough information to resolve an overlap
return -1;
}
b = b->next;
}
if (b == NULL)
{
// Not enough information to resolve an overlap
return -1;
}
hb_buffer_list_t merged_list;
int64_t start, stop, last;
if (b->s.flags & HB_BUF_FLAG_EOF)
{
last = INT64_MAX;
}
else
{
last = b->s.start;
}
hb_buffer_list_clear(&merged_list);
a = hb_buffer_list_head(&sanitizer->list_current);
stop = start = a->s.start;
while (start < last)
{
hb_buffer_t * merge = findOverlap(sanitizer, start, &stop);
if (merge == NULL)
{
break;
}
a = merge;
merged_buf = NULL;
while (a != NULL)
{
hb_buffer_t * tmp;
tmp = merge_ssa(merged_buf, a);
hb_buffer_close(&merged_buf);
merged_buf = tmp;
a = a->next;
}
merged_buf->s.stop = stop;
hb_buffer_close(&merge);
hb_buffer_list_append(&merged_list, merged_buf);
start = stop;
// Remove merged buffers
a = hb_buffer_list_head(&sanitizer->list_current);
while (a != NULL && a->s.start < stop &&
!(a->s.flags & HB_BUF_FLAG_EOF))
{
hb_buffer_t * next = a->next;
if (a->s.stop <= stop)
{
// Buffer consumed
hb_buffer_list_rem(&sanitizer->list_current, a);
hb_buffer_close(&a);
}
else
{
a->s.start = stop;
}
a = next;
}
}
merged_buf = hb_buffer_list_clear(&merged_list);
hb_buffer_list_prepend(&sanitizer->list_current, merged_buf);
return 0;
}
static hb_buffer_t * mergeSubtitles(sync_stream_t * stream)
{
hb_buffer_t * buf;
hb_buffer_list_t list;
subtitle_sanitizer_t * sanitizer = &stream->subtitle.sanitizer;
hb_buffer_list_clear(&list);
if (!sanitizer->merge)
{
// Handle all but the last buffer
// The last buffer may not have been "linked" yet
while (hb_buffer_list_count(&sanitizer->list_current) > 0)
{
buf = hb_buffer_list_head(&sanitizer->list_current);
if (!(buf->s.flags & HB_BUF_FLAG_EOF) &&
buf->s.stop != AV_NOPTS_VALUE)
{
buf = hb_buffer_list_rem_head(&sanitizer->list_current);
buf = setSubDuration(stream, buf);
hb_buffer_list_append(&list, buf);
}
else
{
break;
}
}
return hb_buffer_list_clear(&list);
}
// We only reach here if we are merging subtitles
while (hb_buffer_list_count(&sanitizer->list_current) > 0)
{
buf = hb_buffer_list_head(&sanitizer->list_current);
if (buf->s.flags & HB_BUF_FLAG_EOF)
{
// remove EOF from list, add to output
buf = hb_buffer_list_rem_head(&sanitizer->list_current);
hb_buffer_list_append(&list, buf);
break;
}
int result = mergeSubtitleOverlaps(sanitizer);
if (result < 0)
{
// not enough information available yet to resolve the overlap
break;
}
// Overlap resolved, output a buffer
buf = hb_buffer_list_rem_head(&sanitizer->list_current);
if (buf != NULL && !(buf->s.flags & HB_BUF_FLAG_EOF))
{
buf = setSubDuration(stream, buf);
hb_buffer_list_append(&list, buf);
}
}
return hb_buffer_list_clear(&list);
}
static hb_buffer_t * sanitizeSubtitle(
sync_stream_t * stream,
hb_buffer_t * sub)
{
hb_buffer_list_t list;
subtitle_sanitizer_t * sanitizer = &stream->subtitle.sanitizer;
if (sub == NULL)
{
return NULL;
}
hb_buffer_list_set(&list, sub);
if (!sanitizer->link && !sanitizer->merge)
{
hb_buffer_list_t out_list;
hb_buffer_list_clear(&out_list);
sub = hb_buffer_list_rem_head(&list);
while (sub != NULL && !(sub->s.flags & HB_BUF_FLAG_EOF))
{
sub = setSubDuration(stream, sub);
hb_buffer_list_append(&out_list, sub);
sub = hb_buffer_list_rem_head(&list);
}
return hb_buffer_list_clear(&out_list);
}
sub = hb_buffer_list_rem_head(&list);
while (sub != NULL)
{
if (sub->s.flags & HB_BUF_FLAG_EOF)
{
hb_buffer_list_append(&sanitizer->list_current, sub);
break;
}
hb_buffer_t *last = hb_buffer_list_tail(&sanitizer->list_current);
if (last != NULL && last->s.stop == AV_NOPTS_VALUE)
{
last->s.stop = sub->s.start;
if (last->s.stop <= last->s.start)
{
// Subtitle duration <= 0. Drop it.
hb_log("sync: subtitle 0x%x has no duration, PTS %"PRId64"",
stream->subtitle.subtitle->id, sub->s.start);
hb_buffer_list_rem_tail(&sanitizer->list_current);
hb_buffer_close(&last);
}
}
if (sub->s.flags & HB_BUF_FLAG_EOS)
{
// Used to indicate "clear" subtitles when the duration
// of subtitles is not encoded in the stream
hb_buffer_close(&sub);
}
hb_buffer_list_append(&sanitizer->list_current, sub);
sub = hb_buffer_list_rem_head(&list);
}
return mergeSubtitles(stream);
}
/***********************************************************************
* syncVideoWork
***********************************************************************
*
**********************************************************************/
static int syncVideoWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
hb_buffer_t ** buf_out )
{
hb_work_private_t * pv = w->private_data;
hb_buffer_t * in = *buf_in;
if (pv->stream->done)
{
FlushBuffer(pv->stream->common);
return HB_WORK_DONE;
}
if (in->s.flags & HB_BUF_FLAG_EOF)
{
pv->stream->flush = 1;
flushStreamsLock(pv->common);
// Ideally, we would only do this subtitle scan check in
// syncSubtitleWork, but someone might try to do a subtitle
// scan on a source that has no subtitles :-(
if (pv->common->job->indepth_scan)
{
// When doing subtitle indepth scan, the pipeline ends at sync.
// Terminate job when EOF reached.
*w->done = 1;
}
return HB_WORK_DONE;
}
*buf_in = NULL;
QueueBuffer(pv->stream, in);
Synchronize(pv->stream);
if (pv->stream->done)
{
FlushBuffer(pv->stream->common);
return HB_WORK_DONE;
}
return HB_WORK_OK;
}
hb_work_object_t hb_sync_video =
{
WORK_SYNC_VIDEO,
"Video Synchronization",
syncVideoInit,
syncVideoWork,
syncVideoClose
};
/***********************************************************************
* Close Audio
***********************************************************************
*
**********************************************************************/
static void syncAudioClose( hb_work_object_t * w )
{
hb_work_private_t * pv = w->private_data;
if (pv == NULL)
{
return;
}
// Free samplerate conversion context
if (pv->stream->audio.resample)
{
hb_audio_resample_free(pv->stream->audio.resample);
}
sync_delta_t * delta;
while ((delta = hb_list_item(pv->stream->delta_list, 0)) != NULL)
{
hb_list_rem(pv->stream->delta_list, delta);
free(delta);
}
hb_list_close(&pv->stream->delta_list);
hb_list_empty(&pv->stream->in_queue);
hb_list_empty(&pv->stream->scr_delay_queue);
free(pv);
w->private_data = NULL;
}
static int syncAudioInit( hb_work_object_t * w, hb_job_t * job)
{
return 0;
}
/***********************************************************************
* SyncAudio
***********************************************************************
*
**********************************************************************/
static int syncAudioWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
hb_buffer_t ** buf_out )
{
hb_work_private_t * pv = w->private_data;
hb_buffer_t * in = *buf_in;
if (pv->stream->done)
{
FlushBuffer(pv->stream->common);
return HB_WORK_DONE;
}
if (in->s.flags & HB_BUF_FLAG_EOF)
{
pv->stream->flush = 1;
flushStreamsLock(pv->common);
return HB_WORK_DONE;
}
*buf_in = NULL;
QueueBuffer(pv->stream, in);
Synchronize(pv->stream);
if (pv->stream->done)
{
FlushBuffer(pv->stream->common);
return HB_WORK_DONE;
}
return HB_WORK_OK;
}
hb_work_object_t hb_sync_audio =
{
WORK_SYNC_AUDIO,
"Audio Synchronization",
syncAudioInit,
syncAudioWork,
syncAudioClose
};
// FilterAudioFrame is called after audio timestamp discontinuities
// have all been corrected. So we expect smooth continuous audio
// here.
static hb_buffer_t * FilterAudioFrame( sync_stream_t * stream,
hb_buffer_t *buf )
{
hb_audio_t * audio = stream->audio.audio;
// Can't count of buf->s.stop - buf->s.start for accurate duration
// due to integer rounding, so use buf->s.duration when it is set
// (which should be always if I didn't miss anything)
if (buf->s.duration <= 0)
{
buf->s.duration = buf->s.stop - buf->s.start;
}
if (!(audio->config.out.codec & HB_ACODEC_PASS_FLAG))
{
// TODO: this should all be replaced by an audio filter chain.
// Audio is not passthru. Check if we need to modify the audio
// in any way.
if (stream->audio.resample != NULL)
{
/* do sample rate conversion */
hb_buffer_t * out;
int nsamples, sample_size;
sample_size = hb_mixdown_get_discrete_channel_count(
audio->config.out.mixdown ) * sizeof( float );
nsamples = buf->size / sample_size;
out = hb_audio_resample(stream->audio.resample,
(const uint8_t **)&buf->data,
nsamples);
hb_buffer_close(&buf);
if (out == NULL)
{
return NULL;
}
out->s.start = stream->next_pts;
out->s.stop = stream->next_pts + out->s.duration;
buf = out;
}
if (audio->config.out.gain > 0.0)
{
int count, ii;
count = buf->size / sizeof(float);
for ( ii = 0; ii < count; ii++ )
{
double sample;
sample = (double)*(((float*)buf->data)+ii);
sample *= stream->audio.gain_factor;
if (sample > 0)
sample = MIN(sample, 1.0);
else
sample = MAX(sample, -1.0);
*(((float*)buf->data)+ii) = sample;
}
}
else if( audio->config.out.gain < 0.0 )
{
int count, ii;
count = buf->size / sizeof(float);
for ( ii = 0; ii < count; ii++ )
{
double sample;
sample = (double)*(((float*)buf->data)+ii);
sample *= stream->audio.gain_factor;
*(((float*)buf->data)+ii) = sample;
}
}
}
buf->s.type = AUDIO_BUF;
buf->s.frametype = HB_FRAME_AUDIO;
return buf;
}
static void UpdateState( sync_common_t * common, int frame_count )
{
hb_job_t * job = common->job;
hb_state_t state;
if (job->indepth_scan)
{
// Progress for indept scan is handled by reader
// frame_count is used during indepth_scan
// to find start & end points.
return;
}
if (frame_count == 0)
{
common->st_first = hb_get_date();
}
if (hb_get_date() > common->st_dates[3] + 1000)
{
memmove( &common->st_dates[0], &common->st_dates[1],
3 * sizeof( uint64_t ) );
memmove( &common->st_counts[0], &common->st_counts[1],
3 * sizeof( uint64_t ) );
common->st_dates[3] = hb_get_date();
common->st_counts[3] = frame_count;
}
hb_get_state2(job->h, &state);
state.state = HB_STATE_WORKING;
#define p state.param.working
p.progress = (float)frame_count / common->est_frame_count;
if (p.progress > 1.0)
{
p.progress = 1.0;
}
p.rate_cur = 1000.0 * (common->st_counts[3] - common->st_counts[0]) /
(common->st_dates[3] - common->st_dates[0]);
if (hb_get_date() > common->st_first + 4000)
{
int eta;
p.rate_avg = 1000.0 * common->st_counts[3] /
(common->st_dates[3] - common->st_first - job->st_paused);
eta = (common->est_frame_count - common->st_counts[3]) / p.rate_avg;
p.eta_seconds = eta;
p.hours = eta / 3600;
p.minutes = (eta % 3600) / 60;
p.seconds = eta % 60;
}
else
{
p.rate_avg = 0.0;
p.hours = -1;
p.minutes = -1;
p.seconds = -1;
}
#undef p
hb_set_state(job->h, &state);
}
static void UpdateSearchState( sync_common_t * common, int64_t start,
int frame_count )
{
hb_job_t * job = common->job;
hb_state_t state;
uint64_t now;
double avg;
if (job->indepth_scan)
{
// Progress for indept scan is handled by reader
// frame_count is used during indepth_scan
// to find start & end points.
return;
}
now = hb_get_date();
if (frame_count == 0)
{
common->st_first = now;
}
hb_get_state2(job->h, &state);
state.state = HB_STATE_SEARCHING;
#define p state.param.working
if (common->wait_for_frame)
p.progress = (float)frame_count / job->frame_to_start;
else if (common->wait_for_pts)
p.progress = (float) start / common->pts_to_start;
else
p.progress = 0;
if (p.progress > 1.0)
{
p.progress = 1.0;
}
if (now > common->st_first)
{
int eta = 0;
if (common->wait_for_frame)
{
avg = 1000.0 * frame_count / (now - common->st_first);
eta = (job->frame_to_start - frame_count ) / avg;
}
else if (common->wait_for_pts)
{
avg = 1000.0 * start / (now - common->st_first);
eta = (common->pts_to_start - start) / avg;
}
p.eta_seconds = eta;
p.hours = eta / 3600;
p.minutes = (eta % 3600) / 60;
p.seconds = eta % 60;
}
else
{
p.rate_avg = 0.0;
p.hours = -1;
p.minutes = -1;
p.seconds = -1;
}
#undef p
hb_set_state(job->h, &state);
}
static int syncSubtitleInit( hb_work_object_t * w, hb_job_t * job )
{
return 0;
}
static void syncSubtitleClose( hb_work_object_t * w )
{
hb_work_private_t * pv = w->private_data;
if (pv == NULL)
{
return;
}
sync_delta_t * delta;
while ((delta = hb_list_item(pv->stream->delta_list, 0)) != NULL)
{
hb_list_rem(pv->stream->delta_list, delta);
free(delta);
}
hb_list_close(&pv->stream->delta_list);
hb_list_empty(&pv->stream->in_queue);
hb_list_empty(&pv->stream->scr_delay_queue);
hb_buffer_list_close(&pv->stream->subtitle.sanitizer.list_current);
free(pv);
w->private_data = NULL;
}
static int syncSubtitleWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
hb_buffer_t ** buf_out )
{
hb_work_private_t * pv = w->private_data;
hb_buffer_t * in = *buf_in;
if (pv->stream->done)
{
FlushBuffer(pv->stream->common);
return HB_WORK_DONE;
}
if (in->s.flags & HB_BUF_FLAG_EOF)
{
pv->stream->flush = 1;
// sanitizeSubtitle requires EOF buffer to recognize that
// it needs to flush all subtitles.
hb_list_add(pv->stream->in_queue, hb_buffer_eof_init());
flushStreamsLock(pv->common);
if (pv->common->job->indepth_scan)
{
// When doing subtitle indepth scan, the pipeline ends at sync.
// Terminate job when EOF reached.
*w->done = 1;
}
return HB_WORK_DONE;
}
*buf_in = NULL;
QueueBuffer(pv->stream, in);
Synchronize(pv->stream);
if (pv->stream->done)
{
FlushBuffer(pv->stream->common);
return HB_WORK_DONE;
}
return HB_WORK_OK;
}
hb_work_object_t hb_sync_subtitle =
{
WORK_SYNC_SUBTITLE,
"Subtitle Synchronization",
syncSubtitleInit,
syncSubtitleWork,
syncSubtitleClose
};