oak-team/oak-editor
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

many improvements to the playback algorithm

Browse Source
This commit is contained in:
itsmattkc
2018-10-20 22:58:19 +11:00
parent c9b8775a81
commit 386693d372
11 changed files with 234 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files
playback/cacher.cpp
+47 -32
View File
@@ -98,6 +98,7 @@ void cache_audio_worker(Clip* c, Clip* nest) {
if (c->reverse && !c->audio_just_reset) {
avcodec_flush_buffers(c->codecCtx);
c->reached_end = false;
int64_t backtrack_seek = qMax(c->reverse_target - static_cast<int64_t>(av_q2d(av_inv_q(c->stream->time_base))), static_cast<int64_t>(0));
av_seek_frame(c->formatCtx, c->stream->index, backtrack_seek, AVSEEK_FLAG_BACKWARD);
#ifdef AUDIOWARNINGS
@@ -248,7 +249,7 @@ void cache_audio_worker(Clip* c, Clip* nest) {
if (c->audio_just_reset) {
// get precise sample offset for the elected clip_in from this audio frame
double target_sts = playhead_to_clip_seconds(c, c->audio_target_frame);
double frame_sts = (frame->pts * timebase);
double frame_sts = ((frame->pts - c->stream->start_time) * timebase);
int nb_samples = qRound((target_sts - frame_sts)*c->sequence->audio_frequency);
c->frame_sample_index = nb_samples * 4;
#ifdef AUDIOWARNINGS
@@ -361,16 +362,20 @@ void cache_video_worker(Clip* c, long playhead) {
if (c->reverse) limit *= 2;
if (c->queue.size() < limit) {
int64_t eighth_second = av_q2d(av_inv_q(c->stream->time_base))*0.125;
int64_t smallest_pts = INT64_MAX;
int64_t target_pts = seconds_to_timestamp(c, playhead_to_clip_seconds(c, playhead));
if (c->reverse && c->queue.size() > 0) {
int64_t quarter_sec = qRound(av_q2d(av_inv_q(c->stream->time_base))) >> 2;
for (int i=0;i<c->queue.size();i++) {
smallest_pts = qMin(smallest_pts, c->queue.at(i)->pts);
}
avcodec_flush_buffers(c->codecCtx);
av_seek_frame(c->formatCtx, c->stream->index, qMax(static_cast<int64_t>(0), smallest_pts - quarter_sec), AVSEEK_FLAG_BACKWARD);
c->reached_end = false;
int64_t seek_ts = qMax(static_cast<int64_t>(0), smallest_pts - quarter_sec);
av_seek_frame(c->formatCtx, c->stream->index, seek_ts, AVSEEK_FLAG_BACKWARD);
} else {
smallest_pts = seconds_to_timestamp(c, playhead_to_clip_seconds(c, playhead));
smallest_pts = target_pts;
}
while (true) {
@@ -381,58 +386,67 @@ void cache_video_worker(Clip* c, long playhead) {
read_ret = (c->use_existing_frame) ? 0 : retrieve_next_frame(c, send_frame);
c->use_existing_frame = false;
if (read_ret >= 0) {
// sending NULL to avfiltergraph is supposed to flush it but it'll also no longer accept frames - particularly unhelpful when reversing
// we'll see if it still works completely without doing so
/*if (read_ret == AVERROR_EOF) send_frame = NULL;
if ((send_ret = av_buffersrc_add_frame_flags(c->buffersrc_ctx, send_frame, AV_BUFFERSRC_FLAG_KEEP_REF)) < 0) {
qDebug() << "[ERROR] Failed to add frame to buffer source." << send_ret;
break;
}
if (read_ret >= 0) {
av_frame_unref(c->frame);
}*/
bool send_it = false;
if ((send_ret = av_buffersrc_add_frame_flags(c->buffersrc_ctx, send_frame, AV_BUFFERSRC_FLAG_KEEP_REF)) < 0) {
qDebug() << "[ERROR] Failed to add frame to buffer source." << send_ret;
break;
if (c->reverse) {
send_it = true;
} else if (send_frame->pts > target_pts - eighth_second) {
send_it = true;
} else {
qDebug() << "skipped adding a frame to the queue" << target_pts << "(" << send_frame->pts << send_frame->pkt_duration;
}
if (send_it) {
if ((send_ret = av_buffersrc_add_frame_flags(c->buffersrc_ctx, send_frame, AV_BUFFERSRC_FLAG_KEEP_REF)) < 0) {
qDebug() << "[ERROR] Failed to add frame to buffer source." << send_ret;
break;
}
}
av_frame_unref(c->frame);
} else {
if (read_ret != AVERROR_EOF) qDebug() << "[ERROR] Failed to read frame." << read_ret;
if (read_ret == AVERROR_EOF) {
c->reached_end = true;
} else {
qDebug() << "[ERROR] Failed to read frame." << read_ret;
}
break;
}
}
if (retr_ret < 0) {
if (retr_ret != AVERROR_EOF) qDebug() << "[ERROR] Failed to retrieve frame from buffersink." << retr_ret;
if (retr_ret == AVERROR_EOF) {
c->reached_end = true;
} else {
qDebug() << "[ERROR] Failed to retrieve frame from buffersink." << retr_ret;
}
av_frame_free(&frame);
break;
} else {
if (c->reverse && frame->pts >= smallest_pts) {
if (c->reverse && ((smallest_pts == target_pts && frame->pts >= smallest_pts) || (smallest_pts != target_pts && frame->pts > smallest_pts))) {
av_frame_free(&frame);
break;
} else {
// thread-safety while adding frame to the queue
c->queue_lock.lock();
c->queue.append(frame);
c->queue_lock.unlock();
//break;
if (!c->reverse && c->queue.size() == limit) {
if (rendering) {
// if (rendering) {
// see if we got the frame we needed (used for speed ups primarily)
bool found = false;
for (int i=0;i<c->queue.size();i++) {
// TODO/NOTE: this will not work on sped up AND reversed video
if (c->queue.at(i)->pts >= smallest_pts) {
// TODO/NOTE: this will not work on clips that are sped up AND reversed
if (c->queue.at(i)->pts >= target_pts) {
found = true;
break;
}
}
if (found) {
c->queue_lock.unlock();
break;
} else {
// remove earliest frame and store another
// remove earliest frame and loop to store another
int earliest_frame = 0;
for (int i=1;i<c->queue.size();i++) {
// TODO/NOTE: this will not work on sped up AND reversed video
@@ -440,15 +454,14 @@ void cache_video_worker(Clip* c, long playhead) {
earliest_frame = i;
}
}
c->queue_lock.lock();
av_frame_free(&c->queue[earliest_frame]);
c->queue.removeAt(earliest_frame);
c->queue_lock.unlock();
}
} else {
break;
}
// } else {
// break;
// }
}
c->queue_lock.unlock();
}
}
}
@@ -460,7 +473,6 @@ void reset_cache(Clip* c, long target_frame) {
switch (c->media_type) {
case MEDIA_TYPE_FOOTAGE:
{
c->reached_end = false;
MediaStream* ms = static_cast<Media*>(c->media)->get_stream_from_file_index(c->track < 0, c->media_stream);
if (!ms->infinite_length) {
if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
@@ -476,6 +488,7 @@ void reset_cache(Clip* c, long target_frame) {
while (true) {
// flush ffmpeg codecs
avcodec_flush_buffers(c->codecCtx);
c->reached_end = false;
if (seek_ts > 0) {
av_seek_frame(c->formatCtx, ms->file_index, seek_ts, AVSEEK_FLAG_BACKWARD);
@@ -502,6 +515,7 @@ void reset_cache(Clip* c, long target_frame) {
} else if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
// flush ffmpeg codecs
avcodec_flush_buffers(c->codecCtx);
c->reached_end = false;
// seek (target_frame represents timeline timecode in frames, not clip timecode)
@@ -571,7 +585,8 @@ void open_clip_worker(Clip* clip) {
clip->codecCtx = avcodec_alloc_context3(clip->codec);
avcodec_parameters_to_context(clip->codecCtx, clip->stream->codecpar);
clip->max_queue_size = (ms->infinite_length) ? 1 : 15;
clip->max_queue_size = (ms->infinite_length) ? 1 : qCeil(ms->video_frame_rate*0.25);
if (ms->video_interlacing != VIDEO_PROGRESSIVE) clip->max_queue_size *= 2;
AVDictionary* opts = NULL;
playback/playback.cpp
+51 -137
View File
@@ -25,6 +25,10 @@ extern "C" {
#include <QOpenGLPixelTransferOptions>
#include <QOpenGLFramebufferObject>
#ifdef QT_DEBUG
//#define GCF_DEBUG
#endif
bool texture_failed = false;
bool rendering = false;
@@ -57,6 +61,7 @@ void open_clip(Clip* clip, bool multithreaded) {
void close_clip(Clip* clip) {
// destroy opengl texture in main thread
if (clip->texture != NULL) {
clip->texture->destroy();
delete clip->texture;
clip->texture = NULL;
}
@@ -117,17 +122,26 @@ void get_clip_frame(Clip* c, long playhead) {
AVFrame* target_frame = NULL;
bool reset = false;
bool cache = true;
c->queue_lock.lock();
if (c->queue.size() > 0) {
if (ms->infinite_length) {
target_frame = c->queue.at(0);
#ifdef GCF_DEBUG
qDebug() << "GCF ==> USE PRECISE (INFINITE)";
#endif
} else {
// correct frame may be somewhere else in the queue
int closest_frame = 0;
for (int i=1;i<c->queue.size();i++) {
//qDebug() << "results for" << i << qAbs(c->queue.at(i)->pts - target_pts) << qAbs(c->queue.at(closest_frame)->pts - target_pts) << c->queue.at(i)->pts << target_pts;
if (c->queue.at(i)->pts == target_pts) {
#ifdef GCF_DEBUG
qDebug() << "GCF ==> USE PRECISE";
#endif
closest_frame = i;
break;
} else if (c->queue.at(i)->pts > c->queue.at(closest_frame)->pts && c->queue.at(i)->pts < target_pts) {
@@ -137,24 +151,44 @@ void get_clip_frame(Clip* c, long playhead) {
// remove all frames earlier than this one from the queue
target_frame = c->queue.at(closest_frame);
c->queue_lock.lock();
//qDebug() << "closest frame was" << closest_frame << "with" << target_frame->pts << "/" << target_pts;
for (int i=0;i<c->queue.size();i++) {
if (c->queue.at(i)->pts != target_frame->pts && (c->queue.at(i)->pts < target_frame->pts) == (!c->reverse)) {
if (c->queue.at(i) != target_frame && ((c->queue.at(i)->pts > target_frame->pts) == c->reverse)) {
//qDebug() << "removed frame at" << i << "because its pts was" << c->queue.at(i)->pts << "compared to" << target_frame->pts;
av_frame_free(&c->queue[i]); // may be a little heavy for the UI thread?
c->queue.removeAt(i);
i--;
}
}
c->queue_lock.unlock();
// if this frame is more than one second out, we probably need to reset
if (qAbs(target_pts - target_frame->pts) > second_pts) {
target_frame = NULL;
// qDebug() << "GCF ==> COMPLACENT";
qDebug() << "GCF ==> RESET";
reset = true;
} else {
qDebug() << "GCF ==> USE IMPRECISE";
// we didn't get the exact frame
if (target_frame->pts != target_pts) {
if (target_pts > target_frame->pts && target_pts <= target_frame->pts + target_frame->pkt_duration) {
#ifdef GCF_DEBUG
qDebug() << "GCF ==> USE IMPRECISE";
#endif
} else {
int64_t pts_diff = qAbs(target_pts - target_frame->pts);
if (c->reached_end && target_pts > target_frame->pts) {
#ifdef GCF_DEBUG
qDebug() << "GCF ==> EOF TOLERANT";
#endif
c->reached_end = false;
cache = false;
} else if (target_pts < target_frame->pts || pts_diff > second_pts) {
#ifdef GCF_DEBUG
qDebug() << "GCF ==> RESET" << target_pts << "(" << target_frame->pts << "-" << target_frame->pts+target_frame->pkt_duration << ")";
#endif
target_frame = NULL;
reset = true;
} else {
#ifdef GCF_DEBUG
qDebug() << "GCF ==> WAIT - target:" << target_pts << "closest frame:" << target_frame->pts;
#endif
target_frame = NULL;
}
}
}
}
} else {
@@ -174,133 +208,13 @@ void get_clip_frame(Clip* c, long playhead) {
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
}
c->queue_lock.unlock();
// get more frames
cache_clip(c, playhead, reset, NULL);
if (cache) cache_clip(c, playhead, reset, NULL);
}
}
/*bool get_clip_frame(Clip* c, long playhead) {
if (c->finished_opening) {
// do we need to update the texture?
MediaStream* ms = static_cast<Media*>(c->media)->get_stream_from_file_index(c->track < 0, c->media_stream);
long sequence_clip_time = qMax(0L, playhead - c->timeline_in + c->clip_in);
if (c->reverse && !ms->infinite_length) {
sequence_clip_time = c->getMaximumLength() - sequence_clip_time - 1;
}
double rate = c->getMediaFrameRate();
if (c->skip_type == SKIP_TYPE_DISCARD) rate *= c->speed;
long clip_time = refactor_frame_number(sequence_clip_time, c->sequence->frame_rate, rate);
AVFrame* current_frame = NULL;
bool no_frame = false;
// get frame data
if (ms->infinite_length) { // if clip is a still frame, we only need one
if (c->cache_A.written) {
// retrieve cached frame
current_frame = c->cache_A.frames[0];
} else if (c->lock.tryLock()) {
// grab image
cache_clip(c, 0, true, false, false, NULL);
c->lock.unlock();
}
} else {
// keeping a RAM cache improves performance, however it's detrimental when rendering
// determine which cache contains the requested frame
bool using_cache_A = false;
bool using_cache_B = false;
AVFrame** cache = NULL;
long cache_offset = 0;
bool cache_needs_reset = false;
// TODO just removed a bunch of mutexes - is this safe????
if (c->cache_A.written && clip_time >= c->cache_A.offset && clip_time < c->cache_A.offset + c->cache_size) {
if (clip_time < (c->cache_A.offset + c->cache_A.write_count)) {
using_cache_A = true;
c->cache_A.unread = false;
cache = c->cache_A.frames;
cache_offset = c->cache_A.offset;
} else {
// frame is coming but isn't here yet, no need to reset cache
no_frame = true;
}
} else if (c->cache_B.written && clip_time >= c->cache_B.offset && clip_time < c->cache_B.offset + c->cache_size) {
if (clip_time < (c->cache_B.offset + c->cache_B.write_count)) {
using_cache_B = true;
c->cache_B.unread = false;
cache = c->cache_B.frames;
cache_offset = c->cache_B.offset;
} else {
// frame is coming but isn't here yet, no need to reset cache
no_frame = true;
}
} else {
// this is technically bad, unless we just seeked
c->cache_A.write_count = 0;
c->cache_B.write_count = 0;
c->cache_A.unread = false;
c->cache_B.unread = false;
cache_needs_reset = true;
}
if (!no_frame) {
if (cache != NULL) {
current_frame = cache[clip_time - cache_offset];
}
// determine whether we should start filling the other cache
if (!using_cache_A || !using_cache_B) {
if (c->lock.tryLock()) {
long cache_time;
if (cache_needs_reset) {
cache_time = (c->reverse) ? qMax(clip_time - c->cache_size, 0L) : qMax(clip_time, 0L);
} else if (c->reverse) {
cache_time = (cache_offset - c->cache_size);
} else {
cache_time = (cache_offset + c->cache_size);
}
bool write_A = (!using_cache_A && !c->cache_A.unread);
bool write_B = (!using_cache_B && !c->cache_B.unread);
if (write_A || write_B) {
// if we have no cache and need to seek, start us at the current playhead, otherwise start at the end of the current cache
cache_clip(c, cache_time, write_A, write_B, (cache_needs_reset || c->reverse), NULL);
}
c->lock.unlock();
}
}
}
}
if (playhead >= c->timeline_in) {
if (current_frame != NULL) {
// set up opengl texture
if (c->texture == NULL) {
c->texture = new QOpenGLTexture(QOpenGLTexture::Target2D);
c->texture->setSize(current_frame->width, current_frame->height);
c->texture->setFormat(QOpenGLTexture::RGBA8_UNorm);
c->texture->setMipLevels(c->texture->maximumMipLevels());
c->texture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear);
c->texture->allocateStorage(QOpenGLTexture::RGBA, QOpenGLTexture::UInt8);
}
glPixelStorei(GL_UNPACK_ROW_LENGTH, current_frame->linesize[0]/4);
c->texture->setData(0, QOpenGLTexture::RGBA, QOpenGLTexture::UInt8, current_frame->data[0]);
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
c->texture_frame = clip_time;
return true;
} else {
texture_failed = true;
qDebug() << "[ERROR] Failed to retrieve frame from cache (R:" << clip_time << "| A:" << c->cache_A.offset << "-" << c->cache_A.offset+c->cache_size-1 << "| B:" << c->cache_B.offset << "-" << c->cache_B.offset+c->cache_size-1 << "| WA:" << c->cache_A.written << "| WB:" << c->cache_B.written << ")";
}
}
}
return false;
}*/
long playhead_to_clip_frame(Clip* c, long playhead) {
return (qMax(0L, playhead - c->timeline_in) + c->clip_in);
}
@@ -308,12 +222,12 @@ long playhead_to_clip_frame(Clip* c, long playhead) {
double playhead_to_clip_seconds(Clip* c, long playhead) {
// returns time in seconds
long clip_frame = playhead_to_clip_frame(c, playhead);
if (c->reverse) clip_frame = c->getMaximumLength() - clip_frame;
if (c->reverse) clip_frame = c->getMaximumLength() - clip_frame - 1;
return ((double) clip_frame/c->sequence->frame_rate)*c->speed;
}
int64_t seconds_to_timestamp(Clip* c, double seconds) {
return qFloor((seconds * c->stream->time_base.den)/c->stream->time_base.num);
return qRound((seconds * c->stream->time_base.den)/c->stream->time_base.num) + c->stream->start_time;
}
int64_t playhead_to_timestamp(Clip* c, long playhead) {
@@ -368,7 +282,7 @@ int retrieve_next_frame(Clip* c, AVFrame* f) {
bool is_clip_active(Clip* c, long playhead) {
return c->enabled
&& c->timeline_in < playhead + ceil(c->sequence->frame_rate)
&& c->timeline_in < playhead + ceil(c->sequence->frame_rate*2)
&& c->timeline_out > playhead
&& playhead - c->timeline_in + c->clip_in < c->getMaximumLength();
}