From efa628e88f51cbcfb9b902a5b4860cd09a738326 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Wed, 15 Aug 2018 21:58:29 +1000 Subject: [PATCH] implemented backend for AVFilters --- effects/shakeeffect.cpp | 2 + main.cpp | 4 +- olive.pro | 6 +- playback/cacher.cpp | 175 ++++++++++++++++++++++++++++++---------- playback/playback.cpp | 68 +++++++++------- project/clip.h | 12 ++- 6 files changed, 189 insertions(+), 78 deletions(-) diff --git a/effects/shakeeffect.cpp b/effects/shakeeffect.cpp index 128d2924f..bbc582340 100644 --- a/effects/shakeeffect.cpp +++ b/effects/shakeeffect.cpp @@ -13,6 +13,8 @@ #include "panels/timeline.h" ShakeEffect::ShakeEffect(Clip *c) : Effect(c, EFFECT_TYPE_VIDEO, VIDEO_SHAKE_EFFECT), inside(false) { + enable_pre_gl = true; + EffectRow* intensity_row = add_row("Intensity:"); intensity_val = intensity_row->add_field(EFFECT_FIELD_DOUBLE); intensity_val->set_double_minimum_value(0); diff --git a/main.cpp b/main.cpp index 0634f98c9..0ae2d7fde 100644 --- a/main.cpp +++ b/main.cpp @@ -3,14 +3,14 @@ extern "C" { #include + #include } int main(int argc, char *argv[]) { // init ffmpeg subsystem av_register_all(); - -// QCoreApplication::setAttribute(Qt::AA_ShareOpenGLContexts, true); + avfilter_register_all(); QApplication a(argc, argv); MainWindow w; diff --git a/olive.pro b/olive.pro index e5d1307e9..4478656a9 100644 --- a/olive.pro +++ b/olive.pro @@ -127,16 +127,16 @@ win32 { LIBS += -L../ffmpeg/lib -lopengl32 INCLUDEPATH = ../ffmpeg/include RC_FILE = icons/win.rc - LIBS += -lavutil -lavformat -lavcodec -lswscale -lswresample + LIBS += -lavutil -lavformat -lavcodec -lavfilter -lswscale -lswresample } mac { - LIBS += -L/usr/local/lib -lavutil -lavformat -lavcodec -lswscale -lswresample + LIBS += -L/usr/local/lib -lavutil -lavformat -lavcodec -lavfilter -lswscale -lswresample INCLUDEPATH = /usr/local/include } linux { - LIBS += -lavutil -lavformat -lavcodec -lswscale -lswresample + LIBS += -lavutil -lavformat -lavcodec -lavfilter -lswscale -lswresample } RESOURCES += \ diff --git a/playback/cacher.cpp b/playback/cacher.cpp index 73253552f..ee3ce6398 100644 --- a/playback/cacher.cpp +++ b/playback/cacher.cpp @@ -12,6 +12,10 @@ extern "C" { #include + #include + #include + #include + #include #include #include #include @@ -155,16 +159,9 @@ void cache_audio_worker(Clip* c, Clip* nest) { int upper_byte_index = (c->audio_buffer_write+1)%audio_ibuffer_size; int lower_byte_index = (c->audio_buffer_write)%audio_ibuffer_size; qint16 old_sample = (qint16) ((audio_ibuffer[upper_byte_index] & 0xFF) << 8 | (audio_ibuffer[lower_byte_index] & 0xFF)); - qint16 new_sample = (qint16) ((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF)); - qint32 mixed_sample; - - // peak handling - mixed_sample = old_sample + new_sample; - if (mixed_sample > INT16_MAX) { - mixed_sample = INT16_MAX; - } else if (mixed_sample < INT16_MIN) { - mixed_sample = INT16_MIN; - } + qint16 new_sample = (qint16) ((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF)); + qint32 mixed_sample = old_sample + new_sample; + mixed_sample = qMax(qMin(mixed_sample, (qint32)INT16_MAX), (qint32)INT16_MIN); audio_ibuffer[upper_byte_index] = (quint8) (mixed_sample >> 8); audio_ibuffer[lower_byte_index] = (quint8) mixed_sample; @@ -184,16 +181,52 @@ void cache_video_worker(Clip* c, long playhead, ClipCache* cache) { cache->mutex.lock(); cache->offset = playhead; - if (!c->reached_end) { - for (size_t i=0;icache_size;i++) { - retrieve_next_frame_raw_data(c, cache->frames[i]); - if (c->reached_end) break; - } + + bool error = false; + + int i = 0; + if (!c->reached_end) { + while (i < c->cache_size) { + av_frame_unref(cache->frames[i]); + int ret = av_buffersink_get_frame(c->buffersink_ctx, cache->frames[i]); + if (ret < 0) { + if (ret == AVERROR(EAGAIN)) { + ret = retrieve_next_frame(c, c->frame); + if (ret >= 0) { + if ((ret = av_buffersrc_add_frame_flags(c->buffersrc_ctx, c->frame, AV_BUFFERSRC_FLAG_KEEP_REF)) < 0) { + qDebug() << "[ERROR] Could not feed filtergraph -" << ret; + error = true; + break; + } + } else { + if (ret == AVERROR_EOF) { + c->reached_end = true; + } else { + qDebug() << "[WARNING] Raw frame data could not be retrieved." << ret; + error = true; + } + break; + } + } else { + if (ret != AVERROR_EOF) { + qDebug() << "[ERROR] Could not pull from filtergraph"; + error = true; + } + break; + } + } else { + i++; + } + } } - cache->written = true; - cache->unread = true; - // setting the cache to written even if it hasn't reached_end prevents playback from - // signaling a seek/reset because it wasn't able to find the frame + cache->write_count = i; + + if (!error) { + // setting the cache to written even if it hasn't reached_end prevents playback from + // signaling a seek/reset because it wasn't able to find the frame + cache->written = true; + cache->unread = true; + } cache->mutex.unlock(); } @@ -204,7 +237,7 @@ void reset_cache(Clip* c, long target_frame) { if (ms->infinite_length) { // if this clip is a still image, we only need one frame if (!c->cache_A.written) { - retrieve_next_frame_raw_data(c, c->cache_A.frames[0]); + retrieve_next_frame_raw_data(c, c->sws_frame); c->cache_A.written = true; } } else { @@ -299,14 +332,16 @@ void open_clip_worker(Clip* clip) { if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { // set up swscale context - primarily used for colorspace conversion // as "scaling" is actually done by OpenGL + int dstW = ceil(clip->stream->codecpar->width/2)*2; + int dstH = ceil(clip->stream->codecpar->height/2)*2; int dest_format = AV_PIX_FMT_RGBA; clip->sws_ctx = sws_getContext( clip->stream->codecpar->width, clip->stream->codecpar->height, static_cast(clip->stream->codecpar->format), - ceil(clip->stream->codecpar->width/2)*2, - ceil(clip->stream->codecpar->height/2)*2, + dstW, + dstH, static_cast(dest_format), SWS_FAST_BILINEAR, NULL, @@ -321,31 +356,79 @@ void open_clip_worker(Clip* clip) { clip->cache_size = ceil(av_q2d(av_guess_frame_rate(clip->formatCtx, clip->stream, NULL))/4); // cache is half a second in total // infinite length doesn't need cache B + clip->cache_A.frames = new AVFrame* [clip->cache_size]; clip->cache_B.frames = new AVFrame* [clip->cache_size]; - - } - clip->cache_A.frames = new AVFrame* [clip->cache_size]; - - - for (size_t i=0;icache_size;i++) { - clip->cache_A.frames[i] = av_frame_alloc(); - av_frame_make_writable(clip->cache_A.frames[i]); - clip->cache_A.frames[i]->width = clip->stream->codecpar->width; - clip->cache_A.frames[i]->height = clip->stream->codecpar->height; - clip->cache_A.frames[i]->format = dest_format; - av_frame_get_buffer(clip->cache_A.frames[i], 0); - clip->cache_A.frames[i]->linesize[0] = clip->stream->codecpar->width*4; - - if (!ms->infinite_length) { + for (int i=0;icache_size;i++) { + clip->cache_A.frames[i] = av_frame_alloc(); clip->cache_B.frames[i] = av_frame_alloc(); - av_frame_make_writable(clip->cache_B.frames[i]); - clip->cache_B.frames[i]->width = clip->stream->codecpar->width; - clip->cache_B.frames[i]->height = clip->stream->codecpar->height; - clip->cache_B.frames[i]->format = dest_format; - av_frame_get_buffer(clip->cache_B.frames[i], 0); - clip->cache_B.frames[i]->linesize[0] = clip->stream->codecpar->width*4; } } + + // alloc temporary scale frame + clip->sws_frame = av_frame_alloc(); + av_frame_make_writable(clip->sws_frame); + clip->sws_frame->width = dstW; + clip->sws_frame->height = dstH; + clip->sws_frame->format = dest_format; + if (av_frame_get_buffer(clip->sws_frame, 0)) { + qDebug() << "[ERROR] Could not allocate buffer for sws_frame"; + } + clip->sws_frame->linesize[0] = clip->stream->codecpar->width*4; + + // SET UP FFMPEG FILTERGRAPH + AVFilter *buffersrc = avfilter_get_by_name("buffer"); + AVFilter *buffersink = avfilter_get_by_name("buffersink"); + AVFilterInOut *outputs = avfilter_inout_alloc(); + AVFilterInOut *inputs = avfilter_inout_alloc(); + + clip->filter_graph = avfilter_graph_alloc(); + + if (outputs == NULL || inputs == NULL || clip->filter_graph == NULL) { + qDebug() << "[ERROR] Couldn't alloc filter graphs. Out of memory?"; + } else { + char args[512]; + snprintf(args, sizeof(args), + "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d", + clip->stream->codecpar->width, clip->stream->codecpar->height, clip->stream->codecpar->format, + clip->stream->time_base.num, clip->stream->time_base.den, + clip->stream->codecpar->sample_aspect_ratio.num, clip->stream->codecpar->sample_aspect_ratio.den); + + int ret = avfilter_graph_create_filter(&clip->buffersrc_ctx, buffersrc, "in", args, NULL, clip->filter_graph); + if (ret < 0) { + qDebug() << "[ERROR] Could not create buffer source"; + } else { + ret = avfilter_graph_create_filter(&clip->buffersink_ctx, buffersink, "out", NULL, NULL, clip->filter_graph); + if (ret < 0) { + qDebug() << "[ERROR] Could not create buffer sink"; + } else { + enum AVPixelFormat pix_fmts[] = { static_cast(dest_format), AV_PIX_FMT_NONE }; + av_opt_set_int_list(clip->buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN); + + outputs->name = av_strdup("in"); + outputs->filter_ctx = clip->buffersrc_ctx; + outputs->pad_idx = 0; + outputs->next = NULL; + + inputs->name = av_strdup("out"); + inputs->filter_ctx = clip->buffersink_ctx; + inputs->pad_idx = 0; + inputs->next = 0; + + ret = avfilter_graph_parse_ptr(clip->filter_graph, "null", &inputs, &outputs, NULL); + if (ret < 0) { + qDebug() << "[ERROR] Couldn't set NEGATE filter"; + } else { + ret = avfilter_graph_config(clip->filter_graph, NULL); + if (ret < 0) { + qDebug() << "[ERROR] AVFilter config failed"; + } + } + } + } + } + + avfilter_inout_free(&inputs); + avfilter_inout_free(&outputs); } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { // if FFmpeg can't pick up the channel layout (usually WAV), assume // based on channel count (doesn't support surround sound sources yet) @@ -414,6 +497,10 @@ void close_clip_worker(Clip* clip) { MediaStream* ms = static_cast(clip->media)->get_stream_from_file_index(clip->media_stream); if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { sws_freeContext(clip->sws_ctx); + + // TODO will eventually be in audio too + avfilter_graph_free(&clip->filter_graph); + av_frame_free(&clip->sws_frame); } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { swr_free(&clip->swr_ctx); } @@ -422,7 +509,7 @@ void close_clip_worker(Clip* clip) { avcodec_free_context(&clip->codecCtx); avformat_close_input(&clip->formatCtx); - for (size_t i=0;icache_size;i++) { + for (int i=0;icache_size;i++) { av_frame_free(&clip->cache_A.frames[i]); if (!ms->infinite_length) av_frame_free(&clip->cache_B.frames[i]); } diff --git a/playback/playback.cpp b/playback/playback.cpp index 9f29586a3..7b895e988 100644 --- a/playback/playback.cpp +++ b/playback/playback.cpp @@ -82,12 +82,13 @@ void get_clip_frame(Clip* c, long playhead) { if ((!ms->infinite_length && c->texture_frame != clip_time) || (ms->infinite_length && c->texture_frame == -1)) { AVFrame* current_frame = NULL; + bool no_frame = false; // get frame data if (ms->infinite_length) { // if clip is a still frame, we only need one if (c->cache_A.written) { // retrieve cached frame - current_frame = c->cache_A.frames[0]; + current_frame = c->sws_frame; } else if (c->multithreaded) { if (c->lock.tryLock()) { // grab image (multi-threaded) @@ -108,20 +109,28 @@ void get_clip_frame(Clip* c, long playhead) { bool cache_needs_reset = false; if (c->cache_A.written && clip_time >= c->cache_A.offset && clip_time < c->cache_A.offset + c->cache_size) { - if (c->cache_A.mutex.tryLock()) { // lock in case cacher is still writing to it - using_cache_A = true; - c->cache_A.unread = false; - cache = c->cache_A.frames; - cache_offset = c->cache_A.offset; - c->cache_A.mutex.unlock(); + if (clip_time < c->cache_A.offset + c->cache_A.write_count) { + if (c->cache_A.mutex.tryLock()) { // lock in case cacher is still writing to it + using_cache_A = true; + c->cache_A.unread = false; + cache = c->cache_A.frames; + cache_offset = c->cache_A.offset; + c->cache_A.mutex.unlock(); + } + } else { + no_frame = true; } } else if (c->cache_B.written && clip_time >= c->cache_B.offset && clip_time < c->cache_B.offset + c->cache_size) { - if (c->cache_B.mutex.tryLock()) { // lock in case cacher is still writing to it - using_cache_B = true; - c->cache_B.unread = false; - cache = c->cache_B.frames; - cache_offset = c->cache_B.offset; - c->cache_B.mutex.unlock(); + if (clip_time < c->cache_B.offset + c->cache_B.write_count) { + if (c->cache_B.mutex.tryLock()) { // lock in case cacher is still writing to it + using_cache_B = true; + c->cache_B.unread = false; + cache = c->cache_B.frames; + cache_offset = c->cache_B.offset; + c->cache_B.mutex.unlock(); + } + } else { + no_frame = true; } } else { // this is technically bad, unless we just seeked @@ -129,20 +138,22 @@ void get_clip_frame(Clip* c, long playhead) { cache_needs_reset = true; } - if (cache != NULL) { - current_frame = cache[clip_time - cache_offset]; - } + if (!no_frame) { + if (cache != NULL) { + current_frame = cache[clip_time - cache_offset]; + } - // determine whether we should start filling the other cache - if (!using_cache_A || !using_cache_B) { - if (c->lock.tryLock()) { - bool write_A = (!using_cache_A && !c->cache_A.unread); - bool write_B = (!using_cache_B && !c->cache_B.unread); - if (write_A || write_B) { - // if we have no cache and need to seek, start us at the current playhead, otherwise start at the end of the current cache - cache_clip(c, (cache_needs_reset) ? clip_time : cache_offset + c->cache_size, write_A, write_B, cache_needs_reset, NULL); + // determine whether we should start filling the other cache + if (!using_cache_A || !using_cache_B) { + if (c->lock.tryLock()) { + bool write_A = (!using_cache_A && !c->cache_A.unread); + bool write_B = (!using_cache_B && !c->cache_B.unread); + if (write_A || write_B) { + // if we have no cache and need to seek, start us at the current playhead, otherwise start at the end of the current cache + cache_clip(c, (cache_needs_reset) ? clip_time : cache_offset + c->cache_size, write_A, write_B, cache_needs_reset, NULL); + } + c->lock.unlock(); } - c->lock.unlock(); } } } @@ -151,7 +162,7 @@ void get_clip_frame(Clip* c, long playhead) { // set up opengl texture if (c->texture == NULL) { c->texture = new QOpenGLTexture(QOpenGLTexture::Target2D); - c->texture->setSize(ms->video_width, ms->video_height); + c->texture->setSize(current_frame->width, current_frame->height); c->texture->setFormat(QOpenGLTexture::RGBA8_UNorm); c->texture->setMipLevels(c->texture->maximumMipLevels()); c->texture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear); @@ -160,7 +171,7 @@ void get_clip_frame(Clip* c, long playhead) { c->texture->setData(0, QOpenGLTexture::RGBA, QOpenGLTexture::UInt8, current_frame->data[0]); c->texture_frame = clip_time; - } else { + } else if (!no_frame) { texture_failed = true; qDebug() << "[ERROR] Failed to retrieve frame from cache (R:" << clip_time << "| A:" << c->cache_A.offset << "-" << c->cache_A.offset+c->cache_size-1 << "| B:" << c->cache_B.offset << "-" << c->cache_B.offset+c->cache_size-1 << "| WA:" << c->cache_A.written << "| WB:" << c->cache_B.written << ")"; } @@ -229,7 +240,8 @@ void retrieve_next_frame_raw_data(Clip* c, AVFrame* output) { int ret = retrieve_next_frame(c, c->frame); if (ret >= 0) { if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { - sws_scale(c->sws_ctx, c->frame->data, c->frame->linesize, 0, c->stream->codecpar->height, output->data, output->linesize); + sws_scale(c->sws_ctx, c->frame->data, c->frame->linesize, 0, c->stream->codecpar->height, output->data, output->linesize); +// output->pts = c->frame->best_effort_timestamp; } else if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { output->pts = c->frame->pts; ret = swr_convert_frame(c->swr_ctx, output, c->frame); diff --git a/project/clip.h b/project/clip.h index 5932d7d19..0566a7533 100644 --- a/project/clip.h +++ b/project/clip.h @@ -18,6 +18,8 @@ struct AVCodec; struct AVCodecContext; struct AVFrame; struct AVPacket; +struct AVFilterGraph; +struct AVFilterContext; struct SwsContext; struct SwrContext; class QOpenGLTexture; @@ -27,6 +29,7 @@ struct ClipCache { long offset; bool written; bool unread; + int write_count; QMutex mutex; }; @@ -78,6 +81,13 @@ struct Clip AVCodecContext* codecCtx; AVPacket* pkt; AVFrame* frame; + AVFrame* sws_frame; + + // ffmpeg filters + AVFilterGraph* filter_graph; + AVFilterContext* buffersink_ctx; + AVFilterContext* buffersrc_ctx; + bool pkt_written; bool reached_end; bool open; @@ -88,7 +98,7 @@ struct Clip bool multithreaded; Cacher* cacher; QWaitCondition can_cache; - quint16 cache_size; + int cache_size; ClipCache cache_A; ClipCache cache_B; QMutex lock;