oak-team/oak-editor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implemented backend for AVFilters

Browse Source
This commit is contained in:
itsmattkc
2018-08-15 21:58:29 +10:00
parent 4a96509fbe
commit efa628e88f
6 changed files with 189 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
effects/shakeeffect.cpp
+2
View File
@@ -13,6 +13,8 @@
#include "panels/timeline.h"
ShakeEffect::ShakeEffect(Clip *c) : Effect(c, EFFECT_TYPE_VIDEO, VIDEO_SHAKE_EFFECT), inside(false) {
enable_pre_gl = true;
EffectRow* intensity_row = add_row("Intensity:");
intensity_val = intensity_row->add_field(EFFECT_FIELD_DOUBLE);
intensity_val->set_double_minimum_value(0);
main.cpp
+2 -2
View File
@@ -3,14 +3,14 @@
extern "C" {
#include <libavformat/avformat.h>
#include <libavfilter/avfilter.h>
}
int main(int argc, char *argv[])
{
// init ffmpeg subsystem
av_register_all();
// QCoreApplication::setAttribute(Qt::AA_ShareOpenGLContexts, true);
avfilter_register_all();
QApplication a(argc, argv);
MainWindow w;
olive.pro
+3 -3
View File
@@ -127,16 +127,16 @@ win32 {
LIBS += -L../ffmpeg/lib -lopengl32
INCLUDEPATH = ../ffmpeg/include
RC_FILE = icons/win.rc
LIBS += -lavutil -lavformat -lavcodec -lswscale -lswresample
LIBS += -lavutil -lavformat -lavcodec -lavfilter -lswscale -lswresample
}
mac {
LIBS += -L/usr/local/lib -lavutil -lavformat -lavcodec -lswscale -lswresample
LIBS += -L/usr/local/lib -lavutil -lavformat -lavcodec -lavfilter -lswscale -lswresample
INCLUDEPATH = /usr/local/include
}
linux {
LIBS += -lavutil -lavformat -lavcodec -lswscale -lswresample
LIBS += -lavutil -lavformat -lavcodec -lavfilter -lswscale -lswresample
}
RESOURCES += \
playback/cacher.cpp
+131 -44
View File
@@ -12,6 +12,10 @@
extern "C" {
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavfilter/avfilter.h>
#include <libavfilter/buffersrc.h>
#include <libavfilter/buffersink.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
#include <libswresample/swresample.h>
@@ -155,16 +159,9 @@ void cache_audio_worker(Clip* c, Clip* nest) {
int upper_byte_index = (c->audio_buffer_write+1)%audio_ibuffer_size;
int lower_byte_index = (c->audio_buffer_write)%audio_ibuffer_size;
qint16 old_sample = (qint16) ((audio_ibuffer[upper_byte_index] & 0xFF) << 8 | (audio_ibuffer[lower_byte_index] & 0xFF));
qint16 new_sample = (qint16) ((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF));
qint32 mixed_sample;
// peak handling
mixed_sample = old_sample + new_sample;
if (mixed_sample > INT16_MAX) {
mixed_sample = INT16_MAX;
} else if (mixed_sample < INT16_MIN) {
mixed_sample = INT16_MIN;
}
qint16 new_sample = (qint16) ((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF));
qint32 mixed_sample = old_sample + new_sample;
mixed_sample = qMax(qMin(mixed_sample, (qint32)INT16_MAX), (qint32)INT16_MIN);
audio_ibuffer[upper_byte_index] = (quint8) (mixed_sample >> 8);
audio_ibuffer[lower_byte_index] = (quint8) mixed_sample;
@@ -184,16 +181,52 @@ void cache_video_worker(Clip* c, long playhead, ClipCache* cache) {
cache->mutex.lock();
cache->offset = playhead;
if (!c->reached_end) {
for (size_t i=0;i<c->cache_size;i++) {
retrieve_next_frame_raw_data(c, cache->frames[i]);
if (c->reached_end) break;
}
bool error = false;
int i = 0;
if (!c->reached_end) {
while (i < c->cache_size) {
av_frame_unref(cache->frames[i]);
int ret = av_buffersink_get_frame(c->buffersink_ctx, cache->frames[i]);
if (ret < 0) {
if (ret == AVERROR(EAGAIN)) {
ret = retrieve_next_frame(c, c->frame);
if (ret >= 0) {
if ((ret = av_buffersrc_add_frame_flags(c->buffersrc_ctx, c->frame, AV_BUFFERSRC_FLAG_KEEP_REF)) < 0) {
qDebug() << "[ERROR] Could not feed filtergraph -" << ret;
error = true;
break;
}
} else {
if (ret == AVERROR_EOF) {
c->reached_end = true;
} else {
qDebug() << "[WARNING] Raw frame data could not be retrieved." << ret;
error = true;
}
break;
}
} else {
if (ret != AVERROR_EOF) {
qDebug() << "[ERROR] Could not pull from filtergraph";
error = true;
}
break;
}
} else {
i++;
}
}
}
cache->written = true;
cache->unread = true;
// setting the cache to written even if it hasn't reached_end prevents playback from
// signaling a seek/reset because it wasn't able to find the frame
cache->write_count = i;
if (!error) {
// setting the cache to written even if it hasn't reached_end prevents playback from
// signaling a seek/reset because it wasn't able to find the frame
cache->written = true;
cache->unread = true;
}
cache->mutex.unlock();
}
@@ -204,7 +237,7 @@ void reset_cache(Clip* c, long target_frame) {
if (ms->infinite_length) {
// if this clip is a still image, we only need one frame
if (!c->cache_A.written) {
retrieve_next_frame_raw_data(c, c->cache_A.frames[0]);
retrieve_next_frame_raw_data(c, c->sws_frame);
c->cache_A.written = true;
}
} else {
@@ -299,14 +332,16 @@ void open_clip_worker(Clip* clip) {
if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
// set up swscale context - primarily used for colorspace conversion
// as "scaling" is actually done by OpenGL
int dstW = ceil(clip->stream->codecpar->width/2)*2;
int dstH = ceil(clip->stream->codecpar->height/2)*2;
int dest_format = AV_PIX_FMT_RGBA;
clip->sws_ctx = sws_getContext(
clip->stream->codecpar->width,
clip->stream->codecpar->height,
static_cast<AVPixelFormat>(clip->stream->codecpar->format),
ceil(clip->stream->codecpar->width/2)*2,
ceil(clip->stream->codecpar->height/2)*2,
dstW,
dstH,
static_cast<AVPixelFormat>(dest_format),
SWS_FAST_BILINEAR,
NULL,
@@ -321,31 +356,79 @@ void open_clip_worker(Clip* clip) {
clip->cache_size = ceil(av_q2d(av_guess_frame_rate(clip->formatCtx, clip->stream, NULL))/4); // cache is half a second in total
// infinite length doesn't need cache B
clip->cache_A.frames = new AVFrame* [clip->cache_size];
clip->cache_B.frames = new AVFrame* [clip->cache_size];
}
clip->cache_A.frames = new AVFrame* [clip->cache_size];
for (size_t i=0;i<clip->cache_size;i++) {
clip->cache_A.frames[i] = av_frame_alloc();
av_frame_make_writable(clip->cache_A.frames[i]);
clip->cache_A.frames[i]->width = clip->stream->codecpar->width;
clip->cache_A.frames[i]->height = clip->stream->codecpar->height;
clip->cache_A.frames[i]->format = dest_format;
av_frame_get_buffer(clip->cache_A.frames[i], 0);
clip->cache_A.frames[i]->linesize[0] = clip->stream->codecpar->width*4;
if (!ms->infinite_length) {
for (int i=0;i<clip->cache_size;i++) {
clip->cache_A.frames[i] = av_frame_alloc();
clip->cache_B.frames[i] = av_frame_alloc();
av_frame_make_writable(clip->cache_B.frames[i]);
clip->cache_B.frames[i]->width = clip->stream->codecpar->width;
clip->cache_B.frames[i]->height = clip->stream->codecpar->height;
clip->cache_B.frames[i]->format = dest_format;
av_frame_get_buffer(clip->cache_B.frames[i], 0);
clip->cache_B.frames[i]->linesize[0] = clip->stream->codecpar->width*4;
}
}
// alloc temporary scale frame
clip->sws_frame = av_frame_alloc();
av_frame_make_writable(clip->sws_frame);
clip->sws_frame->width = dstW;
clip->sws_frame->height = dstH;
clip->sws_frame->format = dest_format;
if (av_frame_get_buffer(clip->sws_frame, 0)) {
qDebug() << "[ERROR] Could not allocate buffer for sws_frame";
}
clip->sws_frame->linesize[0] = clip->stream->codecpar->width*4;
// SET UP FFMPEG FILTERGRAPH
AVFilter *buffersrc = avfilter_get_by_name("buffer");
AVFilter *buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut *outputs = avfilter_inout_alloc();
AVFilterInOut *inputs = avfilter_inout_alloc();
clip->filter_graph = avfilter_graph_alloc();
if (outputs == NULL || inputs == NULL || clip->filter_graph == NULL) {
qDebug() << "[ERROR] Couldn't alloc filter graphs. Out of memory?";
} else {
char args[512];
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
clip->stream->codecpar->width, clip->stream->codecpar->height, clip->stream->codecpar->format,
clip->stream->time_base.num, clip->stream->time_base.den,
clip->stream->codecpar->sample_aspect_ratio.num, clip->stream->codecpar->sample_aspect_ratio.den);
int ret = avfilter_graph_create_filter(&clip->buffersrc_ctx, buffersrc, "in", args, NULL, clip->filter_graph);
if (ret < 0) {
qDebug() << "[ERROR] Could not create buffer source";
} else {
ret = avfilter_graph_create_filter(&clip->buffersink_ctx, buffersink, "out", NULL, NULL, clip->filter_graph);
if (ret < 0) {
qDebug() << "[ERROR] Could not create buffer sink";
} else {
enum AVPixelFormat pix_fmts[] = { static_cast<AVPixelFormat>(dest_format), AV_PIX_FMT_NONE };
av_opt_set_int_list(clip->buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
outputs->name = av_strdup("in");
outputs->filter_ctx = clip->buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
inputs->name = av_strdup("out");
inputs->filter_ctx = clip->buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = 0;
ret = avfilter_graph_parse_ptr(clip->filter_graph, "null", &inputs, &outputs, NULL);
if (ret < 0) {
qDebug() << "[ERROR] Couldn't set NEGATE filter";
} else {
ret = avfilter_graph_config(clip->filter_graph, NULL);
if (ret < 0) {
qDebug() << "[ERROR] AVFilter config failed";
}
}
}
}
}
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
} else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
// if FFmpeg can't pick up the channel layout (usually WAV), assume
// based on channel count (doesn't support surround sound sources yet)
@@ -414,6 +497,10 @@ void close_clip_worker(Clip* clip) {
MediaStream* ms = static_cast<Media*>(clip->media)->get_stream_from_file_index(clip->media_stream);
if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
sws_freeContext(clip->sws_ctx);
// TODO will eventually be in audio too
avfilter_graph_free(&clip->filter_graph);
av_frame_free(&clip->sws_frame);
} else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
swr_free(&clip->swr_ctx);
}
@@ -422,7 +509,7 @@ void close_clip_worker(Clip* clip) {
avcodec_free_context(&clip->codecCtx);
avformat_close_input(&clip->formatCtx);
for (size_t i=0;i<clip->cache_size;i++) {
for (int i=0;i<clip->cache_size;i++) {
av_frame_free(&clip->cache_A.frames[i]);
if (!ms->infinite_length) av_frame_free(&clip->cache_B.frames[i]);
}
playback/playback.cpp
+40 -28
View File
@@ -82,12 +82,13 @@ void get_clip_frame(Clip* c, long playhead) {
if ((!ms->infinite_length && c->texture_frame != clip_time) ||
(ms->infinite_length && c->texture_frame == -1)) {
AVFrame* current_frame = NULL;
bool no_frame = false;
// get frame data
if (ms->infinite_length) { // if clip is a still frame, we only need one
if (c->cache_A.written) {
// retrieve cached frame
current_frame = c->cache_A.frames[0];
current_frame = c->sws_frame;
} else if (c->multithreaded) {
if (c->lock.tryLock()) {
// grab image (multi-threaded)
@@ -108,20 +109,28 @@ void get_clip_frame(Clip* c, long playhead) {
bool cache_needs_reset = false;
if (c->cache_A.written && clip_time >= c->cache_A.offset && clip_time < c->cache_A.offset + c->cache_size) {
if (c->cache_A.mutex.tryLock()) { // lock in case cacher is still writing to it
using_cache_A = true;
c->cache_A.unread = false;
cache = c->cache_A.frames;
cache_offset = c->cache_A.offset;
c->cache_A.mutex.unlock();
if (clip_time < c->cache_A.offset + c->cache_A.write_count) {
if (c->cache_A.mutex.tryLock()) { // lock in case cacher is still writing to it
using_cache_A = true;
c->cache_A.unread = false;
cache = c->cache_A.frames;
cache_offset = c->cache_A.offset;
c->cache_A.mutex.unlock();
}
} else {
no_frame = true;
}
} else if (c->cache_B.written && clip_time >= c->cache_B.offset && clip_time < c->cache_B.offset + c->cache_size) {
if (c->cache_B.mutex.tryLock()) { // lock in case cacher is still writing to it
using_cache_B = true;
c->cache_B.unread = false;
cache = c->cache_B.frames;
cache_offset = c->cache_B.offset;
c->cache_B.mutex.unlock();
if (clip_time < c->cache_B.offset + c->cache_B.write_count) {
if (c->cache_B.mutex.tryLock()) { // lock in case cacher is still writing to it
using_cache_B = true;
c->cache_B.unread = false;
cache = c->cache_B.frames;
cache_offset = c->cache_B.offset;
c->cache_B.mutex.unlock();
}
} else {
no_frame = true;
}
} else {
// this is technically bad, unless we just seeked
@@ -129,20 +138,22 @@ void get_clip_frame(Clip* c, long playhead) {
cache_needs_reset = true;
}
if (cache != NULL) {
current_frame = cache[clip_time - cache_offset];
}
if (!no_frame) {
if (cache != NULL) {
current_frame = cache[clip_time - cache_offset];
}
// determine whether we should start filling the other cache
if (!using_cache_A || !using_cache_B) {
if (c->lock.tryLock()) {
bool write_A = (!using_cache_A && !c->cache_A.unread);
bool write_B = (!using_cache_B && !c->cache_B.unread);
if (write_A || write_B) {
// if we have no cache and need to seek, start us at the current playhead, otherwise start at the end of the current cache
cache_clip(c, (cache_needs_reset) ? clip_time : cache_offset + c->cache_size, write_A, write_B, cache_needs_reset, NULL);
// determine whether we should start filling the other cache
if (!using_cache_A || !using_cache_B) {
if (c->lock.tryLock()) {
bool write_A = (!using_cache_A && !c->cache_A.unread);
bool write_B = (!using_cache_B && !c->cache_B.unread);
if (write_A || write_B) {
// if we have no cache and need to seek, start us at the current playhead, otherwise start at the end of the current cache
cache_clip(c, (cache_needs_reset) ? clip_time : cache_offset + c->cache_size, write_A, write_B, cache_needs_reset, NULL);
}
c->lock.unlock();
}
c->lock.unlock();
}
}
}
@@ -151,7 +162,7 @@ void get_clip_frame(Clip* c, long playhead) {
// set up opengl texture
if (c->texture == NULL) {
c->texture = new QOpenGLTexture(QOpenGLTexture::Target2D);
c->texture->setSize(ms->video_width, ms->video_height);
c->texture->setSize(current_frame->width, current_frame->height);
c->texture->setFormat(QOpenGLTexture::RGBA8_UNorm);
c->texture->setMipLevels(c->texture->maximumMipLevels());
c->texture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear);
@@ -160,7 +171,7 @@ void get_clip_frame(Clip* c, long playhead) {
c->texture->setData(0, QOpenGLTexture::RGBA, QOpenGLTexture::UInt8, current_frame->data[0]);
c->texture_frame = clip_time;
} else {
} else if (!no_frame) {
texture_failed = true;
qDebug() << "[ERROR] Failed to retrieve frame from cache (R:" << clip_time << "| A:" << c->cache_A.offset << "-" << c->cache_A.offset+c->cache_size-1 << "| B:" << c->cache_B.offset << "-" << c->cache_B.offset+c->cache_size-1 << "| WA:" << c->cache_A.written << "| WB:" << c->cache_B.written << ")";
}
@@ -229,7 +240,8 @@ void retrieve_next_frame_raw_data(Clip* c, AVFrame* output) {
int ret = retrieve_next_frame(c, c->frame);
if (ret >= 0) {
if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
sws_scale(c->sws_ctx, c->frame->data, c->frame->linesize, 0, c->stream->codecpar->height, output->data, output->linesize);
sws_scale(c->sws_ctx, c->frame->data, c->frame->linesize, 0, c->stream->codecpar->height, output->data, output->linesize);
// output->pts = c->frame->best_effort_timestamp;
} else if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
output->pts = c->frame->pts;
ret = swr_convert_frame(c->swr_ctx, output, c->frame);
project/clip.h
+11 -1
View File
@@ -18,6 +18,8 @@ struct AVCodec;
struct AVCodecContext;
struct AVFrame;
struct AVPacket;
struct AVFilterGraph;
struct AVFilterContext;
struct SwsContext;
struct SwrContext;
class QOpenGLTexture;
@@ -27,6 +29,7 @@ struct ClipCache {
long offset;
bool written;
bool unread;
int write_count;
QMutex mutex;
};
@@ -78,6 +81,13 @@ struct Clip
AVCodecContext* codecCtx;
AVPacket* pkt;
AVFrame* frame;
AVFrame* sws_frame;
// ffmpeg filters
AVFilterGraph* filter_graph;
AVFilterContext* buffersink_ctx;
AVFilterContext* buffersrc_ctx;
bool pkt_written;
bool reached_end;
bool open;
@@ -88,7 +98,7 @@ struct Clip
bool multithreaded;
Cacher* cacher;
QWaitCondition can_cache;
quint16 cache_size;
int cache_size;
ClipCache cache_A;
ClipCache cache_B;
QMutex lock;