From 083954cd063cb57cb8f3ffa39ecde4a1fb08563f Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Tue, 28 Aug 2018 12:52:15 +1000 Subject: [PATCH] added noise effect and drawing nested sequences to texture --- effects/audio/audionoiseeffect.cpp | 42 +++++ effects/audio/audionoiseeffect.h | 15 ++ effects/effect.cpp | 3 + effects/effect.h | 1 + olive.pro | 6 +- playback/cacher.cpp | 24 +-- project/clip.cpp | 3 +- project/clip.h | 5 +- ui/viewerwidget.cpp | 237 +++++++++++++++-------------- ui/viewerwidget.h | 6 +- 10 files changed, 212 insertions(+), 130 deletions(-) create mode 100644 effects/audio/audionoiseeffect.cpp create mode 100644 effects/audio/audionoiseeffect.h diff --git a/effects/audio/audionoiseeffect.cpp b/effects/audio/audionoiseeffect.cpp new file mode 100644 index 000000000..b5c6e79dc --- /dev/null +++ b/effects/audio/audionoiseeffect.cpp @@ -0,0 +1,42 @@ +#include "audionoiseeffect.h" + +#include + +AudioNoiseEffect::AudioNoiseEffect(Clip* c) : Effect(c, EFFECT_TYPE_AUDIO, AUDIO_NOISE_EFFECT) { + amount_val = add_row("Amount:")->add_field(EFFECT_FIELD_DOUBLE); + amount_val->set_double_minimum_value(0); + amount_val->set_double_maximum_value(100); + amount_val->set_double_default_value(20); + + mix_val = add_row("Mix:")->add_field(EFFECT_FIELD_BOOL); + mix_val->set_bool_value(true); + + srand(QDateTime::currentMSecsSinceEpoch()); +} + +void AudioNoiseEffect::process_audio(double timecode_start, double timecode_end, quint8 *samples, int nb_bytes, int) { + double interval = (timecode_end - timecode_start)/nb_bytes; + for (int i=0;iget_double_value(timecode)*0.01; + right_noise_sample *= amount_val->get_double_value(timecode)*0.01; + + // mix with source audio + if (mix_val->get_bool_value(timecode)) { + qint16 left_sample = (qint16) (((samples[i+1] & 0xFF) << 8) | (samples[i] & 0xFF)); + qint16 right_sample = (qint16) (((samples[i+3] & 0xFF) << 8) | (samples[i+2] & 0xFF)); + left_noise_sample += left_sample; + right_noise_sample += right_sample; + } + + samples[i+3] = (quint8) (right_noise_sample >> 8); + samples[i+2] = (quint8) right_noise_sample; + samples[i+1] = (quint8) (left_noise_sample >> 8); + samples[i] = (quint8) left_noise_sample; + } +} diff --git a/effects/audio/audionoiseeffect.h b/effects/audio/audionoiseeffect.h new file mode 100644 index 000000000..0aff80a9c --- /dev/null +++ b/effects/audio/audionoiseeffect.h @@ -0,0 +1,15 @@ +#ifndef AUDIONOISEEFFECT_H +#define AUDIONOISEEFFECT_H + +#include "../effect.h" + +class AudioNoiseEffect : public Effect { +public: + AudioNoiseEffect(Clip* c); + void process_audio(double timecode_start, double timecode_end, quint8* samples, int nb_bytes, int channel_count); + + EffectField* amount_val; + EffectField* mix_val; +}; + +#endif // AUDIONOISEEFFECT_H diff --git a/effects/effect.cpp b/effects/effect.cpp index 59b4ca8fe..d1cca4f23 100644 --- a/effects/effect.cpp +++ b/effects/effect.cpp @@ -29,6 +29,7 @@ #include "effects/audio/paneffect.h" #include "effects/audio/volumeeffect.h" +#include "effects/audio/audionoiseeffect.h" #include #include @@ -55,6 +56,7 @@ void init_effects() { audio_effect_names[AUDIO_VOLUME_EFFECT] = "Volume"; audio_effect_names[AUDIO_PAN_EFFECT] = "Pan"; + audio_effect_names[AUDIO_NOISE_EFFECT] = "Noise"; } Effect* create_effect(int effect_id, Clip* c) { @@ -74,6 +76,7 @@ Effect* create_effect(int effect_id, Clip* c) { switch (effect_id) { case AUDIO_VOLUME_EFFECT: return new VolumeEffect(c); break; case AUDIO_PAN_EFFECT: return new PanEffect(c); break; + case AUDIO_NOISE_EFFECT: return new AudioNoiseEffect(c); break; } } qDebug() << "[ERROR] Invalid effect ID"; diff --git a/effects/effect.h b/effects/effect.h index c8d20d71a..9748c53f1 100644 --- a/effects/effect.h +++ b/effects/effect.h @@ -35,6 +35,7 @@ enum VideoEffects { enum AudioEffects { AUDIO_VOLUME_EFFECT, AUDIO_PAN_EFFECT, + AUDIO_NOISE_EFFECT, AUDIO_EFFECT_COUNT }; diff --git a/olive.pro b/olive.pro index d5b5f0185..264a011ad 100644 --- a/olive.pro +++ b/olive.pro @@ -75,7 +75,8 @@ SOURCES += \ effects/video/chromakeyeffect.cpp \ effects/video/gaussianblureffect.cpp \ effects/video/cropeffect.cpp \ - effects/video/flipeffect.cpp + effects/video/flipeffect.cpp \ + effects/audio/audionoiseeffect.cpp HEADERS += \ mainwindow.h \ @@ -127,7 +128,8 @@ HEADERS += \ effects/video/chromakeyeffect.h \ effects/video/gaussianblureffect.h \ effects/video/cropeffect.h \ - effects/video/flipeffect.h + effects/video/flipeffect.h \ + effects/audio/audionoiseeffect.h FORMS += \ mainwindow.ui \ diff --git a/playback/cacher.cpp b/playback/cacher.cpp index c3479acb5..bc837893e 100644 --- a/playback/cacher.cpp +++ b/playback/cacher.cpp @@ -18,15 +18,17 @@ extern "C" { } #include +#include #include #include int dest_format = AV_PIX_FMT_RGBA; -void apply_audio_effects(Clip* c, AVFrame* frame, int nb_bytes) { +void apply_audio_effects(Clip* c, double timebase, AVFrame* frame, int nb_bytes) { // perform all audio effects - double timecode_start = (frame->pts * av_q2d(c->stream->time_base)); - double timecode_end = timecode_start + ((double) (nb_bytes >> 1) / frame->channels / frame->sample_rate); + double timecode_start, timecode_end; + timecode_start = (frame->pts * timebase); + timecode_end = timecode_start + ((double) (nb_bytes >> 1) / frame->channels / frame->sample_rate); for (int j=0;jeffects.size();j++) { Effect* e = c->effects.at(j); @@ -84,8 +86,9 @@ void cache_audio_worker(Clip* c, Clip* nest) { if (!c->reached_end) { retrieve_next_frame_raw_data(c, frame); int byte_count = av_samples_get_buffer_size(NULL, frame->channels, frame->nb_samples, static_cast(frame->format), 1); - apply_audio_effects(c, frame, byte_count); - if (nest != NULL) apply_audio_effects(nest, frame, byte_count); + double timebase = av_q2d(c->stream->time_base); + apply_audio_effects(c, timebase, frame, byte_count); + if (nest != NULL) apply_audio_effects(nest, timebase, frame, byte_count); } else { // set by retrieve_next_frame_raw_data indicating no more frames in file, // but there still may be samples in swresample @@ -145,7 +148,9 @@ void cache_audio_worker(Clip* c, Clip* nest) { } } if (apply_effects) { - apply_audio_effects(c, frame, nb_bytes); + double timebase = av_q2d(c->stream->time_base); + apply_audio_effects(c, timebase, frame, nb_bytes); + if (nest != NULL) apply_audio_effects(nest, timebase, frame, nb_bytes); } while (c->frame_sample_index < nb_bytes) { @@ -343,7 +348,7 @@ void open_clip_worker(Clip* clip) { clip->cache_B.frames[i]->linesize[0] = dstW*4; } - clip->comp_frame_size = clip->stream->codecpar->width * clip->stream->codecpar->height * 4; + clip->comp_frame_size = dstW * dstH * 4; clip->comp_frame = new uchar[clip->comp_frame_size]; } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { // if FFmpeg can't pick up the channel layout (usually WAV), assume @@ -412,10 +417,11 @@ void close_clip_worker(Clip* clip) { // closes ffmpeg file handle and frees any memory used for caching MediaStream* ms = static_cast(clip->media)->get_stream_from_file_index(clip->track < 0, clip->media_stream); if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { -// sws_freeContext(clip->sws_ctx); + sws_freeContext(clip->sws_ctx); - // TODO will eventually be in audio too delete [] clip->comp_frame; + delete clip->fbo; + clip->fbo = NULL; } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { swr_free(&clip->swr_ctx); } diff --git a/project/clip.cpp b/project/clip.cpp index 0bcdb1505..b99f3b15c 100644 --- a/project/clip.cpp +++ b/project/clip.cpp @@ -28,7 +28,8 @@ Clip::Clip(Sequence* s) : closing_transition(NULL), pkt(new AVPacket()), replaced(false), - texture(NULL) + texture(NULL), + fbo(NULL) { reset(); } diff --git a/project/clip.h b/project/clip.h index a29b7e782..8e112f4ba 100644 --- a/project/clip.h +++ b/project/clip.h @@ -8,6 +8,7 @@ class Cacher; class Effect; class Transition; +class QOpenGLFramebufferObject; struct Sequence; struct Media; struct MediaStream; @@ -80,7 +81,6 @@ struct Clip AVFrame* frame; uchar* comp_frame; int comp_frame_size; - SwsContext* sws_ctx; bool pkt_written; bool reached_end; @@ -99,7 +99,8 @@ struct Clip QMutex open_lock; // video playback variables - //SwsContext* sws_ctx; + SwsContext* sws_ctx; + QOpenGLFramebufferObject* fbo; QOpenGLTexture* texture; long texture_frame; diff --git a/ui/viewerwidget.cpp b/ui/viewerwidget.cpp index ae0de9894..d75baf45e 100644 --- a/ui/viewerwidget.cpp +++ b/ui/viewerwidget.cpp @@ -28,8 +28,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) : multithreaded(true), force_audio(false), enable_paint(true), - flip(false), - fbo(NULL) + flip(false) { QSurfaceFormat format; format.setDepthBufferSize(24); @@ -82,11 +81,11 @@ void ViewerWidget::paintEvent(QPaintEvent *e) { if (enable_paint) QOpenGLWidget::paintEvent(e); } -GLuint ViewerWidget::draw_clip(GLuint texture) { +GLuint ViewerWidget::draw_clip(Clip* clip, GLuint texture) { glPushMatrix(); glLoadIdentity(); glOrtho(0, 1, 0, 1, -1, 1); - fbo->bind(); + clip->fbo->bind(); glBindTexture(GL_TEXTURE_2D, texture); glBegin(GL_QUADS); glTexCoord2f(0, 0); // top left @@ -99,22 +98,20 @@ GLuint ViewerWidget::draw_clip(GLuint texture) { glVertex2f(0, 1); // bottom left glEnd(); glBindTexture(GL_TEXTURE_2D, NULL); - fbo->release(); + clip->fbo->release(); glPopMatrix(); - return fbo->takeTexture(); + return clip->fbo->takeTexture(); } -void ViewerWidget::compose_sequence(QVector& nests, bool render_audio) { - Sequence* s = sequence; +GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { + Sequence* s = sequence; long playhead = panel_timeline->playhead; - Clip* nest = NULL; - if (nests.size() > 0) { - nest = nests.last(); + if (nest != NULL) { s = static_cast(nest->media); - playhead += nest->clip_in - nest->timeline_in; - playhead = refactor_frame_number(playhead, sequence->frame_rate, s->frame_rate); - } + playhead += nest->clip_in - nest->timeline_in; + playhead = refactor_frame_number(playhead, nest->sequence->frame_rate, s->frame_rate); + } QVector current_clips; @@ -175,105 +172,122 @@ void ViewerWidget::compose_sequence(QVector& nests, bool render_audio) { qDebug() << "[WARNING] Tried to display clip" << i << "but it's closed"; texture_failed = true; } else { - switch (c->media_type) { - case MEDIA_TYPE_FOOTAGE: - if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { - // start preparing cache - get_clip_frame(c, playhead); + if (c->track < 0) { + GLuint textureID = 0; + int video_width, video_height; - if (c->texture == NULL) { - qDebug() << "[WARNING] Texture hasn't been created yet"; - texture_failed = true; - } else if (playhead >= c->timeline_in) { - MediaStream* ms = static_cast(c->media)->get_stream_from_file_index(c->track < 0, c->media_stream); - - int half_width = sequence->width/2; - int half_height = sequence->height/2; - if (flip) half_height = -half_height; - glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); - glColor4f(1.0, 1.0, 1.0, 1.0); - glLoadIdentity(); - glOrtho(-half_width, half_width, half_height, -half_height, -1, 1); - glViewport(0, 0, ms->video_width, ms->video_height); - - // TEMP DEBUG CODE - if (fbo != NULL) delete fbo; - fbo = new QOpenGLFramebufferObject(ms->video_width, ms->video_height); - GLuint composite_texture = draw_clip(c->texture->textureId()); - // END - - GLTextureCoords coords; - coords.vertexTopLeftX = coords.vertexBottomLeftX = -ms->video_width/2; - coords.vertexTopLeftY = coords.vertexTopRightY = -ms->video_height/2; - coords.vertexTopRightX = coords.vertexBottomRightX = ms->video_width/2; - coords.vertexBottomLeftY = coords.vertexBottomRightY = ms->video_height/2; - coords.textureTopLeftY = coords.textureTopRightY = coords.textureTopLeftX = coords.textureBottomLeftX = 0; - coords.textureBottomLeftY = coords.textureBottomRightY = coords.textureTopRightX = coords.textureBottomRightX = 1.0; - - // EFFECT CODE START - for (int j=0;jeffects.size();j++) { - if (c->effects.at(j)->enable_opengl && c->effects.at(j)->is_enabled()) { - c->effects.at(j)->process_gl(((double)(panel_timeline->playhead-c->timeline_in+c->clip_in)/(double)sequence->frame_rate), coords); - composite_texture = draw_clip(composite_texture); - } - } - - if (c->opening_transition != NULL) { - int transition_progress = playhead - c->timeline_in; - if (transition_progress < c->opening_transition->length) { - c->opening_transition->process_transition((double)transition_progress/(double)c->opening_transition->length); - } - } - - if (c->closing_transition != NULL) { - int transition_progress = c->closing_transition->length - (playhead - c->timeline_in - c->getLength() + c->closing_transition->length); - if (transition_progress < c->closing_transition->length) { - c->closing_transition->process_transition((double)transition_progress/(double)c->closing_transition->length); - } - } - // EFFECT CODE END - - for (int j=0;jeffects.size();j++) { - if (c->effects.at(j)->enable_opengl && c->effects.at(j)->is_enabled()) { - c->effects.at(j)->clean_gl(); - } - } - - glViewport(0, 0, width(), height()); - - glBindTexture(GL_TEXTURE_2D, composite_texture); - - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - - glBegin(GL_QUADS); - glTexCoord2f(coords.textureTopLeftX, coords.textureTopLeftY); // top left - glVertex2f(coords.vertexTopLeftX, coords.vertexTopLeftY); // top left - glTexCoord2f(coords.textureTopRightX, coords.textureTopRightY); // top right - glVertex2f(coords.vertexTopRightX, coords.vertexTopRightY); // top right - glTexCoord2f(coords.textureBottomRightX, coords.textureBottomRightY); // bottom right - glVertex2f(coords.vertexBottomRightX, coords.vertexBottomRightY); // bottom right - glTexCoord2f(coords.textureBottomLeftX, coords.textureBottomLeftY); // bottom left - glVertex2f(coords.vertexBottomLeftX, coords.vertexBottomLeftY); // bottom left - glEnd(); - - glDeleteTextures(1, &composite_texture); - } - } else if (render_audio && - c->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && - c->lock.tryLock()) { - // clip is not caching, start caching audio - cache_clip(c, playhead, false, false, c->audio_reset, nest); - c->lock.unlock(); + if (c->media_type == MEDIA_TYPE_FOOTAGE) { + get_clip_frame(c, playhead); + MediaStream* ms = static_cast(c->media)->get_stream_from_file_index(c->track < 0, c->media_stream); + video_width = ms->video_width; + video_height = ms->video_height; + if (c->texture != NULL) textureID = c->texture->textureId(); + } else if (c->media_type == MEDIA_TYPE_SEQUENCE) { + video_width = s->width; + video_height = s->height; + textureID = compose_sequence(c, render_audio); } - break; - case MEDIA_TYPE_SEQUENCE: - nests.append(c); - compose_sequence(nests, render_audio); - break; - } + + if (textureID == 0) { + qDebug() << "[WARNING] Texture hasn't been created yet"; + texture_failed = true; + } else if (playhead >= c->timeline_in) { + // start preparing cache + if (c->fbo == NULL) c->fbo = new QOpenGLFramebufferObject(video_width, video_height); + + int half_width = c->sequence->width/2; + int half_height = c->sequence->height/2; + if (flip || nest != NULL) half_height = -half_height; + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + glColor4f(1.0, 1.0, 1.0, 1.0); + glLoadIdentity(); + glOrtho(-half_width, half_width, half_height, -half_height, -1, 1); + glViewport(0, 0, video_width, video_height); + + GLuint composite_texture = draw_clip(c, textureID); + + GLTextureCoords coords; + coords.vertexTopLeftX = coords.vertexBottomLeftX = -video_width/2; + coords.vertexTopLeftY = coords.vertexTopRightY = -video_height/2; + coords.vertexTopRightX = coords.vertexBottomRightX = video_width/2; + coords.vertexBottomLeftY = coords.vertexBottomRightY = video_height/2; + coords.textureTopLeftY = coords.textureTopRightY = coords.textureTopLeftX = coords.textureBottomLeftX = 0; + coords.textureBottomLeftY = coords.textureBottomRightY = coords.textureTopRightX = coords.textureBottomRightX = 1.0; + + // EFFECT CODE START + for (int j=0;jeffects.size();j++) { + if (c->effects.at(j)->enable_opengl && c->effects.at(j)->is_enabled()) { + c->effects.at(j)->process_gl(((double)(panel_timeline->playhead-c->timeline_in+c->clip_in)/(double)sequence->frame_rate), coords); + composite_texture = draw_clip(c, composite_texture); + } + } + + if (c->opening_transition != NULL) { + int transition_progress = playhead - c->timeline_in; + if (transition_progress < c->opening_transition->length) { + c->opening_transition->process_transition((double)transition_progress/(double)c->opening_transition->length); + } + } + + if (c->closing_transition != NULL) { + int transition_progress = c->closing_transition->length - (playhead - c->timeline_in - c->getLength() + c->closing_transition->length); + if (transition_progress < c->closing_transition->length) { + c->closing_transition->process_transition((double)transition_progress/(double)c->closing_transition->length); + } + } + // EFFECT CODE END + + for (int j=0;jeffects.size();j++) { + if (c->effects.at(j)->enable_opengl && c->effects.at(j)->is_enabled()) { + c->effects.at(j)->clean_gl(); + } + } + + if (nest != NULL) { + if (nest->fbo == NULL) nest->fbo = new QOpenGLFramebufferObject(s->width, s->height); + nest->fbo->bind(); + glViewport(0, 0, s->width, s->height); + } else { + glViewport(0, 0, width(), height()); + } + + glBindTexture(GL_TEXTURE_2D, composite_texture); + + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + + glBegin(GL_QUADS); + glTexCoord2f(coords.textureTopLeftX, coords.textureTopLeftY); // top left + glVertex2f(coords.vertexTopLeftX, coords.vertexTopLeftY); // top left + glTexCoord2f(coords.textureTopRightX, coords.textureTopRightY); // top right + glVertex2f(coords.vertexTopRightX, coords.vertexTopRightY); // top right + glTexCoord2f(coords.textureBottomRightX, coords.textureBottomRightY); // bottom right + glVertex2f(coords.vertexBottomRightX, coords.vertexBottomRightY); // bottom right + glTexCoord2f(coords.textureBottomLeftX, coords.textureBottomLeftY); // bottom left + glVertex2f(coords.vertexBottomLeftX, coords.vertexBottomLeftY); // bottom left + glEnd(); + + glDeleteTextures(1, &composite_texture); + + if (nest != NULL) { + nest->fbo->release(); + } + } + } else { + if (c->media_type == MEDIA_TYPE_FOOTAGE + && render_audio + && c->lock.tryLock()) { + // clip is not caching, start caching audio + cache_clip(c, playhead, false, false, c->audio_reset, nest); + c->lock.unlock(); + } else if (c->media_type == MEDIA_TYPE_SEQUENCE) { + compose_sequence(c, render_audio); + } + } } } + + return (nest != NULL && nest->fbo != NULL) ? nest->fbo->takeTexture() : 0; } void ViewerWidget::paintGL() { @@ -291,9 +305,8 @@ void ViewerWidget::paintGL() { glClear(GL_COLOR_BUFFER_BIT); - // compose video preview - QVector nests; - compose_sequence(nests, (panel_timeline->playing || force_audio)); + // compose video preview + compose_sequence(NULL, (panel_timeline->playing || force_audio)); if (texture_failed) { if (multithreaded) { diff --git a/ui/viewerwidget.h b/ui/viewerwidget.h index e58471d8a..489b400b6 100644 --- a/ui/viewerwidget.h +++ b/ui/viewerwidget.h @@ -10,7 +10,6 @@ #include #include -class QOpenGLFramebufferObject; struct Clip; struct Sequence; @@ -31,12 +30,11 @@ protected: // void resizeGL(int w, int h); private: QTimer retry_timer; - QOpenGLFramebufferObject* fbo; private slots: void retry(); void deleteFunction(); - void compose_sequence(QVector& nests, bool render_audio); - GLuint draw_clip(GLuint texture); + GLuint compose_sequence(Clip *nest, bool render_audio); + GLuint draw_clip(Clip *clip, GLuint texture); }; #endif // VIEWERWIDGET_H