oak-team/oak-editor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added noise effect and drawing nested sequences to texture

Browse Source
This commit is contained in:
itsmattkc
2018-08-28 12:52:15 +10:00
parent e3f1d31a2b
commit 083954cd06
10 changed files with 212 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
effects/audio/audionoiseeffect.cpp
+42
View File
@@ -0,0 +1,42 @@
#include "audionoiseeffect.h"
#include <QDateTime>
AudioNoiseEffect::AudioNoiseEffect(Clip* c) : Effect(c, EFFECT_TYPE_AUDIO, AUDIO_NOISE_EFFECT) {
amount_val = add_row("Amount:")->add_field(EFFECT_FIELD_DOUBLE);
amount_val->set_double_minimum_value(0);
amount_val->set_double_maximum_value(100);
amount_val->set_double_default_value(20);
mix_val = add_row("Mix:")->add_field(EFFECT_FIELD_BOOL);
mix_val->set_bool_value(true);
srand(QDateTime::currentMSecsSinceEpoch());
}
void AudioNoiseEffect::process_audio(double timecode_start, double timecode_end, quint8 *samples, int nb_bytes, int) {
double interval = (timecode_end - timecode_start)/nb_bytes;
for (int i=0;i<nb_bytes;i+=4) {
double timecode = timecode_start+(interval*i);
qint16 left_noise_sample = rand();
qint16 right_noise_sample = rand();
// set noise volume
left_noise_sample *= amount_val->get_double_value(timecode)*0.01;
right_noise_sample *= amount_val->get_double_value(timecode)*0.01;
// mix with source audio
if (mix_val->get_bool_value(timecode)) {
qint16 left_sample = (qint16) (((samples[i+1] & 0xFF) << 8) | (samples[i] & 0xFF));
qint16 right_sample = (qint16) (((samples[i+3] & 0xFF) << 8) | (samples[i+2] & 0xFF));
left_noise_sample += left_sample;
right_noise_sample += right_sample;
}
samples[i+3] = (quint8) (right_noise_sample >> 8);
samples[i+2] = (quint8) right_noise_sample;
samples[i+1] = (quint8) (left_noise_sample >> 8);
samples[i] = (quint8) left_noise_sample;
}
}
effects/audio/audionoiseeffect.h
+15
View File
@@ -0,0 +1,15 @@
#ifndef AUDIONOISEEFFECT_H
#define AUDIONOISEEFFECT_H
#include "../effect.h"
class AudioNoiseEffect : public Effect {
public:
AudioNoiseEffect(Clip* c);
void process_audio(double timecode_start, double timecode_end, quint8* samples, int nb_bytes, int channel_count);
EffectField* amount_val;
EffectField* mix_val;
};
#endif // AUDIONOISEEFFECT_H
effects/effect.cpp
+3
View File
@@ -29,6 +29,7 @@
#include "effects/audio/paneffect.h"
#include "effects/audio/volumeeffect.h"
#include "effects/audio/audionoiseeffect.h"
#include <QCheckBox>
#include <QGridLayout>
@@ -55,6 +56,7 @@ void init_effects() {
audio_effect_names[AUDIO_VOLUME_EFFECT] = "Volume";
audio_effect_names[AUDIO_PAN_EFFECT] = "Pan";
audio_effect_names[AUDIO_NOISE_EFFECT] = "Noise";
}
Effect* create_effect(int effect_id, Clip* c) {
@@ -74,6 +76,7 @@ Effect* create_effect(int effect_id, Clip* c) {
switch (effect_id) {
case AUDIO_VOLUME_EFFECT: return new VolumeEffect(c); break;
case AUDIO_PAN_EFFECT: return new PanEffect(c); break;
case AUDIO_NOISE_EFFECT: return new AudioNoiseEffect(c); break;
}
}
qDebug() << "[ERROR] Invalid effect ID";
effects/effect.h
+1
View File
@@ -35,6 +35,7 @@ enum VideoEffects {
enum AudioEffects {
AUDIO_VOLUME_EFFECT,
AUDIO_PAN_EFFECT,
AUDIO_NOISE_EFFECT,
AUDIO_EFFECT_COUNT
};
olive.pro
+4 -2
View File
@@ -75,7 +75,8 @@ SOURCES += \
effects/video/chromakeyeffect.cpp \
effects/video/gaussianblureffect.cpp \
effects/video/cropeffect.cpp \
effects/video/flipeffect.cpp
effects/video/flipeffect.cpp \
effects/audio/audionoiseeffect.cpp
HEADERS += \
mainwindow.h \
@@ -127,7 +128,8 @@ HEADERS += \
effects/video/chromakeyeffect.h \
effects/video/gaussianblureffect.h \
effects/video/cropeffect.h \
effects/video/flipeffect.h
effects/video/flipeffect.h \
effects/audio/audionoiseeffect.h
FORMS += \
mainwindow.ui \
playback/cacher.cpp
+15 -9
View File
@@ -18,15 +18,17 @@ extern "C" {
}
#include <QDebug>
#include <QOpenGLFramebufferObject>
#include <QtMath>
#include <math.h>
int dest_format = AV_PIX_FMT_RGBA;
void apply_audio_effects(Clip* c, AVFrame* frame, int nb_bytes) {
void apply_audio_effects(Clip* c, double timebase, AVFrame* frame, int nb_bytes) {
// perform all audio effects
double timecode_start = (frame->pts * av_q2d(c->stream->time_base));
double timecode_end = timecode_start + ((double) (nb_bytes >> 1) / frame->channels / frame->sample_rate);
double timecode_start, timecode_end;
timecode_start = (frame->pts * timebase);
timecode_end = timecode_start + ((double) (nb_bytes >> 1) / frame->channels / frame->sample_rate);
for (int j=0;j<c->effects.size();j++) {
Effect* e = c->effects.at(j);
@@ -84,8 +86,9 @@ void cache_audio_worker(Clip* c, Clip* nest) {
if (!c->reached_end) {
retrieve_next_frame_raw_data(c, frame);
int byte_count = av_samples_get_buffer_size(NULL, frame->channels, frame->nb_samples, static_cast<AVSampleFormat>(frame->format), 1);
apply_audio_effects(c, frame, byte_count);
if (nest != NULL) apply_audio_effects(nest, frame, byte_count);
double timebase = av_q2d(c->stream->time_base);
apply_audio_effects(c, timebase, frame, byte_count);
if (nest != NULL) apply_audio_effects(nest, timebase, frame, byte_count);
} else {
// set by retrieve_next_frame_raw_data indicating no more frames in file,
// but there still may be samples in swresample
@@ -145,7 +148,9 @@ void cache_audio_worker(Clip* c, Clip* nest) {
}
}
if (apply_effects) {
apply_audio_effects(c, frame, nb_bytes);
double timebase = av_q2d(c->stream->time_base);
apply_audio_effects(c, timebase, frame, nb_bytes);
if (nest != NULL) apply_audio_effects(nest, timebase, frame, nb_bytes);
}
while (c->frame_sample_index < nb_bytes) {
@@ -343,7 +348,7 @@ void open_clip_worker(Clip* clip) {
clip->cache_B.frames[i]->linesize[0] = dstW*4;
}
clip->comp_frame_size = clip->stream->codecpar->width * clip->stream->codecpar->height * 4;
clip->comp_frame_size = dstW * dstH * 4;
clip->comp_frame = new uchar[clip->comp_frame_size];
} else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
// if FFmpeg can't pick up the channel layout (usually WAV), assume
@@ -412,10 +417,11 @@ void close_clip_worker(Clip* clip) {
// closes ffmpeg file handle and frees any memory used for caching
MediaStream* ms = static_cast<Media*>(clip->media)->get_stream_from_file_index(clip->track < 0, clip->media_stream);
if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
// sws_freeContext(clip->sws_ctx);
sws_freeContext(clip->sws_ctx);
// TODO will eventually be in audio too
delete [] clip->comp_frame;
delete clip->fbo;
clip->fbo = NULL;
} else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
swr_free(&clip->swr_ctx);
}
project/clip.cpp
+2 -1
View File
@@ -28,7 +28,8 @@ Clip::Clip(Sequence* s) :
closing_transition(NULL),
pkt(new AVPacket()),
replaced(false),
texture(NULL)
texture(NULL),
fbo(NULL)
{
reset();
}
project/clip.h
+3 -2
View File
@@ -8,6 +8,7 @@
class Cacher;
class Effect;
class Transition;
class QOpenGLFramebufferObject;
struct Sequence;
struct Media;
struct MediaStream;
@@ -80,7 +81,6 @@ struct Clip
AVFrame* frame;
uchar* comp_frame;
int comp_frame_size;
SwsContext* sws_ctx;
bool pkt_written;
bool reached_end;
@@ -99,7 +99,8 @@ struct Clip
QMutex open_lock;
// video playback variables
//SwsContext* sws_ctx;
SwsContext* sws_ctx;
QOpenGLFramebufferObject* fbo;
QOpenGLTexture* texture;
long texture_frame;
ui/viewerwidget.cpp
+125 -112
View File
@@ -28,8 +28,7 @@ ViewerWidget::ViewerWidget(QWidget *parent) :
multithreaded(true),
force_audio(false),
enable_paint(true),
flip(false),
fbo(NULL)
flip(false)
{
QSurfaceFormat format;
format.setDepthBufferSize(24);
@@ -82,11 +81,11 @@ void ViewerWidget::paintEvent(QPaintEvent *e) {
if (enable_paint) QOpenGLWidget::paintEvent(e);
}
GLuint ViewerWidget::draw_clip(GLuint texture) {
GLuint ViewerWidget::draw_clip(Clip* clip, GLuint texture) {
glPushMatrix();
glLoadIdentity();
glOrtho(0, 1, 0, 1, -1, 1);
fbo->bind();
clip->fbo->bind();
glBindTexture(GL_TEXTURE_2D, texture);
glBegin(GL_QUADS);
glTexCoord2f(0, 0); // top left
@@ -99,22 +98,20 @@ GLuint ViewerWidget::draw_clip(GLuint texture) {
glVertex2f(0, 1); // bottom left
glEnd();
glBindTexture(GL_TEXTURE_2D, NULL);
fbo->release();
clip->fbo->release();
glPopMatrix();
return fbo->takeTexture();
return clip->fbo->takeTexture();
}
void ViewerWidget::compose_sequence(QVector<Clip*>& nests, bool render_audio) {
Sequence* s = sequence;
GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) {
Sequence* s = sequence;
long playhead = panel_timeline->playhead;
Clip* nest = NULL;
if (nests.size() > 0) {
nest = nests.last();
if (nest != NULL) {
s = static_cast<Sequence*>(nest->media);
playhead += nest->clip_in - nest->timeline_in;
playhead = refactor_frame_number(playhead, sequence->frame_rate, s->frame_rate);
}
playhead += nest->clip_in - nest->timeline_in;
playhead = refactor_frame_number(playhead, nest->sequence->frame_rate, s->frame_rate);
}
QVector<Clip*> current_clips;
@@ -175,105 +172,122 @@ void ViewerWidget::compose_sequence(QVector<Clip*>& nests, bool render_audio) {
qDebug() << "[WARNING] Tried to display clip" << i << "but it's closed";
texture_failed = true;
} else {
switch (c->media_type) {
case MEDIA_TYPE_FOOTAGE:
if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
// start preparing cache
get_clip_frame(c, playhead);
if (c->track < 0) {
GLuint textureID = 0;
int video_width, video_height;
if (c->texture == NULL) {
qDebug() << "[WARNING] Texture hasn't been created yet";
texture_failed = true;
} else if (playhead >= c->timeline_in) {
MediaStream* ms = static_cast<Media*>(c->media)->get_stream_from_file_index(c->track < 0, c->media_stream);
int half_width = sequence->width/2;
int half_height = sequence->height/2;
if (flip) half_height = -half_height;
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glColor4f(1.0, 1.0, 1.0, 1.0);
glLoadIdentity();
glOrtho(-half_width, half_width, half_height, -half_height, -1, 1);
glViewport(0, 0, ms->video_width, ms->video_height);
// TEMP DEBUG CODE
if (fbo != NULL) delete fbo;
fbo = new QOpenGLFramebufferObject(ms->video_width, ms->video_height);
GLuint composite_texture = draw_clip(c->texture->textureId());
// END
GLTextureCoords coords;
coords.vertexTopLeftX = coords.vertexBottomLeftX = -ms->video_width/2;
coords.vertexTopLeftY = coords.vertexTopRightY = -ms->video_height/2;
coords.vertexTopRightX = coords.vertexBottomRightX = ms->video_width/2;
coords.vertexBottomLeftY = coords.vertexBottomRightY = ms->video_height/2;
coords.textureTopLeftY = coords.textureTopRightY = coords.textureTopLeftX = coords.textureBottomLeftX = 0;
coords.textureBottomLeftY = coords.textureBottomRightY = coords.textureTopRightX = coords.textureBottomRightX = 1.0;
// EFFECT CODE START
for (int j=0;j<c->effects.size();j++) {
if (c->effects.at(j)->enable_opengl && c->effects.at(j)->is_enabled()) {
c->effects.at(j)->process_gl(((double)(panel_timeline->playhead-c->timeline_in+c->clip_in)/(double)sequence->frame_rate), coords);
composite_texture = draw_clip(composite_texture);
}
}
if (c->opening_transition != NULL) {
int transition_progress = playhead - c->timeline_in;
if (transition_progress < c->opening_transition->length) {
c->opening_transition->process_transition((double)transition_progress/(double)c->opening_transition->length);
}
}
if (c->closing_transition != NULL) {
int transition_progress = c->closing_transition->length - (playhead - c->timeline_in - c->getLength() + c->closing_transition->length);
if (transition_progress < c->closing_transition->length) {
c->closing_transition->process_transition((double)transition_progress/(double)c->closing_transition->length);
}
}
// EFFECT CODE END
for (int j=0;j<c->effects.size();j++) {
if (c->effects.at(j)->enable_opengl && c->effects.at(j)->is_enabled()) {
c->effects.at(j)->clean_gl();
}
}
glViewport(0, 0, width(), height());
glBindTexture(GL_TEXTURE_2D, composite_texture);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glBegin(GL_QUADS);
glTexCoord2f(coords.textureTopLeftX, coords.textureTopLeftY); // top left
glVertex2f(coords.vertexTopLeftX, coords.vertexTopLeftY); // top left
glTexCoord2f(coords.textureTopRightX, coords.textureTopRightY); // top right
glVertex2f(coords.vertexTopRightX, coords.vertexTopRightY); // top right
glTexCoord2f(coords.textureBottomRightX, coords.textureBottomRightY); // bottom right
glVertex2f(coords.vertexBottomRightX, coords.vertexBottomRightY); // bottom right
glTexCoord2f(coords.textureBottomLeftX, coords.textureBottomLeftY); // bottom left
glVertex2f(coords.vertexBottomLeftX, coords.vertexBottomLeftY); // bottom left
glEnd();
glDeleteTextures(1, &composite_texture);
}
} else if (render_audio &&
c->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO &&
c->lock.tryLock()) {
// clip is not caching, start caching audio
cache_clip(c, playhead, false, false, c->audio_reset, nest);
c->lock.unlock();
if (c->media_type == MEDIA_TYPE_FOOTAGE) {
get_clip_frame(c, playhead);
MediaStream* ms = static_cast<Media*>(c->media)->get_stream_from_file_index(c->track < 0, c->media_stream);
video_width = ms->video_width;
video_height = ms->video_height;
if (c->texture != NULL) textureID = c->texture->textureId();
} else if (c->media_type == MEDIA_TYPE_SEQUENCE) {
video_width = s->width;
video_height = s->height;
textureID = compose_sequence(c, render_audio);
}
break;
case MEDIA_TYPE_SEQUENCE:
nests.append(c);
compose_sequence(nests, render_audio);
break;
}
if (textureID == 0) {
qDebug() << "[WARNING] Texture hasn't been created yet";
texture_failed = true;
} else if (playhead >= c->timeline_in) {
// start preparing cache
if (c->fbo == NULL) c->fbo = new QOpenGLFramebufferObject(video_width, video_height);
int half_width = c->sequence->width/2;
int half_height = c->sequence->height/2;
if (flip || nest != NULL) half_height = -half_height;
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glColor4f(1.0, 1.0, 1.0, 1.0);
glLoadIdentity();
glOrtho(-half_width, half_width, half_height, -half_height, -1, 1);
glViewport(0, 0, video_width, video_height);
GLuint composite_texture = draw_clip(c, textureID);
GLTextureCoords coords;
coords.vertexTopLeftX = coords.vertexBottomLeftX = -video_width/2;
coords.vertexTopLeftY = coords.vertexTopRightY = -video_height/2;
coords.vertexTopRightX = coords.vertexBottomRightX = video_width/2;
coords.vertexBottomLeftY = coords.vertexBottomRightY = video_height/2;
coords.textureTopLeftY = coords.textureTopRightY = coords.textureTopLeftX = coords.textureBottomLeftX = 0;
coords.textureBottomLeftY = coords.textureBottomRightY = coords.textureTopRightX = coords.textureBottomRightX = 1.0;
// EFFECT CODE START
for (int j=0;j<c->effects.size();j++) {
if (c->effects.at(j)->enable_opengl && c->effects.at(j)->is_enabled()) {
c->effects.at(j)->process_gl(((double)(panel_timeline->playhead-c->timeline_in+c->clip_in)/(double)sequence->frame_rate), coords);
composite_texture = draw_clip(c, composite_texture);
}
}
if (c->opening_transition != NULL) {
int transition_progress = playhead - c->timeline_in;
if (transition_progress < c->opening_transition->length) {
c->opening_transition->process_transition((double)transition_progress/(double)c->opening_transition->length);
}
}
if (c->closing_transition != NULL) {
int transition_progress = c->closing_transition->length - (playhead - c->timeline_in - c->getLength() + c->closing_transition->length);
if (transition_progress < c->closing_transition->length) {
c->closing_transition->process_transition((double)transition_progress/(double)c->closing_transition->length);
}
}
// EFFECT CODE END
for (int j=0;j<c->effects.size();j++) {
if (c->effects.at(j)->enable_opengl && c->effects.at(j)->is_enabled()) {
c->effects.at(j)->clean_gl();
}
}
if (nest != NULL) {
if (nest->fbo == NULL) nest->fbo = new QOpenGLFramebufferObject(s->width, s->height);
nest->fbo->bind();
glViewport(0, 0, s->width, s->height);
} else {
glViewport(0, 0, width(), height());
}
glBindTexture(GL_TEXTURE_2D, composite_texture);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glBegin(GL_QUADS);
glTexCoord2f(coords.textureTopLeftX, coords.textureTopLeftY); // top left
glVertex2f(coords.vertexTopLeftX, coords.vertexTopLeftY); // top left
glTexCoord2f(coords.textureTopRightX, coords.textureTopRightY); // top right
glVertex2f(coords.vertexTopRightX, coords.vertexTopRightY); // top right
glTexCoord2f(coords.textureBottomRightX, coords.textureBottomRightY); // bottom right
glVertex2f(coords.vertexBottomRightX, coords.vertexBottomRightY); // bottom right
glTexCoord2f(coords.textureBottomLeftX, coords.textureBottomLeftY); // bottom left
glVertex2f(coords.vertexBottomLeftX, coords.vertexBottomLeftY); // bottom left
glEnd();
glDeleteTextures(1, &composite_texture);
if (nest != NULL) {
nest->fbo->release();
}
}
} else {
if (c->media_type == MEDIA_TYPE_FOOTAGE
&& render_audio
&& c->lock.tryLock()) {
// clip is not caching, start caching audio
cache_clip(c, playhead, false, false, c->audio_reset, nest);
c->lock.unlock();
} else if (c->media_type == MEDIA_TYPE_SEQUENCE) {
compose_sequence(c, render_audio);
}
}
}
}
return (nest != NULL && nest->fbo != NULL) ? nest->fbo->takeTexture() : 0;
}
void ViewerWidget::paintGL() {
@@ -291,9 +305,8 @@ void ViewerWidget::paintGL() {
glClear(GL_COLOR_BUFFER_BIT);
// compose video preview
QVector<Clip*> nests;
compose_sequence(nests, (panel_timeline->playing || force_audio));
// compose video preview
compose_sequence(NULL, (panel_timeline->playing || force_audio));
if (texture_failed) {
if (multithreaded) {
ui/viewerwidget.h
+2 -4
View File
@@ -10,7 +10,6 @@
#include <QMutex>
#include <QWaitCondition>
class QOpenGLFramebufferObject;
struct Clip;
struct Sequence;
@@ -31,12 +30,11 @@ protected:
// void resizeGL(int w, int h);
private:
QTimer retry_timer;
QOpenGLFramebufferObject* fbo;
private slots:
void retry();
void deleteFunction();
void compose_sequence(QVector<Clip*>& nests, bool render_audio);
GLuint draw_clip(GLuint texture);
GLuint compose_sequence(Clip *nest, bool render_audio);
GLuint draw_clip(Clip *clip, GLuint texture);
};
#endif // VIEWERWIDGET_H