From 323ce263f02ae2a5f5bf15e74269a5add5485d92 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sat, 10 Nov 2018 14:38:51 +1100 Subject: [PATCH] going with the simpler nested audio option --- playback/audio.cpp | 12 +++-------- playback/audio.h | 2 +- playback/cacher.cpp | 40 +++++++++++++++++++++++------------ playback/cacher.h | 5 +++-- playback/playback.cpp | 10 ++++----- playback/playback.h | 2 +- ui/viewerwidget.cpp | 49 ++++++++++++++++++++++++++++--------------- ui/viewerwidget.h | 2 +- 8 files changed, 73 insertions(+), 49 deletions(-) diff --git a/playback/audio.cpp b/playback/audio.cpp index 44e974604..1e2fcb5c2 100644 --- a/playback/audio.cpp +++ b/playback/audio.cpp @@ -86,18 +86,12 @@ void clear_audio_ibuffer() { } int get_buffer_offset_from_frame(Sequence* s, long frame) { - // currently debugging this function. since it has a high potential of failure and isn't actually fatal, we only assert on debug mode -/*#ifdef QT_DEBUG - Q_ASSERT(frame >= audio_ibuffer_frame); - return qFloor(av_samples_get_buffer_size(NULL, av_get_channel_layout_nb_channels(s->audio_layout), qRound(((frame-audio_ibuffer_frame)/s->frame_rate)*s->audio_frequency), AV_SAMPLE_FMT_S16, 1)/4)*4; -#else*/ if (frame >= audio_ibuffer_frame) { - return qFloor(av_samples_get_buffer_size(NULL, av_get_channel_layout_nb_channels(s->audio_layout), qRound(((frame-audio_ibuffer_frame)/s->frame_rate)*s->audio_frequency), AV_SAMPLE_FMT_S16, 1)/4)*4; + return qFloor(((double) (frame - audio_ibuffer_frame)/s->frame_rate)*s->audio_frequency)*av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)*av_get_channel_layout_nb_channels(s->audio_layout); } else { dout << "[WARNING] Invalid values passed to get_buffer_offset_from_frame"; return 0; } -//#endif } AudioSenderThread::AudioSenderThread() : close(false) { @@ -163,14 +157,14 @@ int AudioSenderThread::send_audio_to_output(int offset, int max) { int channel_count = av_get_channel_layout_nb_channels(s->audio_layout); long sample_cache_playhead = panel_timeline->ui->audio_monitor->sample_cache_offset + (panel_timeline->ui->audio_monitor->sample_cache.size()/channel_count); int next_buffer_offset, buffer_offset_adjusted, i; - int buffer_offset = get_buffer_offset_from_frame(s, sample_cache_playhead); + int buffer_offset = get_buffer_offset_from_frame(s, sample_cache_playhead); if (samples.size() != channel_count) samples.resize(channel_count); samples.fill(0); // TODO: I don't like this, but i'm not sure if there's a smarter way to do it while (buffer_offset < audio_ibuffer_limit) { sample_cache_playhead++; - next_buffer_offset = qMin(get_buffer_offset_from_frame(s, sample_cache_playhead), audio_ibuffer_limit); + next_buffer_offset = qMin(get_buffer_offset_from_frame(s, sample_cache_playhead), audio_ibuffer_limit); while (buffer_offset < next_buffer_offset) { for (i=0;i #include +#include #include // temp debug shit @@ -71,12 +73,14 @@ void apply_audio_effects(Clip* c, double timecode_start, AVFrame* frame, int nb_ #define AUDIO_BUFFER_PADDING 2048 -void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { +void cache_audio_worker(Clip* c, bool scrubbing, QVector& nests) { long timeline_in = c->timeline_in; long timeline_out = c->timeline_out; - if (nest != NULL) { - timeline_in = refactor_frame_number(timeline_in, c->sequence->frame_rate, sequence->frame_rate) + nest->timeline_in; - timeline_out = refactor_frame_number(timeline_out, c->sequence->frame_rate, sequence->frame_rate) + nest->timeline_in; + if (!nests.isEmpty()) { + for (int i=nests.size()-1;i>=0;i--) { + timeline_in = refactor_frame_number(timeline_in, c->sequence->frame_rate, nests.at(i)->sequence->frame_rate) + nests.at(i)->timeline_in; + timeline_out = refactor_frame_number(timeline_out, c->sequence->frame_rate, nests.at(i)->sequence->frame_rate) + nests.at(i)->timeline_in; + } } while (true) { @@ -269,7 +273,7 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { #ifdef AUDIOWARNINGS dout << "fsi-post-post:" << c->frame_sample_index; #endif - if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame)); + if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(sequence, qMax(timeline_in, c->audio_target_frame)); int offset = (audio_ibuffer_read + AUDIO_BUFFER_PADDING) - c->audio_buffer_write; if (offset > 0) { @@ -306,7 +310,7 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { apply_audio_effects(c, bytes_to_seconds(frame->pts, frame->channels, frame->sample_rate), frame, nb_bytes); c->frame->pts += nb_bytes; c->frame_sample_index = 0; - if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame)); + if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(sequence, qMax(timeline_in, c->audio_target_frame)); int offset = audio_ibuffer_read - c->audio_buffer_write; if (offset > 0) { c->audio_buffer_write += offset; @@ -323,8 +327,14 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { if (frame->nb_samples == 0) { break; } else { - long buffer_timeline_out = get_buffer_offset_from_frame(c->sequence, timeline_out); + long buffer_timeline_out = get_buffer_offset_from_frame(sequence, timeline_out); audio_write_lock.lock(); + + dout << c->name << "says" << c->frame_sample_index << nb_bytes + << c->audio_buffer_write << audio_ibuffer_read+(audio_ibuffer_size>>1) + << c->audio_buffer_write << buffer_timeline_out; + dout << c->name << "also says" << timeline_in << c->audio_target_frame; + int written = 0; while (c->frame_sample_index < nb_bytes && c->audio_buffer_write < audio_ibuffer_read+(audio_ibuffer_size>>1) && c->audio_buffer_write < buffer_timeline_out) { @@ -339,7 +349,11 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { c->audio_buffer_write+=2; c->frame_sample_index+=2; + + written += 2; } + dout << c->name << "wrote" << written; + #ifdef AUDIOWARNINGS if (c->audio_buffer_write >= buffer_timeline_out) dout << "timeline out at fsi" << c->frame_sample_index << "of frame ts" << c->frame->pts; #endif @@ -744,7 +758,7 @@ void open_clip_worker(Clip* clip) { dout << "[ERROR] Could not set output sample format"; } - int target_sample_rate = clip->sequence->audio_frequency; + int target_sample_rate = sequence->audio_frequency; if (qFuzzyCompare(clip->speed, 1.0)) { avfilter_link(clip->buffersrc_ctx, 0, clip->buffersink_ctx, 0); @@ -779,7 +793,7 @@ void open_clip_worker(Clip* clip) { avfilter_link(last_filter, 0, clip->buffersink_ctx, 0); } else { - target_sample_rate = qRound64(clip->sequence->audio_frequency / clip->speed); + target_sample_rate = qRound64(target_sample_rate / clip->speed); avfilter_link(clip->buffersrc_ctx, 0, clip->buffersink_ctx, 0); } @@ -820,7 +834,7 @@ void open_clip_worker(Clip* clip) { dout << "[INFO] Clip opened on track" << clip->track; } -void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, Clip* nest) { +void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, QVector nests) { if (reset) { // note: for video, playhead is in "internal clip" frames - for audio, it's the timeline playhead reset_cache(clip, playhead); @@ -832,11 +846,11 @@ void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, Cl if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { cache_video_worker(clip, playhead); } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { - cache_audio_worker(clip, scrubbing, nest); + cache_audio_worker(clip, scrubbing, nests); } break; case MEDIA_TYPE_TONE: - cache_audio_worker(clip, scrubbing, nest); + cache_audio_worker(clip, scrubbing, nests); break; } } @@ -875,7 +889,7 @@ void Cacher::run() { if (!caching) { break; } else { - cache_clip_worker(clip, playhead, reset, scrubbing, nest); + cache_clip_worker(clip, playhead, reset, scrubbing, nests); } } diff --git a/playback/cacher.h b/playback/cacher.h index 1a65f6aae..dbfab0793 100644 --- a/playback/cacher.h +++ b/playback/cacher.h @@ -2,6 +2,7 @@ #define CACHER_H #include +#include struct Clip; @@ -18,14 +19,14 @@ public: long playhead; bool reset; bool scrubbing; - Clip* nest; + QVector nests; private: Clip* clip; }; void open_clip_worker(Clip* clip); -void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, Clip *nest); +void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, QVector nest); void close_clip_worker(Clip* clip); #endif // CACHER_H diff --git a/playback/playback.cpp b/playback/playback.cpp index 288207f20..925597528 100644 --- a/playback/playback.cpp +++ b/playback/playback.cpp @@ -26,7 +26,7 @@ extern "C" { #include #ifdef QT_DEBUG -#define GCF_DEBUG +//#define GCF_DEBUG #endif bool texture_failed = false; @@ -95,17 +95,17 @@ void close_clip(Clip* clip) { } } -void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, Clip* nest) { +void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, QVector& nests) { if (clip->media_type == MEDIA_TYPE_FOOTAGE || clip->media_type == MEDIA_TYPE_TONE) { if (clip->multithreaded) { clip->cacher->playhead = playhead; clip->cacher->reset = reset; - clip->cacher->nest = nest; + clip->cacher->nests = nests; clip->cacher->scrubbing = scrubbing; clip->can_cache.wakeAll(); } else { - cache_clip_worker(clip, playhead, reset, scrubbing, nest); + cache_clip_worker(clip, playhead, reset, scrubbing, nests); } } } @@ -225,7 +225,7 @@ void get_clip_frame(Clip* c, long playhead) { c->queue_lock.unlock(); // get more frames - if (cache) cache_clip(c, playhead, reset, false, NULL); + if (cache) cache_clip(c, playhead, reset, false, QVector()); } } diff --git a/playback/playback.h b/playback/playback.h index 4f5067480..af1d742f6 100644 --- a/playback/playback.h +++ b/playback/playback.h @@ -13,7 +13,7 @@ extern bool texture_failed; extern bool rendering; void open_clip(Clip* clip, bool multithreaded); -void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, Clip *nest); +void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, QVector &nests); void close_clip(Clip* clip); void cache_audio_worker(Clip* c, bool write_A); void cache_video_worker(Clip* c, long playhead); diff --git a/ui/viewerwidget.cpp b/ui/viewerwidget.cpp index b8604ed44..4ec1596d6 100644 --- a/ui/viewerwidget.cpp +++ b/ui/viewerwidget.cpp @@ -217,14 +217,22 @@ void ViewerWidget::process_effect(Clip* c, Effect* e, double timecode, GLTexture } } -GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { +GLuint ViewerWidget::compose_sequence(QVector& nests, bool render_audio) { Sequence* s = viewer->seq; long playhead = s->playhead; - if (nest != NULL) { - s = static_cast(nest->media); - playhead += nest->clip_in - nest->timeline_in; - playhead = refactor_frame_number(playhead, nest->sequence->frame_rate, s->frame_rate); + if (!nests.isEmpty()) { + for (int i=0;i(nests.at(i)->media); + playhead += nests.at(i)->clip_in - nests.at(i)->timeline_in; + playhead = refactor_frame_number(playhead, nests.at(i)->sequence->frame_rate, s->frame_rate); + } + + if (nests.last()->fbo != NULL) { + nests.last()->fbo[0]->bind(); + glClear(GL_COLOR_BUFFER_BIT); + nests.last()->fbo[0]->release(); + } } QVector current_clips; @@ -234,7 +242,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { // if clip starts within one second and/or hasn't finished yet if (c != NULL) { - if (!(nest != NULL && !same_sign(c->track, nest->track))) { + if (!(!nests.isEmpty() && !same_sign(c->track, nests.last()->track))) { bool clip_is_active = false; switch (c->media_type) { @@ -288,7 +296,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { int half_width = s->width/2; int half_height = s->height/2; - if (rendering || nest != NULL) half_height = -half_height; // invert vertical + if (rendering || !nests.isEmpty()) half_height = -half_height; // invert vertical glPushMatrix(); glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); glColor4f(1.0, 1.0, 1.0, 1.0); @@ -349,7 +357,9 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { // for nested sequences if (c->media_type == MEDIA_TYPE_SEQUENCE) { - textureID = compose_sequence(c, render_audio); + nests.append(c); + textureID = compose_sequence(nests, render_audio); + nests.removeLast(); fbo_switcher = true; } @@ -406,8 +416,8 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { } // EFFECT CODE END - if (nest != NULL) { - nest->fbo[0]->bind(); + if (!nests.isEmpty()) { + nests.last()->fbo[0]->bind(); glViewport(0, 0, s->width, s->height); } else if (rendering) { glViewport(0, 0, s->width, s->height); @@ -433,8 +443,8 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { glBindTexture(GL_TEXTURE_2D, 0); - if (nest != NULL) { - nest->fbo[0]->release(); + if (!nests.isEmpty()) { + nests.last()->fbo[0]->release(); if (default_fbo != NULL) default_fbo->bind(); } @@ -447,12 +457,14 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { case MEDIA_TYPE_TONE: if (c->lock.tryLock()) { // clip is not caching, start caching audio - cache_clip(c, playhead, c->audio_reset, !render_audio, nest); + cache_clip(c, playhead, c->audio_reset, !render_audio, nests); c->lock.unlock(); } break; case MEDIA_TYPE_SEQUENCE: - compose_sequence(c, render_audio); + nests.append(c); + compose_sequence(nests, render_audio); + nests.removeLast(); break; } } @@ -474,9 +486,9 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { glPopMatrix(); - if (nest != NULL && nest->fbo != NULL) { + if (!nests.isEmpty() && nests.last()->fbo != NULL) { // returns nested clip's texture - return nest->fbo[0]->texture(); + return nests.last()->fbo[0]->texture(); } return 0; @@ -501,7 +513,10 @@ void ViewerWidget::paintGL() { // compose video preview glClearColor(0, 0, 0, 0); - compose_sequence(NULL, render_audio); + + QVector nests; + + compose_sequence(nests, render_audio); if (waveform) { double waveform_zoom = (double) waveform_ms->audio_preview.size() / (double) width(); diff --git a/ui/viewerwidget.h b/ui/viewerwidget.h index cdf1c7f12..5341b21d8 100644 --- a/ui/viewerwidget.h +++ b/ui/viewerwidget.h @@ -42,7 +42,7 @@ private: void drawTitleSafeArea(); bool dragging; void seek_from_click(int x); - GLuint compose_sequence(Clip *nest, bool render_audio); + GLuint compose_sequence(QVector &nests, bool render_audio); GLuint draw_clip(QOpenGLFramebufferObject *clip, GLuint texture); void process_effect(Clip* c, Effect* e, double timecode, GLTextureCoords& coords, GLuint& composite_texture, bool& fbo_switcher); private slots: