diff --git a/playback/audio.cpp b/playback/audio.cpp index 9f3371ce9..5d207b5fb 100644 --- a/playback/audio.cpp +++ b/playback/audio.cpp @@ -86,18 +86,7 @@ void clear_audio_ibuffer() { } int get_buffer_offset_from_frame(Sequence* s, long frame) { - // currently debugging this function. since it has a high potential of failure and isn't actually fatal, we only assert on debug mode -/*#ifdef QT_DEBUG - Q_ASSERT(frame >= audio_ibuffer_frame); return qFloor(av_samples_get_buffer_size(NULL, av_get_channel_layout_nb_channels(s->audio_layout), qRound(((frame-audio_ibuffer_frame)/s->frame_rate)*s->audio_frequency), AV_SAMPLE_FMT_S16, 1)/4)*4; -#else*/ - if (frame >= audio_ibuffer_frame) { - return qFloor(av_samples_get_buffer_size(NULL, av_get_channel_layout_nb_channels(s->audio_layout), qRound(((frame-audio_ibuffer_frame)/s->frame_rate)*s->audio_frequency), AV_SAMPLE_FMT_S16, 1)/4)*4; - } else { - dout << "[WARNING] Invalid values passed to get_buffer_offset_from_frame"; - return 0; - } -//#endif } AudioSenderThread::AudioSenderThread() : close(false) { diff --git a/playback/cacher.cpp b/playback/cacher.cpp index bd3060a12..431927957 100644 --- a/playback/cacher.cpp +++ b/playback/cacher.cpp @@ -74,13 +74,33 @@ int get_read_buffer_index(Clip* nest) { return (nest == NULL) ? audio_ibuffer_read : nest->frame->pts; } -void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { +int get_real_buffer_write(Clip* c, Clip* nest, long frame) { + if (nest == NULL) { + return get_buffer_offset_from_frame(c->sequence, frame); + } else { + return qMax(0, qFloor((((double) frame / c->sequence->frame_rate) * sequence->audio_frequency * 4)/* - nest->frame->pkt_dts*/)); + dout << "ti:" << c->timeline_in << "atf:" << c->audio_target_frame << "abw:" << c->audio_buffer_write << "nest pts:" << nest->frame->pts; + } +} + +void cache_audio_worker(Clip* c, bool scrubbing, QVector nests) { long timeline_in = c->timeline_in; long timeline_out = c->timeline_out; - if (nest != NULL) { - timeline_in = refactor_frame_number(timeline_in, c->sequence->frame_rate, nest->sequence->frame_rate) + nest->timeline_in - nest->clip_in; - timeline_out = refactor_frame_number(timeline_out, c->sequence->frame_rate, nest->sequence->frame_rate) + nest->timeline_in - nest->clip_in; - } + + Clip* nest = NULL; + + long audio_target_frame = c->audio_target_frame; + + if (nests.size() > 0) { + /*double last_fr = c->sequence->frame_rate; + for (int i=nests.size()-1;i>=0;i--) { + timeline_in = refactor_frame_number(timeline_in, last_fr, nests.at(i)->sequence->frame_rate) + nests.at(i)->timeline_in - nests.at(i)->clip_in; + timeline_out = refactor_frame_number(timeline_out, last_fr, nests.at(i)->sequence->frame_rate) + nests.at(i)->timeline_in - nests.at(i)->clip_in; + audio_target_frame = refactor_frame_number(audio_target_frame, last_fr, nests.at(i)->sequence->frame_rate) + nests.at(i)->timeline_in - nests.at(i)->clip_in; + last_fr = nests.at(i)->sequence->frame_rate; + }*/ + nest = nests.last(); + } while (true) { AVFrame* frame; @@ -257,7 +277,7 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { if (c->audio_just_reset) { // get precise sample offset for the elected clip_in from this audio frame - double target_sts = playhead_to_clip_seconds(c, c->audio_target_frame); + double target_sts = playhead_to_clip_seconds(c, audio_target_frame); double frame_sts = ((frame->pts - c->stream->start_time) * timebase); int nb_samples = qRound64((target_sts - frame_sts)*c->sequence->audio_frequency); c->frame_sample_index = nb_samples * 4; @@ -272,7 +292,10 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { #ifdef AUDIOWARNINGS dout << "fsi-post-post:" << c->frame_sample_index; #endif - if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame)); + if (c->audio_buffer_write == 0) { + c->audio_buffer_write = get_real_buffer_write(c, nest, qMax(c->timeline_in, c->audio_target_frame)); + //if (nest != NULL) c->audio_buffer_write -= nest->frame->pkt_dts; + } int offset = get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING - c->audio_buffer_write; if (offset > 0) { @@ -309,7 +332,7 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { apply_audio_effects(c, bytes_to_seconds(frame->pts, frame->channels, frame->sample_rate), frame, nb_bytes); c->frame->pts += nb_bytes; c->frame_sample_index = 0; - if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame)); + if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, audio_target_frame)); int offset = (get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING) - c->audio_buffer_write; if (offset > 0) { c->audio_buffer_write += offset; @@ -319,12 +342,14 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { break; case MEDIA_TYPE_SEQUENCE: frame = c->frame; - nb_bytes = frame->nb_samples * av_get_bytes_per_sample(static_cast(frame->format)) * frame->channels; + nb_bytes = c->frame->linesize[0]; if (c->frame_sample_index == -1 || c->frame_sample_index >= nb_bytes) { c->frame_sample_index = 0; - if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame)); + if (c->audio_buffer_write == 0) { + c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, audio_target_frame)); + } int offset = (get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING) - c->audio_buffer_write; if (offset > 0) { @@ -333,19 +358,6 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { c->frame->pts += offset; } } - - - - //c->audio_buffer_write = 0; - - /*while ((c->frame_sample_index == -1 || c->frame_sample_index >= nb_bytes) && nb_bytes > 0) { - // create "new frame" - int offset = (get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING) - c->audio_buffer_write; - if (offset > 0) { - c->audio_buffer_write += offset; - c->frame_sample_index += offset; - } - }*/ break; } @@ -353,29 +365,29 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { if (frame->nb_samples == 0) { break; } else { - long buffer_timeline_out = get_buffer_offset_from_frame(c->sequence, timeline_out); + long buffer_timeline_out = get_real_buffer_write(c, nest, timeline_out); - int max_write = (nest == NULL) ? audio_ibuffer_size : nest->frame->nb_samples * av_get_bytes_per_sample(static_cast(nest->frame->format)) * nest->frame->channels; - if (nest == NULL) { + int max_write = (nests.isEmpty()) ? audio_ibuffer_size : nest->frame->nb_samples * av_get_bytes_per_sample(static_cast(nest->frame->format)) * nest->frame->channels; + if (nests.size() == 0) { audio_write_lock.lock(); } else { c->queue_lock.lock(); } - uint8_t* buffer = (nest == NULL) ? audio_ibuffer : nest->frame->data[0]; - dout << c->name << "|" << - c->audio_buffer_write << "<" << get_read_buffer_index(nest)+(max_write>>1) << "|" << - c->audio_buffer_write << "<" << buffer_timeline_out << "|" << - c->frame_sample_index << "<" << nb_bytes << "|"; + uint8_t* buffer = (nests.isEmpty()) ? audio_ibuffer : nest->frame->data[0]; + + int written = 0; + + dout << c->name << "reports" << c->frame_sample_index << "(" << nb_bytes << ")" << c->audio_buffer_write << "(" << get_read_buffer_index(nest)+(max_write>>1) << "/" << buffer_timeline_out << ")"; while (c->frame_sample_index < nb_bytes && c->audio_buffer_write < get_read_buffer_index(nest)+(max_write>>1) && c->audio_buffer_write < buffer_timeline_out) { int upper_byte_index = (c->audio_buffer_write+1)%max_write; int lower_byte_index = (c->audio_buffer_write)%max_write; - qint16 old_sample = static_cast((buffer[upper_byte_index] & 0xFF) << 8 | (buffer[lower_byte_index] & 0xFF)); - qint16 new_sample = static_cast((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF)); - qint16 mixed_sample = mix_audio_sample(old_sample, new_sample); + int16_t old_sample = static_cast((buffer[upper_byte_index] & 0xFF) << 8 | (buffer[lower_byte_index] & 0xFF)); + int16_t new_sample = static_cast((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF)); + int16_t mixed_sample = mix_audio_sample(old_sample, new_sample); buffer[upper_byte_index] = static_cast((mixed_sample >> 8) & 0xFF); buffer[lower_byte_index] = static_cast(mixed_sample & 0xFF); @@ -388,15 +400,17 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { c->audio_buffer_write+=2; c->frame_sample_index+=2; + + written += 2; } - dout << c->name << "|" << c->frame_sample_index << "<" << nb_bytes << "LELELEL"; + dout << c->name << "wrote" << written; #ifdef AUDIOWARNINGS if (c->audio_buffer_write >= buffer_timeline_out) dout << "timeline out at fsi" << c->frame_sample_index << "of frame ts" << c->frame->pts; #endif - if (nest == NULL) { + if (nests.isEmpty()) { audio_write_lock.unlock(); } else { c->queue_lock.unlock(); @@ -416,8 +430,6 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { if (c->media_type == MEDIA_TYPE_SEQUENCE) { break; } - -// dout << "ended" << c->frame_sample_index << nb_bytes; } if (c->reached_end) { frame->nb_samples = 0; @@ -479,8 +491,6 @@ void cache_video_worker(Clip* c, long playhead) { send_it = true; } else if (media->get_stream_from_file_index(true, c->media_stream)->infinite_length) { send_it = true; - } else { - dout << "skipped adding a frame to the queue - fpts:" << send_frame->pts << "target:" << target_pts; } if (send_it) { @@ -522,7 +532,6 @@ void cache_video_worker(Clip* c, long playhead) { // see if we got the frame we needed (used for speed ups primarily) bool found = false; for (int i=0;iqueue.size();i++) { - // TODO/NOTE: this will not work on clips that are sped up AND reversed if (c->queue.at(i)->pts >= target_pts) { found = true; break; @@ -618,9 +627,13 @@ void reset_cache(Clip* c, long target_frame) { } break; case MEDIA_TYPE_SEQUENCE: - c->frame->pts = 0; + c->audio_just_reset = true; + c->audio_buffer_write = 0; c->frame_sample_index = -1; c->audio_target_frame = target_frame; + c->frame->pts = qFloor((double) c->clip_in / sequence->audio_frequency); + memset(c->frame->data[0], 0, c->frame->linesize[0]); + c->frame->pkt_dts = qFloor((((double) (c->timeline_in - c->clip_in)/c->sequence->frame_rate) - audio_ibuffer_timecode)*sequence->audio_frequency*4); break; case MEDIA_TYPE_TONE: c->reached_end = false; @@ -872,7 +885,10 @@ void open_clip_worker(Clip* clip) { if (av_frame_get_buffer(clip->frame, 0)) { dout << "[ERROR] Could not allocate buffer for tone clip"; } - clip->audio_reset = true; + clip->reset_audio(); + break; + case MEDIA_TYPE_SEQUENCE: + break; } @@ -885,7 +901,7 @@ void open_clip_worker(Clip* clip) { dout << "[INFO] Clip opened on track" << clip->track; } -void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, Clip* nest) { +void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, QVector nests) { if (reset) { // note: for video, playhead is in "internal clip" frames - for audio, it's the timeline playhead reset_cache(clip, playhead); @@ -897,12 +913,12 @@ void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, Cl if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { cache_video_worker(clip, playhead); } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { - cache_audio_worker(clip, scrubbing, nest); + cache_audio_worker(clip, scrubbing, nests); } break; case MEDIA_TYPE_TONE: case MEDIA_TYPE_SEQUENCE: - cache_audio_worker(clip, scrubbing, nest); + cache_audio_worker(clip, scrubbing, nests); break; } } @@ -941,7 +957,7 @@ void Cacher::run() { if (!caching) { break; } else { - cache_clip_worker(clip, playhead, reset, scrubbing, nest); + cache_clip_worker(clip, playhead, reset, scrubbing, nests); } } diff --git a/playback/cacher.h b/playback/cacher.h index 1a65f6aae..9afe1d0c6 100644 --- a/playback/cacher.h +++ b/playback/cacher.h @@ -2,6 +2,7 @@ #define CACHER_H #include +#include struct Clip; @@ -18,14 +19,14 @@ public: long playhead; bool reset; bool scrubbing; - Clip* nest; + QVector nests; private: Clip* clip; }; void open_clip_worker(Clip* clip); -void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, Clip *nest); +void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, QVector nests); void close_clip_worker(Clip* clip); #endif // CACHER_H diff --git a/playback/playback.cpp b/playback/playback.cpp index 529878c02..83c238c4d 100644 --- a/playback/playback.cpp +++ b/playback/playback.cpp @@ -32,7 +32,7 @@ extern "C" { bool texture_failed = false; bool rendering = false; -void open_clip(Clip* clip, bool multithreaded, Clip* nest) { +void open_clip(Clip* clip, bool multithreaded) { if (clip->media_type == MEDIA_TYPE_FOOTAGE || clip->media_type == MEDIA_TYPE_TONE || (clip->media_type == MEDIA_TYPE_SEQUENCE && clip->track >= 0)) { clip->multithreaded = multithreaded; @@ -40,12 +40,13 @@ void open_clip(Clip* clip, bool multithreaded, Clip* nest) { clip->frame = av_frame_alloc(); clip->frame->format = AV_SAMPLE_FMT_S16; clip->frame->nb_samples = 48000; - clip->frame->channel_layout = AV_CH_LAYOUT_STEREO; - clip->frame->channels = 2; - clip->audio_buffer_offset = ((double) (clip->timeline_in - clip->clip_in) / clip->sequence->frame_rate) - audio_ibuffer_timecode; // may be unnecessary - clip->frame->pkt_dts = clip->audio_buffer_offset * sequence->audio_frequency * av_get_bytes_per_sample(static_cast(clip->frame->format)) * clip->frame->channels; - if (nest != NULL) clip->audio_buffer_offset += nest->audio_buffer_offset; - av_frame_get_buffer(clip->frame, 0); + clip->frame->channel_layout = clip->sequence->audio_layout; + clip->frame->channels = av_get_channel_layout_nb_channels(clip->frame->channel_layout); + + av_frame_make_writable(clip->frame); + if (av_frame_get_buffer(clip->frame, 0)) { + dout << "[ERROR] Could not allocate buffer for sequence audio clip"; + } memset(clip->frame->data[0], 0, clip->frame->nb_samples*av_get_bytes_per_sample(static_cast(clip->frame->format))*clip->frame->channels); clip->reset_audio(); } @@ -103,16 +104,16 @@ void close_clip(Clip* clip) { } } -void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, Clip* nest) { +void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, QVector nests) { if (clip->multithreaded) { clip->cacher->playhead = playhead; clip->cacher->reset = reset; - clip->cacher->nest = nest; + clip->cacher->nests = nests; clip->cacher->scrubbing = scrubbing; clip->can_cache.wakeAll(); } else { - cache_clip_worker(clip, playhead, reset, scrubbing, nest); + cache_clip_worker(clip, playhead, reset, scrubbing, nests); } } @@ -231,7 +232,7 @@ void get_clip_frame(Clip* c, long playhead) { c->queue_lock.unlock(); // get more frames - if (cache) cache_clip(c, playhead, reset, false, NULL); + if (cache) cache_clip(c, playhead, reset, false, QVector()); } } diff --git a/playback/playback.h b/playback/playback.h index 8b7c1d75b..fac079f65 100644 --- a/playback/playback.h +++ b/playback/playback.h @@ -12,8 +12,8 @@ struct AVFrame; extern bool texture_failed; extern bool rendering; -void open_clip(Clip* clip, bool multithreaded, Clip* nest = 0); -void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, Clip *nest); +void open_clip(Clip* clip, bool multithreaded); +void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, QVector nests); void close_clip(Clip* clip); void cache_audio_worker(Clip* c, bool write_A); void cache_video_worker(Clip* c, long playhead); diff --git a/ui/viewerwidget.cpp b/ui/viewerwidget.cpp index d74c5a6b7..3e06ec11b 100644 --- a/ui/viewerwidget.cpp +++ b/ui/viewerwidget.cpp @@ -217,14 +217,14 @@ void ViewerWidget::process_effect(Clip* c, Effect* e, double timecode, GLTexture } } -GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { +GLuint ViewerWidget::compose_sequence(QVector& nests, bool render_audio) { Sequence* s = viewer->seq; long playhead = s->playhead; - if (nest != NULL) { - s = static_cast(nest->media); - playhead += nest->clip_in - nest->timeline_in; - playhead = refactor_frame_number(playhead, nest->sequence->frame_rate, s->frame_rate); + if (nests.size() > 0) { + s = static_cast(nests.last()->media); + playhead += nests.last()->clip_in - nests.last()->timeline_in; + playhead = refactor_frame_number(playhead, nests.last()->sequence->frame_rate, s->frame_rate); } QVector current_clips; @@ -234,7 +234,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { // if clip starts within one second and/or hasn't finished yet if (c != NULL) { - if (!(nest != NULL && !same_sign(c->track, nest->track))) { + if (!(nests.size() > 0 && !same_sign(c->track, nests.last()->track))) { bool clip_is_active = false; switch (c->media_type) { @@ -247,7 +247,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { // if thread is already working, we don't want to touch this, // but we also don't want to hang the UI thread if (!c->open) { - open_clip(c, !rendering, nest); + open_clip(c, !rendering); } clip_is_active = true; } else if (c->open) { @@ -262,7 +262,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { case MEDIA_TYPE_SOLID: case MEDIA_TYPE_TONE: if (is_clip_active(c, playhead)) { - if (!c->open) open_clip(c, !rendering, nest); + if (!c->open) open_clip(c, !rendering); clip_is_active = true; } else if (c->open) { close_clip(c); @@ -288,7 +288,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { int half_width = s->width/2; int half_height = s->height/2; - if (rendering || nest != NULL) half_height = -half_height; // invert vertical + if (rendering || nests.size() > 0) half_height = -half_height; // invert vertical glPushMatrix(); glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); glColor4f(1.0, 1.0, 1.0, 1.0); @@ -349,7 +349,9 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { // for nested sequences if (c->media_type == MEDIA_TYPE_SEQUENCE) { - textureID = compose_sequence(c, render_audio); + nests.append(c); + textureID = compose_sequence(nests, render_audio); + nests.removeLast(); fbo_switcher = true; } @@ -406,8 +408,8 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { } // EFFECT CODE END - if (nest != NULL) { - nest->fbo[0]->bind(); + if (nests.size() > 0) { + nests.last()->fbo[0]->bind(); glViewport(0, 0, s->width, s->height); } else if (rendering) { glViewport(0, 0, s->width, s->height); @@ -433,8 +435,8 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { glBindTexture(GL_TEXTURE_2D, 0); - if (nest != NULL) { - nest->fbo[0]->release(); + if (nests.size() > 0) { + nests.last()->fbo[0]->release(); if (default_fbo != NULL) default_fbo->bind(); } @@ -443,11 +445,13 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { } else { if (render_audio || (config.enable_audio_scrubbing && audio_scrub)) { if (c->media_type == MEDIA_TYPE_SEQUENCE) { - compose_sequence(c, render_audio); + nests.append(c); + compose_sequence(nests, render_audio); + nests.removeLast(); } if (c->lock.tryLock()) { // clip is not caching, start caching audio - cache_clip(c, playhead, c->audio_reset, !render_audio, nest); + cache_clip(c, playhead, c->audio_reset, !render_audio, nests); c->lock.unlock(); } } @@ -469,9 +473,9 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { glPopMatrix(); - if (nest != NULL && nest->fbo != NULL) { + if (nests.size() > 0 && nests.last()->fbo != NULL) { // returns nested clip's texture - return nest->fbo[0]->texture(); + return nests.last()->fbo[0]->texture(); } return 0; @@ -494,9 +498,11 @@ void ViewerWidget::paintGL() { glClear(GL_COLOR_BUFFER_BIT); + QVector nests; + // compose video preview glClearColor(0, 0, 0, 0); - compose_sequence(NULL, render_audio); + compose_sequence(nests, render_audio); if (waveform) { double waveform_zoom = (double) waveform_ms->audio_preview.size() / (double) width(); diff --git a/ui/viewerwidget.h b/ui/viewerwidget.h index cdf1c7f12..5341b21d8 100644 --- a/ui/viewerwidget.h +++ b/ui/viewerwidget.h @@ -42,7 +42,7 @@ private: void drawTitleSafeArea(); bool dragging; void seek_from_click(int x); - GLuint compose_sequence(Clip *nest, bool render_audio); + GLuint compose_sequence(QVector &nests, bool render_audio); GLuint draw_clip(QOpenGLFramebufferObject *clip, GLuint texture); void process_effect(Clip* c, Effect* e, double timecode, GLTextureCoords& coords, GLuint& composite_texture, bool& fbo_switcher); private slots: