diff --git a/playback/audio.cpp b/playback/audio.cpp index 44e974604..9f3371ce9 100644 --- a/playback/audio.cpp +++ b/playback/audio.cpp @@ -29,7 +29,7 @@ QAudioInput* audio_input = NULL; QFile output_recording; bool recording = false; -qint8 audio_ibuffer[audio_ibuffer_size]; +uint8_t audio_ibuffer[audio_ibuffer_size]; int audio_ibuffer_read = 0; long audio_ibuffer_frame = 0; double audio_ibuffer_timecode = 0; diff --git a/playback/audio.h b/playback/audio.h index 344f503b1..b69078344 100644 --- a/playback/audio.h +++ b/playback/audio.h @@ -36,7 +36,7 @@ extern AudioSenderThread* audio_thread; extern QMutex audio_write_lock; #define audio_ibuffer_size 192000 -extern qint8 audio_ibuffer[audio_ibuffer_size]; +extern uint8_t audio_ibuffer[audio_ibuffer_size]; extern int audio_ibuffer_read; extern long audio_ibuffer_frame; extern double audio_ibuffer_timecode; diff --git a/playback/cacher.cpp b/playback/cacher.cpp index abcdaca8f..b8e610cfe 100644 --- a/playback/cacher.cpp +++ b/playback/cacher.cpp @@ -36,8 +36,7 @@ double bytes_to_seconds(int nb_bytes, int nb_channels, int sample_rate) { void apply_audio_effects(Clip* c, double timecode_start, AVFrame* frame, int nb_bytes) { // perform all aud io effects - double timecode_end; - timecode_end = timecode_start + bytes_to_seconds(nb_bytes, frame->channels, frame->sample_rate); + double timecode_end = timecode_start + bytes_to_seconds(nb_bytes, frame->channels, frame->sample_rate); for (int j=0;jeffects.size();j++) { Effect* e = c->effects.at(j); @@ -71,12 +70,16 @@ void apply_audio_effects(Clip* c, double timecode_start, AVFrame* frame, int nb_ #define AUDIO_BUFFER_PADDING 2048 +int get_read_buffer_index(Clip* nest) { + return (nest == NULL) ? audio_ibuffer_read : nest->frame->pts; +} + void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { long timeline_in = c->timeline_in; long timeline_out = c->timeline_out; if (nest != NULL) { - timeline_in = refactor_frame_number(timeline_in, c->sequence->frame_rate, sequence->frame_rate) + nest->timeline_in; - timeline_out = refactor_frame_number(timeline_out, c->sequence->frame_rate, sequence->frame_rate) + nest->timeline_in; + timeline_in = refactor_frame_number(timeline_in, c->sequence->frame_rate, nest->sequence->frame_rate) + nest->timeline_in - nest->clip_in; + timeline_out = refactor_frame_number(timeline_out, c->sequence->frame_rate, nest->sequence->frame_rate) + nest->timeline_in - nest->clip_in; } while (true) { @@ -271,7 +274,7 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { #endif if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame)); - int offset = (audio_ibuffer_read + AUDIO_BUFFER_PADDING) - c->audio_buffer_write; + int offset = get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING - c->audio_buffer_write; if (offset > 0) { c->audio_buffer_write += offset; c->frame_sample_index += offset; @@ -307,44 +310,97 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { c->frame->pts += nb_bytes; c->frame_sample_index = 0; if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame)); - int offset = audio_ibuffer_read - c->audio_buffer_write; + int offset = (get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING) - c->audio_buffer_write; if (offset > 0) { c->audio_buffer_write += offset; c->frame_sample_index += offset; } } break; - default: // shouldn't ever get here - dout << "[ERROR] Tried to cache a non-footage/tone clip"; - return; + case MEDIA_TYPE_SEQUENCE: + frame = c->frame; + nb_bytes = frame->nb_samples * av_get_bytes_per_sample(static_cast(frame->format)) * frame->channels; + + if (c->frame_sample_index == -1 || c->frame_sample_index >= nb_bytes) { + c->frame_sample_index = 0; + + if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame)); + + int offset = (get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING) - c->audio_buffer_write; + if (offset > 0) { + c->audio_buffer_write += offset; + c->frame_sample_index += offset; + c->frame->pts += offset; + } + } + + + + //c->audio_buffer_write = 0; + + /*while ((c->frame_sample_index == -1 || c->frame_sample_index >= nb_bytes) && nb_bytes > 0) { + // create "new frame" + int offset = (get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING) - c->audio_buffer_write; + if (offset > 0) { + c->audio_buffer_write += offset; + c->frame_sample_index += offset; + } + }*/ + break; } - // mix audio into internal buffer + // mix audio into internal buffer if (frame->nb_samples == 0) { break; } else { long buffer_timeline_out = get_buffer_offset_from_frame(c->sequence, timeline_out); - audio_write_lock.lock(); + + int max_write = (nest == NULL) ? audio_ibuffer_size : nest->frame->nb_samples * av_get_bytes_per_sample(static_cast(nest->frame->format)) * nest->frame->channels; + if (nest == NULL) { + audio_write_lock.lock(); + } else { + c->queue_lock.lock(); + } + uint8_t* buffer = (nest == NULL) ? audio_ibuffer : nest->frame->data[0]; + + dout << c->name << "|" << + c->audio_buffer_write << "<" << get_read_buffer_index(nest)+(max_write>>1) << "|" << + c->audio_buffer_write << "<" << buffer_timeline_out << "|" << + c->frame_sample_index << "<" << nb_bytes << "|"; + while (c->frame_sample_index < nb_bytes - && c->audio_buffer_write < audio_ibuffer_read+(audio_ibuffer_size>>1) + && c->audio_buffer_write < get_read_buffer_index(nest)+(max_write>>1) && c->audio_buffer_write < buffer_timeline_out) { - int upper_byte_index = (c->audio_buffer_write+1)%audio_ibuffer_size; - int lower_byte_index = (c->audio_buffer_write)%audio_ibuffer_size; - qint16 old_sample = static_cast((audio_ibuffer[upper_byte_index] & 0xFF) << 8 | (audio_ibuffer[lower_byte_index] & 0xFF)); - qint16 new_sample = static_cast((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF)); + int upper_byte_index = (c->audio_buffer_write+1)%max_write; + int lower_byte_index = (c->audio_buffer_write)%max_write; + qint16 old_sample = static_cast((buffer[upper_byte_index] & 0xFF) << 8 | (buffer[lower_byte_index] & 0xFF)); + qint16 new_sample = static_cast((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF)); qint16 mixed_sample = mix_audio_sample(old_sample, new_sample); - audio_ibuffer[upper_byte_index] = static_cast((mixed_sample >> 8) & 0xFF); - audio_ibuffer[lower_byte_index] = static_cast(mixed_sample & 0xFF); + buffer[upper_byte_index] = static_cast((mixed_sample >> 8) & 0xFF); + buffer[lower_byte_index] = static_cast(mixed_sample & 0xFF); + + if (c->media_type == MEDIA_TYPE_SEQUENCE) { + frame->data[0][c->frame_sample_index+1] = 0; + frame->data[0][c->frame_sample_index] = 0; + c->frame->pts += 2; + } c->audio_buffer_write+=2; c->frame_sample_index+=2; } + + dout << c->name << "|" << c->frame_sample_index << "<" << nb_bytes << "LELELEL"; + #ifdef AUDIOWARNINGS if (c->audio_buffer_write >= buffer_timeline_out) dout << "timeline out at fsi" << c->frame_sample_index << "of frame ts" << c->frame->pts; #endif - audio_write_lock.unlock(); + if (nest == NULL) { + audio_write_lock.unlock(); + } else { + c->queue_lock.unlock(); + } if (scrubbing) { audio_thread->notifyReceiver(); @@ -357,6 +413,10 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) { break; } + if (c->media_type == MEDIA_TYPE_SEQUENCE) { + break; + } + // dout << "ended" << c->frame_sample_index << nb_bytes; } if (c->reached_end) { @@ -557,6 +617,11 @@ void reset_cache(Clip* c, long target_frame) { } } break; + case MEDIA_TYPE_SEQUENCE: + c->frame->pts = 0; + c->frame_sample_index = -1; + c->audio_target_frame = target_frame; + break; case MEDIA_TYPE_TONE: c->reached_end = false; c->audio_target_frame = target_frame; @@ -819,6 +884,7 @@ void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, Cl } break; case MEDIA_TYPE_TONE: + case MEDIA_TYPE_SEQUENCE: cache_audio_worker(clip, scrubbing, nest); break; } @@ -849,7 +915,7 @@ void Cacher::run() { clip->lock.lock(); clip->finished_opening = false; clip->open = true; - caching = true; + caching = true; open_clip_worker(clip); diff --git a/playback/playback.cpp b/playback/playback.cpp index 288207f20..529878c02 100644 --- a/playback/playback.cpp +++ b/playback/playback.cpp @@ -26,17 +26,30 @@ extern "C" { #include #ifdef QT_DEBUG -#define GCF_DEBUG +//#define GCF_DEBUG #endif bool texture_failed = false; bool rendering = false; -void open_clip(Clip* clip, bool multithreaded) { - switch (clip->media_type) { - case MEDIA_TYPE_FOOTAGE: - case MEDIA_TYPE_TONE: +void open_clip(Clip* clip, bool multithreaded, Clip* nest) { + if (clip->media_type == MEDIA_TYPE_FOOTAGE || clip->media_type == MEDIA_TYPE_TONE || (clip->media_type == MEDIA_TYPE_SEQUENCE && clip->track >= 0)) { clip->multithreaded = multithreaded; + + if (clip->media_type == MEDIA_TYPE_SEQUENCE) { + clip->frame = av_frame_alloc(); + clip->frame->format = AV_SAMPLE_FMT_S16; + clip->frame->nb_samples = 48000; + clip->frame->channel_layout = AV_CH_LAYOUT_STEREO; + clip->frame->channels = 2; + clip->audio_buffer_offset = ((double) (clip->timeline_in - clip->clip_in) / clip->sequence->frame_rate) - audio_ibuffer_timecode; // may be unnecessary + clip->frame->pkt_dts = clip->audio_buffer_offset * sequence->audio_frequency * av_get_bytes_per_sample(static_cast(clip->frame->format)) * clip->frame->channels; + if (nest != NULL) clip->audio_buffer_offset += nest->audio_buffer_offset; + av_frame_get_buffer(clip->frame, 0); + memset(clip->frame->data[0], 0, clip->frame->nb_samples*av_get_bytes_per_sample(static_cast(clip->frame->format))*clip->frame->channels); + clip->reset_audio(); + } + if (multithreaded) { if (clip->open_lock.tryLock()) { // maybe keep cacher instance in memory while clip exists for performance? @@ -50,11 +63,8 @@ void open_clip(Clip* clip, bool multithreaded) { open_clip_worker(clip); } - break; - case MEDIA_TYPE_SEQUENCE: - case MEDIA_TYPE_SOLID: + } else if (clip->media_type == MEDIA_TYPE_SEQUENCE || clip->media_type == MEDIA_TYPE_SOLID) { clip->open = true; - break; } } @@ -77,36 +87,32 @@ void close_clip(Clip* clip) { clip->fbo = NULL; } - switch (clip->media_type) { - case MEDIA_TYPE_FOOTAGE: - case MEDIA_TYPE_TONE: + if (clip->media_type == MEDIA_TYPE_SEQUENCE) { + closeActiveClips(static_cast(clip->media), false); + } + + if (clip->media_type == MEDIA_TYPE_FOOTAGE || clip->media_type == MEDIA_TYPE_TONE || (clip->media_type == MEDIA_TYPE_SEQUENCE && clip->track >= 0)) { if (clip->multithreaded) { clip->cacher->caching = false; clip->can_cache.wakeAll(); } else { close_clip_worker(clip); } - break; - case MEDIA_TYPE_SEQUENCE: - closeActiveClips(static_cast(clip->media), false); - case MEDIA_TYPE_SOLID: + } else if (clip->media_type == MEDIA_TYPE_SEQUENCE || clip->media_type == MEDIA_TYPE_SOLID) { clip->open = false; - break; } } void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, Clip* nest) { - if (clip->media_type == MEDIA_TYPE_FOOTAGE || clip->media_type == MEDIA_TYPE_TONE) { - if (clip->multithreaded) { - clip->cacher->playhead = playhead; - clip->cacher->reset = reset; - clip->cacher->nest = nest; - clip->cacher->scrubbing = scrubbing; + if (clip->multithreaded) { + clip->cacher->playhead = playhead; + clip->cacher->reset = reset; + clip->cacher->nest = nest; + clip->cacher->scrubbing = scrubbing; - clip->can_cache.wakeAll(); - } else { - cache_clip_worker(clip, playhead, reset, scrubbing, nest); - } + clip->can_cache.wakeAll(); + } else { + cache_clip_worker(clip, playhead, reset, scrubbing, nest); } } diff --git a/playback/playback.h b/playback/playback.h index 4f5067480..8b7c1d75b 100644 --- a/playback/playback.h +++ b/playback/playback.h @@ -12,7 +12,7 @@ struct AVFrame; extern bool texture_failed; extern bool rendering; -void open_clip(Clip* clip, bool multithreaded); +void open_clip(Clip* clip, bool multithreaded, Clip* nest = 0); void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, Clip *nest); void close_clip(Clip* clip); void cache_audio_worker(Clip* c, bool write_A); diff --git a/project/clip.cpp b/project/clip.cpp index 66b493129..29ceabd64 100644 --- a/project/clip.cpp +++ b/project/clip.cpp @@ -89,23 +89,17 @@ void Clip::reset() { } void Clip::reset_audio() { - switch (media_type) { - case MEDIA_TYPE_FOOTAGE: - case MEDIA_TYPE_TONE: - audio_reset = true; - frame_sample_index = -1; - audio_buffer_write = 0; - break; - case MEDIA_TYPE_SEQUENCE: - { - Sequence* nested_sequence = static_cast(media); - for (int i=0;iclips.size();i++) { - Clip* c = nested_sequence->clips.at(i); - if (c != NULL) c->reset_audio(); - } - } - break; - } + audio_reset = true; + frame_sample_index = -1; + audio_buffer_write = 0; + + if (media_type == MEDIA_TYPE_SEQUENCE) { + Sequence* nested_sequence = static_cast(media); + for (int i=0;iclips.size();i++) { + Clip* c = nested_sequence->clips.at(i); + if (c != NULL) c->reset_audio(); + } + } } void Clip::refresh() { diff --git a/project/clip.h b/project/clip.h index 70ab4d2b7..cb0218085 100644 --- a/project/clip.h +++ b/project/clip.h @@ -120,6 +120,7 @@ struct Clip int64_t reverse_target; int frame_sample_index; int audio_buffer_write; + double audio_buffer_offset; bool audio_reset; bool audio_just_reset; long audio_target_frame; diff --git a/ui/viewerwidget.cpp b/ui/viewerwidget.cpp index b8604ed44..d74c5a6b7 100644 --- a/ui/viewerwidget.cpp +++ b/ui/viewerwidget.cpp @@ -247,7 +247,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { // if thread is already working, we don't want to touch this, // but we also don't want to hang the UI thread if (!c->open) { - open_clip(c, !rendering); + open_clip(c, !rendering, nest); } clip_is_active = true; } else if (c->open) { @@ -262,7 +262,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { case MEDIA_TYPE_SOLID: case MEDIA_TYPE_TONE: if (is_clip_active(c, playhead)) { - if (!c->open) open_clip(c, !rendering); + if (!c->open) open_clip(c, !rendering, nest); clip_is_active = true; } else if (c->open) { close_clip(c); @@ -442,18 +442,13 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { } } else { if (render_audio || (config.enable_audio_scrubbing && audio_scrub)) { - switch (c->media_type) { - case MEDIA_TYPE_FOOTAGE: - case MEDIA_TYPE_TONE: - if (c->lock.tryLock()) { - // clip is not caching, start caching audio - cache_clip(c, playhead, c->audio_reset, !render_audio, nest); - c->lock.unlock(); - } - break; - case MEDIA_TYPE_SEQUENCE: + if (c->media_type == MEDIA_TYPE_SEQUENCE) { compose_sequence(c, render_audio); - break; + } + if (c->lock.tryLock()) { + // clip is not caching, start caching audio + cache_clip(c, playhead, c->audio_reset, !render_audio, nest); + c->lock.unlock(); } }