From a2e73fd87ed32803e14547aa03bd8c60d383bb98 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Mon, 10 Sep 2018 15:00:09 +1000 Subject: [PATCH] progress on NULL audio --- effects/audio/toneeffect.cpp | 4 + panels/timeline.cpp | 20 +- playback/audio.cpp | 8 +- playback/cacher.cpp | 550 +++++++++++++++++++---------------- playback/playback.cpp | 4 +- project/clip.cpp | 11 +- project/clip.h | 3 +- ui/timelinewidget.cpp | 15 +- ui/viewerwidget.cpp | 20 +- 9 files changed, 350 insertions(+), 285 deletions(-) diff --git a/effects/audio/toneeffect.cpp b/effects/audio/toneeffect.cpp index 530d00b44..b6a213112 100644 --- a/effects/audio/toneeffect.cpp +++ b/effects/audio/toneeffect.cpp @@ -51,6 +51,10 @@ void ToneEffect::process_audio(double timecode_start, double timecode_end, quint samples[i+1] = (quint8) (left_tone_sample >> 8); samples[i] = (quint8) left_tone_sample; + int presin = sinX; sinX++; + if (sinX < presin) { + qDebug() << "overflowed"; + } } } diff --git a/panels/timeline.cpp b/panels/timeline.cpp index 462d0e139..2dc3177ed 100644 --- a/panels/timeline.cpp +++ b/panels/timeline.cpp @@ -144,17 +144,17 @@ void Timeline::next_cut() { void Timeline::reset_all_audio() { // reset all clip audio - if (sequence != NULL) { - for (int i=0;iclip_count();i++) { - Clip* c = sequence->get_clip(i); - if (c != NULL) { - c->reset_audio(); - } - } - } - ui->audio_monitor->reset(); - clear_audio_ibuffer(); audio_ibuffer_frame = playhead; + if (sequence != NULL) { + for (int i=0;iclip_count();i++) { + Clip* c = sequence->get_clip(i); + if (c != NULL) { + c->reset_audio(); + } + } + } + ui->audio_monitor->reset(); + clear_audio_ibuffer(); } void Timeline::seek(long p) { diff --git a/playback/audio.cpp b/playback/audio.cpp index 6de4ae866..148ce90ac 100644 --- a/playback/audio.cpp +++ b/playback/audio.cpp @@ -74,12 +74,8 @@ void clear_audio_ibuffer() { } int get_buffer_offset_from_frame(long frame) { - if (frame >= audio_ibuffer_frame) { - return qFloor(av_samples_get_buffer_size(NULL, av_get_channel_layout_nb_channels(sequence->audio_layout), qRound(((frame-audio_ibuffer_frame)/sequence->frame_rate)*sequence->audio_frequency), AV_SAMPLE_FMT_S16, 1)/4)*4; - } else { - qDebug() << "[WARNING] get_buffer_offset_from_frame called incorrectly"; - return 0; - } + Q_ASSERT(frame >= audio_ibuffer_frame); + return qFloor(av_samples_get_buffer_size(NULL, av_get_channel_layout_nb_channels(sequence->audio_layout), qRound(((frame-audio_ibuffer_frame)/sequence->frame_rate)*sequence->audio_frequency), AV_SAMPLE_FMT_S16, 1)/4)*4; } AudioSenderThread::AudioSenderThread() : close(false) { diff --git a/playback/cacher.cpp b/playback/cacher.cpp index 78b7c7594..699a041e2 100644 --- a/playback/cacher.cpp +++ b/playback/cacher.cpp @@ -24,11 +24,10 @@ extern "C" { int dest_format = AV_PIX_FMT_RGBA; -void apply_audio_effects(Clip* c, double timebase, AVFrame* frame, int nb_bytes) { - // perform all audio effects - double timecode_start, timecode_end; - timecode_start = (frame->pts * timebase); - timecode_end = timecode_start + ((double) (nb_bytes >> 1) / frame->channels / frame->sample_rate); +void apply_audio_effects(Clip* c, double timecode_start, AVFrame* frame, int nb_bytes) { + // perform all audio effects + double timecode_end; + timecode_end = timecode_start + ((double) (nb_bytes >> 1) / frame->channels / frame->sample_rate); // converts bytes to seconds for (int j=0;jeffects.size();j++) { Effect* e = c->effects.at(j); @@ -72,80 +71,89 @@ void cache_audio_worker(Clip* c, Clip* nest) { } while (written < max_write) { - // gets one frame worth of audio and sends it to the audio buffer - AVFrame* frame = c->cache_A.frames[0]; + qDebug() << "written:" << written << max_write; + // gets one frame worth of audio and sends it to the audio buffer + AVFrame* frame; + int nb_bytes = INT_MAX; - if (c->need_new_audio_frame) { - // no more audio left in frame, get a new one - if (!c->reached_end) { - retrieve_next_frame_raw_data(c, frame); - int byte_count = av_samples_get_buffer_size(NULL, frame->channels, frame->nb_samples, static_cast(frame->format), 1); - double timebase = av_q2d(c->stream->time_base); - apply_audio_effects(c, timebase, frame, byte_count); - if (nest != NULL) apply_audio_effects(nest, timebase, frame, byte_count); - } else { - // set by retrieve_next_frame_raw_data indicating no more frames in file, - // but there still may be samples in swresample - swr_convert_frame(c->swr_ctx, frame, NULL); - } + switch (c->media_type) { + case MEDIA_TYPE_FOOTAGE: + { + frame = c->cache_A.frames[0]; - c->need_new_audio_frame = false; - if (c->audio_just_reset) { - // get precise sample offset for the elected clip_in from this audio frame - double target_sts = 0; - if (c->audio_target_frame < timeline_in) { - target_sts = clip_frame_to_seconds(c, c->clip_in); - } else { - target_sts = playhead_to_seconds(c, c->audio_target_frame); - } - double frame_sts = (frame->pts * av_q2d(c->stream->time_base)); - int nb_samples = qRound((target_sts - frame_sts)*c->sequence->audio_frequency); - if (nb_samples == 0) { - c->frame_sample_index = 0; - } else { - c->frame_sample_index = av_samples_get_buffer_size(NULL, av_get_channel_layout_nb_channels(c->sequence->audio_layout), nb_samples, AV_SAMPLE_FMT_S16, 1); - } - c->audio_just_reset = false; - } else { - c->frame_sample_index = 0; - } - } + // retrieve frame + bool new_frame = false; + while ((c->frame_sample_index < 0 || c->frame_sample_index >= nb_bytes) && nb_bytes > 0) { + // no more audio left in frame, get a new one + if (!c->reached_end) { + retrieve_next_frame_raw_data(c, frame); + } else { + // if there is no more data in the file, we flush the remainder out of swresample + swr_convert_frame(c->swr_ctx, frame, NULL); + } + new_frame = true; - if (frame->nb_samples == 0) { - written = max_write; - } else { - int nb_bytes = av_samples_get_buffer_size(NULL, frame->channels, frame->nb_samples, static_cast(frame->format), 1); + nb_bytes = av_samples_get_buffer_size(NULL, frame->channels, frame->nb_samples, static_cast(frame->format), 1); - if (c->audio_buffer_write == 0) { - c->audio_buffer_write = get_buffer_offset_from_frame(qMax(timeline_in, c->audio_target_frame)); + if (c->audio_just_reset) { + // get precise sample offset for the elected clip_in from this audio frame + double target_sts = 0; + if (c->audio_target_frame < timeline_in) { + target_sts = clip_frame_to_seconds(c, c->clip_in); + } else { + target_sts = playhead_to_seconds(c, c->audio_target_frame); + } + double frame_sts = (frame->pts * av_q2d(c->stream->time_base)); + int nb_samples = qRound((target_sts - frame_sts)*c->sequence->audio_frequency); + if (nb_samples == 0) { + c->frame_sample_index = 0; + } else { + c->frame_sample_index = av_samples_get_buffer_size(NULL, av_get_channel_layout_nb_channels(c->sequence->audio_layout), nb_samples, AV_SAMPLE_FMT_S16, 1); + } + c->audio_just_reset = false; + } else { + c->frame_sample_index = 0; + } + + qDebug() << "atf:" << c->audio_target_frame << audio_ibuffer_frame << c->frame_sample_index; + if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(qMax(timeline_in, c->audio_target_frame)); int offset = audio_ibuffer_read - c->audio_buffer_write; if (offset > 0) { - c->audio_buffer_write += offset; - c->frame_sample_index += offset; + c->audio_buffer_write += offset; + c->frame_sample_index += offset; } - } - bool apply_effects = false; - while (c->frame_sample_index > nb_bytes) { - // get new frame - retrieve_next_frame_raw_data(c, frame); - apply_effects = true; - nb_bytes = av_samples_get_buffer_size(NULL, frame->channels, frame->nb_samples, static_cast(frame->format), 1); - c->frame_sample_index -= nb_bytes; + } - // code DISGUSTINGLY copy/pasted from above - int offset = audio_ibuffer_read - c->audio_buffer_write; - if (offset > 0) { - c->audio_buffer_write += offset; - c->frame_sample_index += offset; - } - } - if (apply_effects) { - double timebase = av_q2d(c->stream->time_base); - apply_audio_effects(c, timebase, frame, nb_bytes); - if (nest != NULL) apply_audio_effects(nest, timebase, frame, nb_bytes); - } + // apply any audio effects to the data + if (nb_bytes == INT_MAX) nb_bytes = av_samples_get_buffer_size(NULL, frame->channels, frame->nb_samples, static_cast(frame->format), 1); + if (new_frame) { + apply_audio_effects(c, (frame->pts * av_q2d(c->stream->time_base)), frame, nb_bytes); + } + } + break; + case MEDIA_TYPE_TONE: + frame = c->frame; + nb_bytes = av_samples_get_buffer_size(NULL, frame->channels, frame->nb_samples, static_cast(frame->format), 1); + if (c->frame_sample_index < 0) { + // create "new frame" + memset(c->frame->data[0], 0, nb_bytes); + apply_audio_effects(c, 0, frame, nb_bytes); + c->frame_sample_index = 0; + if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(qMax(timeline_in, c->audio_target_frame)); + } + break; + default: // shouldn't ever get here + qDebug() << "[ERROR] Tried to cache a non-footage/tone clip"; + return; + } + qDebug() << c->audio_buffer_write << c->frame_sample_index; + + // mix audio into internal buffer + if (frame->nb_samples == 0) { + break; + } else { long buffer_timeline_out = get_buffer_offset_from_frame(timeline_out); audio_write_lock.lock(); while (c->frame_sample_index < nb_bytes @@ -163,15 +171,15 @@ void cache_audio_worker(Clip* c, Clip* nest) { c->audio_buffer_write+=2; c->frame_sample_index+=2; written+=2; - } + } audio_write_lock.unlock(); - if (c->frame_sample_index == nb_bytes) { - c->need_new_audio_frame = true; + if (c->frame_sample_index == nb_bytes) { + c->frame_sample_index = -1; } else { // assume we have no more data to send break; } - } + } } } @@ -207,184 +215,213 @@ void cache_video_worker(Clip* c, long playhead, ClipCache* cache) { void reset_cache(Clip* c, long target_frame) { // if we seek to a whole other place in the timeline, we'll need to reset the cache with new values - MediaStream* ms = static_cast(c->media)->get_stream_from_file_index(c->track < 0, c->media_stream); - if (!ms->infinite_length) { - // flush ffmpeg codecs - avcodec_flush_buffers(c->codecCtx); + switch (c->media_type) { + case MEDIA_TYPE_FOOTAGE: + { + MediaStream* ms = static_cast(c->media)->get_stream_from_file_index(c->track < 0, c->media_stream); + if (!ms->infinite_length) { + // flush ffmpeg codecs + avcodec_flush_buffers(c->codecCtx); - double timebase = av_q2d(c->stream->time_base); + double timebase = av_q2d(c->stream->time_base); - if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { - // seeks to nearest keyframe (target_frame represents internal clip frame) + if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + // seeks to nearest keyframe (target_frame represents internal clip frame) - av_seek_frame(c->formatCtx, ms->file_index, (int64_t) qFloor(clip_frame_to_seconds(c, target_frame) / timebase), AVSEEK_FLAG_BACKWARD); + av_seek_frame(c->formatCtx, ms->file_index, (int64_t) qFloor(clip_frame_to_seconds(c, target_frame) / timebase), AVSEEK_FLAG_BACKWARD); - // play up to the frame we actually want - long retrieved_frame = 0; - AVFrame* temp = av_frame_alloc(); - do { - retrieve_next_frame(c, temp); - if (retrieved_frame == 0) { - if (target_frame != 0) retrieved_frame = floor(temp->pts * timebase * av_q2d(av_guess_frame_rate(c->formatCtx, c->stream, temp))); - } else { - retrieved_frame++; - } - } while (retrieved_frame < target_frame); + // play up to the frame we actually want + long retrieved_frame = 0; + AVFrame* temp = av_frame_alloc(); + do { + retrieve_next_frame(c, temp); + if (retrieved_frame == 0) { + if (target_frame != 0) retrieved_frame = floor(temp->pts * timebase * av_q2d(av_guess_frame_rate(c->formatCtx, c->stream, temp))); + } else { + retrieved_frame++; + } + } while (retrieved_frame < target_frame); - av_frame_free(&temp); - } else if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { - // seek (target_frame represents timeline timecode in frames, not clip timecode) - swr_drop_output(c->swr_ctx, swr_get_out_samples(c->swr_ctx, 0)); - av_seek_frame(c->formatCtx, ms->file_index, playhead_to_seconds(c, target_frame) / timebase, AVSEEK_FLAG_BACKWARD); - c->audio_target_frame = target_frame; - c->need_new_audio_frame = true; - c->audio_just_reset = true; + av_frame_free(&temp); + } else if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + // seek (target_frame represents timeline timecode in frames, not clip timecode) + swr_drop_output(c->swr_ctx, swr_get_out_samples(c->swr_ctx, 0)); + av_seek_frame(c->formatCtx, ms->file_index, playhead_to_seconds(c, target_frame) / timebase, AVSEEK_FLAG_BACKWARD); + c->audio_target_frame = target_frame; + c->frame_sample_index = -1; + c->audio_just_reset = true; + } } } + break; + case MEDIA_TYPE_TONE: + c->audio_target_frame = target_frame; + c->frame_sample_index = -1; + break; + } } Cacher::Cacher(Clip* c) : clip(c) {} +int sample_format = AV_SAMPLE_FMT_S16; + void open_clip_worker(Clip* clip) { - // opens file resource for FFmpeg and prepares Clip struct for playback - Media* m = static_cast(clip->media); - QByteArray ba = m->url.toUtf8(); - const char* filename = ba.constData(); - MediaStream* ms = m->get_stream_from_file_index(clip->track < 0, clip->media_stream); + switch (clip->media_type) { + case MEDIA_TYPE_FOOTAGE: + { + // opens file resource for FFmpeg and prepares Clip struct for playback + Media* m = static_cast(clip->media); + QByteArray ba = m->url.toUtf8(); + const char* filename = ba.constData(); + MediaStream* ms = m->get_stream_from_file_index(clip->track < 0, clip->media_stream); - int errCode = avformat_open_input( - &clip->formatCtx, - filename, - NULL, - NULL - ); - if (errCode != 0) { - char err[1024]; - av_strerror(errCode, err, 1024); - qDebug() << "[ERROR] Could not open" << filename << "-" << err; - } - - errCode = avformat_find_stream_info(clip->formatCtx, NULL); - if (errCode < 0) { - char err[1024]; - av_strerror(errCode, err, 1024); - qDebug() << "[ERROR] Could not open" << filename << "-" << err; - } - - av_dump_format(clip->formatCtx, 0, filename, 0); - - clip->stream = clip->formatCtx->streams[ms->file_index]; - clip->codec = avcodec_find_decoder(clip->stream->codecpar->codec_id); - clip->codecCtx = avcodec_alloc_context3(clip->codec); - avcodec_parameters_to_context(clip->codecCtx, clip->stream->codecpar); - - AVDictionary* opts = NULL; - - // decoding optimization configuration - if (clip->stream->codecpar->codec_id != AV_CODEC_ID_PNG && - clip->stream->codecpar->codec_id != AV_CODEC_ID_APNG && - clip->stream->codecpar->codec_id != AV_CODEC_ID_TIFF && - clip->stream->codecpar->codec_id != AV_CODEC_ID_PSD) { - av_dict_set(&opts, "threads", "auto", 0); - } - if (clip->stream->codecpar->codec_id == AV_CODEC_ID_H264) { - av_dict_set(&opts, "tune", "fastdecode", 0); - av_dict_set(&opts, "tune", "zerolatency", 0); - } - - // Open codec - if (avcodec_open2(clip->codecCtx, clip->codec, &opts) < 0) { - qDebug() << "[ERROR] Could not open codec"; - } - - if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { - // set up swscale context - primarily used for colorspace conversion - // as "scaling" is actually done by OpenGL - int dstW = ceil(clip->stream->codecpar->width/2)*2; - int dstH = ceil(clip->stream->codecpar->height/2)*2; - - clip->sws_ctx = sws_getContext( - clip->stream->codecpar->width, - clip->stream->codecpar->height, - static_cast(clip->stream->codecpar->format), - dstW, - dstH, - static_cast(dest_format), - SWS_FAST_BILINEAR, - NULL, + int errCode = avformat_open_input( + &clip->formatCtx, + filename, NULL, NULL ); - - // create memory cache for video - clip->cache_size = (ms->infinite_length) ? 1 : ceil(av_q2d(av_guess_frame_rate(clip->formatCtx, clip->stream, NULL))/4); // cache is half a second in total - - // infinite length doesn't need cache B - clip->cache_A.frames = new AVFrame* [clip->cache_size]; - clip->cache_B.frames = new AVFrame* [clip->cache_size]; - for (int i=0;icache_size;i++) { - clip->cache_A.frames[i] = av_frame_alloc(); - av_frame_make_writable(clip->cache_A.frames[i]); - clip->cache_A.frames[i]->width = dstW; - clip->cache_A.frames[i]->height = dstH; - clip->cache_A.frames[i]->format = dest_format; - if (av_frame_get_buffer(clip->cache_A.frames[i], 0)) { - qDebug() << "[ERROR] Could not allocate buffer for sws_frame"; - } -// clip->cache_A.frames[i]->linesize[0] = dstW*4; - - clip->cache_B.frames[i] = av_frame_alloc(); - av_frame_make_writable(clip->cache_B.frames[i]); - clip->cache_B.frames[i]->width = dstW; - clip->cache_B.frames[i]->height = dstH; - clip->cache_B.frames[i]->format = dest_format; - if (av_frame_get_buffer(clip->cache_B.frames[i], 0)) { - qDebug() << "[ERROR] Could not allocate buffer for sws_frame"; - } -// clip->cache_B.frames[i]->linesize[0] = dstW*4; - } - } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { - // if FFmpeg can't pick up the channel layout (usually WAV), assume - // based on channel count (doesn't support surround sound sources yet) - if (clip->codecCtx->channel_layout == 0) { - clip->codecCtx->channel_layout = guess_layout_from_channels(clip->stream->codecpar->channels); + if (errCode != 0) { + char err[1024]; + av_strerror(errCode, err, 1024); + qDebug() << "[ERROR] Could not open" << filename << "-" << err; } - int sample_format = AV_SAMPLE_FMT_S16; + errCode = avformat_find_stream_info(clip->formatCtx, NULL); + if (errCode < 0) { + char err[1024]; + av_strerror(errCode, err, 1024); + qDebug() << "[ERROR] Could not open" << filename << "-" << err; + } - // init resampling context - clip->swr_ctx = swr_alloc_set_opts( - NULL, - sequence->audio_layout, - static_cast(sample_format), - sequence->audio_frequency, - clip->codecCtx->channel_layout, - static_cast(clip->stream->codecpar->format), - clip->stream->codecpar->sample_rate, - 0, - NULL - ); - swr_init(clip->swr_ctx); + av_dump_format(clip->formatCtx, 0, filename, 0); - // set up cache - clip->cache_A.frames = new AVFrame* [1]; - clip->cache_A.frames[0] = av_frame_alloc(); - clip->cache_A.frames[0]->format = sample_format; - clip->cache_A.frames[0]->channel_layout = sequence->audio_layout; - clip->cache_A.frames[0]->channels = av_get_channel_layout_nb_channels(clip->cache_A.frames[0]->channel_layout); - clip->cache_A.frames[0]->sample_rate = sequence->audio_frequency; - av_frame_make_writable(clip->cache_A.frames[0]); + clip->stream = clip->formatCtx->streams[ms->file_index]; + clip->codec = avcodec_find_decoder(clip->stream->codecpar->codec_id); + clip->codecCtx = avcodec_alloc_context3(clip->codec); + avcodec_parameters_to_context(clip->codecCtx, clip->stream->codecpar); - clip->audio_reset = true; + AVDictionary* opts = NULL; + + // decoding optimization configuration + if (clip->stream->codecpar->codec_id != AV_CODEC_ID_PNG && + clip->stream->codecpar->codec_id != AV_CODEC_ID_APNG && + clip->stream->codecpar->codec_id != AV_CODEC_ID_TIFF && + clip->stream->codecpar->codec_id != AV_CODEC_ID_PSD) { + av_dict_set(&opts, "threads", "auto", 0); + } + if (clip->stream->codecpar->codec_id == AV_CODEC_ID_H264) { + av_dict_set(&opts, "tune", "fastdecode", 0); + av_dict_set(&opts, "tune", "zerolatency", 0); + } + + // Open codec + if (avcodec_open2(clip->codecCtx, clip->codec, &opts) < 0) { + qDebug() << "[ERROR] Could not open codec"; + } + + if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + // set up swscale context - primarily used for colorspace conversion + // as "scaling" is actually done by OpenGL + int dstW = ceil(clip->stream->codecpar->width/2)*2; + int dstH = ceil(clip->stream->codecpar->height/2)*2; + + clip->sws_ctx = sws_getContext( + clip->stream->codecpar->width, + clip->stream->codecpar->height, + static_cast(clip->stream->codecpar->format), + dstW, + dstH, + static_cast(dest_format), + SWS_FAST_BILINEAR, + NULL, + NULL, + NULL + ); + + // create memory cache for video + clip->cache_size = (ms->infinite_length) ? 1 : ceil(av_q2d(av_guess_frame_rate(clip->formatCtx, clip->stream, NULL))/4); // cache is half a second in total + + // infinite length doesn't need cache B + clip->cache_A.frames = new AVFrame* [clip->cache_size]; + clip->cache_B.frames = new AVFrame* [clip->cache_size]; + for (int i=0;icache_size;i++) { + clip->cache_A.frames[i] = av_frame_alloc(); + av_frame_make_writable(clip->cache_A.frames[i]); + clip->cache_A.frames[i]->width = dstW; + clip->cache_A.frames[i]->height = dstH; + clip->cache_A.frames[i]->format = dest_format; + if (av_frame_get_buffer(clip->cache_A.frames[i], 0)) { + qDebug() << "[ERROR] Could not allocate buffer for sws_frame"; + } + // clip->cache_A.frames[i]->linesize[0] = dstW*4; + + clip->cache_B.frames[i] = av_frame_alloc(); + av_frame_make_writable(clip->cache_B.frames[i]); + clip->cache_B.frames[i]->width = dstW; + clip->cache_B.frames[i]->height = dstH; + clip->cache_B.frames[i]->format = dest_format; + if (av_frame_get_buffer(clip->cache_B.frames[i], 0)) { + qDebug() << "[ERROR] Could not allocate buffer for sws_frame"; + } + // clip->cache_B.frames[i]->linesize[0] = dstW*4; + } + } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + // if FFmpeg can't pick up the channel layout (usually WAV), assume + // based on channel count (doesn't support surround sound sources yet) + if (clip->codecCtx->channel_layout == 0) { + clip->codecCtx->channel_layout = guess_layout_from_channels(clip->stream->codecpar->channels); + } + + // init resampling context + clip->swr_ctx = swr_alloc_set_opts( + NULL, + sequence->audio_layout, + static_cast(sample_format), + sequence->audio_frequency, + clip->codecCtx->channel_layout, + static_cast(clip->stream->codecpar->format), + clip->stream->codecpar->sample_rate, + 0, + NULL + ); + swr_init(clip->swr_ctx); + + // set up cache + clip->cache_A.frames = new AVFrame* [1]; + clip->cache_A.frames[0] = av_frame_alloc(); + clip->cache_A.frames[0]->format = sample_format; + clip->cache_A.frames[0]->channel_layout = sequence->audio_layout; + clip->cache_A.frames[0]->channels = av_get_channel_layout_nb_channels(clip->cache_A.frames[0]->channel_layout); + clip->cache_A.frames[0]->sample_rate = sequence->audio_frequency; + av_frame_make_writable(clip->cache_A.frames[0]); + + clip->audio_reset = true; + } + + clip->frame = av_frame_alloc(); + } + break; + case MEDIA_TYPE_TONE: + clip->frame = av_frame_alloc(); + clip->frame->format = sample_format; + clip->frame->channel_layout = sequence->audio_layout; + clip->frame->channels = av_get_channel_layout_nb_channels(clip->frame->channel_layout); + clip->frame->sample_rate = sequence->audio_frequency; + clip->frame->nb_samples = 2048; + av_frame_make_writable(clip->frame); + if (av_frame_get_buffer(clip->frame, 0)) { + qDebug() << "[ERROR] Could not allocate buffer for tone clip"; + } + clip->audio_reset = true; + break; } for (int i=0;ieffects.size();i++) { clip->effects.at(i)->open(); } - clip->frame = av_frame_alloc(); - - clip->finished_opening = true; + clip->finished_opening = true; qDebug() << "[INFO] Clip opened on track" << clip->track; } @@ -393,44 +430,53 @@ void cache_clip_worker(Clip* clip, long playhead, bool write_A, bool write_B, bo if (reset) { // note: for video, playhead is in "internal clip" frames - for audio, it's the timeline playhead reset_cache(clip, playhead); - clip->audio_reset = false; + clip->audio_reset = false; } - if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { - if (write_A) { - cache_video_worker(clip, playhead, &clip->cache_A); - playhead += clip->cache_size; - } + switch (clip->media_type) { + case MEDIA_TYPE_FOOTAGE: + if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + if (write_A) { + cache_video_worker(clip, playhead, &clip->cache_A); + playhead += clip->cache_size; + } - if (write_B) { - cache_video_worker(clip, playhead, &clip->cache_B); + if (write_B) { + cache_video_worker(clip, playhead, &clip->cache_B); + } + } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + cache_audio_worker(clip, nest); } - } else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { - cache_audio_worker(clip, nest); + break; + case MEDIA_TYPE_TONE: + cache_audio_worker(clip, nest); + break; } } void close_clip_worker(Clip* clip) { - // closes ffmpeg file handle and frees any memory used for caching - MediaStream* ms = static_cast(clip->media)->get_stream_from_file_index(clip->track < 0, clip->media_stream); - if (clip->track < 0) { - sws_freeContext(clip->sws_ctx); - } else { - swr_free(&clip->swr_ctx); + if (clip->media_type == MEDIA_TYPE_FOOTAGE) { + // closes ffmpeg file handle and frees any memory used for caching + MediaStream* ms = static_cast(clip->media)->get_stream_from_file_index(clip->track < 0, clip->media_stream); + if (clip->track < 0) { + sws_freeContext(clip->sws_ctx); + } else { + swr_free(&clip->swr_ctx); + } + + avcodec_close(clip->codecCtx); + avcodec_free_context(&clip->codecCtx); + avformat_close_input(&clip->formatCtx); + + for (int i=0;icache_size;i++) { + av_frame_free(&clip->cache_A.frames[i]); + if (!ms->infinite_length) av_frame_free(&clip->cache_B.frames[i]); + } + delete [] clip->cache_A.frames; + if (!ms->infinite_length) delete [] clip->cache_B.frames; } - avcodec_close(clip->codecCtx); - avcodec_free_context(&clip->codecCtx); - avformat_close_input(&clip->formatCtx); - - for (int i=0;icache_size;i++) { - av_frame_free(&clip->cache_A.frames[i]); - if (!ms->infinite_length) av_frame_free(&clip->cache_B.frames[i]); - } - delete [] clip->cache_A.frames; - if (!ms->infinite_length) delete [] clip->cache_B.frames; - - av_frame_free(&clip->frame); + av_frame_free(&clip->frame); clip->reset(); diff --git a/playback/playback.cpp b/playback/playback.cpp index 3bb03883d..6e9a79914 100644 --- a/playback/playback.cpp +++ b/playback/playback.cpp @@ -29,6 +29,7 @@ bool texture_failed = false; void open_clip(Clip* clip, bool multithreaded) { switch (clip->media_type) { case MEDIA_TYPE_FOOTAGE: + case MEDIA_TYPE_TONE: clip->multithreaded = multithreaded; if (multithreaded) { if (clip->open_lock.tryLock()) { @@ -76,6 +77,7 @@ void close_clip(Clip* clip) { switch (clip->media_type) { case MEDIA_TYPE_FOOTAGE: + case MEDIA_TYPE_TONE: if (clip->multithreaded) { clip->cacher->caching = false; clip->can_cache.wakeAll(); @@ -92,7 +94,7 @@ void close_clip(Clip* clip) { } void cache_clip(Clip* clip, long playhead, bool write_A, bool write_B, bool reset, Clip* nest) { - if (clip->media_type == MEDIA_TYPE_FOOTAGE) { + if (clip->media_type == MEDIA_TYPE_FOOTAGE || clip->media_type == MEDIA_TYPE_TONE) { if (clip->multithreaded) { clip->cacher->playhead = playhead; clip->cacher->write_A = write_A; diff --git a/project/clip.cpp b/project/clip.cpp index 0ec266fc4..207f3bb55 100644 --- a/project/clip.cpp +++ b/project/clip.cpp @@ -75,9 +75,8 @@ void Clip::reset() { cache_B.unread = false; reached_end = false; audio_reset = false; - frame_sample_index = false; - audio_buffer_write = false; - need_new_audio_frame = false; + frame_sample_index = -1; + audio_buffer_write = false; texture_frame = -1; formatCtx = NULL; stream = NULL; @@ -90,9 +89,10 @@ void Clip::reset() { void Clip::reset_audio() { switch (media_type) { - case MEDIA_TYPE_FOOTAGE: + case MEDIA_TYPE_FOOTAGE: + case MEDIA_TYPE_TONE: audio_reset = true; - frame_sample_index = 0; + frame_sample_index = -1; audio_buffer_write = 0; reached_end = false; break; @@ -182,6 +182,7 @@ long Clip::getMediaLength(double framerate) { } break; case MEDIA_TYPE_SOLID: + case MEDIA_TYPE_TONE: return timeline_out - timeline_in; break; } diff --git a/project/clip.h b/project/clip.h index 3a3ce2e30..3612fa8db 100644 --- a/project/clip.h +++ b/project/clip.h @@ -105,8 +105,7 @@ struct Clip long texture_frame; // audio playback variables - SwrContext* swr_ctx; - bool need_new_audio_frame; + SwrContext* swr_ctx; int frame_sample_index; int audio_buffer_write; bool audio_reset; diff --git a/ui/timelinewidget.cpp b/ui/timelinewidget.cpp index 13eecc9cf..d3ab8cce5 100644 --- a/ui/timelinewidget.cpp +++ b/ui/timelinewidget.cpp @@ -552,7 +552,6 @@ void TimelineWidget::mouseReleaseEvent(QMouseEvent *event) { Clip* c = new Clip(sequence); c->media = NULL; - c->media_type = MEDIA_TYPE_SOLID; c->timeline_in = qMin(g.in, g.out); c->timeline_out = qMax(g.in, g.out); c->clip_in = 0; @@ -566,7 +565,12 @@ void TimelineWidget::mouseReleaseEvent(QMouseEvent *event) { ta->add_clips(sequence, add); int clipIndex = sequence->clip_count(); - ta->add_effect(sequence, clipIndex, VIDEO_TRANSFORM_EFFECT); + if (c->track < 0) { + // default video effects (before custom effects) + ta->add_effect(sequence, clipIndex, VIDEO_TRANSFORM_EFFECT); + c->media_type = MEDIA_TYPE_SOLID; + } + switch (panel_timeline->creatingObject) { case ADD_OBJ_TITLE: c->name = "Title"; @@ -586,6 +590,13 @@ void TimelineWidget::mouseReleaseEvent(QMouseEvent *event) { break; } + if (c->track >= 0) { + // default audio effects (after custom effects) + ta->add_effect(sequence, clipIndex, AUDIO_VOLUME_EFFECT); + ta->add_effect(sequence, clipIndex, AUDIO_PAN_EFFECT); + c->media_type = MEDIA_TYPE_TONE; + } + Selection s; s.in = c->timeline_in; s.out = c->timeline_out; diff --git a/ui/viewerwidget.cpp b/ui/viewerwidget.cpp index 4612c371a..e5ac70814 100644 --- a/ui/viewerwidget.cpp +++ b/ui/viewerwidget.cpp @@ -136,6 +136,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { break; case MEDIA_TYPE_SEQUENCE: case MEDIA_TYPE_SOLID: + case MEDIA_TYPE_TONE: if (is_clip_active(c, playhead)) { if (!c->open) open_clip(c, !rendering); clip_is_active = true; @@ -309,14 +310,19 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) { glPopMatrix(); } else { - if ((c->media_type == MEDIA_TYPE_FOOTAGE || c->media_type == MEDIA_TYPE_SOLID) - && render_audio - && c->lock.tryLock()) { - // clip is not caching, start caching audio - cache_clip(c, playhead, false, false, c->audio_reset, nest); - c->lock.unlock(); - } else if (c->media_type == MEDIA_TYPE_SEQUENCE) { + switch (c->media_type) { + case MEDIA_TYPE_FOOTAGE: + case MEDIA_TYPE_TONE: + if (render_audio + && c->lock.tryLock()) { + // clip is not caching, start caching audio + cache_clip(c, playhead, false, false, c->audio_reset, nest); + c->lock.unlock(); + } + break; + case MEDIA_TYPE_SEQUENCE: compose_sequence(c, render_audio); + break; } } }