diff --git a/app/audio/sampleformat.h b/app/audio/sampleformat.h index 6d5951866..f19b8e7e8 100644 --- a/app/audio/sampleformat.h +++ b/app/audio/sampleformat.h @@ -28,11 +28,14 @@ namespace olive { */ enum SampleFormat { SAMPLE_FMT_INVALID = -1, + SAMPLE_FMT_U8, SAMPLE_FMT_S16, SAMPLE_FMT_S32, + SAMPLE_FMT_S64, SAMPLE_FMT_FLT, SAMPLE_FMT_DBL, + SAMPLE_FMT_COUNT }; diff --git a/app/common/timecodefunctions.cpp b/app/common/timecodefunctions.cpp index 5bed5ab3a..2783fd237 100644 --- a/app/common/timecodefunctions.cpp +++ b/app/common/timecodefunctions.cpp @@ -35,8 +35,24 @@ QString olive::timestamp_to_timecode(const int64_t ×tamp, { double timestamp_dbl = (rational(timestamp) * timebase).toDouble(); + // Determine what symbol to separate frames (";" is used for drop frame, ":" is non-drop frame) + QString frame_token = ";"; + switch (display) { - case kTimecodeFrames: + case kTimecodeNonDropFrame: + frame_token = ":"; + + // Convert timestamp from drop frame to non-drop frame + // FIXME: There's probably a better way to do this + if (timebase == rational(1001, 30000)) { + timestamp_dbl = timestamp_dbl / (30000.0/1001.0) * 30.0; + } else if (timebase == rational(1001, 60000)) { + timestamp_dbl = timestamp_dbl / (60000.0/1001.0) * 60.0; + } else if (timebase == rational(1001, 24000)) { + timestamp_dbl = timestamp_dbl / (24000.0/1001.0) * 24.0; + } + /* fall-through */ + case kTimecodeDropFrame: case kTimecodeSeconds: { QString prefix; @@ -68,11 +84,12 @@ QString olive::timestamp_to_timecode(const int64_t ×tamp, int frames = qRound((timestamp_dbl - total_seconds) * frame_rate.toDouble()); - return QString("%1%2:%3:%4;%5").arg(prefix, - padded(hours, 2), - padded(mins, 2), - padded(secs, 2), - padded(frames, 2)); + return QString("%1%2:%3:%4%5%6").arg(prefix, + padded(hours, 2), + padded(mins, 2), + padded(secs, 2), + frame_token, + padded(frames, 2)); } } case kFrames: diff --git a/app/common/timecodefunctions.h b/app/common/timecodefunctions.h index 4dcd2344f..3bd06a72e 100644 --- a/app/common/timecodefunctions.h +++ b/app/common/timecodefunctions.h @@ -28,7 +28,8 @@ namespace olive { enum TimecodeDisplay { - kTimecodeFrames, + kTimecodeDropFrame, + kTimecodeNonDropFrame, kTimecodeSeconds, kFrames, kMilliseconds diff --git a/app/config/config.cpp b/app/config/config.cpp index 28e9cfbec..b87db7a89 100644 --- a/app/config/config.cpp +++ b/app/config/config.cpp @@ -49,7 +49,7 @@ Config &Config::Current() void Config::SetDefaults() { config_map_.clear(); - config_map_["TimecodeDisplay"] = olive::kTimecodeFrames; + config_map_["TimecodeDisplay"] = olive::kTimecodeNonDropFrame; config_map_["DefaultStillLength"] = QVariant::fromValue(rational(2)); config_map_["HoverFocus"] = false; config_map_["AudioScrubbing"] = true; diff --git a/app/decoder/CMakeLists.txt b/app/decoder/CMakeLists.txt index 272ef91bb..12716d4fc 100644 --- a/app/decoder/CMakeLists.txt +++ b/app/decoder/CMakeLists.txt @@ -23,5 +23,7 @@ set(OLIVE_SOURCES decoder/decoder.cpp decoder/frame.h decoder/frame.cpp + decoder/wave.h + decoder/wave.cpp PARENT_SCOPE ) diff --git a/app/decoder/ffmpeg/ffmpegdecoder.cpp b/app/decoder/ffmpeg/ffmpegdecoder.cpp index 84f2ecbce..9769623de 100644 --- a/app/decoder/ffmpeg/ffmpegdecoder.cpp +++ b/app/decoder/ffmpeg/ffmpegdecoder.cpp @@ -32,6 +32,7 @@ extern "C" { #include "common/filefunctions.h" #include "common/timecodefunctions.h" +#include "decoder/wave.h" #include "render/pixelservice.h" FFmpegDecoder::FFmpegDecoder() : @@ -412,7 +413,7 @@ bool FFmpegDecoder::Probe(Footage *f) str->set_type(Stream::kAttachment); break; - // We should never realistically get here, but we make an "invalid" stream just in case + // We should never realistically get here, but we make an "invalid" stream just in case default: str->set_type(Stream::kUnknown); break; @@ -501,27 +502,104 @@ void FFmpegDecoder::Index() // Reset state Seek(0); - // This should be unnecessary, but just in case... - frame_index_.clear(); - - // Iterate through every single frame and get each timestamp - // NOTE: Expects no frames to have been read so far - int ret = 0; - while (true) { - ret = GetFrame(); + if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + // This should be unnecessary, but just in case... + frame_index_.clear(); - if (ret < 0) { - break; + // Iterate through every single frame and get each timestamp + // NOTE: Expects no frames to have been read so far + + while (true) { + ret = GetFrame(); + + if (ret < 0) { + break; + } else { + frame_index_.append(frame_->pts); + } + } + + // Save index to file + SaveFrameIndex(); + } else if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + // Iterate through each audio frame and extract the PCM data + + WaveOutput wave_out("C:\\Users\\Matt\\AppData\\Local\\Temp\\temporary.wav", // FIXME: Hardcoded path + AudioRenderingParams(avstream_->codecpar->sample_rate, + avstream_->codecpar->channel_layout, + GetNativeSampleRate(static_cast(avstream_->codecpar->format)))); + + SwrContext* resampler = nullptr; + AVSampleFormat src_sample_fmt = static_cast(avstream_->codecpar->format); + AVSampleFormat dst_sample_fmt; + + // We don't use planar types internally, so if this is a planar format convert it now + if (av_sample_fmt_is_planar(src_sample_fmt)) { + dst_sample_fmt = av_get_packed_sample_fmt(src_sample_fmt); + + // Bizarrely, swr_alloc_set_opts() uses a signed int64 while most of FFmpeg uses unsigned. We cast here. + int64_t channel_layout = static_cast(avstream_->codecpar->channel_layout); + + resampler = swr_alloc_set_opts(nullptr, + channel_layout, + dst_sample_fmt, + avstream_->codecpar->sample_rate, + channel_layout, + src_sample_fmt, + avstream_->codecpar->sample_rate, + 0, + nullptr); } else { - frame_index_.append(frame_->pts); + dst_sample_fmt = src_sample_fmt; + } + + if (wave_out.open()) { + while (true) { + ret = GetFrame(); + + if (ret < 0) { + break; + } else { + // Calculate the byte size for this audio buffer + int buffer_size = av_samples_get_buffer_size(nullptr, + avstream_->codecpar->channels, + frame_->nb_samples, + dst_sample_fmt, + 0); // FIXME: Documentation unclear - should this be 0 or 1? + + uint8_t* resampler_output; + + if (resampler != nullptr) { + // We must need to resample this (mainly just convert from planar to packed if necessary) + resampler_output = new uint8_t[buffer_size]; + swr_convert(resampler, &resampler_output, frame_->nb_samples, frame_->data, frame_->nb_samples); + } else { + // No resampling required, we can write directly from te frame buffer + resampler_output = frame_->data[0]; + } + + // Write packed WAV data to the disk cache + wave_out.write(resampler_output, buffer_size); + + // If we allocated an output for the resampler, delete it here + if (resampler_output != frame_->data[0]) { + delete [] resampler_output; + } + } + } + + wave_out.close(); + } else { + qWarning() << "Failed to open WAVE output for indexing"; + } + + if (resampler != nullptr) { + swr_free(&resampler); } } - // Save index to file - SaveFrameIndex(); - // Reset state Seek(0); } @@ -529,7 +607,7 @@ void FFmpegDecoder::Index() QString FFmpegDecoder::GetIndexFilename() { if (!open_) { - qWarning() << tr("GetIndexFilename tried to run while decoder was closed"); + qWarning() << "GetIndexFilename tried to run while decoder was closed"; return QString(); } @@ -636,6 +714,35 @@ AVPixelFormat FFmpegDecoder::GetCompatiblePixelFormat(const AVPixelFormat &pix_f nullptr); } +olive::SampleFormat FFmpegDecoder::GetNativeSampleRate(const AVSampleFormat &smp_fmt) +{ + switch (smp_fmt) { + case AV_SAMPLE_FMT_U8: + return olive::SAMPLE_FMT_U8; + case AV_SAMPLE_FMT_S16: + return olive::SAMPLE_FMT_S16; + case AV_SAMPLE_FMT_S32: + return olive::SAMPLE_FMT_S32; + case AV_SAMPLE_FMT_S64: + return olive::SAMPLE_FMT_S64; + case AV_SAMPLE_FMT_FLT: + return olive::SAMPLE_FMT_FLT; + case AV_SAMPLE_FMT_DBL: + return olive::SAMPLE_FMT_DBL; + case AV_SAMPLE_FMT_U8P : + case AV_SAMPLE_FMT_S16P: + case AV_SAMPLE_FMT_S32P: + case AV_SAMPLE_FMT_S64P: + case AV_SAMPLE_FMT_FLTP: + case AV_SAMPLE_FMT_DBLP: + case AV_SAMPLE_FMT_NONE: + case AV_SAMPLE_FMT_NB: + break; + } + + return olive::SAMPLE_FMT_INVALID; +} + int64_t FFmpegDecoder::GetClosestTimestampInIndex(const int64_t &ts) { // Index now if we haven't already diff --git a/app/decoder/ffmpeg/ffmpegdecoder.h b/app/decoder/ffmpeg/ffmpegdecoder.h index 7300727be..dfe76e3fb 100644 --- a/app/decoder/ffmpeg/ffmpegdecoder.h +++ b/app/decoder/ffmpeg/ffmpegdecoder.h @@ -30,6 +30,7 @@ extern "C" { #include #include "decoder/decoder.h" +#include "audio/sampleformat.h" /** * @brief A Decoder derivative that wraps FFmpeg functions as on Olive decoder @@ -130,6 +131,8 @@ private: */ AVPixelFormat GetCompatiblePixelFormat(const AVPixelFormat& pix_fmt); + olive::SampleFormat GetNativeSampleRate(const AVSampleFormat& smp_fmt); + AVFormatContext* fmt_ctx_; AVCodecContext* codec_ctx_; AVStream* avstream_; diff --git a/app/decoder/wave.cpp b/app/decoder/wave.cpp new file mode 100644 index 000000000..85027f465 --- /dev/null +++ b/app/decoder/wave.cpp @@ -0,0 +1,143 @@ +#include "wave.h" + +const int16_t kWAVIntegerFormat = 1; +const int16_t kWAVFloatFormat = 3; + +WaveOutput::WaveOutput(const QString &f, + const AudioRenderingParams& params) : + file_(f), + params_(params) +{ +} + +WaveOutput::~WaveOutput() +{ + close(); +} + +bool WaveOutput::open() +{ + data_length_ = 0; + + if (file_.open(QFile::WriteOnly)) { + // RIFF header + file_.write("RIFF"); + + // Total file size minus RIFF and this integer (minus 8 bytes, filled in later) + write_int(&file_, 0); + + // File type header + file_.write("WAVE"); + + // Begin format descriptor chunk + file_.write("fmt "); + + // Format chunk size + write_int(&file_, 16); + + // Type of format + switch (params_.format()) { + case olive::SAMPLE_FMT_U8: + case olive::SAMPLE_FMT_S16: + case olive::SAMPLE_FMT_S32: + case olive::SAMPLE_FMT_S64: + write_int(&file_, kWAVIntegerFormat); + break; + case olive::SAMPLE_FMT_FLT: + case olive::SAMPLE_FMT_DBL: + write_int(&file_, kWAVFloatFormat); + break; + case olive::SAMPLE_FMT_INVALID: + case olive::SAMPLE_FMT_COUNT: + qWarning() << "Invalid sample format for WAVE audio"; + file_.close(); + return false; + } + + // Number of channels + write_int(&file_, static_cast(params_.channel_count())); + + // Sample rate + write_int(&file_, params_.sample_rate()); + + // Bytes per second + write_int(&file_, (params_.sample_rate() * params_.bits_per_sample() * params_.channel_count())/8); + + // Bytes per sample + write_int(&file_, static_cast((params_.bits_per_sample() * params_.channel_count())/8)); + + // Bits per sample + write_int(&file_, static_cast(params_.bits_per_sample())); + + // Data chunk header + file_.write("data"); + + // Size of data chunk (filled in later) + write_int(&file_, 0); + + return true; + } + + return false; +} + +void WaveOutput::write(const QByteArray &bytes) +{ + if (file_.isOpen()) { + file_.write(bytes); + + data_length_ += bytes.size(); + } +} + +void WaveOutput::write(const char *bytes, int length) +{ + if (file_.isOpen()) { + file_.write(bytes, length); + + data_length_ += length; + } +} + +void WaveOutput::close() +{ + if (file_.isOpen()) { + + // Write file sizes + file_.seek(4); + write_int(&file_, data_length_ + 36); + + file_.seek(40); + write_int(&file_, data_length_); + + file_.close(); + } +} + +void WaveOutput::switch_endianness(QByteArray& array) +{ + int half_sz = array.size()/2; + + for (int i=0;i +void WaveOutput::write_int(QFile *file, T integer) +{ + QByteArray bytes; + bytes.resize(sizeof(T)); + memcpy(bytes.data(), &integer, static_cast(bytes.size())); + + // WAV expects little-endian, so if the integer is big endian we need to switch + if (QSysInfo::ByteOrder == QSysInfo::BigEndian) { + switch_endianness(bytes); + } + + file->write(bytes); +} diff --git a/app/decoder/wave.h b/app/decoder/wave.h new file mode 100644 index 000000000..73566f3f3 --- /dev/null +++ b/app/decoder/wave.h @@ -0,0 +1,39 @@ +#ifndef WAVEAUDIO_H +#define WAVEAUDIO_H + +#include +#include + +#include "audio/sampleformat.h" +#include "render/audio/audioparams.h" + +class WaveOutput +{ +public: + WaveOutput(const QString& f, + const AudioRenderingParams& params); + + ~WaveOutput(); + + bool open(); + + void write(const QByteArray& bytes); + void write(const char* bytes, int length); + + void close(); + +private: + template + void write_int(QFile* file, T integer); + + void switch_endianness(QByteArray &array); + + QFile file_; + + AudioRenderingParams params_; + + int data_length_; + +}; + +#endif // WAVEAUDIO_H diff --git a/app/node/input/media/media.cpp b/app/node/input/media/media.cpp index a4518f722..866fb8ad7 100644 --- a/app/node/input/media/media.cpp +++ b/app/node/input/media/media.cpp @@ -269,9 +269,6 @@ QVariant MediaInput::Value(NodeOutput *output, const rational &in, const rationa transform.scale(static_cast(frame_->width()), static_cast(frame_->height())); transform.scale(0.5f, 0.5f); - //float media_size = static_cast(frame_->height()) / static_cast(renderer->height() * renderer->divider()); - //transform.scale(media_size, media_size); - // Use pipeline to blit using transformation matrix from input if (renderer->params().mode() == olive::RenderMode::kOffline) { olive::gl::OCIOBlit(pipeline_, ocio_texture_, false, transform); diff --git a/app/render/audio/audioparams.cpp b/app/render/audio/audioparams.cpp index facaaf457..e3a3d59ec 100644 --- a/app/render/audio/audioparams.cpp +++ b/app/render/audio/audioparams.cpp @@ -52,7 +52,7 @@ int AudioRenderingParams::time_to_bytes(const rational &time) const { Q_ASSERT(is_valid()); - return qFloor(time.toDouble() * sample_rate()) * channel_count() * sample_size(); + return qFloor(time.toDouble() * sample_rate()) * channel_count() * bytes_per_sample(); } int AudioRenderingParams::channel_count() const @@ -60,7 +60,7 @@ int AudioRenderingParams::channel_count() const return av_get_channel_layout_nb_channels(channel_layout()); } -int AudioRenderingParams::sample_size() const +int AudioRenderingParams::bytes_per_sample() const { switch (format_) { case olive::SAMPLE_FMT_U8: @@ -71,6 +71,7 @@ int AudioRenderingParams::sample_size() const case olive::SAMPLE_FMT_FLT: return 4; case olive::SAMPLE_FMT_DBL: + case olive::SAMPLE_FMT_S64: return 8; case olive::SAMPLE_FMT_INVALID: case olive::SAMPLE_FMT_COUNT: @@ -80,6 +81,11 @@ int AudioRenderingParams::sample_size() const return 0; } +int AudioRenderingParams::bits_per_sample() const +{ + return bytes_per_sample() * 8; +} + bool AudioRenderingParams::is_valid() const { return (sample_rate() > 0 diff --git a/app/render/audio/audioparams.h b/app/render/audio/audioparams.h index 9f657108c..447fe2317 100644 --- a/app/render/audio/audioparams.h +++ b/app/render/audio/audioparams.h @@ -30,7 +30,8 @@ public: int time_to_bytes(const rational& time) const; int channel_count() const; - int sample_size() const; + int bytes_per_sample() const; + int bits_per_sample() const; bool is_valid() const; const olive::SampleFormat& format() const; diff --git a/app/render/video/videorenderer.cpp b/app/render/video/videorenderer.cpp index 3bda31620..2d414ff9f 100644 --- a/app/render/video/videorenderer.cpp +++ b/app/render/video/videorenderer.cpp @@ -344,14 +344,27 @@ void VideoRendererProcessor::ThreadCallback(RenderTexturePtr texture, const rati // If the connected output is using this time, signal it to update if (last_time_requested_ == time) { + copy_buffer_.Bind(); - texture->Bind(); - QOpenGLContext::currentContext()->functions()->glViewport(0, 0, master_texture_->width(), master_texture_->height()); + QOpenGLFunctions* f = QOpenGLContext::currentContext()->functions(); - olive::gl::Blit(copy_pipeline_); + if (texture == nullptr) { + + // No texture, clear the master and push it + f->glClearColor(0.0f, 0.0f, 0.0f, 0.0f); + f->glClear(GL_COLOR_BUFFER_BIT); + + } else { + texture->Bind(); + + f->glViewport(0, 0, master_texture_->width(), master_texture_->height()); + + olive::gl::Blit(copy_pipeline_); + + texture->Release(); + } - texture->Release(); copy_buffer_.Release(); push_time_ = time; diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 51faab723..eccab0b0b 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -281,9 +281,11 @@ void ViewerWidget::NextFrame() void ViewerWidget::GoToEnd() { - Pause(); + if (viewer_node_ != nullptr) { + Pause(); - qWarning() << "No end frame support yet"; + SetTime(olive::time_to_timestamp(viewer_node_->Length(), time_base_)); + } } void ViewerWidget::ShuttleLeft()