renderer: switched audio rendering over to planar buffers
While planar audio cannot be used for playback, it's much easier for processing. This should lead to cleaner and easier to write code in the future.
This commit is contained in:
@@ -62,7 +62,7 @@ void AudioBackend::ThreadCompletedCache(NodeDependency dep, NodeValueTable data,
|
||||
if (job_time == render_job_info_.value(dep.range())) {
|
||||
render_job_info_.remove(dep.range());
|
||||
|
||||
QByteArray cached_samples = data.Get(NodeParam::kSamples).toByteArray();
|
||||
QByteArray cached_samples = data.Get(NodeParam::kSamples).value<SampleBufferPtr>()->toPackedData();
|
||||
|
||||
int offset = params().time_to_bytes(dep.in());
|
||||
int length = params().time_to_bytes(dep.range().length());
|
||||
|
||||
@@ -12,11 +12,10 @@ void AudioWorker::FrameToValue(DecoderPtr decoder, StreamPtr stream, const TimeR
|
||||
}
|
||||
|
||||
if (decoder->HasConformedVersion(audio_params())) {
|
||||
FramePtr frame = nullptr;
|
||||
frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params());
|
||||
SampleBufferPtr frame = decoder->RetrieveAudio(range.in(), range.out() - range.in(), audio_params());
|
||||
|
||||
if (frame) {
|
||||
table->Push(NodeParam::kSamples, frame->ToByteArray());
|
||||
table->Push(NodeParam::kSamples, QVariant::fromValue(frame));
|
||||
}
|
||||
} else {
|
||||
emit ConformUnavailable(decoder->stream(), CurrentPath().range(), range.out(), audio_params());
|
||||
@@ -41,10 +40,10 @@ void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, c
|
||||
return;
|
||||
}
|
||||
|
||||
QByteArray input_buffer = samples_var.toByteArray();
|
||||
QByteArray output_buffer(input_buffer.size(), 0);
|
||||
SampleBufferPtr input_buffer = samples_var.value<SampleBufferPtr>();
|
||||
SampleBufferPtr output_buffer = SampleBuffer::CreateAllocated(input_buffer->audio_params(), input_buffer->sample_count_per_channel());
|
||||
|
||||
int sample_count = input_buffer.size() / audio_params().bytes_per_sample_per_channel();
|
||||
int sample_count = input_buffer->sample_count_per_channel();
|
||||
|
||||
// FIXME: Hardcoded float sample format
|
||||
for (int i=0;i<sample_count;i++) {
|
||||
@@ -69,10 +68,10 @@ void AudioWorker::RunNodeAccelerated(const Node *node, const TimeRange &range, c
|
||||
|
||||
node->ProcessSamples(&input_params,
|
||||
audio_params(),
|
||||
reinterpret_cast<const float*>(input_buffer.constData()),
|
||||
reinterpret_cast<float*>(output_buffer.data()),
|
||||
input_buffer,
|
||||
output_buffer,
|
||||
i);
|
||||
}
|
||||
|
||||
output_params->Push(NodeParam::kSamples, output_buffer);
|
||||
output_params->Push(NodeParam::kSamples, QVariant::fromValue(output_buffer));
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
|
||||
QList<Block*> active_blocks = track->BlocksAtTimeRange(range);
|
||||
|
||||
// All these blocks will need to output to a buffer so we create one here
|
||||
QByteArray block_range_buffer(audio_params_.time_to_bytes(range.length()), 0);
|
||||
SampleBufferPtr block_range_buffer = SampleBuffer::CreateAllocated(audio_params_, audio_params_.time_to_samples(range.length()));
|
||||
|
||||
NodeValueTable merged_table;
|
||||
|
||||
@@ -46,69 +46,56 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
|
||||
|
||||
// Destination buffer
|
||||
NodeValueTable table = ProcessNode(NodeDependency(b, range_for_block));
|
||||
QByteArray samples_from_this_block = table.Take(NodeParam::kSamples).toByteArray();
|
||||
QVariant sample_val = table.Take(NodeParam::kSamples);
|
||||
|
||||
if (!samples_from_this_block.isEmpty()) {
|
||||
// Stretch samples here
|
||||
rational abs_speed = qAbs(b->speed());
|
||||
if (sample_val.isNull()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (abs_speed != 1) {
|
||||
QByteArray speed_adjusted_samples;
|
||||
SampleBufferPtr samples_from_this_block = sample_val.value<SampleBufferPtr>();
|
||||
|
||||
double clip_speed = abs_speed.toDouble();
|
||||
// Stretch samples here
|
||||
rational abs_speed = qAbs(b->speed());
|
||||
|
||||
int sample_count = audio_params_.bytes_to_samples(samples_from_this_block.size());
|
||||
if (abs_speed != 1) {
|
||||
samples_from_this_block->speed(abs_speed.toDouble());
|
||||
}
|
||||
|
||||
for (double i=0;i<sample_count;i+=clip_speed) {
|
||||
int sample_index = qFloor(i);
|
||||
int byte_index = audio_params_.samples_to_bytes(sample_index);
|
||||
if (b->is_reversed()) {
|
||||
// Reverse the audio buffer
|
||||
samples_from_this_block->reverse();
|
||||
}
|
||||
|
||||
QByteArray sample_at_this_index = samples_from_this_block.mid(byte_index, audio_params_.samples_to_bytes(1));
|
||||
speed_adjusted_samples.append(sample_at_this_index);
|
||||
}
|
||||
int destination_offset = audio_params_.time_to_samples(range_for_block.in() - range.in()) * sizeof(float);
|
||||
int max_dest_sz = audio_params_.time_to_samples(range_for_block.length()) * sizeof(float);
|
||||
int input_sz = samples_from_this_block->sample_count_per_channel() * sizeof(float);
|
||||
int actual_copy_size = qMin(max_dest_sz, input_sz);
|
||||
|
||||
samples_from_this_block = speed_adjusted_samples;
|
||||
for (int i=0;i<audio_params_.channel_count();i++) {
|
||||
memcpy(block_range_buffer->data()[i] + destination_offset,
|
||||
samples_from_this_block->data()[i],
|
||||
actual_copy_size);
|
||||
|
||||
if (input_sz < max_dest_sz) {
|
||||
memset(block_range_buffer->data()[i] + destination_offset + input_sz, 0, max_dest_sz - input_sz);
|
||||
}
|
||||
}
|
||||
|
||||
if (b->is_reversed()) {
|
||||
// Reverse the audio buffer
|
||||
AudioManager::ReverseBuffer(samples_from_this_block.data(), samples_from_this_block.size(), audio_params_.samples_to_bytes(1));
|
||||
}
|
||||
{
|
||||
// Save waveform to file
|
||||
Block* src_block = static_cast<Block*>(copy_map_->value(b));
|
||||
QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString());
|
||||
QDir waveform_loc = local_appdata_dir.filePath("waveform");
|
||||
waveform_loc.mkpath(".");
|
||||
QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast<quintptr>(src_block))));
|
||||
|
||||
int destination_offset = audio_params_.time_to_bytes(range_for_block.in() - range.in());
|
||||
int actual_copy_size = qMin(audio_params_.time_to_bytes(range_for_block.length()), samples_from_this_block.size());
|
||||
if (wave_file.open(QFile::ReadWrite)) {
|
||||
// We use S32 as a size-compatible substitute for SampleSummer::Sum which is 4 bytes in size
|
||||
AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S32);
|
||||
int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate());
|
||||
|
||||
if (destination_offset < 0
|
||||
|| destination_offset + actual_copy_size > block_range_buffer.size()) {
|
||||
qCritical() << "Tried to copy audio beyond the size of the destination buffer";
|
||||
qCritical() << " Destination size:" << block_range_buffer.size();
|
||||
qCritical() << " Destination offset:" << destination_offset;
|
||||
qCritical() << " Copy size:" << actual_copy_size;
|
||||
abort();
|
||||
} else {
|
||||
memcpy(block_range_buffer.data()+destination_offset,
|
||||
samples_from_this_block.data(),
|
||||
actual_copy_size);
|
||||
|
||||
// Save waveform to file
|
||||
Block* src_block = static_cast<Block*>(copy_map_->value(b));
|
||||
QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString());
|
||||
QDir waveform_loc = local_appdata_dir.filePath("waveform");
|
||||
waveform_loc.mkpath(".");
|
||||
QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast<quintptr>(src_block))));
|
||||
wave_file.open(QFile::ReadWrite);
|
||||
|
||||
// FIXME: Assumes 32-bit float
|
||||
float* flt_samples = reinterpret_cast<float*>(samples_from_this_block.data());
|
||||
int nb_sample = samples_from_this_block.size() / sizeof(float);
|
||||
|
||||
// FIXME: Hardcoded sample rate
|
||||
// We use S16 here as a size-compatible substitute for qfloat16/half
|
||||
AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S16);
|
||||
int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()) * waveform_params.channel_count();
|
||||
|
||||
qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in()) * 2;
|
||||
qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()) * 2;
|
||||
qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in());
|
||||
qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length());
|
||||
qint64 end_offset = start_offset + length_offset;
|
||||
|
||||
if (wave_file.size() < end_offset) {
|
||||
@@ -117,10 +104,10 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
|
||||
|
||||
wave_file.seek(start_offset);
|
||||
|
||||
for (int i=0;i<nb_sample;i+=chunk_size) {
|
||||
QVector<SampleSummer::Sum> summary = SampleSummer::SumSamples(&flt_samples[i],
|
||||
qMin(chunk_size, nb_sample - i),
|
||||
audio_params().channel_count());
|
||||
for (int i=0;i<samples_from_this_block->sample_count_per_channel();i+=chunk_size) {
|
||||
QVector<SampleSummer::Sum> summary = SampleSummer::SumSamples(samples_from_this_block,
|
||||
i,
|
||||
qMin(chunk_size, samples_from_this_block->sample_count_per_channel() - i));
|
||||
|
||||
wave_file.write(reinterpret_cast<const char*>(summary.constData()),
|
||||
summary.size() * sizeof(SampleSummer::Sum));
|
||||
@@ -137,7 +124,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
|
||||
NodeValueTable::Merge({merged_table, table});
|
||||
}
|
||||
|
||||
merged_table.Push(NodeParam::kSamples, block_range_buffer);
|
||||
merged_table.Push(NodeParam::kSamples, QVariant::fromValue(block_range_buffer));
|
||||
|
||||
return merged_table;
|
||||
}
|
||||
|
||||
@@ -317,9 +317,7 @@ void VideoRenderWorker::Download(const rational& time, QVariant texture, QString
|
||||
} else {
|
||||
|
||||
FramePtr frame = Frame::Create();
|
||||
frame->set_width(video_params().width());
|
||||
frame->set_height(video_params().height());
|
||||
frame->set_format(video_params().format());
|
||||
frame->set_video_params(video_params());
|
||||
frame->allocate();
|
||||
|
||||
TextureToBuffer(texture, frame->data());
|
||||
|
||||
@@ -237,10 +237,8 @@ FramePtr PixelFormat::ConvertPixelFormat(FramePtr frame, const PixelFormat::Form
|
||||
FramePtr converted = Frame::Create();
|
||||
|
||||
// Copy parameters
|
||||
converted->set_width(frame->width());
|
||||
converted->set_height(frame->height());
|
||||
converted->set_video_params(frame->video_params());
|
||||
converted->set_timestamp(frame->timestamp());
|
||||
converted->set_format(dest_format);
|
||||
converted->allocate();
|
||||
|
||||
OIIO::ImageBuf src(OIIO::ImageSpec(frame->width(), frame->height(), ChannelCount(frame->format()), GetOIIOTypeDesc(frame->format())), frame->data());
|
||||
|
||||
@@ -34,6 +34,12 @@ VideoRenderingParams::VideoRenderingParams() :
|
||||
{
|
||||
}
|
||||
|
||||
VideoRenderingParams::VideoRenderingParams(const int &width, const int &height, const PixelFormat::Format &format) :
|
||||
VideoParams(width, height, rational()),
|
||||
format_(format)
|
||||
{
|
||||
}
|
||||
|
||||
VideoRenderingParams::VideoRenderingParams(const int &width, const int &height, const rational &time_base, const PixelFormat::Format &format, const RenderMode::Mode& mode, const int ÷r) :
|
||||
VideoParams(width, height, time_base),
|
||||
format_(format),
|
||||
|
||||
@@ -25,6 +25,7 @@ private:
|
||||
class VideoRenderingParams : public VideoParams {
|
||||
public:
|
||||
VideoRenderingParams();
|
||||
VideoRenderingParams(const int& width, const int& height, const PixelFormat::Format& format);
|
||||
VideoRenderingParams(const int& width, const int& height, const rational& time_base, const PixelFormat::Format& format, const RenderMode::Mode& mode, const int& divider = 1);
|
||||
VideoRenderingParams(const VideoParams& params, const PixelFormat::Format& format, const RenderMode::Mode& mode, const int& divider = 1);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user