oak-team/oak-editor
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

renderer: switched audio rendering over to planar buffers

Browse Source 
While planar audio cannot be used for playback, it's much easier for processing.
This should lead to cleaner and easier to write code in the future.
This commit is contained in:
itsmattkc
2020-03-31 18:01:20 +11:00
parent 8413886c17
commit 6f0d72c05d
30 changed files with 436 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/render/backend/audiorenderworker.cpp
+45 -58
View File
@@ -35,7 +35,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
QList<Block*> active_blocks = track->BlocksAtTimeRange(range);
// All these blocks will need to output to a buffer so we create one here
QByteArray block_range_buffer(audio_params_.time_to_bytes(range.length()), 0);
SampleBufferPtr block_range_buffer = SampleBuffer::CreateAllocated(audio_params_, audio_params_.time_to_samples(range.length()));
NodeValueTable merged_table;
@@ -46,69 +46,56 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
// Destination buffer
NodeValueTable table = ProcessNode(NodeDependency(b, range_for_block));
QByteArray samples_from_this_block = table.Take(NodeParam::kSamples).toByteArray();
QVariant sample_val = table.Take(NodeParam::kSamples);
if (!samples_from_this_block.isEmpty()) {
// Stretch samples here
rational abs_speed = qAbs(b->speed());
if (sample_val.isNull()) {
continue;
}
if (abs_speed != 1) {
QByteArray speed_adjusted_samples;
SampleBufferPtr samples_from_this_block = sample_val.value<SampleBufferPtr>();
double clip_speed = abs_speed.toDouble();
// Stretch samples here
rational abs_speed = qAbs(b->speed());
int sample_count = audio_params_.bytes_to_samples(samples_from_this_block.size());
if (abs_speed != 1) {
samples_from_this_block->speed(abs_speed.toDouble());
}
for (double i=0;i<sample_count;i+=clip_speed) {
int sample_index = qFloor(i);
int byte_index = audio_params_.samples_to_bytes(sample_index);
if (b->is_reversed()) {
// Reverse the audio buffer
samples_from_this_block->reverse();
}
QByteArray sample_at_this_index = samples_from_this_block.mid(byte_index, audio_params_.samples_to_bytes(1));
speed_adjusted_samples.append(sample_at_this_index);
}
int destination_offset = audio_params_.time_to_samples(range_for_block.in() - range.in()) * sizeof(float);
int max_dest_sz = audio_params_.time_to_samples(range_for_block.length()) * sizeof(float);
int input_sz = samples_from_this_block->sample_count_per_channel() * sizeof(float);
int actual_copy_size = qMin(max_dest_sz, input_sz);
samples_from_this_block = speed_adjusted_samples;
for (int i=0;i<audio_params_.channel_count();i++) {
memcpy(block_range_buffer->data()[i] + destination_offset,
samples_from_this_block->data()[i],
actual_copy_size);
if (input_sz < max_dest_sz) {
memset(block_range_buffer->data()[i] + destination_offset + input_sz, 0, max_dest_sz - input_sz);
}
}
if (b->is_reversed()) {
// Reverse the audio buffer
AudioManager::ReverseBuffer(samples_from_this_block.data(), samples_from_this_block.size(), audio_params_.samples_to_bytes(1));
}
{
// Save waveform to file
Block* src_block = static_cast<Block*>(copy_map_->value(b));
QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString());
QDir waveform_loc = local_appdata_dir.filePath("waveform");
waveform_loc.mkpath(".");
QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast<quintptr>(src_block))));
int destination_offset = audio_params_.time_to_bytes(range_for_block.in() - range.in());
int actual_copy_size = qMin(audio_params_.time_to_bytes(range_for_block.length()), samples_from_this_block.size());
if (wave_file.open(QFile::ReadWrite)) {
// We use S32 as a size-compatible substitute for SampleSummer::Sum which is 4 bytes in size
AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S32);
int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate());
if (destination_offset < 0
|| destination_offset + actual_copy_size > block_range_buffer.size()) {
qCritical() << "Tried to copy audio beyond the size of the destination buffer";
qCritical() << " Destination size:" << block_range_buffer.size();
qCritical() << " Destination offset:" << destination_offset;
qCritical() << " Copy size:" << actual_copy_size;
abort();
} else {
memcpy(block_range_buffer.data()+destination_offset,
samples_from_this_block.data(),
actual_copy_size);
// Save waveform to file
Block* src_block = static_cast<Block*>(copy_map_->value(b));
QDir local_appdata_dir(Config::Current()["DiskCachePath"].toString());
QDir waveform_loc = local_appdata_dir.filePath("waveform");
waveform_loc.mkpath(".");
QFile wave_file(waveform_loc.filePath(QString::number(reinterpret_cast<quintptr>(src_block))));
wave_file.open(QFile::ReadWrite);
// FIXME: Assumes 32-bit float
float* flt_samples = reinterpret_cast<float*>(samples_from_this_block.data());
int nb_sample = samples_from_this_block.size() / sizeof(float);
// FIXME: Hardcoded sample rate
// We use S16 here as a size-compatible substitute for qfloat16/half
AudioRenderingParams waveform_params(SampleSummer::kSumSampleRate, audio_params_.channel_layout(), SampleFormat::SAMPLE_FMT_S16);
int chunk_size = (audio_params().sample_rate() / waveform_params.sample_rate()) * waveform_params.channel_count();
qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in()) * 2;
qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length()) * 2;
qint64 start_offset = waveform_params.time_to_bytes(range_for_block.in() - b->in());
qint64 length_offset = waveform_params.time_to_bytes(range_for_block.length());
qint64 end_offset = start_offset + length_offset;
if (wave_file.size() < end_offset) {
@@ -117,10 +104,10 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
wave_file.seek(start_offset);
for (int i=0;i<nb_sample;i+=chunk_size) {
QVector<SampleSummer::Sum> summary = SampleSummer::SumSamples(&flt_samples[i],
qMin(chunk_size, nb_sample - i),
audio_params().channel_count());
for (int i=0;i<samples_from_this_block->sample_count_per_channel();i+=chunk_size) {
QVector<SampleSummer::Sum> summary = SampleSummer::SumSamples(samples_from_this_block,
i,
qMin(chunk_size, samples_from_this_block->sample_count_per_channel() - i));
wave_file.write(reinterpret_cast<const char*>(summary.constData()),
summary.size() * sizeof(SampleSummer::Sum));
@@ -137,7 +124,7 @@ NodeValueTable AudioRenderWorker::RenderBlock(const TrackOutput *track, const Ti
NodeValueTable::Merge({merged_table, table});
}
merged_table.Push(NodeParam::kSamples, block_range_buffer);
merged_table.Push(NodeParam::kSamples, QVariant::fromValue(block_range_buffer));
return merged_table;
}