oak-team/oak-editor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

began work on audio buffering for #115

Browse Source
This commit is contained in:
itsmattkc
2018-11-09 22:13:13 +11:00
parent dd55b97203
commit bfc584edb1
8 changed files with 142 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
playback/audio.cpp
+1 -1
View File
@@ -29,7 +29,7 @@ QAudioInput* audio_input = NULL;
QFile output_recording;
bool recording = false;
qint8 audio_ibuffer[audio_ibuffer_size];
uint8_t audio_ibuffer[audio_ibuffer_size];
int audio_ibuffer_read = 0;
long audio_ibuffer_frame = 0;
double audio_ibuffer_timecode = 0;
playback/audio.h
+1 -1
View File
@@ -36,7 +36,7 @@ extern AudioSenderThread* audio_thread;
extern QMutex audio_write_lock;
#define audio_ibuffer_size 192000
extern qint8 audio_ibuffer[audio_ibuffer_size];
extern uint8_t audio_ibuffer[audio_ibuffer_size];
extern int audio_ibuffer_read;
extern long audio_ibuffer_frame;
extern double audio_ibuffer_timecode;
playback/cacher.cpp
+86 -20
View File
@@ -36,8 +36,7 @@ double bytes_to_seconds(int nb_bytes, int nb_channels, int sample_rate) {
void apply_audio_effects(Clip* c, double timecode_start, AVFrame* frame, int nb_bytes) {
// perform all aud io effects
double timecode_end;
timecode_end = timecode_start + bytes_to_seconds(nb_bytes, frame->channels, frame->sample_rate);
double timecode_end = timecode_start + bytes_to_seconds(nb_bytes, frame->channels, frame->sample_rate);
for (int j=0;j<c->effects.size();j++) {
Effect* e = c->effects.at(j);
@@ -71,12 +70,16 @@ void apply_audio_effects(Clip* c, double timecode_start, AVFrame* frame, int nb_
#define AUDIO_BUFFER_PADDING 2048
int get_read_buffer_index(Clip* nest) {
return (nest == NULL) ? audio_ibuffer_read : nest->frame->pts;
}
void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) {
long timeline_in = c->timeline_in;
long timeline_out = c->timeline_out;
if (nest != NULL) {
timeline_in = refactor_frame_number(timeline_in, c->sequence->frame_rate, sequence->frame_rate) + nest->timeline_in;
timeline_out = refactor_frame_number(timeline_out, c->sequence->frame_rate, sequence->frame_rate) + nest->timeline_in;
timeline_in = refactor_frame_number(timeline_in, c->sequence->frame_rate, nest->sequence->frame_rate) + nest->timeline_in - nest->clip_in;
timeline_out = refactor_frame_number(timeline_out, c->sequence->frame_rate, nest->sequence->frame_rate) + nest->timeline_in - nest->clip_in;
}
while (true) {
@@ -271,7 +274,7 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) {
#endif
if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame));
int offset = (audio_ibuffer_read + AUDIO_BUFFER_PADDING) - c->audio_buffer_write;
int offset = get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING - c->audio_buffer_write;
if (offset > 0) {
c->audio_buffer_write += offset;
c->frame_sample_index += offset;
@@ -307,44 +310,97 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) {
c->frame->pts += nb_bytes;
c->frame_sample_index = 0;
if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame));
int offset = audio_ibuffer_read - c->audio_buffer_write;
int offset = (get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING) - c->audio_buffer_write;
if (offset > 0) {
c->audio_buffer_write += offset;
c->frame_sample_index += offset;
}
}
break;
default: // shouldn't ever get here
dout << "[ERROR] Tried to cache a non-footage/tone clip";
return;
case MEDIA_TYPE_SEQUENCE:
frame = c->frame;
nb_bytes = frame->nb_samples * av_get_bytes_per_sample(static_cast<AVSampleFormat>(frame->format)) * frame->channels;
if (c->frame_sample_index == -1 || c->frame_sample_index >= nb_bytes) {
c->frame_sample_index = 0;
if (c->audio_buffer_write == 0) c->audio_buffer_write = get_buffer_offset_from_frame(c->sequence, qMax(timeline_in, c->audio_target_frame));
int offset = (get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING) - c->audio_buffer_write;
if (offset > 0) {
c->audio_buffer_write += offset;
c->frame_sample_index += offset;
c->frame->pts += offset;
}
}
//c->audio_buffer_write = 0;
/*while ((c->frame_sample_index == -1 || c->frame_sample_index >= nb_bytes) && nb_bytes > 0) {
// create "new frame"
int offset = (get_read_buffer_index(nest) + AUDIO_BUFFER_PADDING) - c->audio_buffer_write;
if (offset > 0) {
c->audio_buffer_write += offset;
c->frame_sample_index += offset;
}
}*/
break;
}
// mix audio into internal buffer
// mix audio into internal buffer
if (frame->nb_samples == 0) {
break;
} else {
long buffer_timeline_out = get_buffer_offset_from_frame(c->sequence, timeline_out);
audio_write_lock.lock();
int max_write = (nest == NULL) ? audio_ibuffer_size : nest->frame->nb_samples * av_get_bytes_per_sample(static_cast<AVSampleFormat>(nest->frame->format)) * nest->frame->channels;
if (nest == NULL) {
audio_write_lock.lock();
} else {
c->queue_lock.lock();
}
uint8_t* buffer = (nest == NULL) ? audio_ibuffer : nest->frame->data[0];
dout << c->name << "|" <<
c->audio_buffer_write << "<" << get_read_buffer_index(nest)+(max_write>>1) << "|" <<
c->audio_buffer_write << "<" << buffer_timeline_out << "|" <<
c->frame_sample_index << "<" << nb_bytes << "|";
while (c->frame_sample_index < nb_bytes
&& c->audio_buffer_write < audio_ibuffer_read+(audio_ibuffer_size>>1)
&& c->audio_buffer_write < get_read_buffer_index(nest)+(max_write>>1)
&& c->audio_buffer_write < buffer_timeline_out) {
int upper_byte_index = (c->audio_buffer_write+1)%audio_ibuffer_size;
int lower_byte_index = (c->audio_buffer_write)%audio_ibuffer_size;
qint16 old_sample = static_cast<qint16>((audio_ibuffer[upper_byte_index] & 0xFF) << 8 | (audio_ibuffer[lower_byte_index] & 0xFF));
qint16 new_sample = static_cast<qint16>((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF));
int upper_byte_index = (c->audio_buffer_write+1)%max_write;
int lower_byte_index = (c->audio_buffer_write)%max_write;
qint16 old_sample = static_cast<int16_t>((buffer[upper_byte_index] & 0xFF) << 8 | (buffer[lower_byte_index] & 0xFF));
qint16 new_sample = static_cast<int16_t>((frame->data[0][c->frame_sample_index+1] & 0xFF) << 8 | (frame->data[0][c->frame_sample_index] & 0xFF));
qint16 mixed_sample = mix_audio_sample(old_sample, new_sample);
audio_ibuffer[upper_byte_index] = static_cast<quint8>((mixed_sample >> 8) & 0xFF);
audio_ibuffer[lower_byte_index] = static_cast<quint8>(mixed_sample & 0xFF);
buffer[upper_byte_index] = static_cast<uint8_t>((mixed_sample >> 8) & 0xFF);
buffer[lower_byte_index] = static_cast<uint8_t>(mixed_sample & 0xFF);
if (c->media_type == MEDIA_TYPE_SEQUENCE) {
frame->data[0][c->frame_sample_index+1] = 0;
frame->data[0][c->frame_sample_index] = 0;
c->frame->pts += 2;
}
c->audio_buffer_write+=2;
c->frame_sample_index+=2;
}
dout << c->name << "|" << c->frame_sample_index << "<" << nb_bytes << "LELELEL";
#ifdef AUDIOWARNINGS
if (c->audio_buffer_write >= buffer_timeline_out) dout << "timeline out at fsi" << c->frame_sample_index << "of frame ts" << c->frame->pts;
#endif
audio_write_lock.unlock();
if (nest == NULL) {
audio_write_lock.unlock();
} else {
c->queue_lock.unlock();
}
if (scrubbing) {
audio_thread->notifyReceiver();
@@ -357,6 +413,10 @@ void cache_audio_worker(Clip* c, bool scrubbing, Clip* nest) {
break;
}
if (c->media_type == MEDIA_TYPE_SEQUENCE) {
break;
}
// dout << "ended" << c->frame_sample_index << nb_bytes;
}
if (c->reached_end) {
@@ -557,6 +617,11 @@ void reset_cache(Clip* c, long target_frame) {
}
}
break;
case MEDIA_TYPE_SEQUENCE:
c->frame->pts = 0;
c->frame_sample_index = -1;
c->audio_target_frame = target_frame;
break;
case MEDIA_TYPE_TONE:
c->reached_end = false;
c->audio_target_frame = target_frame;
@@ -819,6 +884,7 @@ void cache_clip_worker(Clip* clip, long playhead, bool reset, bool scrubbing, Cl
}
break;
case MEDIA_TYPE_TONE:
case MEDIA_TYPE_SEQUENCE:
cache_audio_worker(clip, scrubbing, nest);
break;
}
@@ -849,7 +915,7 @@ void Cacher::run() {
clip->lock.lock();
clip->finished_opening = false;
clip->open = true;
caching = true;
caching = true;
open_clip_worker(clip);
playback/playback.cpp
+33 -27
View File
@@ -26,17 +26,30 @@ extern "C" {
#include <QOpenGLFramebufferObject>
#ifdef QT_DEBUG
#define GCF_DEBUG
//#define GCF_DEBUG
#endif
bool texture_failed = false;
bool rendering = false;
void open_clip(Clip* clip, bool multithreaded) {
switch (clip->media_type) {
case MEDIA_TYPE_FOOTAGE:
case MEDIA_TYPE_TONE:
void open_clip(Clip* clip, bool multithreaded, Clip* nest) {
if (clip->media_type == MEDIA_TYPE_FOOTAGE || clip->media_type == MEDIA_TYPE_TONE || (clip->media_type == MEDIA_TYPE_SEQUENCE && clip->track >= 0)) {
clip->multithreaded = multithreaded;
if (clip->media_type == MEDIA_TYPE_SEQUENCE) {
clip->frame = av_frame_alloc();
clip->frame->format = AV_SAMPLE_FMT_S16;
clip->frame->nb_samples = 48000;
clip->frame->channel_layout = AV_CH_LAYOUT_STEREO;
clip->frame->channels = 2;
clip->audio_buffer_offset = ((double) (clip->timeline_in - clip->clip_in) / clip->sequence->frame_rate) - audio_ibuffer_timecode; // may be unnecessary
clip->frame->pkt_dts = clip->audio_buffer_offset * sequence->audio_frequency * av_get_bytes_per_sample(static_cast<AVSampleFormat>(clip->frame->format)) * clip->frame->channels;
if (nest != NULL) clip->audio_buffer_offset += nest->audio_buffer_offset;
av_frame_get_buffer(clip->frame, 0);
memset(clip->frame->data[0], 0, clip->frame->nb_samples*av_get_bytes_per_sample(static_cast<AVSampleFormat>(clip->frame->format))*clip->frame->channels);
clip->reset_audio();
}
if (multithreaded) {
if (clip->open_lock.tryLock()) {
// maybe keep cacher instance in memory while clip exists for performance?
@@ -50,11 +63,8 @@ void open_clip(Clip* clip, bool multithreaded) {
open_clip_worker(clip);
}
break;
case MEDIA_TYPE_SEQUENCE:
case MEDIA_TYPE_SOLID:
} else if (clip->media_type == MEDIA_TYPE_SEQUENCE || clip->media_type == MEDIA_TYPE_SOLID) {
clip->open = true;
break;
}
}
@@ -77,36 +87,32 @@ void close_clip(Clip* clip) {
clip->fbo = NULL;
}
switch (clip->media_type) {
case MEDIA_TYPE_FOOTAGE:
case MEDIA_TYPE_TONE:
if (clip->media_type == MEDIA_TYPE_SEQUENCE) {
closeActiveClips(static_cast<Sequence*>(clip->media), false);
}
if (clip->media_type == MEDIA_TYPE_FOOTAGE || clip->media_type == MEDIA_TYPE_TONE || (clip->media_type == MEDIA_TYPE_SEQUENCE && clip->track >= 0)) {
if (clip->multithreaded) {
clip->cacher->caching = false;
clip->can_cache.wakeAll();
} else {
close_clip_worker(clip);
}
break;
case MEDIA_TYPE_SEQUENCE:
closeActiveClips(static_cast<Sequence*>(clip->media), false);
case MEDIA_TYPE_SOLID:
} else if (clip->media_type == MEDIA_TYPE_SEQUENCE || clip->media_type == MEDIA_TYPE_SOLID) {
clip->open = false;
break;
}
}
void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, Clip* nest) {
if (clip->media_type == MEDIA_TYPE_FOOTAGE || clip->media_type == MEDIA_TYPE_TONE) {
if (clip->multithreaded) {
clip->cacher->playhead = playhead;
clip->cacher->reset = reset;
clip->cacher->nest = nest;
clip->cacher->scrubbing = scrubbing;
if (clip->multithreaded) {
clip->cacher->playhead = playhead;
clip->cacher->reset = reset;
clip->cacher->nest = nest;
clip->cacher->scrubbing = scrubbing;
clip->can_cache.wakeAll();
} else {
cache_clip_worker(clip, playhead, reset, scrubbing, nest);
}
clip->can_cache.wakeAll();
} else {
cache_clip_worker(clip, playhead, reset, scrubbing, nest);
}
}
playback/playback.h
+1 -1
View File
@@ -12,7 +12,7 @@ struct AVFrame;
extern bool texture_failed;
extern bool rendering;
void open_clip(Clip* clip, bool multithreaded);
void open_clip(Clip* clip, bool multithreaded, Clip* nest = 0);
void cache_clip(Clip* clip, long playhead, bool reset, bool scrubbing, Clip *nest);
void close_clip(Clip* clip);
void cache_audio_worker(Clip* c, bool write_A);
project/clip.cpp
+11 -17
View File
@@ -89,23 +89,17 @@ void Clip::reset() {
}
void Clip::reset_audio() {
switch (media_type) {
case MEDIA_TYPE_FOOTAGE:
case MEDIA_TYPE_TONE:
audio_reset = true;
frame_sample_index = -1;
audio_buffer_write = 0;
break;
case MEDIA_TYPE_SEQUENCE:
{
Sequence* nested_sequence = static_cast<Sequence*>(media);
for (int i=0;i<nested_sequence->clips.size();i++) {
Clip* c = nested_sequence->clips.at(i);
if (c != NULL) c->reset_audio();
}
}
break;
}
audio_reset = true;
frame_sample_index = -1;
audio_buffer_write = 0;
if (media_type == MEDIA_TYPE_SEQUENCE) {
Sequence* nested_sequence = static_cast<Sequence*>(media);
for (int i=0;i<nested_sequence->clips.size();i++) {
Clip* c = nested_sequence->clips.at(i);
if (c != NULL) c->reset_audio();
}
}
}
void Clip::refresh() {
project/clip.h
+1
View File
@@ -120,6 +120,7 @@ struct Clip
int64_t reverse_target;
int frame_sample_index;
int audio_buffer_write;
double audio_buffer_offset;
bool audio_reset;
bool audio_just_reset;
long audio_target_frame;
ui/viewerwidget.cpp
+8 -13
View File
@@ -247,7 +247,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) {
// if thread is already working, we don't want to touch this,
// but we also don't want to hang the UI thread
if (!c->open) {
open_clip(c, !rendering);
open_clip(c, !rendering, nest);
}
clip_is_active = true;
} else if (c->open) {
@@ -262,7 +262,7 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) {
case MEDIA_TYPE_SOLID:
case MEDIA_TYPE_TONE:
if (is_clip_active(c, playhead)) {
if (!c->open) open_clip(c, !rendering);
if (!c->open) open_clip(c, !rendering, nest);
clip_is_active = true;
} else if (c->open) {
close_clip(c);
@@ -442,18 +442,13 @@ GLuint ViewerWidget::compose_sequence(Clip* nest, bool render_audio) {
}
} else {
if (render_audio || (config.enable_audio_scrubbing && audio_scrub)) {
switch (c->media_type) {
case MEDIA_TYPE_FOOTAGE:
case MEDIA_TYPE_TONE:
if (c->lock.tryLock()) {
// clip is not caching, start caching audio
cache_clip(c, playhead, c->audio_reset, !render_audio, nest);
c->lock.unlock();
}
break;
case MEDIA_TYPE_SEQUENCE:
if (c->media_type == MEDIA_TYPE_SEQUENCE) {
compose_sequence(c, render_audio);
break;
}
if (c->lock.tryLock()) {
// clip is not caching, start caching audio
cache_clip(c, playhead, c->audio_reset, !render_audio, nest);
c->lock.unlock();
}
}