From a2d70257760389bcc1c2fa1afbd85414b9a63d35 Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Sun, 7 Apr 2019 11:31:37 +1000 Subject: [PATCH] optimized waveform generation --- olive.pro | 6 ++-- project/footage.h | 2 +- project/previewgenerator.cpp | 33 +++++++++-------- ui/timelineview.cpp | 61 ++----------------------------- ui/timelineview.h | 2 -- ui/viewerwidget.cpp | 3 +- ui/waveform.cpp | 70 ++++++++++++++++++++++++++++++++++++ ui/waveform.h | 25 +++++++++++++ 8 files changed, 120 insertions(+), 82 deletions(-) create mode 100644 ui/waveform.cpp create mode 100644 ui/waveform.h diff --git a/olive.pro b/olive.pro index a58ad5db1..a3db45409 100644 --- a/olive.pro +++ b/olive.pro @@ -183,7 +183,8 @@ SOURCES += \ timeline/selection.cpp \ global/clipboard.cpp \ timeline/timelinetools.cpp \ - timeline/ghost.cpp + timeline/ghost.cpp \ + ui/waveform.cpp HEADERS += \ ui/mainwindow.h \ @@ -323,7 +324,8 @@ HEADERS += \ ui/timelineview.h \ ui/timelinelabel.h \ global/clipboard.h \ - timeline/timelinetools.h + timeline/timelinetools.h \ + ui/waveform.h FORMS += diff --git a/project/footage.h b/project/footage.h index e77ee6af1..8b250b9e4 100644 --- a/project/footage.h +++ b/project/footage.h @@ -62,7 +62,7 @@ struct FootageStream { // preview thumbnail/waveform bool preview_done; QImage video_preview; - QVector audio_preview; + QVector audio_preview; }; class Footage { diff --git a/project/previewgenerator.cpp b/project/previewgenerator.cpp index 173bc1572..36895599b 100644 --- a/project/previewgenerator.cpp +++ b/project/previewgenerator.cpp @@ -168,10 +168,7 @@ bool PreviewGenerator::retrieve_preview(const QString& hash) { f.open(QFile::ReadOnly); QByteArray data = f.readAll(); ms.audio_preview.resize(data.size()); - for (int j=0;jnb_streams]{0}; // stores samples while scanning before they get sent to preview file - qint16*** waveform_cache_data = new qint16** [fmt_ctx_->nb_streams]; + qint8*** waveform_cache_data = new qint8** [fmt_ctx_->nb_streams]; int waveform_cache_count = 0; // defaults to false, sets to true if we find a valid stream to make a preview of @@ -275,11 +272,11 @@ void PreviewGenerator::generate_waveform() { if (fmt_ctx_->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { // allocate sample cache for this stream - waveform_cache_data[i] = new qint16* [fmt_ctx_->streams[i]->codecpar->channels]; + waveform_cache_data[i] = new qint8* [fmt_ctx_->streams[i]->codecpar->channels]; // each channel gets a min and a max value so we allocate two ints for each one for (int j=0;jstreams[i]->codecpar->channels;j++) { - waveform_cache_data[i][j] = new qint16[2]; + waveform_cache_data[i][j] = new qint8[2]; } // if codec context has no defined channel layout, guess it from the channel count @@ -381,7 +378,7 @@ void PreviewGenerator::generate_waveform() { AVFrame* swr_frame = av_frame_alloc(); swr_frame->channel_layout = temp_frame->channel_layout; swr_frame->sample_rate = temp_frame->sample_rate; - swr_frame->format = AV_SAMPLE_FMT_S16P; + swr_frame->format = AV_SAMPLE_FMT_U8P; swr_ctx = swr_alloc_set_opts( nullptr, @@ -419,11 +416,11 @@ void PreviewGenerator::generate_waveform() { // if so, we dump our cached values into the preview and reset them // for the next interval for (int j=0;jchannels;j++) { - qint16& min = waveform_cache_data[packet->stream_index][j][0]; - qint16& max = waveform_cache_data[packet->stream_index][j][1]; + qint8& min = waveform_cache_data[packet->stream_index][j][0]; + qint8& max = waveform_cache_data[packet->stream_index][j][1]; - s->audio_preview.append(min >> 8); - s->audio_preview.append(max >> 8); + s->audio_preview.append(min); + s->audio_preview.append(max); } waveform_cache_count = 0; @@ -431,8 +428,8 @@ void PreviewGenerator::generate_waveform() { // standard processing for each channel of information for (int j=0;jchannels;j++) { - qint16& min = waveform_cache_data[packet->stream_index][j][0]; - qint16& max = waveform_cache_data[packet->stream_index][j][1]; + qint8& min = waveform_cache_data[packet->stream_index][j][0]; + qint8& max = waveform_cache_data[packet->stream_index][j][1]; // if we're starting over, reset cache to zero if (waveform_cache_count == 0) { @@ -440,8 +437,10 @@ void PreviewGenerator::generate_waveform() { max = 0; } - // store most minimum and most maximum samples of this interval - qint16 sample = qint16((swr_frame->data[j][i+1] << 8) | swr_frame->data[j][i]); + // Convert unsigned 8-bit PCM sample to signed + qint8 sample = qint8(int(swr_frame->data[j][i]-128)); + + // Store most minimum and most maximum samples of this interval min = qMin(min, sample); max = qMax(max, sample); } @@ -590,7 +589,7 @@ void PreviewGenerator::run() { FootageStream& ms = footage_->audio_tracks[i]; QFile f(get_waveform_path(hash, ms)); f.open(QFile::WriteOnly); - f.write(ms.audio_preview.constData(), ms.audio_preview.size()); + f.write(reinterpret_cast(ms.audio_preview.constData()), ms.audio_preview.size()); f.close(); //dout << "saved" << ms->file_index << "waveform to" << get_waveform_path(hash, ms); } diff --git a/ui/timelineview.cpp b/ui/timelineview.cpp index 6b9685da3..c18d30af5 100644 --- a/ui/timelineview.cpp +++ b/ui/timelineview.cpp @@ -61,6 +61,7 @@ #include "timeline/track.h" #include "global/math.h" #include "project/projectfunctions.h" +#include "ui/waveform.h" #define MAX_TEXT_WIDTH 20 #define TRANSITION_BETWEEN_RANGE 40 @@ -2784,64 +2785,6 @@ void TimelineView::leaveEvent(QEvent*) { tooltip_timer.stop(); } -void draw_waveform(Clip* clip, const FootageStream* ms, long media_length, QPainter *p, const QRect& clip_rect, int waveform_start, int waveform_limit, double zoom) { - // audio channels multiplied by the number of bytes in a 16-bit audio sample - int divider = ms->audio_channels*2; - - int channel_height = clip_rect.height()/ms->audio_channels; - - int last_waveform_index = -1; - - for (int i=waveform_start;iclip_in() + (double(i)/zoom))/media_length) * ms->audio_preview.size())/divider)*divider; - - if (clip->reversed()) { - waveform_index = ms->audio_preview.size() - waveform_index - (ms->audio_channels * 2); - } - - if (last_waveform_index < 0) last_waveform_index = waveform_index; - - for (int j=0;jaudio_channels;j++) { - int mid = (olive::config.rectified_waveforms) ? clip_rect.top()+channel_height*(j+1) : clip_rect.top()+channel_height*j+(channel_height/2); - - int offset_range_start = last_waveform_index+(j*2); - int offset_range_end = waveform_index+(j*2); - int offset_range_min = qMin(offset_range_start, offset_range_end); - int offset_range_max = qMax(offset_range_start, offset_range_end); - - // Break if we're about to draw from an index that doesn't exist - if (offset_range_min+1 >= ms->audio_preview.size()) { - break; - } - - qint8 min = qint8(qRound(double(ms->audio_preview.at(offset_range_min)) / 128.0 * (channel_height/2))); - qint8 max = qint8(qRound(double(ms->audio_preview.at(offset_range_min+1)) / 128.0 * (channel_height/2))); - - if ((offset_range_max + 1) < ms->audio_preview.size()) { - - // for waveform drawings, we get the maximum below 0 and maximum above 0 for this waveform range - for (int k=offset_range_min+2;k<=offset_range_max;k+=2) { - min = qMin(min, qint8(qRound(double(ms->audio_preview.at(k)) / 128.0 * (channel_height/2)))); - max = qMax(max, qint8(qRound(double(ms->audio_preview.at(k+1)) / 128.0 * (channel_height/2)))); - } - - // draw waveforms - if (olive::config.rectified_waveforms) { - - // rectified waveforms start from the bottom and draw upwards - p->drawLine(clip_rect.left()+i, mid, clip_rect.left()+i, mid - (max - min)); - } else { - - // non-rectified waveforms start from the center and draw outwards - p->drawLine(clip_rect.left()+i, mid+min, clip_rect.left()+i, mid+max); - - } - } - } - last_waveform_index = waveform_index; - } -} - void TimelineView::draw_transition(QPainter& p, Clip* c, const QRect& clip_rect, QRect& text_rect, int transition_type) { TransitionPtr t = (transition_type == kTransitionOpening) ? c->opening_transition : c->closing_transition; if (t != nullptr) { @@ -3033,7 +2976,7 @@ void TimelineView::paintEvent(QPaintEvent*) { if (waveform_limit > 0) checkerboard_rect.setLeft(checkerboard_rect.left() + waveform_limit); } - draw_waveform(clip, ms, media_length, &p, clip_rect, waveform_start, waveform_limit, ParentTimeline()->zoom); + olive::ui::DrawWaveform(clip, ms, media_length, &p, clip_rect, waveform_start, waveform_limit, ParentTimeline()->zoom); } } if (draw_checkerboard) { diff --git a/ui/timelineview.h b/ui/timelineview.h index 50be9c63a..6a9ab16c8 100644 --- a/ui/timelineview.h +++ b/ui/timelineview.h @@ -38,8 +38,6 @@ class Timeline; -void draw_waveform(Clip* clip, const FootageStream *ms, long media_length, QPainter* p, const QRect& clip_rect, int waveform_start, int waveform_limit, double zoom); - class TimelineView : public QWidget { Q_OBJECT public: diff --git a/ui/viewerwidget.cpp b/ui/viewerwidget.cpp index b773de2ec..186fcc02a 100644 --- a/ui/viewerwidget.cpp +++ b/ui/viewerwidget.cpp @@ -61,6 +61,7 @@ extern "C" { #include "rendering/shadergenerators.h" #include "ui/viewerwindow.h" #include "ui/menu.h" +#include "ui/waveform.h" #include "mainwindow.h" const int kTitleActionSafeVertexSize = 84; @@ -436,7 +437,7 @@ void ViewerWidget::draw_waveform_func() { wr.setX(wr.x() - waveform_scroll); p.setPen(Qt::green); - draw_waveform(waveform_clip.get(), waveform_ms, waveform_clip->timeline_out(), &p, wr, waveform_scroll, width()+waveform_scroll, waveform_zoom); + olive::ui::DrawWaveform(waveform_clip.get(), waveform_ms, waveform_clip->timeline_out(), &p, wr, waveform_scroll, width()+waveform_scroll, waveform_zoom); p.setPen(Qt::red); int playhead_x = getScreenPointFromFrame(waveform_zoom, viewer->seq->playhead) - waveform_scroll; p.drawLine(playhead_x, 0, playhead_x, height()); diff --git a/ui/waveform.cpp b/ui/waveform.cpp new file mode 100644 index 000000000..40933453b --- /dev/null +++ b/ui/waveform.cpp @@ -0,0 +1,70 @@ +#include "waveform.h" + +#include + +#include "global/config.h" + +int ConvertSampleToHeight(qint8 signed_sample, int channel_height) { + int half_channel_height = channel_height >> 1; + return (half_channel_height) + qRound(((double(signed_sample) / 128.0)) * half_channel_height); +} + +void olive::ui::DrawWaveform(Clip* clip, + const FootageStream* ms, + long media_length, + QPainter *p, + const QRect& clip_rect, + int waveform_start, + int waveform_limit, + double zoom) { + + int divider = ms->audio_channels; + + int channel_height = clip_rect.height()/ms->audio_channels; + + int last_waveform_index = -1; + + for (int i=waveform_start;iclip_in() + (double(i)/zoom))/media_length) * ms->audio_preview.size())/divider)*divider; + + if (clip->reversed()) { + waveform_index = ms->audio_preview.size() - waveform_index - (ms->audio_channels * 2); + } + + if (last_waveform_index < 0) last_waveform_index = waveform_index; + + for (int j=0;jaudio_channels;j++) { + int bottom = clip_rect.top()+channel_height*(j+1); + + int offset_range_start = last_waveform_index+(j*2); + int offset_range_end = waveform_index+(j*2); + int offset_range_min = qMin(offset_range_start, offset_range_end); + int offset_range_max = qMax(offset_range_start, offset_range_end); + + // Break if we're about to draw from an index that doesn't exist + if (offset_range_min+1 >= ms->audio_preview.size()) { + break; + } + + int min = ConvertSampleToHeight(ms->audio_preview.at(offset_range_min), channel_height); + int max = ConvertSampleToHeight(ms->audio_preview.at(offset_range_min+1), channel_height); + + if ((offset_range_max + 1) < ms->audio_preview.size()) { + + // for waveform drawings, we get the maximum below 0 and maximum above 0 for this waveform range + for (int k=offset_range_min+2;k<=offset_range_max;k+=2) { + min = ConvertSampleToHeight(ms->audio_preview.at(k), channel_height); + max = ConvertSampleToHeight(ms->audio_preview.at(k+1), channel_height); + } + + // draw waveforms + if (olive::config.rectified_waveforms) { + p->drawLine(clip_rect.left()+i, bottom, clip_rect.left()+i, bottom - (max - min)); + } else { + p->drawLine(clip_rect.left()+i, bottom - min, clip_rect.left()+i, bottom - max); + } + } + } + last_waveform_index = waveform_index; + } +} diff --git a/ui/waveform.h b/ui/waveform.h new file mode 100644 index 000000000..d7dc92040 --- /dev/null +++ b/ui/waveform.h @@ -0,0 +1,25 @@ +#ifndef WAVEFORM_H +#define WAVEFORM_H + +#include +#include + +#include "timeline/clip.h" +#include "project/footage.h" + +namespace olive { +namespace ui { + +void DrawWaveform(Clip* clip, + const FootageStream *ms, + long media_length, + QPainter* p, + const QRect& clip_rect, + int waveform_start, + int waveform_limit, + double zoom); + +} +} + +#endif // WAVEFORM_H