ffmpegframepool: implemented functional frame pool

Includes splitting off core functionality to a base MemoryPool class that can
be re-used later if necessary.
This commit is contained in:
itsmattkc
2020-04-14 02:43:12 +10:00
parent 2eb03bf386
commit 7bfdf8af7c
10 changed files with 396 additions and 260 deletions
-2
View File
@@ -25,8 +25,6 @@ set(OLIVE_SOURCES
codec/encoder.cpp
codec/frame.h
codec/frame.cpp
codec/framepool.h
codec/framepool.cpp
codec/samplebuffer.h
codec/samplebuffer.cpp
codec/waveinput.h
+2
View File
@@ -23,5 +23,7 @@ set(OLIVE_SOURCES
codec/ffmpeg/ffmpegdecoder.cpp
codec/ffmpeg/ffmpegencoder.h
codec/ffmpeg/ffmpegencoder.cpp
codec/ffmpeg/ffmpegframepool.h
codec/ffmpeg/ffmpegframepool.cpp
PARENT_SCOPE
)
+72 -53
View File
@@ -48,11 +48,11 @@ QHash< Stream*, QList<FFmpegDecoderInstance*> > FFmpegDecoder::instances_;
QMutex FFmpegDecoder::instance_lock_;
// FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints
const int FFmpegDecoder::kMaxFrameLife = 5000;
const int FFmpegDecoder::kMaxFrameLife = 2000;
FFmpegDecoder::FFmpegDecoder() :
scale_ctx_(nullptr),
scale_divider_(-1)
scale_divider_(0)
{
clear_timer_.setInterval(kMaxFrameLife);
clear_timer_.moveToThread(qApp->thread());
@@ -172,7 +172,7 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid
int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_;
FFmpegDecoderInstance* working_instance = nullptr;
AVFramePtr return_frame = nullptr;
FFmpegFramePool::ElementPtr return_frame = nullptr;
// Find instance
do {
@@ -276,11 +276,12 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid
if (return_frame) {
if (divider != scale_divider_) {
FreeScaler();
SetupScaler(divider);
InitScaler(divider);
}
VideoStream* vs = static_cast<VideoStream*>(stream().get());
// Create frame to return
FramePtr copy = Frame::Create();
copy->set_video_params(VideoRenderingParams(vs->width() / divider,
vs->height() / divider,
@@ -289,15 +290,27 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid
copy->set_sample_aspect_ratio(aspect_ratio_);
copy->allocate();
// Convert frame to RGBA for the rest of the pipeline
// Align buffer to data/linesize points that can be passed to sws_scale
uint8_t* input_data[4];
int input_linesize[4];
av_image_fill_arrays(input_data,
input_linesize,
reinterpret_cast<const uint8_t*>(return_frame->data()),
src_pix_fmt_,
vs->width(),
vs->height(),
1);
// Convert frame to RGB/A for the rest of the pipeline
uint8_t* output_data = reinterpret_cast<uint8_t*>(copy->data());
int output_linesize = copy->width() * PixelFormat::ChannelCount(native_pix_fmt_) * PixelFormat::BytesPerChannel(native_pix_fmt_);
int output_linesize = copy->width() * PixelFormat::BytesPerPixel(native_pix_fmt_);
sws_scale(scale_ctx_,
return_frame->frame()->data,
return_frame->frame()->linesize,
input_data,
input_linesize,
0,
return_frame->frame()->height,
vs->height(),
&output_data,
&output_linesize);
@@ -789,6 +802,8 @@ void FFmpegDecoderInstance::Seek(int64_t timestamp)
av_seek_frame(fmt_ctx_, avstream_->index, timestamp, AVSEEK_FLAG_BACKWARD);
}
/* OLD UNUSED CODE: Keeping this around in case the code proves useful
void FFmpegDecoder::CacheFrameToDisk(AVFrame *f)
{
QFile save_frame(GetIndexFilename().append(QString::number(f->pts)));
@@ -818,7 +833,7 @@ void FFmpegDecoder::CacheFrameToDisk(AVFrame *f)
}
// See if we stored this frame in the disk cache
/*
QByteArray frame_loader;
if (!got_frame) {
QFile compressed_frame(GetIndexFilename().append(QString::number(target_ts)));
@@ -841,30 +856,8 @@ void FFmpegDecoder::CacheFrameToDisk(AVFrame *f)
got_frame = true;
}
}
*/
}
/*void FFmpegDecoder::RemoveFirstFromFrameCache()
{
if (cached_frames_.isEmpty()) {
return;
}
AVFrame* first = cached_frames_.takeFirst();
av_frame_free(&first);
cache_at_zero_ = false;
}
void FFmpegDecoder::RemoveLastFromFrameCache()
{
if (cached_frames_.isEmpty()) {
return;
}
AVFrame* last = cached_frames_.takeLast();
av_frame_free(&last);
cache_at_eof_ = false;
}*/
*/
void FFmpegDecoderInstance::ClearFrameCache()
{
@@ -873,7 +866,7 @@ void FFmpegDecoderInstance::ClearFrameCache()
cache_at_zero_ = false;
}
FramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked)
FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked)
{
if (!cache_is_locked) {
cache_lock_.lock();
@@ -898,7 +891,7 @@ FramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target
int ret;
AVPacket* pkt = av_packet_alloc();
FramePool::ElementPtr return_frame = nullptr;
FFmpegFramePool::ElementPtr return_frame = nullptr;
// Allocate a new frame
AVFrameWrapper working_frame;
@@ -954,9 +947,23 @@ FramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target
} else {
// Whatever it is, keep this frame in memory for the time being just in case
FramePool::ElementPtr cached = frame_pool_.Get();
FFmpegFramePool::ElementPtr cached = frame_pool_.Get(working_frame.frame());
Q_ASSERT(cached);
// Set timestamp so this frame can be identified later
cached->set_timestamp(working_frame.frame()->pts);
// Store frame before just in case
FFmpegFramePool::ElementPtr previous;
if (cached_frames_.isEmpty()) {
previous = nullptr;
} else {
previous = cached_frames_.last();
}
// Append this frame and signal to other threads that a new frame has arrived
cached_frames_.append(cached);
cache_wait_cond_.wakeAll();
cache_lock_.unlock();
@@ -965,11 +972,11 @@ FramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target
return_frame = cached;
break;
} else if (cached->timestamp() > target_ts) {
if (cached_frames_.isEmpty() && cache_at_zero_) {
if (!previous && cache_at_zero_) {
return_frame = cached;
break;
} else {
return_frame = cached_frames_.at(cached_frames_.size() - 2);
return_frame = previous;
break;
}
}
@@ -988,7 +995,7 @@ void FFmpegDecoder::ClearResources()
open_ = false;
}
void FFmpegDecoder::SetupScaler(const int &divider)
void FFmpegDecoder::InitScaler(int divider)
{
VideoStream* vs = static_cast<VideoStream*>(stream().get());
@@ -1003,10 +1010,10 @@ void FFmpegDecoder::SetupScaler(const int &divider)
nullptr,
nullptr);
if (!scale_ctx_) {
Error(QStringLiteral("Failed to allocate SwsContext"));
} else {
if (scale_ctx_) {
scale_divider_ = divider;
} else {
scale_divider_ = 0;
}
}
@@ -1015,7 +1022,8 @@ void FFmpegDecoder::FreeScaler()
if (scale_ctx_) {
sws_freeContext(scale_ctx_);
scale_ctx_ = nullptr;
scale_divider_ = -1;
scale_divider_ = 0;
}
}
@@ -1058,17 +1066,19 @@ bool FFmpegDecoderInstance::CacheIsEmpty() const
return cached_frames_.isEmpty();
}
FramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const
FFmpegFramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const
{
if (t < cached_frames_.first()->timestamp()) {
if (cache_at_zero_) {
cached_frames_.first()->access();
return cached_frames_.first();
}
} else if (t > cached_frames_.last()->timestamp()) {
if (cache_at_eof_) {
cached_frames_.last()->access();
return cached_frames_.last();
}
@@ -1076,11 +1086,12 @@ FramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t)
// We already have this frame in the cache, find it
for (int i=0;i<cached_frames_.size();i++) {
FramePool::ElementPtr this_frame = cached_frames_.at(i);
FFmpegFramePool::ElementPtr this_frame = cached_frames_.at(i);
if (this_frame->timestamp() == t // Test for an exact match
|| (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->timestamp() > t)) { // Or for this frame to be the "closest"
this_frame->access();
return this_frame;
}
@@ -1092,6 +1103,7 @@ FramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t)
void FFmpegDecoderInstance::RemoveFramesBefore(const qint64 &t)
{
// We keep one frame in memory as an identifier for what pts the decoder is up to
while (cached_frames_.size() > 1 && cached_frames_.first()->last_accessed() < t) {
cached_frames_.removeFirst();
cache_at_zero_ = false;
@@ -1182,12 +1194,19 @@ FFmpegDecoderInstance::FFmpegDecoderInstance(const char *filename, int stream_in
return;
}
// Create buffer pool
if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO
&& !frame_pool_.Allocate()) {
qDebug() << "Failed to allocate frame pool";
ClearResources();
return;
// Create frame pool
if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
frame_pool_.SetParams(avstream_->codecpar->width,
avstream_->codecpar->height,
static_cast<AVPixelFormat>(avstream_->codecpar->format));
if (!frame_pool_.Allocate(64)) {
qDebug() << "Failed to allocate frame pool";
ClearResources();
return;
}
}
// Store one second in the source's timebase
@@ -1208,6 +1227,8 @@ void FFmpegDecoderInstance::ClearResources()
{
ClearFrameCache();
frame_pool_.Destroy();
if (opts_) {
av_dict_free(&opts_);
opts_ = nullptr;
@@ -1222,8 +1243,6 @@ void FFmpegDecoderInstance::ClearResources()
avformat_close_input(&fmt_ctx_);
fmt_ctx_ = nullptr;
}
frame_pool_.Destroy();
}
void FFmpegDecoder::ClearTimerEvent()
+8 -11
View File
@@ -35,8 +35,8 @@ extern "C" {
#include "audio/sampleformat.h"
#include "avframeptr.h"
#include "codec/decoder.h"
#include "codec/framepool.h"
#include "codec/waveoutput.h"
#include "ffmpegframepool.h"
#include "project/item/footage/videostream.h"
OLIVE_NAMESPACE_ENTER
@@ -56,7 +56,7 @@ public:
bool CacheWillContainTime(const int64_t& t) const;
bool CacheCouldContainTime(const int64_t& t) const;
bool CacheIsEmpty() const;
FramePool::ElementPtr GetFrameFromCache(const int64_t& t) const;
FFmpegFramePool::ElementPtr GetFrameFromCache(const int64_t& t) const;
void RemoveFramesBefore(const qint64& t);
@@ -65,7 +65,7 @@ public:
void ClearFrameCache();
FramePool::ElementPtr RetrieveFrame(const int64_t &target_ts, bool cache_is_locked);
FFmpegFramePool::ElementPtr RetrieveFrame(const int64_t &target_ts, bool cache_is_locked);
/**
* @brief Uses the FFmpeg API to retrieve a packet (stored in pkt_) and decode it (stored in frame_)
@@ -96,8 +96,8 @@ private:
QWaitCondition cache_wait_cond_;
QMutex cache_lock_;
QList<FramePool::ElementPtr> cached_frames_;
FramePool frame_pool_;
QList<FFmpegFramePool::ElementPtr> cached_frames_;
FFmpegFramePool frame_pool_;
int64_t cache_target_time_;
@@ -160,13 +160,13 @@ private:
void UnconditionalAudioIndex(const QAtomicInt* cancelled);
void CacheFrameToDisk(AVFrame* f);
void ClearResources();
void SetupScaler(const int& divider);
void InitScaler(int divider);
void FreeScaler();
SwsContext* scale_ctx_;
int scale_divider_;
AVPixelFormat src_pix_fmt_;
AVPixelFormat ideal_pix_fmt_;
PixelFormat::Format native_pix_fmt_;
@@ -175,9 +175,6 @@ private:
rational aspect_ratio_;
int64_t start_time_;
SwsContext* scale_ctx_;
int scale_divider_;
QTimer clear_timer_;
FFmpegDecoderInstance* our_instance_;
+95
View File
@@ -0,0 +1,95 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2019 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "ffmpegframepool.h"
extern "C" {
#include <libavutil/imgutils.h>
}
OLIVE_NAMESPACE_ENTER
FFmpegFramePool::FFmpegFramePool() :
width_(0),
height_(0),
format_(AV_PIX_FMT_NONE)
{
}
FFmpegFramePool::ElementPtr FFmpegFramePool::Get(AVFrame *copy)
{
ElementPtr ele = MemoryPool::Get();
if (ele) {
av_image_copy_to_buffer(ele->data(),
GetElementSize(),
copy->data,
copy->linesize,
format_,
width_,
height_,
1);
}
return ele;
}
void FFmpegFramePool::SetParams(int width, int height, AVPixelFormat format)
{
int old_nb_elements;
if (IsAllocated()) {
old_nb_elements = GetElementCount();
Destroy();
} else {
old_nb_elements = 0;
}
width_ = width;
height_ = height;
format_ = format;
if (old_nb_elements) {
// Re-allocate automatically
Allocate(old_nb_elements);
}
}
size_t FFmpegFramePool::GetElementSize()
{
if (width_ == 0 || height_ == 0 || format_ == AV_PIX_FMT_NONE) {
return 0;
}
int buf_sz = av_image_get_buffer_size(static_cast<AVPixelFormat>(format_),
width_,
height_,
1);
if (buf_sz < 0) {
qDebug() << "Failed to find buffer size:" << buf_sz;
return 0;
}
return buf_sz;
}
OLIVE_NAMESPACE_EXIT
+53
View File
@@ -0,0 +1,53 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2019 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef FFMPEGFRAMEPOOL_H
#define FFMPEGFRAMEPOOL_H
#include "common/memorypool.h"
#include "render/pixelformat.h"
#include "render/videoparams.h"
OLIVE_NAMESPACE_ENTER
class FFmpegFramePool : public MemoryPool<uint8_t>
{
public:
FFmpegFramePool();
ElementPtr Get(AVFrame* copy);
void SetParams(int width, int height, AVPixelFormat format);
protected:
virtual size_t GetElementSize() override;
private:
int width_;
int height_;
AVPixelFormat format_;
};
OLIVE_NAMESPACE_EXIT
#endif // FFMPEGFRAMEPOOL_H
-97
View File
@@ -1,97 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2019 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#include "framepool.h"
OLIVE_NAMESPACE_ENTER
FramePool::FramePool()
{
data_ = nullptr;
}
bool FramePool::Allocate(int width, int height, PixelFormat::Format format, int nb_elements)
{
delete [] data_;
element_sz_ = width * height * PixelFormat::BytesPerPixel(format);
if ((data_ = new char[element_sz_ * nb_elements])) {
// Only allocate available array if data allocation succeeded
available_.resize(nb_elements);
available_.fill(true);
return true;
} else {
available_.clear();
return false;
}
}
void FramePool::Destroy()
{
delete [] data_;
available_.clear();
}
FramePool::~FramePool()
{
delete [] data_;
}
FramePool::ElementPtr FramePool::Get()
{
for (int i=0;i<available_.size();i++) {
if (available_.at(i)) {
// This buffer is available
available_.replace(i, false);
return std::make_shared<Element>(this, data_ + i * element_sz_);
}
}
return nullptr;
}
void FramePool::Release(FramePool::Element *e)
{
quintptr offs = reinterpret_cast<quintptr>(e->data());
quintptr start = reinterpret_cast<quintptr>(data_);
quintptr diff = offs - start;
int index = diff / element_sz_;
available_.replace(index, true);
}
FramePool::Element::Element(FramePool *parent, char *data)
{
parent_ = parent;
data_ = data;
accessed_ = QDateTime::currentMSecsSinceEpoch();
}
FramePool::Element::~Element()
{
parent_->Release(this);
}
OLIVE_NAMESPACE_EXIT
-97
View File
@@ -1,97 +0,0 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2019 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef FRAMEPOOL_H
#define FRAMEPOOL_H
#include <QDateTime>
#include <memory>
#include "render/pixelformat.h"
OLIVE_NAMESPACE_ENTER
class FramePool
{
public:
FramePool();
~FramePool();
bool Allocate(int width, int height, PixelFormat::Format format, int nb_elements);
void Destroy();
DISABLE_COPY_MOVE(FramePool)
class Element {
public:
Element(FramePool* parent, char* data);
~Element();
inline char* data() const {
return data_;
}
inline const int64_t& timestamp() const {
return timestamp_;
}
inline void set_timestamp(const int64_t& timestamp) {
timestamp_ = timestamp;
}
inline void access() {
accessed_ = QDateTime::currentMSecsSinceEpoch();
}
inline const int64_t& last_accessed() const {
return accessed_;
}
private:
FramePool* parent_;
char* data_;
int64_t timestamp_;
int64_t accessed_;
};
using ElementPtr = std::shared_ptr<Element>;
ElementPtr Get();
void Release(Element* e);
private:
char* data_;
int element_sz_;
QVector<bool> available_;
};
OLIVE_NAMESPACE_EXIT
#endif // FRAMEPOOL_H
+1
View File
@@ -32,6 +32,7 @@ set(OLIVE_SOURCES
common/flipmodifiers.cpp
common/functiontimer.h
common/lerp.h
common/memorypool.h
common/qtutils.h
common/qtutils.cpp
common/range.h
+165
View File
@@ -0,0 +1,165 @@
/***
Olive - Non-Linear Video Editor
Copyright (C) 2019 Olive Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
***/
#ifndef MEMORYPOOL_H
#define MEMORYPOOL_H
#include <memory>
#include <QDateTime>
#include <stdint.h>
#include <QDebug>
#include "common/define.h"
OLIVE_NAMESPACE_ENTER
template <typename T>
class MemoryPool
{
public:
MemoryPool() {
data_ = nullptr;
}
~MemoryPool() {
delete [] data_;
}
DISABLE_COPY_MOVE(MemoryPool)
bool Allocate(int nb_elements) {
delete [] data_;
size_t ele_sz = GetElementSize();
if (!ele_sz) {
return false;
}
if ((data_ = new char[ele_sz * nb_elements])) {
available_.resize(nb_elements);
available_.fill(true);
return true;
} else {
available_.clear();
return false;
}
}
void Destroy() {
delete [] data_;
data_ = nullptr;
available_.clear();
}
inline bool IsAllocated() const {
return data_;
}
inline int GetElementCount() const {
return available_.size();
}
class Element {
public:
Element(MemoryPool* parent, T* data) {
parent_ = parent;
data_ = data;
accessed_ = QDateTime::currentMSecsSinceEpoch();
}
~Element() {
parent_->Release(this);
}
inline T* data() const {
return data_;
}
inline const int64_t& timestamp() const {
return timestamp_;
}
inline void set_timestamp(const int64_t& timestamp) {
timestamp_ = timestamp;
}
inline void access() {
accessed_ = QDateTime::currentMSecsSinceEpoch();
}
inline const int64_t& last_accessed() const {
return accessed_;
}
private:
MemoryPool* parent_;
T* data_;
int64_t timestamp_;
int64_t accessed_;
};
using ElementPtr = std::shared_ptr<Element>;
ElementPtr Get() {
for (int i=0;i<available_.size();i++) {
if (available_.at(i)) {
// This buffer is available
available_.replace(i, false);
return std::make_shared<Element>(this, reinterpret_cast<T*>(data_ + i * GetElementSize()));
}
}
// FIXME: Allocate a new "arena"
return nullptr;
}
void Release(Element* e) {
quintptr diff = reinterpret_cast<quintptr>(e->data()) - reinterpret_cast<quintptr>(data_);
int index = diff / GetElementSize();
available_.replace(index, true);
}
protected:
virtual size_t GetElementSize() {
return sizeof(T);
}
private:
char* data_;
QVector<bool> available_;
};
OLIVE_NAMESPACE_EXIT
#endif // MEMORYPOOL_H