oak-team/oak-editor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

decoder/timeline: improve decoding of still images

Browse Source
This commit is contained in:
itsmattkc
2020-07-12 18:25:51 +10:00
parent 4d6003bb55
commit 11afdf32fe
8 changed files with 313 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/codec/ffmpeg/ffmpegdecoder.cpp
+236 -188
View File
@@ -86,12 +86,12 @@ bool FFmpegDecoder::Open()
return false;
}
if (stream()->type() == Stream::kVideo) {
if (stream()->type() == Stream::kImage || stream()->type() == Stream::kVideo) {
// Get an Olive compatible AVPixelFormat
src_pix_fmt_ = static_cast<AVPixelFormat>(our_instance->stream()->codecpar->format);
ideal_pix_fmt_ = FFmpegCommon::GetCompatiblePixelFormat(src_pix_fmt_);
{
if (stream()->type() == Stream::kVideo) {
QMutexLocker map_locker(&instance_map_lock_);
// FIXME: Test code, this should be changed later
@@ -144,165 +144,177 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid
return nullptr;
}
if (stream()->type() != Stream::kVideo) {
if (stream()->type() != Stream::kImage && stream()->type() != Stream::kVideo) {
return nullptr;
}
int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_;
ImageStreamPtr is = std::static_pointer_cast<ImageStream>(stream());
VideoStreamPtr vs = std::static_pointer_cast<VideoStream>(stream());
if (stream()->type() == Stream::kImage) {
FFmpegDecoderInstance* working_instance = nullptr;
FFmpegFramePool::ElementPtr return_frame = nullptr;
// FIXME: Hacky
FFmpegDecoderInstance i(stream()->footage()->filename().toUtf8(), stream()->index());
// Find instance
do {
QMutexLocker list_locker(&instance_map_lock_);
AVPacket* pkt = av_packet_alloc();
AVFrame* frame = av_frame_alloc();
FramePtr output_frame = nullptr;
QList<FFmpegDecoderInstance*> non_ideal_contenders;
int ret = i.GetFrame(pkt, frame);
QList<FFmpegDecoderInstance*> instances = instance_map_.value(stream().get());
if (ret >= 0) {
output_frame = BuffersToNativeFrame(divider,
is->width(),
is->height(),
0,
frame->data,
frame->linesize);
} else {
qWarning() << "Failed to retrieve still image from decoder";
}
foreach (FFmpegDecoderInstance* i, instances) {
av_frame_free(&frame);
av_packet_free(&pkt);
i->cache_lock()->lock();
return output_frame;
if (i->CacheContainsTime(target_ts)) {
} else {
// Found our instance, allow others to enter the list
FFmpegFramePool::ElementPtr return_frame = nullptr;
list_locker.unlock();
int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_;
// Get the frame from this cache
return_frame = i->GetFrameFromCache(target_ts);
VideoStreamPtr vs = std::static_pointer_cast<VideoStream>(stream());
// Got our frame, allow cache to continue
i->cache_lock()->unlock();
break;
FFmpegDecoderInstance* working_instance = nullptr;
} else if (i->CacheWillContainTime(target_ts) || i->CacheCouldContainTime(target_ts)) {
// Find instance
do {
QMutexLocker list_locker(&instance_map_lock_);
// Found our instance, allow others to enter the list
list_locker.unlock();
QList<FFmpegDecoderInstance*> non_ideal_contenders;
// If the instance is currently in use, enter into a loop of seeing from frames come up next in case one is ours
if (i->IsWorking()) {
QList<FFmpegDecoderInstance*> instances = instance_map_.value(stream().get());
do {
// Allow instance to continue to the next frame
i->cache_wait_cond()->wait(i->cache_lock());
foreach (FFmpegDecoderInstance* i, instances) {
// See if the cache now contains this frame, if so we'll exit this loop
if (i->CacheContainsTime(target_ts)) {
i->cache_lock()->lock();
// Grab the frame
return_frame = i->GetFrameFromCache(target_ts);
if (i->CacheContainsTime(target_ts)) {
// We can release this worker now since we don't need it anymore
i->cache_lock()->unlock();
// Found our instance, allow others to enter the list
} else if (!i->IsWorking()) {
list_locker.unlock();
// This instance finished and we didn't get our frame, we'll take it and continue it
working_instance = i;
break;
// Get the frame from this cache
return_frame = i->GetFrameFromCache(target_ts);
}
} while (!return_frame);
// Got our frame, allow cache to continue
i->cache_lock()->unlock();
break;
} else if (i->CacheWillContainTime(target_ts) || i->CacheCouldContainTime(target_ts)) {
// Found our instance, allow others to enter the list
list_locker.unlock();
// If the instance is currently in use, enter into a loop of seeing from frames come up next in case one is ours
if (i->IsWorking()) {
do {
// Allow instance to continue to the next frame
i->cache_wait_cond()->wait(i->cache_lock());
// See if the cache now contains this frame, if so we'll exit this loop
if (i->CacheContainsTime(target_ts)) {
// Grab the frame
return_frame = i->GetFrameFromCache(target_ts);
// We can release this worker now since we don't need it anymore
i->cache_lock()->unlock();
} else if (!i->IsWorking()) {
// This instance finished and we didn't get our frame, we'll take it and continue it
working_instance = i;
break;
}
} while (!return_frame);
} else {
// Otherwise, we'll grab this instance and continue it ourselves
working_instance = i;
}
break;
} else if (i->IsWorking()) {
// Ignore currently working instances
i->cache_lock()->unlock();
} else if (i->CacheIsEmpty()) {
// Prioritize this cache over others (leaves this instance LOCKED in case we end up using it later)
non_ideal_contenders.prepend(i);
} else {
// Otherwise, we'll grab this instance and continue it ourselves
working_instance = i;
// De-prioritize this cache (leaves this instance LOCKED in case we end up using it later)
non_ideal_contenders.append(i);
}
break;
} else if (i->IsWorking()) {
// Ignore currently working instances
i->cache_lock()->unlock();
} else if (i->CacheIsEmpty()) {
// Prioritize this cache over others (leaves this instance LOCKED in case we end up using it later)
non_ideal_contenders.prepend(i);
} else {
// De-prioritize this cache (leaves this instance LOCKED in case we end up using it later)
non_ideal_contenders.append(i);
}
// If we didn't find a suitable contender, grab the first non-suitable and roll with that
if (!return_frame && !working_instance && !non_ideal_contenders.isEmpty()) {
working_instance = non_ideal_contenders.takeFirst();
}
// For all instances we left locked but didn't end up using, lock them now
foreach (FFmpegDecoderInstance* unsuitable_instance, non_ideal_contenders) {
unsuitable_instance->cache_lock()->unlock();
}
} while (!return_frame && !working_instance);
if (!return_frame && working_instance) {
// This instance SHOULD remain locked from our earlier loop, making this operation safe
working_instance->SetWorking(true);
// Retrieve frame
return_frame = working_instance->RetrieveFrame(target_ts, true);
// Set working to false and wake any threads waiting
working_instance->cache_lock()->lock();
working_instance->SetWorking(false);
working_instance->cache_wait_cond()->wakeAll();
working_instance->cache_lock()->unlock();
}
// If we didn't find a suitable contender, grab the first non-suitable and roll with that
if (!return_frame && !working_instance && !non_ideal_contenders.isEmpty()) {
working_instance = non_ideal_contenders.takeFirst();
// We found the frame, we'll return a copy
if (return_frame) {
// Align buffer to data/linesize points that can be passed to sws_scale
uint8_t* input_data[4];
int input_linesize[4];
av_image_fill_arrays(input_data,
input_linesize,
reinterpret_cast<const uint8_t*>(return_frame->data()),
src_pix_fmt_,
vs->width(),
vs->height(),
1);
return BuffersToNativeFrame(divider,
vs->width(),
vs->height(),
target_ts,
input_data,
input_linesize);
}
// For all instances we left locked but didn't end up using, lock them now
foreach (FFmpegDecoderInstance* unsuitable_instance, non_ideal_contenders) {
unsuitable_instance->cache_lock()->unlock();
}
} while (!return_frame && !working_instance);
if (!return_frame && working_instance) {
// This instance SHOULD remain locked from our earlier loop, making this operation safe
working_instance->SetWorking(true);
// Retrieve frame
return_frame = working_instance->RetrieveFrame(target_ts, true);
// Set working to false and wake any threads waiting
working_instance->cache_lock()->lock();
working_instance->SetWorking(false);
working_instance->cache_wait_cond()->wakeAll();
working_instance->cache_lock()->unlock();
}
// We found the frame, we'll return a copy
if (return_frame) {
if (divider != scale_divider_) {
FreeScaler();
InitScaler(divider);
}
// Create frame to return
FramePtr copy = Frame::Create();
copy->set_video_params(VideoParams(vs->width(),
vs->height(),
native_pix_fmt_,
divider));
copy->set_timestamp(Timecode::timestamp_to_time(target_ts, time_base_));
copy->set_sample_aspect_ratio(aspect_ratio_);
copy->allocate();
// Align buffer to data/linesize points that can be passed to sws_scale
uint8_t* input_data[4];
int input_linesize[4];
av_image_fill_arrays(input_data,
input_linesize,
reinterpret_cast<const uint8_t*>(return_frame->data()),
src_pix_fmt_,
vs->width(),
vs->height(),
1);
// Convert frame to RGB/A for the rest of the pipeline
uint8_t* output_data = reinterpret_cast<uint8_t*>(copy->data());
int output_linesize = copy->linesize_bytes();
sws_scale(scale_ctx_,
input_data,
input_linesize,
0,
vs->height(),
&output_data,
&output_linesize);
return copy;
}
return nullptr;
@@ -436,8 +448,6 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
AVFormatContext* fmt_ctx = nullptr;
error_code = avformat_open_input(&fmt_ctx, filename, nullptr, nullptr);
QList<Stream*> streams_that_need_manual_duration;
// Handle format context error
if (error_code == 0) {
@@ -453,16 +463,72 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
if (avstream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
// Create a video stream object
VideoStreamPtr video_stream = std::make_shared<VideoStream>();
bool image_is_still = false;
ImageStream::Interlacing interlacing = ImageStream::kInterlaceNone;
video_stream->set_width(avstream->codecpar->width);
video_stream->set_height(avstream->codecpar->height);
video_stream->set_format(GetNativePixelFormat(FFmpegCommon::GetCompatiblePixelFormat(static_cast<AVPixelFormat>(avstream->codecpar->format))));
video_stream->set_frame_rate(av_guess_frame_rate(fmt_ctx, avstream, nullptr));
video_stream->set_start_time(avstream->start_time);
{
// Read at least two frames to get more information about this video stream
AVPacket* pkt = av_packet_alloc();
AVFrame* frame = av_frame_alloc();
str = video_stream;
{
FFmpegDecoderInstance instance(filename, i);
// Read first frame and retrieve some metadata
if (instance.GetFrame(pkt, frame) >= 0) {
// Check if video is interlaced and what field dominance it has if so
if (frame->interlaced_frame) {
if (frame->top_field_first) {
interlacing = ImageStream::kInterlacedTopFirst;
} else {
interlacing = ImageStream::kInterlacedBottomFirst;
}
}
}
// Read second frame
int ret = instance.GetFrame(pkt, frame);
if (ret >= 0) {
// Check if we need a manual duration
if (avstream->duration == AV_NOPTS_VALUE) {
int64_t new_dur;
do {
new_dur = frame->pts;
} while (instance.GetFrame(pkt, frame) >= 0);
avstream->duration = new_dur;
}
} else if (ret == AVERROR_EOF) {
// Video has only one frame in it, treat it like a still image
image_is_still = true;
}
}
av_frame_free(&frame);
av_packet_free(&pkt);
}
ImageStreamPtr image_stream;
if (image_is_still) {
image_stream = std::make_shared<ImageStream>();
} else {
VideoStreamPtr video_stream = std::make_shared<VideoStream>();
video_stream->set_frame_rate(av_guess_frame_rate(fmt_ctx, avstream, nullptr));
video_stream->set_start_time(avstream->start_time);
image_stream = video_stream;
}
image_stream->set_width(avstream->codecpar->width);
image_stream->set_height(avstream->codecpar->height);
image_stream->set_format(GetNativePixelFormat(FFmpegCommon::GetCompatiblePixelFormat(static_cast<AVPixelFormat>(avstream->codecpar->format))));
image_stream->set_interlacing(interlacing);
str = image_stream;
} else if (avstream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
@@ -511,11 +577,6 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
str->set_timebase(avstream->time_base);
str->set_duration(avstream->duration);
// The container/stream info may not contain a duration, so we'll need to manually retrieve it
if (avstream->duration == AV_NOPTS_VALUE) {
streams_that_need_manual_duration.append(str.get());
}
f->add_stream(str);
}
@@ -523,51 +584,6 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
result = true;
}
// If the metadata did not contain a duration, we'll need to loop through the file to retrieve it
if (!streams_that_need_manual_duration.isEmpty()) {
AVPacket* pkt = av_packet_alloc();
QVector<int64_t> durations(streams_that_need_manual_duration.size());
durations.fill(0);
while (true) {
if (cancelled && *cancelled) {
break;
}
// Ensure previous buffers are cleared
av_packet_unref(pkt);
// Read packet from file
int ret = av_read_frame(fmt_ctx, pkt);
if (ret < 0) {
// Handle errors that aren't EOF (which simply means the file is finished)
if (ret != AVERROR_EOF) {
qWarning() << "Error while finding duration";
}
break;
} else {
for (int i=0;i<streams_that_need_manual_duration.size();i++) {
if (streams_that_need_manual_duration.at(i)->index() == pkt->stream_index
&& pkt->pts > durations.at(i)) {
durations.replace(i, pkt->pts);
}
}
}
}
av_packet_free(&pkt);
if (!cancelled || !*cancelled) {
for (int i=0;i<streams_that_need_manual_duration.size();i++) {
streams_that_need_manual_duration.at(i)->set_duration(durations.at(i));
}
}
}
// Free all memory
avformat_close_input(&fmt_ctx);
@@ -795,6 +811,38 @@ uint64_t FFmpegDecoder::ValidateChannelLayout(AVStream* stream)
return av_get_default_channel_layout(stream->codecpar->channels);
}
FramePtr FFmpegDecoder::BuffersToNativeFrame(int divider, int width, int height, int64_t ts, uint8_t** input_data, int* input_linesize)
{
if (divider != scale_divider_) {
FreeScaler();
InitScaler(divider);
}
// Create frame to return
FramePtr copy = Frame::Create();
copy->set_video_params(VideoParams(width,
height,
native_pix_fmt_,
divider));
copy->set_timestamp(Timecode::timestamp_to_time(ts, time_base_));
copy->set_sample_aspect_ratio(aspect_ratio_);
copy->allocate();
// Convert frame to RGB/A for the rest of the pipeline
uint8_t* output_data = reinterpret_cast<uint8_t*>(copy->data());
int output_linesize = copy->linesize_bytes();
sws_scale(scale_ctx_,
input_data,
input_linesize,
0,
height,
&output_data,
&output_linesize);
return copy;
}
int FFmpegDecoderInstance::GetFrame(AVPacket *pkt, AVFrame *frame)
{
bool eof = false;
app/codec/ffmpeg/ffmpegdecoder.h
+2
View File
@@ -182,6 +182,8 @@ private:
static uint64_t ValidateChannelLayout(AVStream *stream);
FramePtr BuffersToNativeFrame(int divider, int width, int height, int64_t ts, uint8_t **input_data, int* input_linesize);
SwsContext* scale_ctx_;
int scale_divider_;
AVPixelFormat src_pix_fmt_;
app/dialog/footageproperties/streamproperties/videostreamproperties.cpp
+22 -1
View File
@@ -41,6 +41,21 @@ VideoStreamProperties::VideoStreamProperties(ImageStreamPtr stream) :
int row = 0;
video_layout->addWidget(new QLabel(tr("Interlacing:")), row, 0);
video_interlace_combo_ = new QComboBox();
// These must match the Interlacing enum in ImageStream
video_interlace_combo_->addItem(tr("None (Progressive)"));
video_interlace_combo_->addItem(tr("Top-Field First"));
video_interlace_combo_->addItem(tr("Bottom-Field First"));
video_interlace_combo_->setCurrentIndex(stream->interlacing());
video_layout->addWidget(video_interlace_combo_, row, 1);
row++;
video_layout->addWidget(new QLabel(tr("Color Space:")), row, 0);
video_color_space_ = new QComboBox();
@@ -109,6 +124,7 @@ void VideoStreamProperties::Accept(QUndoCommand *parent)
new VideoStreamChangeCommand(stream_,
video_premultiply_alpha_->isChecked(),
set_colorspace,
static_cast<ImageStream::Interlacing>(video_interlace_combo_->currentIndex()),
parent);
}
@@ -150,11 +166,13 @@ bool VideoStreamProperties::IsImageSequence(ImageStream *stream)
VideoStreamProperties::VideoStreamChangeCommand::VideoStreamChangeCommand(ImageStreamPtr stream,
bool premultiplied,
QString colorspace,
ImageStream::Interlacing interlacing,
QUndoCommand *parent) :
UndoCommand(parent),
stream_(stream),
new_premultiplied_(premultiplied),
new_colorspace_(colorspace)
new_colorspace_(colorspace),
new_interlacing_(interlacing)
{
}
@@ -167,15 +185,18 @@ void VideoStreamProperties::VideoStreamChangeCommand::redo_internal()
{
old_premultiplied_ = stream_->premultiplied_alpha();
old_colorspace_ = stream_->colorspace(false);
old_interlacing_ = stream_->interlacing();
stream_->set_premultiplied_alpha(new_premultiplied_);
stream_->set_colorspace(new_colorspace_);
stream_->set_interlacing(new_interlacing_);
}
void VideoStreamProperties::VideoStreamChangeCommand::undo_internal()
{
stream_->set_premultiplied_alpha(old_premultiplied_);
stream_->set_colorspace(old_colorspace_);
stream_->set_interlacing(old_interlacing_);
}
VideoStreamProperties::ImageSequenceChangeCommand::ImageSequenceChangeCommand(VideoStreamPtr video_stream, int64_t start_index, int64_t duration, QUndoCommand *parent) :
app/dialog/footageproperties/streamproperties/videostreamproperties.h
+8
View File
@@ -58,6 +58,11 @@ private:
*/
QComboBox* video_color_space_;
/**
* @brief Setting for video interlacing
*/
QComboBox* video_interlace_combo_;
/**
* @brief Sets the start index for image sequences
*/
@@ -73,6 +78,7 @@ private:
VideoStreamChangeCommand(ImageStreamPtr stream,
bool premultiplied,
QString colorspace,
ImageStream::Interlacing interlacing,
QUndoCommand* parent = nullptr);
virtual Project* GetRelevantProject() const override;
@@ -86,9 +92,11 @@ private:
bool new_premultiplied_;
QString new_colorspace_;
ImageStream::Interlacing new_interlacing_;
bool old_premultiplied_;
QString old_colorspace_;
ImageStream::Interlacing old_interlacing_;
};
app/project/item/footage/imagestream.cpp
+2 -1
View File
@@ -28,7 +28,8 @@
OLIVE_NAMESPACE_ENTER
ImageStream::ImageStream() :
premultiplied_alpha_(false)
premultiplied_alpha_(false),
interlacing_(kInterlaceNone)
{
set_type(kImage);
}
app/project/item/footage/imagestream.h
+20
View File
@@ -75,6 +75,24 @@ public:
QString get_colorspace_match_string() const;
enum Interlacing {
kInterlaceNone,
kInterlacedTopFirst,
kInterlacedBottomFirst
};
Interlacing interlacing() const
{
return interlacing_;
}
void set_interlacing(Interlacing i)
{
interlacing_ = i;
emit ParametersChanged();
}
protected:
virtual void FootageSetEvent(Footage*) override;
@@ -87,6 +105,7 @@ private:
int height_;
bool premultiplied_alpha_;
QString colorspace_;
Interlacing interlacing_;
PixelFormat::Format format_;
@@ -94,6 +113,7 @@ private slots:
void ColorConfigChanged();
void DefaultColorSpaceChanged();
};
using ImageStreamPtr = std::shared_ptr<ImageStream>;
app/widget/timelinewidget/tool/import.cpp
+20 -7
View File
@@ -216,7 +216,9 @@ void TimelineWidget::ImportTool::FootageToGhosts(rational ghost_start, const QLi
QVector<int> track_offsets(Timeline::kTrackTypeCount);
track_offsets.fill(track_start);
QVector<TimelineViewGhostItem*> footage_ghosts;
rational footage_duration;
bool contains_image_stream = false;
quint64 enabled_streams = footage.streams();
@@ -236,22 +238,21 @@ void TimelineWidget::ImportTool::FootageToGhosts(rational ghost_start, const QLi
TimelineViewGhostItem* ghost = new TimelineViewGhostItem();
if (stream->type() == Stream::kImage) {
// Stream is essentially length-less - use config's default image length
footage_duration = Config::Current()["DefaultStillLength"].value<rational>();
// Stream is essentially length-less - we may use the default still image length in config,
// or we may use another stream's length depending on the circumstance
contains_image_stream = true;
} else {
// Rescale stream duration to timeline timebase
// Convert to rational time
if (footage.footage()->workarea()->enabled()) {
footage_duration = footage.footage()->workarea()->range().length();
footage_duration = qMax(footage_duration, footage.footage()->workarea()->range().length());
ghost->SetMediaIn(footage.footage()->workarea()->in());
} else {
int64_t stream_duration = Timecode::rescale_timestamp_ceil(stream->duration(), stream->timebase(), dest_tb);
footage_duration = Timecode::timestamp_to_time(stream_duration, dest_tb);
footage_duration = qMax(footage_duration, Timecode::timestamp_to_time(stream_duration, dest_tb));
}
}
ghost->SetIn(ghost_start);
ghost->SetOut(ghost_start + footage_duration);
ghost->SetTrack(TrackReference(track_type, track_offsets.at(track_type)));
// Increment track count for this track type
@@ -263,10 +264,22 @@ void TimelineWidget::ImportTool::FootageToGhosts(rational ghost_start, const QLi
ghost->setData(TimelineViewGhostItem::kAttachedFootage, QVariant::fromValue(stream));
ghost->SetMode(Timeline::kMove);
parent()->AddGhost(ghost);
footage_ghosts.append(ghost);
}
if (contains_image_stream && footage_duration.isNull()) {
// Footage must ONLY be image streams so no duration value was found, use default in config
footage_duration = Config::Current()["DefaultStillLength"].value<rational>();
}
foreach (TimelineViewGhostItem* ghost, footage_ghosts) {
ghost->SetIn(ghost_start);
ghost->SetOut(ghost_start + footage_duration);
parent()->AddGhost(ghost);
}
// Stack each ghost one after the other
ghost_start += footage_duration;
app/widget/timelinewidget/tool/pointer.cpp
+3
View File
@@ -548,6 +548,9 @@ rational TimelineWidget::PointerTool::ValidateInTrimming(rational movement,
}
prev = prev->previous();
}
// Limit in point at 0 on the timeline
earliest_in = qMax(rational(), earliest_in);
}
}