decoder/timeline: improve decoding of still images
This commit is contained in:
itsmattkc
+236
-188
@@ -86,12 +86,12 @@ bool FFmpegDecoder::Open()
|
||||
return false;
|
||||
}
|
||||
|
||||
if (stream()->type() == Stream::kVideo) {
|
||||
if (stream()->type() == Stream::kImage || stream()->type() == Stream::kVideo) {
|
||||
// Get an Olive compatible AVPixelFormat
|
||||
src_pix_fmt_ = static_cast<AVPixelFormat>(our_instance->stream()->codecpar->format);
|
||||
ideal_pix_fmt_ = FFmpegCommon::GetCompatiblePixelFormat(src_pix_fmt_);
|
||||
|
||||
{
|
||||
if (stream()->type() == Stream::kVideo) {
|
||||
QMutexLocker map_locker(&instance_map_lock_);
|
||||
|
||||
// FIXME: Test code, this should be changed later
|
||||
@@ -144,165 +144,177 @@ FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int &divid
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (stream()->type() != Stream::kVideo) {
|
||||
if (stream()->type() != Stream::kImage && stream()->type() != Stream::kVideo) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_;
|
||||
ImageStreamPtr is = std::static_pointer_cast<ImageStream>(stream());
|
||||
|
||||
VideoStreamPtr vs = std::static_pointer_cast<VideoStream>(stream());
|
||||
if (stream()->type() == Stream::kImage) {
|
||||
|
||||
FFmpegDecoderInstance* working_instance = nullptr;
|
||||
FFmpegFramePool::ElementPtr return_frame = nullptr;
|
||||
// FIXME: Hacky
|
||||
FFmpegDecoderInstance i(stream()->footage()->filename().toUtf8(), stream()->index());
|
||||
|
||||
// Find instance
|
||||
do {
|
||||
QMutexLocker list_locker(&instance_map_lock_);
|
||||
AVPacket* pkt = av_packet_alloc();
|
||||
AVFrame* frame = av_frame_alloc();
|
||||
FramePtr output_frame = nullptr;
|
||||
|
||||
QList<FFmpegDecoderInstance*> non_ideal_contenders;
|
||||
int ret = i.GetFrame(pkt, frame);
|
||||
|
||||
QList<FFmpegDecoderInstance*> instances = instance_map_.value(stream().get());
|
||||
if (ret >= 0) {
|
||||
output_frame = BuffersToNativeFrame(divider,
|
||||
is->width(),
|
||||
is->height(),
|
||||
0,
|
||||
frame->data,
|
||||
frame->linesize);
|
||||
} else {
|
||||
qWarning() << "Failed to retrieve still image from decoder";
|
||||
}
|
||||
|
||||
foreach (FFmpegDecoderInstance* i, instances) {
|
||||
av_frame_free(&frame);
|
||||
av_packet_free(&pkt);
|
||||
|
||||
i->cache_lock()->lock();
|
||||
return output_frame;
|
||||
|
||||
if (i->CacheContainsTime(target_ts)) {
|
||||
} else {
|
||||
|
||||
// Found our instance, allow others to enter the list
|
||||
FFmpegFramePool::ElementPtr return_frame = nullptr;
|
||||
|
||||
list_locker.unlock();
|
||||
int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_;
|
||||
|
||||
// Get the frame from this cache
|
||||
return_frame = i->GetFrameFromCache(target_ts);
|
||||
VideoStreamPtr vs = std::static_pointer_cast<VideoStream>(stream());
|
||||
|
||||
// Got our frame, allow cache to continue
|
||||
i->cache_lock()->unlock();
|
||||
break;
|
||||
FFmpegDecoderInstance* working_instance = nullptr;
|
||||
|
||||
} else if (i->CacheWillContainTime(target_ts) || i->CacheCouldContainTime(target_ts)) {
|
||||
// Find instance
|
||||
do {
|
||||
QMutexLocker list_locker(&instance_map_lock_);
|
||||
|
||||
// Found our instance, allow others to enter the list
|
||||
list_locker.unlock();
|
||||
QList<FFmpegDecoderInstance*> non_ideal_contenders;
|
||||
|
||||
// If the instance is currently in use, enter into a loop of seeing from frames come up next in case one is ours
|
||||
if (i->IsWorking()) {
|
||||
QList<FFmpegDecoderInstance*> instances = instance_map_.value(stream().get());
|
||||
|
||||
do {
|
||||
// Allow instance to continue to the next frame
|
||||
i->cache_wait_cond()->wait(i->cache_lock());
|
||||
foreach (FFmpegDecoderInstance* i, instances) {
|
||||
|
||||
// See if the cache now contains this frame, if so we'll exit this loop
|
||||
if (i->CacheContainsTime(target_ts)) {
|
||||
i->cache_lock()->lock();
|
||||
|
||||
// Grab the frame
|
||||
return_frame = i->GetFrameFromCache(target_ts);
|
||||
if (i->CacheContainsTime(target_ts)) {
|
||||
|
||||
// We can release this worker now since we don't need it anymore
|
||||
i->cache_lock()->unlock();
|
||||
// Found our instance, allow others to enter the list
|
||||
|
||||
} else if (!i->IsWorking()) {
|
||||
list_locker.unlock();
|
||||
|
||||
// This instance finished and we didn't get our frame, we'll take it and continue it
|
||||
working_instance = i;
|
||||
break;
|
||||
// Get the frame from this cache
|
||||
return_frame = i->GetFrameFromCache(target_ts);
|
||||
|
||||
}
|
||||
} while (!return_frame);
|
||||
// Got our frame, allow cache to continue
|
||||
i->cache_lock()->unlock();
|
||||
break;
|
||||
|
||||
} else if (i->CacheWillContainTime(target_ts) || i->CacheCouldContainTime(target_ts)) {
|
||||
|
||||
// Found our instance, allow others to enter the list
|
||||
list_locker.unlock();
|
||||
|
||||
// If the instance is currently in use, enter into a loop of seeing from frames come up next in case one is ours
|
||||
if (i->IsWorking()) {
|
||||
|
||||
do {
|
||||
// Allow instance to continue to the next frame
|
||||
i->cache_wait_cond()->wait(i->cache_lock());
|
||||
|
||||
// See if the cache now contains this frame, if so we'll exit this loop
|
||||
if (i->CacheContainsTime(target_ts)) {
|
||||
|
||||
// Grab the frame
|
||||
return_frame = i->GetFrameFromCache(target_ts);
|
||||
|
||||
// We can release this worker now since we don't need it anymore
|
||||
i->cache_lock()->unlock();
|
||||
|
||||
} else if (!i->IsWorking()) {
|
||||
|
||||
// This instance finished and we didn't get our frame, we'll take it and continue it
|
||||
working_instance = i;
|
||||
break;
|
||||
|
||||
}
|
||||
} while (!return_frame);
|
||||
|
||||
} else {
|
||||
// Otherwise, we'll grab this instance and continue it ourselves
|
||||
working_instance = i;
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
} else if (i->IsWorking()) {
|
||||
|
||||
// Ignore currently working instances
|
||||
i->cache_lock()->unlock();
|
||||
|
||||
} else if (i->CacheIsEmpty()) {
|
||||
|
||||
// Prioritize this cache over others (leaves this instance LOCKED in case we end up using it later)
|
||||
non_ideal_contenders.prepend(i);
|
||||
|
||||
} else {
|
||||
// Otherwise, we'll grab this instance and continue it ourselves
|
||||
working_instance = i;
|
||||
|
||||
// De-prioritize this cache (leaves this instance LOCKED in case we end up using it later)
|
||||
non_ideal_contenders.append(i);
|
||||
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
} else if (i->IsWorking()) {
|
||||
|
||||
// Ignore currently working instances
|
||||
i->cache_lock()->unlock();
|
||||
|
||||
} else if (i->CacheIsEmpty()) {
|
||||
|
||||
// Prioritize this cache over others (leaves this instance LOCKED in case we end up using it later)
|
||||
non_ideal_contenders.prepend(i);
|
||||
|
||||
} else {
|
||||
|
||||
// De-prioritize this cache (leaves this instance LOCKED in case we end up using it later)
|
||||
non_ideal_contenders.append(i);
|
||||
|
||||
}
|
||||
|
||||
// If we didn't find a suitable contender, grab the first non-suitable and roll with that
|
||||
if (!return_frame && !working_instance && !non_ideal_contenders.isEmpty()) {
|
||||
working_instance = non_ideal_contenders.takeFirst();
|
||||
}
|
||||
|
||||
// For all instances we left locked but didn't end up using, lock them now
|
||||
foreach (FFmpegDecoderInstance* unsuitable_instance, non_ideal_contenders) {
|
||||
unsuitable_instance->cache_lock()->unlock();
|
||||
}
|
||||
} while (!return_frame && !working_instance);
|
||||
|
||||
if (!return_frame && working_instance) {
|
||||
|
||||
// This instance SHOULD remain locked from our earlier loop, making this operation safe
|
||||
working_instance->SetWorking(true);
|
||||
|
||||
// Retrieve frame
|
||||
return_frame = working_instance->RetrieveFrame(target_ts, true);
|
||||
|
||||
// Set working to false and wake any threads waiting
|
||||
working_instance->cache_lock()->lock();
|
||||
working_instance->SetWorking(false);
|
||||
working_instance->cache_wait_cond()->wakeAll();
|
||||
working_instance->cache_lock()->unlock();
|
||||
}
|
||||
|
||||
// If we didn't find a suitable contender, grab the first non-suitable and roll with that
|
||||
if (!return_frame && !working_instance && !non_ideal_contenders.isEmpty()) {
|
||||
working_instance = non_ideal_contenders.takeFirst();
|
||||
// We found the frame, we'll return a copy
|
||||
if (return_frame) {
|
||||
// Align buffer to data/linesize points that can be passed to sws_scale
|
||||
uint8_t* input_data[4];
|
||||
int input_linesize[4];
|
||||
|
||||
av_image_fill_arrays(input_data,
|
||||
input_linesize,
|
||||
reinterpret_cast<const uint8_t*>(return_frame->data()),
|
||||
src_pix_fmt_,
|
||||
vs->width(),
|
||||
vs->height(),
|
||||
1);
|
||||
|
||||
return BuffersToNativeFrame(divider,
|
||||
vs->width(),
|
||||
vs->height(),
|
||||
target_ts,
|
||||
input_data,
|
||||
input_linesize);
|
||||
}
|
||||
|
||||
// For all instances we left locked but didn't end up using, lock them now
|
||||
foreach (FFmpegDecoderInstance* unsuitable_instance, non_ideal_contenders) {
|
||||
unsuitable_instance->cache_lock()->unlock();
|
||||
}
|
||||
} while (!return_frame && !working_instance);
|
||||
|
||||
if (!return_frame && working_instance) {
|
||||
|
||||
// This instance SHOULD remain locked from our earlier loop, making this operation safe
|
||||
working_instance->SetWorking(true);
|
||||
|
||||
// Retrieve frame
|
||||
return_frame = working_instance->RetrieveFrame(target_ts, true);
|
||||
|
||||
// Set working to false and wake any threads waiting
|
||||
working_instance->cache_lock()->lock();
|
||||
working_instance->SetWorking(false);
|
||||
working_instance->cache_wait_cond()->wakeAll();
|
||||
working_instance->cache_lock()->unlock();
|
||||
}
|
||||
|
||||
// We found the frame, we'll return a copy
|
||||
if (return_frame) {
|
||||
if (divider != scale_divider_) {
|
||||
FreeScaler();
|
||||
InitScaler(divider);
|
||||
}
|
||||
|
||||
// Create frame to return
|
||||
FramePtr copy = Frame::Create();
|
||||
copy->set_video_params(VideoParams(vs->width(),
|
||||
vs->height(),
|
||||
native_pix_fmt_,
|
||||
divider));
|
||||
copy->set_timestamp(Timecode::timestamp_to_time(target_ts, time_base_));
|
||||
copy->set_sample_aspect_ratio(aspect_ratio_);
|
||||
copy->allocate();
|
||||
|
||||
// Align buffer to data/linesize points that can be passed to sws_scale
|
||||
uint8_t* input_data[4];
|
||||
int input_linesize[4];
|
||||
|
||||
av_image_fill_arrays(input_data,
|
||||
input_linesize,
|
||||
reinterpret_cast<const uint8_t*>(return_frame->data()),
|
||||
src_pix_fmt_,
|
||||
vs->width(),
|
||||
vs->height(),
|
||||
1);
|
||||
|
||||
// Convert frame to RGB/A for the rest of the pipeline
|
||||
uint8_t* output_data = reinterpret_cast<uint8_t*>(copy->data());
|
||||
int output_linesize = copy->linesize_bytes();
|
||||
|
||||
sws_scale(scale_ctx_,
|
||||
input_data,
|
||||
input_linesize,
|
||||
0,
|
||||
vs->height(),
|
||||
&output_data,
|
||||
&output_linesize);
|
||||
|
||||
return copy;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
@@ -436,8 +448,6 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
|
||||
AVFormatContext* fmt_ctx = nullptr;
|
||||
error_code = avformat_open_input(&fmt_ctx, filename, nullptr, nullptr);
|
||||
|
||||
QList<Stream*> streams_that_need_manual_duration;
|
||||
|
||||
// Handle format context error
|
||||
if (error_code == 0) {
|
||||
|
||||
@@ -453,16 +463,72 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
|
||||
|
||||
if (avstream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
|
||||
|
||||
// Create a video stream object
|
||||
VideoStreamPtr video_stream = std::make_shared<VideoStream>();
|
||||
bool image_is_still = false;
|
||||
ImageStream::Interlacing interlacing = ImageStream::kInterlaceNone;
|
||||
|
||||
video_stream->set_width(avstream->codecpar->width);
|
||||
video_stream->set_height(avstream->codecpar->height);
|
||||
video_stream->set_format(GetNativePixelFormat(FFmpegCommon::GetCompatiblePixelFormat(static_cast<AVPixelFormat>(avstream->codecpar->format))));
|
||||
video_stream->set_frame_rate(av_guess_frame_rate(fmt_ctx, avstream, nullptr));
|
||||
video_stream->set_start_time(avstream->start_time);
|
||||
{
|
||||
// Read at least two frames to get more information about this video stream
|
||||
AVPacket* pkt = av_packet_alloc();
|
||||
AVFrame* frame = av_frame_alloc();
|
||||
|
||||
str = video_stream;
|
||||
{
|
||||
FFmpegDecoderInstance instance(filename, i);
|
||||
|
||||
// Read first frame and retrieve some metadata
|
||||
if (instance.GetFrame(pkt, frame) >= 0) {
|
||||
// Check if video is interlaced and what field dominance it has if so
|
||||
if (frame->interlaced_frame) {
|
||||
if (frame->top_field_first) {
|
||||
interlacing = ImageStream::kInterlacedTopFirst;
|
||||
} else {
|
||||
interlacing = ImageStream::kInterlacedBottomFirst;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Read second frame
|
||||
int ret = instance.GetFrame(pkt, frame);
|
||||
|
||||
if (ret >= 0) {
|
||||
// Check if we need a manual duration
|
||||
if (avstream->duration == AV_NOPTS_VALUE) {
|
||||
int64_t new_dur;
|
||||
|
||||
do {
|
||||
new_dur = frame->pts;
|
||||
} while (instance.GetFrame(pkt, frame) >= 0);
|
||||
|
||||
avstream->duration = new_dur;
|
||||
}
|
||||
} else if (ret == AVERROR_EOF) {
|
||||
// Video has only one frame in it, treat it like a still image
|
||||
image_is_still = true;
|
||||
}
|
||||
}
|
||||
|
||||
av_frame_free(&frame);
|
||||
av_packet_free(&pkt);
|
||||
}
|
||||
|
||||
ImageStreamPtr image_stream;
|
||||
|
||||
if (image_is_still) {
|
||||
image_stream = std::make_shared<ImageStream>();
|
||||
} else {
|
||||
VideoStreamPtr video_stream = std::make_shared<VideoStream>();
|
||||
|
||||
video_stream->set_frame_rate(av_guess_frame_rate(fmt_ctx, avstream, nullptr));
|
||||
video_stream->set_start_time(avstream->start_time);
|
||||
|
||||
image_stream = video_stream;
|
||||
}
|
||||
|
||||
image_stream->set_width(avstream->codecpar->width);
|
||||
image_stream->set_height(avstream->codecpar->height);
|
||||
image_stream->set_format(GetNativePixelFormat(FFmpegCommon::GetCompatiblePixelFormat(static_cast<AVPixelFormat>(avstream->codecpar->format))));
|
||||
image_stream->set_interlacing(interlacing);
|
||||
|
||||
str = image_stream;
|
||||
|
||||
} else if (avstream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
|
||||
|
||||
@@ -511,11 +577,6 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
|
||||
str->set_timebase(avstream->time_base);
|
||||
str->set_duration(avstream->duration);
|
||||
|
||||
// The container/stream info may not contain a duration, so we'll need to manually retrieve it
|
||||
if (avstream->duration == AV_NOPTS_VALUE) {
|
||||
streams_that_need_manual_duration.append(str.get());
|
||||
}
|
||||
|
||||
f->add_stream(str);
|
||||
}
|
||||
|
||||
@@ -523,51 +584,6 @@ bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled)
|
||||
result = true;
|
||||
}
|
||||
|
||||
// If the metadata did not contain a duration, we'll need to loop through the file to retrieve it
|
||||
if (!streams_that_need_manual_duration.isEmpty()) {
|
||||
|
||||
AVPacket* pkt = av_packet_alloc();
|
||||
|
||||
QVector<int64_t> durations(streams_that_need_manual_duration.size());
|
||||
durations.fill(0);
|
||||
|
||||
while (true) {
|
||||
if (cancelled && *cancelled) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Ensure previous buffers are cleared
|
||||
av_packet_unref(pkt);
|
||||
|
||||
// Read packet from file
|
||||
int ret = av_read_frame(fmt_ctx, pkt);
|
||||
|
||||
if (ret < 0) {
|
||||
// Handle errors that aren't EOF (which simply means the file is finished)
|
||||
if (ret != AVERROR_EOF) {
|
||||
qWarning() << "Error while finding duration";
|
||||
}
|
||||
break;
|
||||
} else {
|
||||
for (int i=0;i<streams_that_need_manual_duration.size();i++) {
|
||||
if (streams_that_need_manual_duration.at(i)->index() == pkt->stream_index
|
||||
&& pkt->pts > durations.at(i)) {
|
||||
durations.replace(i, pkt->pts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
av_packet_free(&pkt);
|
||||
|
||||
if (!cancelled || !*cancelled) {
|
||||
for (int i=0;i<streams_that_need_manual_duration.size();i++) {
|
||||
streams_that_need_manual_duration.at(i)->set_duration(durations.at(i));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Free all memory
|
||||
avformat_close_input(&fmt_ctx);
|
||||
|
||||
@@ -795,6 +811,38 @@ uint64_t FFmpegDecoder::ValidateChannelLayout(AVStream* stream)
|
||||
return av_get_default_channel_layout(stream->codecpar->channels);
|
||||
}
|
||||
|
||||
FramePtr FFmpegDecoder::BuffersToNativeFrame(int divider, int width, int height, int64_t ts, uint8_t** input_data, int* input_linesize)
|
||||
{
|
||||
if (divider != scale_divider_) {
|
||||
FreeScaler();
|
||||
InitScaler(divider);
|
||||
}
|
||||
|
||||
// Create frame to return
|
||||
FramePtr copy = Frame::Create();
|
||||
copy->set_video_params(VideoParams(width,
|
||||
height,
|
||||
native_pix_fmt_,
|
||||
divider));
|
||||
copy->set_timestamp(Timecode::timestamp_to_time(ts, time_base_));
|
||||
copy->set_sample_aspect_ratio(aspect_ratio_);
|
||||
copy->allocate();
|
||||
|
||||
// Convert frame to RGB/A for the rest of the pipeline
|
||||
uint8_t* output_data = reinterpret_cast<uint8_t*>(copy->data());
|
||||
int output_linesize = copy->linesize_bytes();
|
||||
|
||||
sws_scale(scale_ctx_,
|
||||
input_data,
|
||||
input_linesize,
|
||||
0,
|
||||
height,
|
||||
&output_data,
|
||||
&output_linesize);
|
||||
|
||||
return copy;
|
||||
}
|
||||
|
||||
int FFmpegDecoderInstance::GetFrame(AVPacket *pkt, AVFrame *frame)
|
||||
{
|
||||
bool eof = false;
|
||||
|
||||
Reference in New Issue
Block a user