/*** Olive - Non-Linear Video Editor Copyright (C) 2019 Olive Team This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ***/ #include "ffmpegdecoder.h" extern "C" { #include #include #include #include } #include #include #include #include #include #include #include #include #include "codec/waveinput.h" #include "common/define.h" #include "common/filefunctions.h" #include "common/functiontimer.h" #include "common/timecodefunctions.h" #include "ffmpegcommon.h" #include "render/framehashcache.h" #include "render/diskmanager.h" #include "render/pixelformat.h" OLIVE_NAMESPACE_ENTER QHash< Stream*, QList > FFmpegDecoder::instance_map_; QMutex FFmpegDecoder::instance_map_lock_; QHash< Stream*, FFmpegFramePool* > FFmpegDecoder::frame_pool_map_; // FIXME: Hardcoded, ideally this value is dynamically chosen based on memory restraints const int FFmpegDecoderInstance::kMaxFrameLife = 2000; FFmpegDecoder::FFmpegDecoder() : scale_ctx_(nullptr), scale_divider_(0) { } FFmpegDecoder::~FFmpegDecoder() { Close(); } bool FFmpegDecoder::Open() { QMutexLocker locker(&mutex_); if (open_) { return true; } Q_ASSERT(stream()); // Convert QString to a C string QByteArray fn_bytes = stream()->footage()->filename().toUtf8(); FFmpegDecoderInstance* our_instance = new FFmpegDecoderInstance(fn_bytes.constData(), stream()->index()); if (!our_instance->IsValid()) { delete our_instance; return false; } if (stream()->type() == Stream::kImage || stream()->type() == Stream::kVideo) { // Get an Olive compatible AVPixelFormat src_pix_fmt_ = static_cast(our_instance->stream()->codecpar->format); ideal_pix_fmt_ = FFmpegCommon::GetCompatiblePixelFormat(src_pix_fmt_); if (stream()->type() == Stream::kVideo) { QMutexLocker map_locker(&instance_map_lock_); // FIXME: Test code, this should be changed later FFmpegFramePool* frame_pool = frame_pool_map_.value(stream().get()); if (!frame_pool) { frame_pool = new FFmpegFramePool(256, our_instance->stream()->codecpar->width, our_instance->stream()->codecpar->height, static_cast(our_instance->stream()->codecpar->format)); frame_pool_map_.insert(stream().get(), frame_pool); } our_instance->SetFramePool(frame_pool); // End test code } // Determine which Olive native pixel format we retrieved // Note that FFmpeg doesn't support float formats native_pix_fmt_ = GetNativePixelFormat(ideal_pix_fmt_); Q_ASSERT(native_pix_fmt_ != PixelFormat::PIX_FMT_INVALID); aspect_ratio_ = our_instance->sample_aspect_ratio(); } time_base_ = our_instance->stream()->time_base; start_time_ = our_instance->stream()->start_time; // All allocation succeeded so we set the state to open open_ = true; { QMutexLocker l(&instance_map_lock_); QList list = instance_map_.value(stream().get()); list.append(our_instance); instance_map_.insert(stream().get(), list); } return true; } FramePtr FFmpegDecoder::RetrieveVideo(const rational &timecode, const int ÷r) { QMutexLocker locker(&mutex_); if (!open_) { qWarning() << "Tried to retrieve video on a decoder that's still closed"; return nullptr; } if (stream()->type() != Stream::kImage && stream()->type() != Stream::kVideo) { return nullptr; } ImageStreamPtr is = std::static_pointer_cast(stream()); if (stream()->type() == Stream::kImage) { // FIXME: Hacky FFmpegDecoderInstance i(stream()->footage()->filename().toUtf8(), stream()->index()); AVPacket* pkt = av_packet_alloc(); AVFrame* frame = av_frame_alloc(); FramePtr output_frame = nullptr; int ret = i.GetFrame(pkt, frame); if (ret >= 0) { output_frame = BuffersToNativeFrame(divider, is->width(), is->height(), 0, frame->data, frame->linesize); } else { qWarning() << "Failed to retrieve still image from decoder"; } av_frame_free(&frame); av_packet_free(&pkt); return output_frame; } else { FFmpegFramePool::ElementPtr return_frame = nullptr; int64_t target_ts = Timecode::time_to_timestamp(timecode, time_base_) + start_time_; VideoStreamPtr vs = std::static_pointer_cast(stream()); FFmpegDecoderInstance* working_instance = nullptr; // Find instance do { QMutexLocker list_locker(&instance_map_lock_); QList non_ideal_contenders; QList instances = instance_map_.value(stream().get()); foreach (FFmpegDecoderInstance* i, instances) { i->cache_lock()->lock(); if (i->CacheContainsTime(target_ts)) { // Found our instance, allow others to enter the list list_locker.unlock(); // Get the frame from this cache return_frame = i->GetFrameFromCache(target_ts); // Got our frame, allow cache to continue i->cache_lock()->unlock(); break; } else if (i->CacheWillContainTime(target_ts) || i->CacheCouldContainTime(target_ts)) { // Found our instance, allow others to enter the list list_locker.unlock(); // If the instance is currently in use, enter into a loop of seeing from frames come up next in case one is ours if (i->IsWorking()) { do { // Allow instance to continue to the next frame i->cache_wait_cond()->wait(i->cache_lock()); // See if the cache now contains this frame, if so we'll exit this loop if (i->CacheContainsTime(target_ts)) { // Grab the frame return_frame = i->GetFrameFromCache(target_ts); // We can release this worker now since we don't need it anymore i->cache_lock()->unlock(); } else if (!i->IsWorking()) { // This instance finished and we didn't get our frame, we'll take it and continue it working_instance = i; break; } } while (!return_frame); } else { // Otherwise, we'll grab this instance and continue it ourselves working_instance = i; } break; } else if (i->IsWorking()) { // Ignore currently working instances i->cache_lock()->unlock(); } else if (i->CacheIsEmpty()) { // Prioritize this cache over others (leaves this instance LOCKED in case we end up using it later) non_ideal_contenders.prepend(i); } else { // De-prioritize this cache (leaves this instance LOCKED in case we end up using it later) non_ideal_contenders.append(i); } } // If we didn't find a suitable contender, grab the first non-suitable and roll with that if (!return_frame && !working_instance && !non_ideal_contenders.isEmpty()) { working_instance = non_ideal_contenders.takeFirst(); } // For all instances we left locked but didn't end up using, lock them now foreach (FFmpegDecoderInstance* unsuitable_instance, non_ideal_contenders) { unsuitable_instance->cache_lock()->unlock(); } } while (!return_frame && !working_instance); if (!return_frame && working_instance) { // This instance SHOULD remain locked from our earlier loop, making this operation safe working_instance->SetWorking(true); // Retrieve frame return_frame = working_instance->RetrieveFrame(target_ts, true); // Set working to false and wake any threads waiting working_instance->cache_lock()->lock(); working_instance->SetWorking(false); working_instance->cache_wait_cond()->wakeAll(); working_instance->cache_lock()->unlock(); } // We found the frame, we'll return a copy if (return_frame) { // Align buffer to data/linesize points that can be passed to sws_scale uint8_t* input_data[4]; int input_linesize[4]; av_image_fill_arrays(input_data, input_linesize, reinterpret_cast(return_frame->data()), src_pix_fmt_, vs->width(), vs->height(), 1); return BuffersToNativeFrame(divider, vs->width(), vs->height(), target_ts, input_data, input_linesize); } } return nullptr; } SampleBufferPtr FFmpegDecoder::RetrieveAudio(const rational &timecode, const rational &length, const AudioParams ¶ms) { QMutexLocker locker(&mutex_); if (!open_) { qWarning() << "Tried to retrieve audio on a decoder that's still closed"; return nullptr; } if (stream()->type() != Stream::kAudio) { return nullptr; } QString wav_fn = GetConformedFilename(params); WaveInput input(wav_fn); if (input.open()) { const AudioParams& input_params = input.params(); // Read bytes from wav QByteArray packed_data = input.read(input_params.time_to_bytes(timecode), input_params.time_to_bytes(length)); input.close(); // Create sample buffer SampleBufferPtr sample_buffer = SampleBuffer::CreateFromPackedData(input_params, packed_data); return sample_buffer; } qCritical() << "Failed to open cached file" << wav_fn; return nullptr; } void FFmpegDecoder::Close() { QMutexLocker locker(&mutex_); { // Clear whichever instance is not in use and is least useful (there are only ever as many instances as there are // threads so if this thread is closing, an instance MUST be inactive) QMutexLocker l(&instance_map_lock_); QList list = instance_map_.value(stream().get()); if (!list.isEmpty()) { // Rank the instances by least useful (the top one should be one that isn't working and isn't in use) QList least_useful; foreach (FFmpegDecoderInstance* i, list) { i->cache_lock()->lock(); if (i->IsWorking()) { // Don't bother any currently working instances i->cache_lock()->unlock(); continue; } if (i->CacheIsEmpty()) { least_useful.prepend(i); } else { least_useful.append(i); } } // Remove the least useful from the list and re-insert it into the map FFmpegDecoderInstance* least_useful_instance = least_useful.first(); list.removeOne(least_useful_instance); instance_map_.insert(stream().get(), list); // If there are no more instances, destroy frame pool if (list.isEmpty()) { FFmpegFramePool* frame_pool = frame_pool_map_.take(stream().get()); delete frame_pool; } // We're done with the list now, we can unlock it and allow others to use it l.unlock(); // Unlock all the instances we locked foreach (FFmpegDecoderInstance* i, least_useful) { i->cache_lock()->unlock(); } // Delete this least useful instance now that we've definitely taken ownership of it least_useful_instance->deleteLater(); } } ClearResources(); } QString FFmpegDecoder::id() { return QStringLiteral("ffmpeg"); } bool FFmpegDecoder::SupportsVideo() { return true; } bool FFmpegDecoder::SupportsAudio() { return true; } bool FFmpegDecoder::Probe(Footage *f, const QAtomicInt* cancelled) { if (open_) { qWarning() << "Probe must be called while the Decoder is closed"; return false; } // Variable for receiving errors from FFmpeg int error_code; // Result to return bool result = false; // Convert QString to a C string QByteArray ba = f->filename().toUtf8(); const char* filename = ba.constData(); // Open file in a format context AVFormatContext* fmt_ctx = nullptr; error_code = avformat_open_input(&fmt_ctx, filename, nullptr, nullptr); // Handle format context error if (error_code == 0) { // Retrieve metadata about the media avformat_find_stream_info(fmt_ctx, nullptr); // Dump it into the Footage object for (unsigned int i=0;inb_streams;i++) { AVStream* avstream = fmt_ctx->streams[i]; StreamPtr str; if (avstream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { bool image_is_still = false; ImageStream::Interlacing interlacing = ImageStream::kInterlaceNone; { // Read at least two frames to get more information about this video stream AVPacket* pkt = av_packet_alloc(); AVFrame* frame = av_frame_alloc(); { FFmpegDecoderInstance instance(filename, i); // Read first frame and retrieve some metadata if (instance.GetFrame(pkt, frame) >= 0) { // Check if video is interlaced and what field dominance it has if so if (frame->interlaced_frame) { if (frame->top_field_first) { interlacing = ImageStream::kInterlacedTopFirst; } else { interlacing = ImageStream::kInterlacedBottomFirst; } } } // Read second frame int ret = instance.GetFrame(pkt, frame); if (ret >= 0) { // Check if we need a manual duration if (avstream->duration == AV_NOPTS_VALUE) { int64_t new_dur; do { new_dur = frame->pts; } while (instance.GetFrame(pkt, frame) >= 0); avstream->duration = new_dur; } } else if (ret == AVERROR_EOF) { // Video has only one frame in it, treat it like a still image image_is_still = true; } } av_frame_free(&frame); av_packet_free(&pkt); } ImageStreamPtr image_stream; if (image_is_still) { image_stream = std::make_shared(); } else { VideoStreamPtr video_stream = std::make_shared(); video_stream->set_frame_rate(av_guess_frame_rate(fmt_ctx, avstream, nullptr)); video_stream->set_start_time(avstream->start_time); image_stream = video_stream; } image_stream->set_width(avstream->codecpar->width); image_stream->set_height(avstream->codecpar->height); image_stream->set_format(GetNativePixelFormat(FFmpegCommon::GetCompatiblePixelFormat(static_cast(avstream->codecpar->format)))); image_stream->set_interlacing(interlacing); str = image_stream; } else if (avstream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { // Create an audio stream object AudioStreamPtr audio_stream = std::make_shared(); uint64_t channel_layout = avstream->codecpar->channel_layout; if (!channel_layout) { channel_layout = static_cast(av_get_default_channel_layout(avstream->codecpar->channels)); } audio_stream->set_channel_layout(channel_layout); audio_stream->set_channels(avstream->codecpar->channels); audio_stream->set_sample_rate(avstream->codecpar->sample_rate); str = audio_stream; } else { // This is data we can't utilize at the moment, but we make a Stream object anyway to keep parity with the file str = std::make_shared(); // Set the correct codec type based on FFmpeg's result switch (avstream->codecpar->codec_type) { case AVMEDIA_TYPE_UNKNOWN: str->set_type(Stream::kUnknown); break; case AVMEDIA_TYPE_DATA: str->set_type(Stream::kData); break; case AVMEDIA_TYPE_SUBTITLE: str->set_type(Stream::kSubtitle); break; case AVMEDIA_TYPE_ATTACHMENT: str->set_type(Stream::kAttachment); break; default: // We should never realistically get here, but we make an "invalid" stream just in case str->set_type(Stream::kUnknown); break; } } str->set_index(avstream->index); str->set_timebase(avstream->time_base); str->set_duration(avstream->duration); f->add_stream(str); } // As long as we can open the container and retrieve information, this was a successful probe result = true; } // Free all memory avformat_close_input(&fmt_ctx); return result; } void FFmpegDecoder::FFmpegError(int error_code) { char err[1024]; av_strerror(error_code, err, 1024); Error(QStringLiteral("Error decoding %1 - %2 %3").arg(stream()->footage()->filename(), QString::number(error_code), err)); } void FFmpegDecoder::Error(const QString &s) { qWarning() << s; ClearResources(); } QMutex scaler_lock; void SaveCacheFrame(FFmpegDecoder* decoder, SwsContext* scaler, AVFrame* frame, VideoParams params, QString dst_fn) { QByteArray converted_buffer(PixelFormat::GetBufferSize(params.format(), params.width(), params.height()), Qt::Uninitialized); uint8_t* converted_data = reinterpret_cast(converted_buffer.data()); int converted_linesize = PixelFormat::GetBufferSize(params.format(), params.width(), 1); scaler_lock.lock(); sws_scale(scaler, frame->data, frame->linesize, 0, frame->height, &converted_data, &converted_linesize); scaler_lock.unlock(); if (!FrameHashCache::SaveCacheFrame(dst_fn, converted_buffer.data(), params, converted_linesize)) { qCritical() <<" Failed to save cache frame" << dst_fn; } av_frame_free(&frame); } bool FFmpegDecoder::ConformAudio(const QAtomicInt *cancelled, const AudioParams &p) { // Iterate through each audio frame and extract the PCM data AudioStreamPtr audio_stream = std::static_pointer_cast(stream()); // Check if we already have a conform of this type QString conformed_fn = GetConformedFilename(p); if (QFileInfo::exists(conformed_fn)) { // If we have one, and we can open it correctly, we can use it as-is WaveInput input(conformed_fn); if (input.open()) { audio_stream->append_conformed_version(p); input.close(); return true; } } // Conform doesn't exist, we'll have to produce one FFmpegDecoderInstance index_instance(stream()->footage()->filename().toUtf8(), stream()->index()); // Handle NULL channel layout uint64_t channel_layout = ValidateChannelLayout(index_instance.stream()); if (!channel_layout) { qCritical() << "Failed to determine channel layout of audio file, could not conform"; return false; } // Create resampling context SwrContext* resampler = swr_alloc_set_opts(nullptr, p.channel_layout(), FFmpegCommon::GetFFmpegSampleFormat(p.format()), p.sample_rate(), channel_layout, static_cast(index_instance.stream()->codecpar->format), index_instance.stream()->codecpar->sample_rate, 0, nullptr); swr_init(resampler); WaveOutput wave_out(conformed_fn, p); AVPacket* pkt = av_packet_alloc(); AVFrame* frame = av_frame_alloc(); int ret; bool success = false; if (wave_out.open()) { while (true) { // Check if we have a `cancelled` ptr and its value if (cancelled && *cancelled) { break; } ret = index_instance.GetFrame(pkt, frame); if (ret < 0) { if (ret == AVERROR_EOF) { success = true; } else { char err_str[50]; av_strerror(ret, err_str, 50); qWarning() << "Failed to conform:" << ret << err_str; } break; } // Allocate buffers int nb_samples = swr_get_out_samples(resampler, frame->nb_samples); char* data = new char[p.samples_to_bytes(nb_samples)]; // Resample audio to our destination parameters nb_samples = swr_convert(resampler, reinterpret_cast(&data), nb_samples, const_cast(frame->data), frame->nb_samples); if (nb_samples < 0) { char err_str[50]; av_strerror(nb_samples, err_str, 50); qWarning() << "libswresample failed with error:" << nb_samples << err_str; break; } // Write packed WAV data to the disk cache wave_out.write(data, p.samples_to_bytes(nb_samples)); // If we allocated an output for the resampler, delete it here if (data != reinterpret_cast(frame->data[0])) { delete [] data; } SignalProcessingProgress(frame->pts); } wave_out.close(); if (success) { // If our conform succeeded, add it audio_stream->append_conformed_version(p); } else { // Audio index didn't complete, delete it QFile(conformed_fn).remove(); } } else { qWarning() << "Failed to open WAVE output for indexing"; } swr_free(&resampler); av_frame_free(&frame); av_packet_free(&pkt); return success; } QString FFmpegDecoder::GetIndexFilename() const { return FileFunctions::GetMediaIndexFilename(FileFunctions::GetUniqueFileIdentifier(stream()->footage()->filename())) .append(QString::number(stream()->index())); } QString FFmpegDecoder::GetProxyFilename(int divider) const { return GetIndexFilename().append('d').append(QString::number(divider)); } int FFmpegDecoder::GetScaledDimension(int dim, int divider) { return dim / divider; } PixelFormat::Format FFmpegDecoder::GetNativePixelFormat(AVPixelFormat pix_fmt) { switch (pix_fmt) { case AV_PIX_FMT_RGB24: return PixelFormat::PIX_FMT_RGB8; case AV_PIX_FMT_RGBA: return PixelFormat::PIX_FMT_RGBA8; case AV_PIX_FMT_RGB48: return PixelFormat::PIX_FMT_RGB16U; case AV_PIX_FMT_RGBA64: return PixelFormat::PIX_FMT_RGBA16U; default: return PixelFormat::PIX_FMT_INVALID; } } uint64_t FFmpegDecoder::ValidateChannelLayout(AVStream* stream) { if (stream->codecpar->channel_layout) { return stream->codecpar->channel_layout; } return av_get_default_channel_layout(stream->codecpar->channels); } FramePtr FFmpegDecoder::BuffersToNativeFrame(int divider, int width, int height, int64_t ts, uint8_t** input_data, int* input_linesize) { if (divider != scale_divider_) { FreeScaler(); InitScaler(divider); } // Create frame to return FramePtr copy = Frame::Create(); copy->set_video_params(VideoParams(width, height, native_pix_fmt_, divider)); copy->set_timestamp(Timecode::timestamp_to_time(ts, time_base_)); copy->set_sample_aspect_ratio(aspect_ratio_); copy->allocate(); // Convert frame to RGB/A for the rest of the pipeline uint8_t* output_data = reinterpret_cast(copy->data()); int output_linesize = copy->linesize_bytes(); sws_scale(scale_ctx_, input_data, input_linesize, 0, height, &output_data, &output_linesize); return copy; } int FFmpegDecoderInstance::GetFrame(AVPacket *pkt, AVFrame *frame) { bool eof = false; int ret; // Clear any previous frames av_frame_unref(frame); while ((ret = avcodec_receive_frame(codec_ctx_, frame)) == AVERROR(EAGAIN) && !eof) { // Find next packet in the correct stream index do { // Free buffer in packet if there is one av_packet_unref(pkt); // Read packet from file ret = av_read_frame(fmt_ctx_, pkt); } while (pkt->stream_index != avstream_->index && ret >= 0); if (ret == AVERROR_EOF) { // Don't break so that receive gets called again, but don't try to read again eof = true; // Send a null packet to signal end of avcodec_send_packet(codec_ctx_, nullptr); } else if (ret < 0) { // Handle other error by breaking loop and returning the code we received break; } else { // Successful read, send the packet ret = avcodec_send_packet(codec_ctx_, pkt); // We don't need the packet anymore, so free it av_packet_unref(pkt); if (ret < 0) { break; } } } return ret; } QMutex *FFmpegDecoderInstance::cache_lock() { return &cache_lock_; } QWaitCondition *FFmpegDecoderInstance::cache_wait_cond() { return &cache_wait_cond_; } bool FFmpegDecoderInstance::IsWorking() const { return is_working_; } void FFmpegDecoderInstance::SetWorking(bool working) { is_working_ = working; } void FFmpegDecoderInstance::Seek(int64_t timestamp) { avcodec_flush_buffers(codec_ctx_); av_seek_frame(fmt_ctx_, avstream_->index, timestamp, AVSEEK_FLAG_BACKWARD); } /* OLD UNUSED CODE: Keeping this around in case the code proves useful void FFmpegDecoder::CacheFrameToDisk(AVFrame *f) { QFile save_frame(GetIndexFilename().append(QString::number(f->pts))); if (save_frame.open(QFile::WriteOnly)) { // Save frame to media index int cached_buffer_sz = av_image_get_buffer_size(static_cast(f->format), f->width, f->height, 1); QByteArray cached_frame(cached_buffer_sz, Qt::Uninitialized); av_image_copy_to_buffer(reinterpret_cast(cached_frame.data()), cached_frame.size(), f->data, f->linesize, static_cast(f->format), f->width, f->height, 1); save_frame.write(qCompress(cached_frame, 1)); save_frame.close(); DiskManager::instance()->CreatedFile(save_frame.fileName(), QByteArray()); } // See if we stored this frame in the disk cache QByteArray frame_loader; if (!got_frame) { QFile compressed_frame(GetIndexFilename().append(QString::number(target_ts))); if (compressed_frame.exists() && compressed_frame.size() > 0 && compressed_frame.open(QFile::ReadOnly)) { DiskManager::instance()->Accessed(compressed_frame.fileName()); // Read data frame_loader = qUncompress(compressed_frame.readAll()); av_image_fill_arrays(input_data, input_linesize, reinterpret_cast(frame_loader.data()), static_cast(avstream_->codecpar->format), avstream_->codecpar->width, avstream_->codecpar->height, 1); got_frame = true; } } } */ void FFmpegDecoderInstance::ClearFrameCache() { cached_frames_.clear(); cache_at_eof_ = false; cache_at_zero_ = false; } FFmpegFramePool::ElementPtr FFmpegDecoderInstance::RetrieveFrame(const int64_t& target_ts, bool cache_is_locked) { if (!cache_is_locked) { cache_lock_.lock(); } int64_t seek_ts = target_ts; bool still_seeking = false; // CacheCouldContainTime uses cache_target_time_, so we'll temporarily set it to the last frame's TS if (!cached_frames_.isEmpty()) { cache_target_time_ = cached_frames_.last()->timestamp(); } // If the frame wasn't in the frame cache, see if this frame cache is too old to use if (!CacheCouldContainTime(target_ts)) { ClearFrameCache(); Seek(seek_ts); if (seek_ts == 0) { cache_at_zero_ = true; } still_seeking = true; } cache_target_time_ = target_ts; int ret; AVPacket* pkt = av_packet_alloc(); FFmpegFramePool::ElementPtr return_frame = nullptr; // Allocate a new frame AVFrameWrapper working_frame; bool unlocked = false; while (true) { // Pull from the decoder ret = GetFrame(pkt, working_frame.frame()); // Handle any errors that aren't EOF (EOF is handled later on) if (ret < 0 && ret != AVERROR_EOF) { cache_lock_.unlock(); qCritical() << "Failed to retrieve frame:" << ret; break; } if (still_seeking) { // Handle a failure to seek (occurs on some media) // We'll only be here if the frame cache was emptied earlier if (!cache_at_zero_ && (ret == AVERROR_EOF || working_frame.frame()->pts > target_ts)) { seek_ts = qMax(static_cast(0), seek_ts - second_ts_); Seek(seek_ts); if (seek_ts == 0) { cache_at_zero_ = true; } continue; } else { still_seeking = false; } } if (cache_is_locked) { cache_is_locked = false; } else if (unlocked) { cache_lock_.lock(); } if (ret == AVERROR_EOF) { // Handle an "expected" EOF by using the last frame of our cache cache_at_eof_ = true; return_frame = cached_frames_.last(); cache_wait_cond_.wakeAll(); cache_lock_.unlock(); break; } else { // Whatever it is, keep this frame in memory for the time being just in case if (!frame_pool_) { qCritical() << "Cannot retrieve video without a valid frame pool"; cache_lock_.unlock(); break; } FFmpegFramePool::ElementPtr cached = frame_pool_->Get(working_frame.frame()); if (!cached) { qCritical() << "Frame pool failed to return a valid frame - out of memory?"; cache_lock_.unlock(); break; } // Set timestamp so this frame can be identified later cached->set_timestamp(working_frame.frame()->pts); // Store frame before just in case FFmpegFramePool::ElementPtr previous; if (cached_frames_.isEmpty()) { previous = nullptr; } else { previous = cached_frames_.last(); } // Clear early frames // FIXME: Hardcoded value (only stores a maximum of 2 seconds in the cache at any time) TruncateCacheRangeTo(2*second_ts_); // Append this frame and signal to other threads that a new frame has arrived cached_frames_.append(cached); cache_wait_cond_.wakeAll(); cache_lock_.unlock(); unlocked = true; // If this is a valid frame, see if this or the frame before it are the one we need if (cached->timestamp() == target_ts) { return_frame = cached; break; } else if (cached->timestamp() > target_ts) { if (!previous && cache_at_zero_) { return_frame = cached; break; } else { return_frame = previous; break; } } } } av_packet_free(&pkt); return return_frame; } void FFmpegDecoder::ClearResources() { FreeScaler(); open_ = false; } void FFmpegDecoder::InitScaler(int divider) { VideoStream* vs = static_cast(stream().get()); scale_ctx_ = sws_getContext(vs->width(), vs->height(), src_pix_fmt_, GetScaledDimension(vs->width(), divider), GetScaledDimension(vs->height(), divider), ideal_pix_fmt_, SWS_FAST_BILINEAR, nullptr, nullptr, nullptr); if (scale_ctx_) { scale_divider_ = divider; } else { scale_divider_ = 0; } } void FFmpegDecoder::FreeScaler() { if (scale_ctx_) { sws_freeContext(scale_ctx_); scale_ctx_ = nullptr; scale_divider_ = 0; } } QString FFmpegDecoder::GetProxyFrameFilename(const int64_t ×tamp, const int& divider) const { QString dst_fn = GetProxyFilename(divider); dst_fn.append(QString::number(timestamp)); dst_fn.append(FrameHashCache::GetFormatExtension()); return dst_fn; } int64_t FFmpegDecoderInstance::RangeStart() const { if (cached_frames_.isEmpty()) { return AV_NOPTS_VALUE; } return cached_frames_.first()->timestamp(); } int64_t FFmpegDecoderInstance::RangeEnd() const { if (cached_frames_.isEmpty()) { return AV_NOPTS_VALUE; } return cached_frames_.last()->timestamp(); } bool FFmpegDecoderInstance::CacheContainsTime(const int64_t &t) const { return !cached_frames_.isEmpty() && ((RangeStart() <= t && RangeEnd() >= t) || (cache_at_zero_ && t < cached_frames_.first()->timestamp()) || (cache_at_eof_ && t > cached_frames_.last()->timestamp())); } bool FFmpegDecoderInstance::CacheWillContainTime(const int64_t &t) const { return !cached_frames_.isEmpty() && t >= cached_frames_.first()->timestamp() && t <= cache_target_time_; } bool FFmpegDecoderInstance::CacheCouldContainTime(const int64_t &t) const { return !cached_frames_.isEmpty() && t >= cached_frames_.first()->timestamp() && t <= (cache_target_time_ + 2*second_ts_); } bool FFmpegDecoderInstance::CacheIsEmpty() const { return cached_frames_.isEmpty(); } FFmpegFramePool::ElementPtr FFmpegDecoderInstance::GetFrameFromCache(const int64_t &t) const { if (t < cached_frames_.first()->timestamp()) { if (cache_at_zero_) { cached_frames_.first()->access(); return cached_frames_.first(); } } else if (t > cached_frames_.last()->timestamp()) { if (cache_at_eof_) { cached_frames_.last()->access(); return cached_frames_.last(); } } else { // We already have this frame in the cache, find it for (int i=0;itimestamp() == t // Test for an exact match || (i < cached_frames_.size() - 1 && cached_frames_.at(i+1)->timestamp() > t)) { // Or for this frame to be the "closest" this_frame->access(); return this_frame; } } } return nullptr; } void FFmpegDecoderInstance::RemoveFramesBefore(const qint64 &t) { // We keep one frame in memory as an identifier for what pts the decoder is up to while (cached_frames_.size() > 1 && cached_frames_.first()->last_accessed() < t) { cached_frames_.removeFirst(); cache_at_zero_ = false; } } void FFmpegDecoderInstance::TruncateCacheRangeTo(const qint64 &t) { // We keep one frame in memory as an identifier for what pts the decoder is up to while (cached_frames_.size() > 1 && (RangeEnd() - RangeStart()) > t) { cached_frames_.removeFirst(); cache_at_zero_ = false; } } rational FFmpegDecoderInstance::sample_aspect_ratio() const { return av_guess_sample_aspect_ratio(fmt_ctx_, avstream_, nullptr); } AVStream *FFmpegDecoderInstance::stream() const { return avstream_; } FFmpegDecoderInstance::FFmpegDecoderInstance(const char *filename, int stream_index) : fmt_ctx_(nullptr), opts_(nullptr), frame_pool_(nullptr), is_working_(false), cache_at_zero_(false), cache_at_eof_(false), clear_timer_(nullptr) { // Open file in a format context int error_code = avformat_open_input(&fmt_ctx_, filename, nullptr, nullptr); // Handle format context error if (error_code != 0) { qCritical() << "Failed to open input:" << filename << error_code; ClearResources(); return; } // Get stream information from format error_code = avformat_find_stream_info(fmt_ctx_, nullptr); // Handle get stream information error if (error_code < 0) { qCritical() << "Failed to find stream info:" << error_code; ClearResources(); return; } // Get reference to correct AVStream avstream_ = fmt_ctx_->streams[stream_index]; // Find decoder AVCodec* codec = avcodec_find_decoder(avstream_->codecpar->codec_id); // Handle failure to find decoder if (codec == nullptr) { qCritical() << "Failed to find appropriate decoder for this codec:" << filename << stream_index << avstream_->codecpar->codec_id; ClearResources(); return; } // Allocate context for the decoder codec_ctx_ = avcodec_alloc_context3(codec); if (codec_ctx_ == nullptr) { qCritical() << "Failed to allocate codec context"; ClearResources(); return; } // Copy parameters from the AVStream to the AVCodecContext error_code = avcodec_parameters_to_context(codec_ctx_, avstream_->codecpar); // Handle failure to copy parameters if (error_code < 0) { qCritical() << "Failed to copy parameters from AVStream to AVCodecContext"; ClearResources(); return; } // Set multithreading setting error_code = av_dict_set(&opts_, "threads", "auto", 0); // Handle failure to set multithreaded decoding if (error_code < 0) { qCritical() << "Failed to set codec options, performance may suffer"; } // Open codec error_code = avcodec_open2(codec_ctx_, codec, &opts_); if (error_code < 0) { char buf[50]; av_strerror(error_code, buf, 50); qCritical() << "Failed to open codec" << codec->id << error_code << buf; ClearResources(); return; } // Create frame pool if (avstream_->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { // Start clear timer clear_timer_ = new QTimer(); clear_timer_->setInterval(kMaxFrameLife); //clear_timer_->moveToThread(qApp->thread()); connect(clear_timer_, &QTimer::timeout, this, &FFmpegDecoderInstance::ClearTimerEvent); QMetaObject::invokeMethod(clear_timer_, "start", Qt::QueuedConnection); } // Store one second in the source's timebase second_ts_ = qRound64(av_q2d(av_inv_q(avstream_->time_base))); } FFmpegDecoderInstance::~FFmpegDecoderInstance() { ClearResources(); } bool FFmpegDecoderInstance::IsValid() const { return codec_ctx_; } void FFmpegDecoderInstance::SetFramePool(FFmpegFramePool *frame_pool) { frame_pool_ = frame_pool; } void FFmpegDecoderInstance::ClearResources() { ClearFrameCache(); // Stop timer if (clear_timer_) { if (clear_timer_->thread() == QThread::currentThread()) { clear_timer_->stop(); } else { QMetaObject::invokeMethod(clear_timer_, "stop", Qt::BlockingQueuedConnection); } QMetaObject::invokeMethod(clear_timer_, "deleteLater", Qt::QueuedConnection); clear_timer_ = nullptr; } if (opts_) { av_dict_free(&opts_); opts_ = nullptr; } if (codec_ctx_) { avcodec_free_context(&codec_ctx_); codec_ctx_ = nullptr; } if (fmt_ctx_) { avformat_close_input(&fmt_ctx_); fmt_ctx_ = nullptr; } } void FFmpegDecoderInstance::ClearTimerEvent() { cache_lock()->lock(); RemoveFramesBefore(QDateTime::currentMSecsSinceEpoch() - kMaxFrameLife); cache_lock()->unlock(); } OLIVE_NAMESPACE_EXIT