First commit of brand new rewrite

playback/audio.cpp

@@ -0,0 +1,72 @@
+#include "audio.h"
+
+#include "project/sequence.h"
+
+#include <QAudioOutput>
+#include <QDebug>
+
+extern "C" {
+	#include <libavcodec/avcodec.h>
+}
+
+QAudioOutput* audio_output;
+QIODevice* audio_io_device;
+bool audio_device_set = false;
+
+uint8_t* audio_cache_A = NULL;
+uint8_t* audio_cache_B = NULL;
+int audio_cache_size = 0;
+bool switch_audio_cache = true;
+bool reading_audio_cache_A = false;
+
+void init_audio(Sequence* s) {
+	if (audio_cache_A != NULL) {
+		delete [] audio_cache_A;
+		audio_cache_A = NULL;
+	}
+
+	if (audio_cache_B != NULL) {
+		delete [] audio_cache_B;
+		audio_cache_B = NULL;
+	}
+
+	if (audio_device_set) {
+		audio_output->stop();
+		delete audio_output;
+		audio_device_set = false;
+	}
+
+	if (s != NULL) {
+		QAudioFormat audio_format;
+		audio_format.setSampleRate(s->audio_frequency);
+		audio_format.setChannelCount(av_get_channel_layout_nb_channels(s->audio_layout));
+		audio_format.setSampleSize(16);
+		audio_format.setCodec("audio/pcm");
+		audio_format.setByteOrder(QAudioFormat::LittleEndian);
+		audio_format.setSampleType(QAudioFormat::SignedInt);
+
+		QAudioDeviceInfo info(QAudioDeviceInfo::defaultOutputDevice());
+		if (!info.isFormatSupported(audio_format)) {
+			qWarning() << "[WARNING] Couldn't initialize audio. Audio format is not supported by backend";
+		} else {
+			audio_output = new QAudioOutput(audio_format);
+			// connect
+			audio_io_device = audio_output->start();
+			audio_device_set = true;
+
+			audio_cache_size = av_samples_get_buffer_size(NULL, av_get_channel_layout_nb_channels(s->audio_layout), s->audio_frequency/8, AV_SAMPLE_FMT_S16, 1);
+			audio_cache_A = new uint8_t[audio_cache_size];
+			audio_cache_B = new uint8_t[audio_cache_size];
+			clear_cache(true, true);
+		}
+	}
+}
+
+void clear_cache(bool clear_A, bool clear_B) {
+	if (clear_A) {
+		memset(audio_cache_A, 0, audio_cache_size);
+	}
+	if (clear_B) {
+		memset(audio_cache_B, 0, audio_cache_size);
+	}
+}

playback/audio.h

@@ -0,0 +1,19 @@
+#ifndef AUDIO_H
+#define AUDIO_H
+
+#include <QIODevice>
+#include <QMutex>
+
+struct Sequence;
+
+extern QIODevice* audio_io_device;
+
+extern uint8_t* audio_cache_A;
+extern uint8_t* audio_cache_B;
+extern int audio_cache_size;
+extern bool switch_audio_cache;
+extern bool reading_audio_cache_A;
+void init_audio(Sequence* s);
+void clear_cache(bool clear_A, bool clear_B);
+
+#endif // AUDIO_H

playback/cacher.cpp

@@ -0,0 +1,339 @@
+#include "cacher.h"
+
+#include "project/clip.h"
+#include "project/sequence.h"
+#include "io/media.h"
+#include "playback/audio.h"
+#include "playback/playback.h"
+#include "effects/effects.h"
+
+extern "C" {
+	#include <libavformat/avformat.h>
+	#include <libavcodec/avcodec.h>
+	#include <libswscale/swscale.h>
+	#include <libswresample/swresample.h>
+}
+
+#include <QDebug>
+
+void cache_audio_worker(Clip* c, bool write_A) {
+	// gets one frame worth of audio and sends it to the audio buffer
+	AVFrame* frame = c->cache_A.frames[0];
+	uint8_t* cache = (write_A) ? audio_cache_A : audio_cache_B;
+
+	int bytes_written = 0;
+	int j = 0;
+	while (bytes_written < audio_cache_size && !c->reached_end) {
+		// is there audio left in the frame
+		if (c->frame_sample_index == 0) {
+			// no more audio left in frame, get a new one
+			retrieve_next_frame_raw_data(c, frame);
+		}
+		if (!c->reached_end) {
+			int nb_bytes = av_samples_get_buffer_size(NULL, frame->channels, frame->nb_samples, static_cast<AVSampleFormat>(frame->format), 1);
+			int limit = std::min(c->frame_sample_index + audio_cache_size - bytes_written, nb_bytes);
+			// perform all audio effects
+			for (unsigned int j=0;j<c->effects.size();j++) {
+				c->effects[j]->process_audio(frame->data[0], limit);
+			}
+			// mix audio into cache
+			for (int i=c->frame_sample_index;i<limit;i++) {
+				cache[j] += frame->data[0][i];
+				j++;
+			}
+			bytes_written += limit - c->frame_sample_index;
+			c->frame_sample_index = (limit == nb_bytes) ? 0 : limit;
+		}
+	}
+}
+
+void cache_video_worker(Clip* c, long playhead, ClipCache* cache) {
+	cache->mutex.lock();
+
+	cache->offset = playhead;
+	for (size_t i=0;i<c->cache_size;i++) {
+		retrieve_next_frame_raw_data(c, cache->frames[i]);
+	}
+	cache->written = true;
+	cache->unread = true;
+
+	cache->mutex.unlock();
+}
+
+void reset_cache(Clip* c, long target_frame) {
+	// if we seek to a whole other place in the timeline, we'll need to reset the cache with new values
+	if (c->media_stream->infinite_length) {
+		// if this clip is a still image, we only need one frame
+		if (!c->cache_A.written) {
+			retrieve_next_frame_raw_data(c, c->cache_A.frames[0]);
+			c->cache_A.written = true;
+		}
+	} else {
+		// flush ffmpeg codecs
+		avcodec_flush_buffers(c->codecCtx);
+
+		double timebase = av_q2d(c->stream->time_base);
+
+		if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+			// seeks to nearest keyframe (target_frame represents internal clip frame)
+			av_seek_frame(c->formatCtx, c->media_stream->file_index, clip_frame_to_seconds(c, target_frame) / timebase, AVSEEK_FLAG_BACKWARD);
+
+			// play up to the frame we actually want
+			long retrieved_frame = 0;
+			AVFrame* temp = av_frame_alloc();
+			do {
+				retrieve_next_frame(c, temp);
+				if (retrieved_frame == 0) {
+					if (target_frame != 0) {
+//						retrieved_frame = floor(av_frame_get_best_effort_timestamp(temp) * timebase * av_q2d(c->stream->avg_frame_rate));
+						retrieved_frame = floor(temp->pts * timebase * av_q2d(av_guess_frame_rate(c->formatCtx, c->stream, temp)));
+					}
+				} else {
+					retrieved_frame++;
+				}
+			} while (retrieved_frame < target_frame);
+			av_frame_free(&temp);
+		} else if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+			// seek (target_frame represents timeline timecode in frames, not clip timecode)
+			swr_drop_output(c->swr_ctx, swr_get_out_samples(c->swr_ctx, 0));
+			av_seek_frame(c->formatCtx, c->media_stream->file_index, playhead_to_seconds(c, target_frame) / timebase, AVSEEK_FLAG_ANY);
+		}
+	}
+}
+
+Cacher::Cacher(Clip* c) : clip(c) {}
+
+void open_clip_worker(Clip* clip) {
+	// opens file resource for FFmpeg and prepares Clip struct for playback
+	QByteArray ba = clip->media->url.toUtf8();
+	const char* filename = ba.constData();
+
+	int errCode = avformat_open_input(
+			&clip->formatCtx,
+			filename,
+			NULL,
+			NULL
+		);
+	if (errCode != 0) {
+		char err[1024];
+		av_strerror(errCode, err, 1024);
+		qDebug() << "[ERROR] Could not open" << filename << "-" << err;
+	}
+
+	errCode = avformat_find_stream_info(clip->formatCtx, NULL);
+	if (errCode < 0) {
+		char err[1024];
+		av_strerror(errCode, err, 1024);
+		qDebug() << "[ERROR] Could not open" << filename << "-" << err;
+	}
+
+	av_dump_format(clip->formatCtx, 0, filename, 0);
+
+	clip->stream = clip->formatCtx->streams[clip->media_stream->file_index];
+	clip->codec = avcodec_find_decoder(clip->stream->codecpar->codec_id);
+	clip->codecCtx = avcodec_alloc_context3(clip->codec);
+	avcodec_parameters_to_context(clip->codecCtx, clip->stream->codecpar);
+
+	AVDictionary* opts = NULL;
+
+	// decoding optimization configuration
+	if (clip->stream->codecpar->codec_id != AV_CODEC_ID_PNG &&
+		clip->stream->codecpar->codec_id != AV_CODEC_ID_APNG &&
+		clip->stream->codecpar->codec_id != AV_CODEC_ID_TIFF &&
+		clip->stream->codecpar->codec_id != AV_CODEC_ID_PSD) {
+		av_dict_set(&opts, "threads", "auto", 0);
+	}
+	if (clip->stream->codecpar->codec_id == AV_CODEC_ID_H264) {
+		av_dict_set(&opts, "tune", "fastdecode", 0);
+		av_dict_set(&opts, "tune", "zerolatency", 0);
+	}
+
+	// Open codec
+	if (avcodec_open2(clip->codecCtx, clip->codec, &opts) < 0) {
+		qDebug() << "[ERROR] Could not open codec";
+	}
+
+	if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+		// set up swscale context - primarily used for colorspace conversion
+		// as "scaling" is actually done by OpenGL
+		int dest_format = AV_PIX_FMT_RGBA;
+
+		clip->sws_ctx = sws_getContext(
+				clip->stream->codecpar->width,
+				clip->stream->codecpar->height,
+				static_cast<AVPixelFormat>(clip->stream->codecpar->format),
+				ceil(clip->stream->codecpar->width/2)*2,
+				ceil(clip->stream->codecpar->height/2)*2,
+				static_cast<AVPixelFormat>(dest_format),
+				SWS_FAST_BILINEAR,
+				NULL,
+				NULL,
+				NULL
+			);
+
+		// create memory cache for video
+		if (clip->media_stream->infinite_length) {
+			clip->cache_size = 1;
+		} else {
+			clip->cache_size = ceil(av_q2d(av_guess_frame_rate(clip->formatCtx, clip->stream, NULL))/4); // cache is half a second in total
+
+			// infinite length doesn't need cache B
+			clip->cache_B.frames = new AVFrame* [clip->cache_size];
+
+		}
+		clip->cache_A.frames = new AVFrame* [clip->cache_size];
+
+
+		for (size_t i=0;i<clip->cache_size;i++) {
+			clip->cache_A.frames[i] = av_frame_alloc();
+			av_frame_make_writable(clip->cache_A.frames[i]);
+			clip->cache_A.frames[i]->width = clip->stream->codecpar->width;
+			clip->cache_A.frames[i]->height = clip->stream->codecpar->height;
+			clip->cache_A.frames[i]->format = dest_format;
+			av_frame_get_buffer(clip->cache_A.frames[i], 0);
+			clip->cache_A.frames[i]->linesize[0] = clip->stream->codecpar->width*4;
+
+			if (!clip->media_stream->infinite_length) {
+				clip->cache_B.frames[i] = av_frame_alloc();
+				av_frame_make_writable(clip->cache_B.frames[i]);
+				clip->cache_B.frames[i]->width = clip->stream->codecpar->width;
+				clip->cache_B.frames[i]->height = clip->stream->codecpar->height;
+				clip->cache_B.frames[i]->format = dest_format;
+				av_frame_get_buffer(clip->cache_B.frames[i], 0);
+				clip->cache_B.frames[i]->linesize[0] = clip->stream->codecpar->width*4;
+			}
+		}
+	} else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+		// if FFmpeg can't pick up the channel layout (usually WAV), assume
+		// based on channel count (doesn't support surround sound sources yet)
+		if (clip->codecCtx->channel_layout == 0) {
+			switch (clip->stream->codecpar->channels) {
+				case 1: clip->codecCtx->channel_layout = AV_CH_LAYOUT_MONO;
+				default:
+					clip->codecCtx->channel_layout = AV_CH_LAYOUT_STEREO;
+					qDebug() << "[WARNING] Could not detect audio channel layout - assuming stereo";
+					break;
+			}
+		}
+
+		int sample_format = AV_SAMPLE_FMT_S16;
+
+		// init resampling context
+		clip->swr_ctx = swr_alloc_set_opts(
+				NULL,
+				clip->sequence->audio_layout,
+				static_cast<AVSampleFormat>(sample_format),
+				clip->sequence->audio_frequency,
+				clip->stream->codecpar->channel_layout,
+				static_cast<AVSampleFormat>(clip->stream->codecpar->format),
+				clip->stream->codecpar->sample_rate,
+				0,
+				NULL
+			);
+		swr_init(clip->swr_ctx);
+
+		// set up cache
+		clip->cache_A.frames = new AVFrame* [1];
+		clip->cache_A.frames[0] = av_frame_alloc();
+		clip->cache_A.frames[0]->format = sample_format;
+		clip->cache_A.frames[0]->channel_layout = clip->sequence->audio_layout;
+		clip->cache_A.frames[0]->channels = av_get_channel_layout_nb_channels(clip->cache_A.frames[0]->channel_layout);
+		clip->cache_A.frames[0]->sample_rate = clip->sequence->audio_frequency;
+		av_frame_make_writable(clip->cache_A.frames[0]);
+
+		clip->reset_audio = true;
+	}
+
+	clip->frame = av_frame_alloc();
+
+	qDebug() << "[INFO] Clip opened on track" << clip->track;
+
+	clip->open = true;
+}
+
+void cache_clip_worker(Clip* clip, long playhead, bool write_A, bool write_B, bool reset) {
+	if (reset) {
+		// note: for video, playhead is in "internal clip" frames - for audio, it's the timeline playhead
+		reset_cache(clip, playhead);
+		clip->reset_audio = false;
+	}
+
+	if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+		if (write_A) {
+			cache_video_worker(clip, playhead, &clip->cache_A);
+			playhead += clip->cache_size;
+		}
+
+		if (write_B) {
+			cache_video_worker(clip, playhead, &clip->cache_B);
+		}
+	} else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+		if (write_A) {
+			cache_audio_worker(clip, true);
+		}
+		if (write_B) {
+			cache_audio_worker(clip, false);
+		}
+	}
+}
+
+void close_clip_worker(Clip* clip) {
+	// closes ffmpeg file handle and frees any memory used for caching
+	if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+		sws_freeContext(clip->sws_ctx);
+	} else if (clip->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+		swr_free(&clip->swr_ctx);
+	}
+
+	avcodec_close(clip->codecCtx);
+	avcodec_free_context(&clip->codecCtx);
+	avformat_close_input(&clip->formatCtx);
+
+	for (size_t i=0;i<clip->cache_size;i++) {
+		av_frame_free(&clip->cache_A.frames[i]);
+		if (!clip->media_stream->infinite_length) av_frame_free(&clip->cache_B.frames[i]);
+	}
+	delete [] clip->cache_A.frames;
+	if (!clip->media_stream->infinite_length) delete [] clip->cache_B.frames;
+
+	av_frame_free(&clip->frame);
+
+	clip->reset();
+
+	// remove clip from current_clips
+	cc_lock.lock();
+	bool found = false;
+	for (int i=0;i<current_clips.count();i++) {
+		if (current_clips[i] == clip) {
+			current_clips.erase(current_clips.begin()+i);
+			found = true;
+			i = current_clips.count();
+		}
+	}
+	cc_lock.unlock();
+	if (!found) qDebug() << "[WARNING] Could not remove clip from current clips";
+
+	qDebug() << "[INFO] Clip closed on track" << clip->track;
+}
+
+void Cacher::run() {
+	// open_lock is used to prevent the clip from being destroyed before the cacher has closed it properly
+	clip->open_lock.lock();
+
+	open_clip_worker(clip);
+
+	caching = true;
+	while (true) {
+		clip->can_cache.wait(&clip->lock);
+		if (!caching) {
+			break;
+		} else {
+			cache_clip_worker(clip, playhead, write_A, write_B, reset);
+		}
+	}
+
+	close_clip_worker(clip);
+	clip->lock.unlock();
+
+	clip->open_lock.unlock();
+}

playback/cacher.h

@@ -0,0 +1,31 @@
+#ifndef CACHER_H
+#define CACHER_H
+
+#include <QThread>
+
+struct Clip;
+
+class Cacher : public QThread
+{
+//	Q_OBJECT
+public:
+	Cacher(Clip* c);
+	void run() override;
+
+	bool caching;
+
+	// must be set before caching
+	long playhead;
+	bool write_A;
+	bool write_B;
+	bool reset;
+
+private:
+	Clip* clip;
+};
+
+void open_clip_worker(Clip* clip);
+void cache_clip_worker(Clip* clip, long playhead, bool write_A, bool write_B, bool reset);
+void close_clip_worker(Clip* clip);
+
+#endif // CACHER_H

playback/playback.cpp

@@ -0,0 +1,288 @@
+#include "playback.h"
+
+#include "project/clip.h"
+#include "project/sequence.h"
+#include "io/media.h"
+#include "playback/audio.h"
+#include "playback/cacher.h"
+#include "panels/timeline.h"
+#include <algorithm>
+
+extern "C" {
+	#include <libavformat/avformat.h>
+	#include <libavcodec/avcodec.h>
+	#include <libswscale/swscale.h>
+	#include <libswresample/swresample.h>
+}
+
+#include <QObject>
+#include <QOpenGLTexture>
+#include <QDebug>
+#include <QOpenGLPixelTransferOptions>
+
+QList<Clip*> current_clips;
+bool texture_failed = false;
+
+QMutex cc_lock;
+
+void handle_media(Sequence* sequence, long playhead, bool multithreaded) {
+	for (int i=0;i<sequence->clip_count();i++) {
+		Clip& c = sequence->get_clip(i);
+
+		// if clip starts within one second and/or hasn't finished yet
+		if (is_clip_active(&c, playhead)) {
+			// if thread is already working, we don't want to touch this,
+			// but we also don't want to hang the UI thread
+			if (!c.open) {
+				if (c.lock.tryLock()) {
+					open_clip(&c, multithreaded);
+
+					// add to current_clips, (insertion) sorted by track so composite them in order
+					cc_lock.lock();
+					bool found = false;
+					for (int j=0;j<current_clips.size();j++) {
+						if (current_clips[j]->track < c.track) {
+							current_clips.insert(current_clips.begin()+j, &c);
+							found = true;
+							break;
+						}
+					}
+					if (!found) {
+						current_clips.push_back(&c);
+					}
+					cc_lock.unlock();
+				}
+			}
+		} else if (c.open) {			
+			close_clip(&c);
+		}
+	}
+}
+
+void open_clip(Clip* clip, bool multithreaded) {
+	clip->multithreaded = multithreaded;
+	if (multithreaded) {
+		// maybe keep cacher instance in memory while clip exists for performance?
+		clip->cacher = new Cacher(clip);
+		QObject::connect(clip->cacher, SIGNAL(finished()), clip->cacher, SLOT(deleteLater()));
+
+		clip->cacher->start(QThread::LowPriority);
+	} else {
+		open_clip_worker(clip);
+		clip->lock.unlock();
+	}
+}
+
+void close_clip(Clip* clip) {
+	// destroy opengl texture in main thread
+	if (clip->texture != NULL) {
+		clip->texture->destroy();
+		clip->texture = NULL;
+	}
+
+	if (clip->multithreaded) {
+		clip->cacher->caching = false;
+		clip->can_cache.wakeAll();
+	} else {
+		close_clip_worker(clip);
+	}
+}
+
+void cache_clip(Clip* clip, long playhead, bool write_A, bool write_B, bool reset) {
+	if (clip->multithreaded) {
+		clip->cacher->playhead = playhead;
+		clip->cacher->write_A = write_A;
+		clip->cacher->write_B = write_B;
+		clip->cacher->reset = reset;
+
+		clip->can_cache.wakeAll();
+	} else {
+		cache_clip_worker(clip, playhead, write_A, write_B, reset);
+	}
+}
+
+void get_clip_frame(Clip* c, long playhead) {
+	if (c->open) {
+		long clip_time = seconds_to_clip_frame(c, playhead_to_seconds(c, playhead));
+
+		// do we need to update the texture?
+		if ((!c->media_stream->infinite_length && c->texture_frame != clip_time) ||
+				(c->media_stream->infinite_length && c->texture_frame == -1)) {
+			AVFrame* current_frame = NULL;
+
+			// get frame data
+			if (c->media_stream->infinite_length) { // if clip is a still frame, we only need one
+				if (c->cache_A.written) {
+					// retrieve cached frame
+					current_frame = c->cache_A.frames[0];
+				} else if (c->multithreaded) {
+					if (c->lock.tryLock()) {
+						// grab image (multi-threaded)
+						cache_clip(c, 0, false, false, true);
+						c->lock.unlock();
+					}
+				} else {
+					// grab image (single-threaded)
+					reset_cache(c, playhead);
+				}
+			} else {
+				// keeping a RAM cache improves performance, however it's detrimental when rendering
+				// determine which cache contains the requested frame
+				bool using_cache_A = false;
+				bool using_cache_B = false;
+				AVFrame** cache = NULL;
+				long cache_offset = 0;
+				bool cache_needs_reset = false;
+
+				if (c->cache_A.written && clip_time >= c->cache_A.offset && clip_time < c->cache_A.offset + c->cache_size) {
+					if (c->cache_A.mutex.tryLock()) { // lock in case cacher is still writing to it
+						using_cache_A = true;
+						c->cache_A.unread = false;
+						cache = c->cache_A.frames;
+						cache_offset = c->cache_A.offset;
+						c->cache_A.mutex.unlock();
+					}
+				} else if (c->cache_B.written && clip_time >= c->cache_B.offset && clip_time < c->cache_B.offset + c->cache_size) {
+					if (c->cache_B.mutex.tryLock()) { // lock in case cacher is still writing to it
+						using_cache_B = true;
+						c->cache_B.unread = false;
+						cache = c->cache_B.frames;
+						cache_offset = c->cache_B.offset;
+						c->cache_B.mutex.unlock();
+					}
+				} else {
+					// this is technically bad, unless we just seeked
+					c->cache_A.unread = c->cache_B.unread = false;
+					cache_needs_reset = true;
+				}
+
+				if (cache != NULL) {
+					current_frame = cache[clip_time - cache_offset];
+				}
+
+				// determine whether we should start filling the other cache
+				if (!using_cache_A || !using_cache_B) {
+					if (c->lock.tryLock()) {
+						bool write_A = (!using_cache_A && !c->cache_A.unread);
+						bool write_B = (!using_cache_B && !c->cache_B.unread);
+						if (write_A || write_B) {
+							long playhead;
+							if (cache_needs_reset) {
+								// if we have no cache and need to seek, start us at the current playhead...
+								playhead = clip_time;
+							} else {
+								// ...otherwise start at the end of the current cache
+								playhead = cache_offset + c->cache_size;
+							}
+							cache_clip(c, playhead, write_A, write_B, cache_needs_reset);
+						}
+						c->lock.unlock();
+					}
+				}
+			}
+
+			if (current_frame != NULL) {
+				// set up opengl texture
+				if (c->texture == NULL) {
+					c->texture = new QOpenGLTexture(QOpenGLTexture::Target2D);
+					c->texture->setSize(c->media_stream->video_width, c->media_stream->video_height);
+					c->texture->setFormat(QOpenGLTexture::RGBA8_UNorm);
+					c->texture->setMipLevels(c->texture->maximumMipLevels());
+					c->texture->setMinMagFilters(QOpenGLTexture::Linear, QOpenGLTexture::Linear);
+					c->texture->allocateStorage(QOpenGLTexture::RGBA, QOpenGLTexture::UInt8);
+				}
+
+				c->texture->setData(0, QOpenGLTexture::RGBA, QOpenGLTexture::UInt8, current_frame->data[0]);
+				c->texture_frame = clip_time;
+			} else {
+				texture_failed = true;
+				qDebug() << "[ERROR] Failed to retrieve frame from cache (R:" << clip_time << "| A:" << c->cache_A.offset << "-" << c->cache_A.offset+c->cache_size-1 << "| B:" << c->cache_B.offset << "-" << c->cache_B.offset+c->cache_size-1 << "| WA:" << c->cache_A.written << "| WB:" << c->cache_B.written << ")";
+			}
+		}
+	}
+}
+
+float playhead_to_seconds(Clip* c, long playhead) {
+	// returns time in seconds
+	return (std::max((long) 0, playhead - c->timeline_in) + c->clip_in)/c->sequence->frame_rate;
+}
+
+long seconds_to_clip_frame(Clip* c, float seconds) {
+	// returns time as frame number (according to clip's frame rate)
+	if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+		return floor(seconds*av_q2d(av_guess_frame_rate(c->formatCtx, c->stream, c->frame)));
+	} else {
+		qDebug() << "[ERROR] seconds_to_clip_frame only works on video streams";
+		return 0;
+	}
+}
+
+float clip_frame_to_seconds(Clip* c, long clip_frame) {
+	// returns frame number as seconds (decimal)
+	if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+		return (double) clip_frame / av_q2d(c->stream->avg_frame_rate);
+	} else {
+		qDebug() << "[ERROR] clip_frame_to_seconds only works on video streams";
+		return 0;
+	}
+}
+
+int retrieve_next_frame(Clip* c, AVFrame* f) {
+	int result = 0;
+	int receive_ret;
+
+	// could be optimized if "linked stream" shares timeline_in and clip_in, only use one read_frame?
+	// depends how much time av_read_frame needs, the decoding is probably far more consuming anyway
+
+	// do we need to retrieve a new packet for a new frame?
+	while ((receive_ret = avcodec_receive_frame(c->codecCtx, f)) == AVERROR(EAGAIN)) {
+		int read_ret = 0;
+		do {
+			if (c->pkt_written) {
+				av_packet_unref(c->pkt);
+			}
+			read_ret = av_read_frame(c->formatCtx, c->pkt);
+			c->pkt_written = true;
+		} while (read_ret >= 0 && c->pkt->stream_index != c->media_stream->file_index);
+
+		if (read_ret >= 0) {
+			int send_ret = avcodec_send_packet(c->codecCtx, c->pkt);
+			if (send_ret < 0) {
+				qDebug() << "[ERROR] Failed to send packet to decoder." << send_ret;
+				result = send_ret;
+				break;
+			}
+		} else {
+			if (read_ret != AVERROR_EOF) qDebug() << "[ERROR] Could not read frame." << read_ret;
+			return read_ret; // skips trying to find a frame at all
+		}
+	}
+	if (receive_ret < 0) {
+		qDebug() << "[ERROR] Failed to receive packet from decoder." << receive_ret;
+		result = receive_ret;
+	}
+
+	return result;
+}
+
+void retrieve_next_frame_raw_data(Clip* c, AVFrame* output) {
+	int ret = retrieve_next_frame(c, c->frame);
+	if (ret >= 0) {
+		if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+			sws_scale(c->sws_ctx, c->frame->data, c->frame->linesize, 0, c->stream->codecpar->height, output->data, output->linesize);
+		} else if (c->stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+			ret = swr_convert_frame(c->swr_ctx, output, c->frame);
+			if (ret < 0) {
+				qDebug() << "[ERROR] Failed to resample audio." << ret;
+			}
+		}
+	} else if (ret == AVERROR_EOF) {
+		c->reached_end = true;
+	} else {
+		qDebug() << "[WARNING] Raw frame data could not be retrieved." << ret;
+	}
+}
+
+bool is_clip_active(Clip* c, long playhead) {
+	return c->timeline_in < playhead + ceil(c->sequence->frame_rate) && c->timeline_out > playhead;
+}

playback/playback.h

@@ -0,0 +1,41 @@
+#ifndef PLAYBACK_H
+#define PLAYBACK_H
+
+#include <QVector>
+#include <QMutex>
+
+struct Clip;
+struct ClipCache;
+struct Sequence;
+struct AVFrame;
+
+extern QList<Clip*> current_clips;
+extern QMutex cc_lock;
+
+extern bool texture_failed;
+
+void open_clip(Clip* clip, bool multithreaded);
+void cache_clip(Clip* clip, long playhead, bool write_A, bool write_B, bool reset);
+void close_clip(Clip* clip);
+void cache_audio_worker(Clip* c, bool write_A);
+void cache_video_worker(Clip* c, long playhead, ClipCache* cache);
+void handle_media(Sequence* sequence, long playhead, bool multithreaded);
+void reset_cache(Clip* c, long target_frame);
+void get_clip_frame(Clip* c, long playhead);
+float playhead_to_seconds(Clip* c, long playhead);
+long seconds_to_clip_frame(Clip* c, float seconds);
+float clip_frame_to_seconds(Clip* c, long clip_frame);
+int retrieve_next_frame(Clip* c, AVFrame* f);
+void retrieve_next_frame_raw_data(Clip* c, AVFrame* output);
+bool is_clip_active(Clip* c, long playhead);
+void get_next_audio(Clip* c, bool mix);
+
+struct ClipCacheData {
+	Clip& clip;
+	long playhead;
+	bool write_A;
+	bool write_B;
+	bool reset;
+};
+
+#endif // PLAYBACK_H