From 3be8c3e8fd0f9d6e5bbedb9f7f0014daafdb31eb Mon Sep 17 00:00:00 2001
From: Mike Solar <iam@mikesolar.cn>
Date: Sun, 30 Aug 2026 00:31:32 +0800
Subject: [PATCH] codec: prefer native NVDEC over VAAPI on Linux

On multi-GPU boxes VAAPI's default render node can point at a device
with no VA driver (NVIDIA without libva-nvidia-driver), failing before
the working AMD/Intel node is ever tried; trying CUDA (NVDEC) first is
both the discrete-GPU path and sidesteps that misdirection. VAAPI stays
as the fallback for AMD/Intel-only machines.

Adds a hwcheck example printing which hw device a stream opens with and
the raw av_hwdevice_ctx_create result codes per device type.
---
 crates/oak-codec/examples/hwcheck.rs | 44 ++++++++++++++++++++++++++++
 crates/oak-codec/src/hwdecode.rs     | 11 +++++--
 2 files changed, 53 insertions(+), 2 deletions(-)
 create mode 100644 crates/oak-codec/examples/hwcheck.rs

diff --git a/crates/oak-codec/examples/hwcheck.rs b/crates/oak-codec/examples/hwcheck.rs
new file mode 100644
index 000000000..a1062f3a5
--- /dev/null
+++ b/crates/oak-codec/examples/hwcheck.rs
@@ -0,0 +1,44 @@
+// Temporary diagnostic: print the hardware decoder a video stream opens
+// with (None = software fallback).
+// Usage: hwcheck <mediafile> [stream_index]
+
+use oak_codec::decoder::{CodecStream, Decoder};
+use oak_codec::ffmpeg::FFmpegDecoder;
+
+fn probe(device_type: ffmpeg_next::ffi::AVHWDeviceType, name: &str) {
+	let mut dev: *mut ffmpeg_next::ffi::AVBufferRef = std::ptr::null_mut();
+	let rc = unsafe {
+		ffmpeg_next::ffi::av_hwdevice_ctx_create(
+			&mut dev,
+			device_type,
+			std::ptr::null(),
+			std::ptr::null_mut(),
+			0,
+		)
+	};
+	eprintln!("{name}: av_hwdevice_ctx_create rc = {rc}");
+	if rc >= 0 && !dev.is_null() {
+		unsafe { ffmpeg_next::ffi::av_buffer_unref(&mut dev) };
+	}
+}
+
+fn main() {
+	ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Debug);
+	probe(
+		ffmpeg_next::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA,
+		"CUDA",
+	);
+	probe(
+		ffmpeg_next::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
+		"VAAPI",
+	);
+	let args: Vec<String> = std::env::args().collect();
+	if args.len() < 2 {
+		return;
+	}
+	let stream: i32 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(0);
+	let d = FFmpegDecoder::new();
+	let s = CodecStream::with_block(args[1].clone(), stream, None);
+	d.open(&s).expect("open video stream");
+	println!("hw_decoder_name = {:?}", d.hw_decoder_name());
+}
diff --git a/crates/oak-codec/src/hwdecode.rs b/crates/oak-codec/src/hwdecode.rs
index 47c9c7213..62cecaefc 100644
--- a/crates/oak-codec/src/hwdecode.rs
+++ b/crates/oak-codec/src/hwdecode.rs
@@ -31,7 +31,7 @@
 //! hardware surface.
 //!
 //! - **macOS**: `AV_HWDEVICE_TYPE_VIDEOTOOLBOX`
-//! - **Linux**: `VAAPI`, then `CUDA` (NVDEC)
+//! - **Linux**: `CUDA` (NVDEC, the discrete-GPU path), then `VAAPI`
 //! - **Windows**: `D3D11VA`, then `CUDA` (NVDEC)
 //!
 //! Device creation can fail on machines without the device/driver (a
@@ -131,9 +131,16 @@ pub fn device_type_candidates() -> &'static [sys::AVHWDeviceType] {
 	}
 	#[cfg(all(unix, not(target_os = "macos")))]
 	{
+		// CUDA (native NVDEC) first: it is the discrete-GPU path, and on
+		// multi-GPU boxes VAAPI's default render node can point at a
+		// device with no VA driver (e.g. NVIDIA without
+		// libva-nvidia-driver) while the AMD/Intel node would work —
+		// trying NVDEC first sidesteps that misdirection. VAAPI stays as
+		// the fallback for AMD/Intel-only machines (CUDA device creation
+		// fails fast without libcuda).
 		&[
-			sys::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
 			sys::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA,
+			sys::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
 		]
 	}
 }