name: CI

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

concurrency:
  group: ci-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  linux:
    name: Build & test (Linux)
    runs-on: warp-ubuntu-latest-x64-8x
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # gpui/ is a git submodule; its crates are workspace members of
          # their own repo and build as path dependencies of oakapp.
          submodules: true

      - name: Install Rust (stable)
        uses: dtolnay/rust-toolchain@stable

      # ------------------------------------------------------------------
      # System dependencies
      # ------------------------------------------------------------------
      - name: Install system dependencies
        run: |
          sudo apt-get update
          # The codec/filter libraries behind FFmpeg come from the vcpkg
          # manifest (root vcpkg.json; see docs/build.md) — this list is
          # the toolchain vcpkg itself needs, cmake/make for the vendored
          # OpenColorIO build (ocio-sys `bundled`; Ubuntu's
          # libopencolorio-dev is 2.1, older than the bridge's API floor),
          # and the headless test infra gpui needs: X11, software Mesa
          # Vulkan (lavapipe) and xvfb.
          sudo apt-get install -y \
            build-essential clang libclang-dev cmake pkg-config nasm \
            git curl zip unzip tar python3 \
            libpipewire-0.3-dev libspa-0.2-dev libjack-jackd2-dev \
            libasound2-dev libpulse-dev libsndfile1-dev \
            libgl1-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers \
            libvulkan-dev libxkbcommon-dev libxkbcommon-x11-dev xvfb

      # ------------------------------------------------------------------
      # vcpkg (manifest mode) + caches
      # ------------------------------------------------------------------
      # The runner has no vcpkg preinstalled; bootstrap a fresh clone.
      # The manifest at the repo root pins the dependency set (FFmpeg 9.0.1
      # via `overrides`, everything else via `builtin-baseline`) and the
      # resolved tree lands in vcpkg_installed/.
      - name: Bootstrap vcpkg
        run: |
          git clone --depth 1 https://github.com/microsoft/vcpkg.git .cache/vcpkg
          .cache/vcpkg/bootstrap-vcpkg.sh -disableMetrics
          echo "$PWD/.cache/vcpkg" >> "$GITHUB_PATH"
          echo "VCPKG_ROOT=$PWD/.cache/vcpkg" >> "$GITHUB_ENV"

      # The archives dir is vcpkg's binary cache: a manifest bump then
      # rebuilds only what changed. Same key scheme as the Windows job.
      - name: Cache vcpkg artifacts
        uses: actions/cache@v4
        with:
          path: |
            vcpkg_installed
            ~/.cache/vcpkg/archives
          key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json') }}
          save-always: true

      - name: Install dependencies (vcpkg manifest)
        run: vcpkg install --triplet x64-linux

      # ------------------------------------------------------------------
      # Build environment
      # ------------------------------------------------------------------
      # ocio-sys builds a stub bridge unless these are set; the oak-core
      # ocioutils tests need the real library.
      # tooling/ocio-env.sh: vendored static OCIO (the [patch.crates-io]
      # ocio-sys tracks shaloong/ocio-rs main, whose vendored sources build
      # on GCC >= 16).
      - name: Configure build environment
        run: |
          bash tooling/ocio-env.sh >> "$GITHUB_ENV"
          prefix="$PWD/vcpkg_installed/x64-linux"
          echo "FFMPEG_DIR=$prefix" >> "$GITHUB_ENV"
          echo "PKG_CONFIG_PATH=$prefix/lib/pkgconfig" >> "$GITHUB_ENV"
          echo "$prefix/tools/pkgconf" >> "$GITHUB_PATH"

      # ------------------------------------------------------------------
      # Caches
      # ------------------------------------------------------------------
      # Covers the whole target/ dir plus ~/.cargo; shared across branches
      # of the same OS.
      - name: Cache cargo artifacts
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: oak-ci-linux
          cache-on-failure: true

      # ------------------------------------------------------------------
      # Build & test
      # ------------------------------------------------------------------
      # `cargo check` (not build): the Test step links the test binaries
      # anyway, and a full build would codegen every workspace crate twice
      # (once without and once with cfg(test)).
      - name: Build
        run: cargo check --workspace --locked

      # xvfb + 24-bit screen: the gpui #[gpui::test] tests open real windows
      # and render through wgpu on Mesa's software Vulkan (lavapipe).
      # The watchdog bounds the step: a deadlocked test produces no output
      # and no failure, so after 1800 s it dumps every hung process's
      # thread stacks and kills the suite.
      - name: Test
        run: |
          sudo apt-get install -y gdb
          run_suite() {
            xvfb-run -a -s "-screen 0 1920x1080x24" cargo test --workspace --locked &
            TEST_PID=$!
            # The watchdog inherits the step's stdout/stderr; detach it so
            # it cannot keep the runner's I/O pipes open after the step
            # ends ("WaitDelay expired before I/O complete" otherwise).
            (
              sleep 1800
              echo "::warning::test suite exceeded 1800s; dumping hung-process stacks"
              for p in $(pgrep -f 'target/debug/deps/|target/debug/oak-worker'); do
                echo "===== thread stacks of pid $p ($(readlink /proc/$p/exe 2>/dev/null)) ====="
                sudo gdb -batch -ex 'thread apply all bt' -p "$p" || true
              done
              pkill -9 -f 'target/debug/deps/' || true
              pkill -9 -f 'target/debug/oak-worker' || true
            ) >/dev/null 2>&1 &
            WATCHDOG_PID=$!
            wait $TEST_PID
            rc=$?
            kill $WATCHDOG_PID 2>/dev/null || true
            # Reap the watchdog so no child holds the step's pipes.
            wait $WATCHDOG_PID 2>/dev/null || true
            return $rc
          }
          # Retry once (same policy as the Windows job): worker-pool
          # startup under full-suite parallelism has flaked once
          # (full_res_worker_outlives_a_dropped_project: the render
          # manager's process dispatcher failed to start while every other
          # test passed). A real regression fails both passes.
          if ! run_suite; then
            echo "first pass failed; retrying once for worker-pool flakes"
            run_suite
          fi

      # A crashing (SIGSEGV) test gives no Rust backtrace; rerun the
      # crashing test binaries under gdb to capture the native stack.
      # `--args` is required — plain `--` makes gdb treat the test args as
      # a core file. The extra probes target loader-stage crashes (the
      # copier_test SIGSEGV happens inside ld.so's dl_main): si_addr/si_code
      # pin down the fault type, the dynsym dump exposes symbols the
      # executable exports for interposition, strace shows the last loader
      # syscalls, and valgrind catches a corrupting static initializer.
      - name: Backtrace on test failure
        if: failure()
        run: |
          sudo apt-get install -y gdb strace valgrind
          for name in node_e2e_test suites_test copier_test; do
            BIN=$(ls -t target/debug/deps/$name-* | grep -v '\.d$' | head -1)
            [ -n "$BIN" ] || continue
            echo "===== $BIN ====="
            file "$BIN" || true
            echo "--- exported defined dynsyms:"
            readelf --dyn-syms -W "$BIN" 2>/dev/null | grep -v ' UND ' | tail -n +4 | head -30 || true
            echo "--- strace tail:"
            strace -f "$BIN" --list 2>&1 | tail -15 || true
            echo "--- valgrind tail:"
            valgrind -q "$BIN" --list 2>&1 | tail -25 || true
            echo "--- gdb:"
            xvfb-run -a gdb -batch \
              -ex run \
              -ex 'bt' \
              -ex 'p $_siginfo.si_code' \
              -ex 'p/x $_siginfo._sifields._sigfault.si_addr' \
              -ex 'x/6i $rip' \
              --args "$BIN" --nocapture || true
          done

      # ------------------------------------------------------------------
      # OFX plugin discovery end-to-end
      # ------------------------------------------------------------------
      # Build a minimal but real OFX plugin into a .ofx.bundle, point
      # OFX_PLUGIN_PATH at it and let the scan_probe example run the full
      # host path (directory scan -> dlopen -> setHost -> load -> describe
      # -> register). The assertion is the plugin's registration line; CI
      # machines have no system-wide OFX plugins, so the fixture is the
      # only discovery.
      - name: Build OFX fixture plugin
        run: crates/oak-plugin/tests/fixtures/build_fixture.sh .cache/ofx-fixture

      - name: Probe OFX plugin discovery
        run: |
          OFX_PLUGIN_PATH="$PWD/.cache/ofx-fixture" \
            cargo run --locked -p oak-plugin --example scan_probe > probe.log 2>&1
          grep -q 'type_id=rs.oak.CiTestPlugin' probe.log
          # A project carrying a plugin node must survive save/load (the
          # serializer resolves plugin types via the dynamic factory).
          OAK_OFX_FIXTURE_DIR="$PWD/.cache/ofx-fixture" \
            cargo test --locked -p oak-plugin --test ofx_roundtrip

  windows:
    name: Build & test (Windows)
    runs-on: warp-windows-2025-vs2026-x64-16x
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # gpui/ is a git submodule; its crates are workspace members of
          # their own repo and build as path dependencies of oakapp.
          submodules: true

      - name: Install Rust (stable, MSVC)
        uses: dtolnay/rust-toolchain@stable
        with:
          # The Windows build is MSVC-ABI (the runner carries VS 2026):
          # vcpkg's FFmpeg and the vendored OCIO build both want it.
          toolchain: stable-x86_64-pc-windows-msvc

      # ------------------------------------------------------------------
      # vcpkg (manifest mode) + caches
      # ------------------------------------------------------------------
      # vcpkg.json at the repo root pins the dependency set (FFmpeg with
      # every free codec + hwaccel, pkgconf, librsvg); the resolved tree
      # lands in vcpkg_installed/ and is keyed on the manifest. The
      # binary-cache archives dir makes a manifest bump rebuild cheap.
      # Bootstrap a fresh clone rather than leaning on whatever vcpkg the
      # image carries: `builtin-baseline`/`overrides` are only honored by
      # a recent vcpkg-tool, and all three OS jobs must behave alike.
      - name: Bootstrap vcpkg
        run: |
          git clone --depth 1 https://github.com/microsoft/vcpkg.git "$env:GITHUB_WORKSPACE\.cache\vcpkg"
          & "$env:GITHUB_WORKSPACE\.cache\vcpkg\bootstrap-vcpkg.bat" -disableMetrics
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          "$env:GITHUB_WORKSPACE\.cache\vcpkg" >> $env:GITHUB_PATH
          "VCPKG_ROOT=$env:GITHUB_WORKSPACE\.cache\vcpkg" >> $env:GITHUB_ENV

      - name: Cache vcpkg artifacts
        uses: actions/cache@v4
        with:
          path: |
            vcpkg_installed
            ~/AppData/Local/vcpkg/archives
          key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json') }}
          save-always: true

      - name: Install dependencies (vcpkg manifest)
        run: vcpkg install --triplet x64-windows

      # ------------------------------------------------------------------
      # Build environment
      # ------------------------------------------------------------------
      - name: Configure build environment
        run: |
          $prefix = "$env:GITHUB_WORKSPACE\vcpkg_installed\x64-windows"
          "FFMPEG_DIR=$prefix" >> $env:GITHUB_ENV
          "PKG_CONFIG_PATH=$prefix\lib\pkgconfig" >> $env:GITHUB_ENV
          "$prefix\tools\pkgconf" >> $env:GITHUB_PATH
          # Bundled OCIO: ocio-sys' vendored sources build with the MSVC
          # toolchain (what they need — the MSYS2 package was the
          # workaround, not the preference), so no OCIO_INSTALL_DIR and
          # no OCIO_RS_NO_MSVC_INCLUDES anywhere.
          "OCIO_RS_ENABLE_REAL=1" >> $env:GITHUB_ENV
          "OCIO_RS_LINK=static" >> $env:GITHUB_ENV
          # Record the resolved versions in the build log (the manifest
          # pins them via overrides + builtin-baseline).
          vcpkg list

      # Covers the whole target/ dir plus ~/.cargo; shared across branches
      # of the same OS.
      - name: Cache cargo artifacts
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: oak-ci-windows
          cache-on-failure: true

      # ------------------------------------------------------------------
      # Build & test
      # ------------------------------------------------------------------
      # `cargo check` (not build): the Test step links the test binaries
      # anyway, and a full build would codegen every workspace crate twice
      # (once without and once with cfg(test)).
      - name: Build
        run: cargo check --workspace --locked

      - name: Test
        run: |
          cargo test --workspace --locked
          if ($LASTEXITCODE -ne 0) {
            # Retry once: a few gpui keystroke tests flake on Windows CI —
            # a synthetic keystroke is occasionally never delivered (the
            # undo/redo pair and a plain 's' toggle both failed once,
            # each identically to its pass state). A real regression
            # fails both passes.
            Write-Host "first pass failed; retrying once for gpui keystroke flakes"
            cargo test --workspace --locked
            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          }

  macos:
    name: Build & test (macOS)
    runs-on: warp-macos-26-arm64-6x
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # gpui/ is a git submodule; its crates are workspace members of
          # their own repo and build as path dependencies of oakapp.
          submodules: true

      - name: Install Rust (stable)
        uses: dtolnay/rust-toolchain@stable

      # ------------------------------------------------------------------
      # System dependencies
      # ------------------------------------------------------------------
      - name: Install system dependencies
        run: |
          # Homebrew's pkgconf installs a `pkg-config` symlink, which is
          # the name crates/oak-ffmpeg-link/build.rs invokes; nasm is what
          # vcpkg's ffmpeg port requires to build (same split as the other
          # platforms: FFmpeg libraries come from the vcpkg manifest).
          brew install cmake pkg-config nasm

      # ------------------------------------------------------------------
      # vcpkg (manifest mode) + caches
      # ------------------------------------------------------------------
      # The runner has no vcpkg preinstalled; bootstrap a fresh clone.
      - name: Bootstrap vcpkg
        run: |
          git clone --depth 1 https://github.com/microsoft/vcpkg.git .cache/vcpkg
          .cache/vcpkg/bootstrap-vcpkg.sh -disableMetrics
          echo "$PWD/.cache/vcpkg" >> "$GITHUB_PATH"
          echo "VCPKG_ROOT=$PWD/.cache/vcpkg" >> "$GITHUB_ENV"

      # The archives dir is vcpkg's binary cache: a manifest bump then
      # rebuilds only what changed. Same key scheme as the Windows job.
      - name: Cache vcpkg artifacts
        uses: actions/cache@v4
        with:
          path: |
            vcpkg_installed
            ~/.cache/vcpkg/archives
          key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json') }}
          save-always: true

      - name: Install dependencies (vcpkg manifest)
        run: vcpkg install --triplet arm64-osx

      # ------------------------------------------------------------------
      # Build environment
      # ------------------------------------------------------------------
      - name: Configure build environment
        run: |
          # Vendored static OCIO (same as every non-Windows platform via
          # tooling/ocio-env.sh); no OCIO_INSTALL_DIR override.
          bash tooling/ocio-env.sh >> "$GITHUB_ENV"
          prefix="$PWD/vcpkg_installed/arm64-osx"
          echo "FFMPEG_DIR=$prefix" >> "$GITHUB_ENV"
          echo "PKG_CONFIG_PATH=$prefix/lib/pkgconfig" >> "$GITHUB_ENV"
          echo "$prefix/tools/pkgconf" >> "$GITHUB_PATH"

      # ------------------------------------------------------------------
      # Caches
      # ------------------------------------------------------------------
      # Covers the whole target/ dir plus ~/.cargo; shared across branches
      # of the same OS.
      - name: Cache cargo artifacts
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: oak-ci-macos
          cache-on-failure: true

      # ------------------------------------------------------------------
      # Build & test
      # ------------------------------------------------------------------
      # `cargo check` (not build): the Test step links the test binaries
      # anyway, and a full build would codegen every workspace crate twice
      # (once without and once with cfg(test)).
      - name: Build
        run: cargo check --workspace --locked

      # The runner's GUI session doubles as the display for the gpui
      # #[gpui::test] windows; wgpu renders through Metal.
      - name: Test
        run: |
          # Retry once (same policy as the other platforms): worker-pool
          # startup under full-suite parallelism has flaked on Linux. A
          # real regression fails both passes.
          if ! cargo test --workspace --locked; then
            echo "first pass failed; retrying once for worker-pool flakes"
            cargo test --workspace --locked
          fi

  # second upstream target: openKylin (Debian/Ubuntu-derived) in its own
  # image. The base image is bare (no sudo, no make, no python3) and the
  # steps run as root, so there is no sudo prefix anywhere below.
  openkylin:
    name: Build & test (openKylin ${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
    container: openkylin/openkylin:latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: x64
            runner: warp-ubuntu-latest-x64-8x
            triplet: x64-linux
          - arch: arm64
            runner: warp-ubuntu-latest-arm64-16x
            triplet: arm64-linux
    # The Test step's watchdog caps a hung suite at 30 min; give a cold
    # vcpkg install + full compile + test room beyond that.
    timeout-minutes: 90
    steps:
      # The image ships neither git nor curl (checkout and vcpkg need
      # both). First step of the job, so the package lists are still fresh.
      - name: Install git and curl
        run: apt-get update && apt-get install -y git curl ca-certificates

      - name: Checkout
        uses: actions/checkout@v4
        with:
          # gpui/ is a git submodule; its crates are workspace members of
          # their own repo and build as path dependencies of oakapp.
          submodules: true

      - name: Install Rust (stable)
        uses: dtolnay/rust-toolchain@stable

      # ------------------------------------------------------------------
      # System dependencies
      # ------------------------------------------------------------------
      # Same package set as the Linux job (both distros carry the Ubuntu
      # names); patch/xz-utils are openKylin's own packaging brought in by
      # zip/unzip. nasm and zip come from the kylinsoft "anything" PPA,
      # which the image enables by default.
      - name: Install system dependencies
        run: |
          apt-get update
          apt-get install -y \
            build-essential clang libclang-dev cmake pkg-config nasm \
            git curl zip unzip tar python3 patch xz-utils \
            libpipewire-0.3-dev libspa-0.2-dev libjack-jackd2-dev \
            libasound2-dev libpulse-dev libsndfile1-dev \
            libgl1-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers \
            libvulkan-dev libxkbcommon-dev libxkbcommon-x11-dev xvfb \
            gdb file

      # ------------------------------------------------------------------
      # vcpkg (manifest mode) + caches
      # ------------------------------------------------------------------
      # The image has no vcpkg preinstalled; bootstrap a fresh clone.
      - name: Bootstrap vcpkg
        run: |
          git clone --depth 1 https://github.com/microsoft/vcpkg.git .cache/vcpkg
          .cache/vcpkg/bootstrap-vcpkg.sh -disableMetrics
          echo "$PWD/.cache/vcpkg" >> "$GITHUB_PATH"
          echo "VCPKG_ROOT=$PWD/.cache/vcpkg" >> "$GITHUB_ENV"

      # The archives dir is vcpkg's binary cache: a manifest bump then
      # rebuilds only what changed.
      - name: Cache vcpkg artifacts
        uses: actions/cache@v4
        with:
          path: |
            vcpkg_installed
            ~/.cache/vcpkg/archives
          # The arch prefix matters here: both jobs run the same distro
          # image through the same cache scope, unlike the OS jobs.
          key: vcpkg-${{ runner.os }}-${{ matrix.arch }}-openkylin-${{ hashFiles('vcpkg.json') }}
          save-always: true

      - name: Install dependencies (vcpkg manifest)
        run: vcpkg install --triplet ${{ matrix.triplet }}

      # ------------------------------------------------------------------
      # Build environment
      # ------------------------------------------------------------------
      - name: Configure build environment
        run: |
          {
            echo "CC=clang"
            echo "CXX=clang++"
          } >> "$GITHUB_ENV"
          bash tooling/ocio-env.sh >> "$GITHUB_ENV"
          prefix="$PWD/vcpkg_installed/${{ matrix.triplet }}"
          echo "FFMPEG_DIR=$prefix" >> "$GITHUB_ENV"
          echo "PKG_CONFIG_PATH=$prefix/lib/pkgconfig" >> "$GITHUB_ENV"
          echo "$prefix/tools/pkgconf" >> "$GITHUB_PATH"

      # Covers the whole target/ dir plus ~/.cargo.
      - name: Cache cargo artifacts
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: oak-ci-openkylin-${{ matrix.arch }}
          cache-on-failure: true

      # ------------------------------------------------------------------
      # Build & test
      # ------------------------------------------------------------------
      - name: Build
        run: cargo check --workspace --locked

      # xvfb + 24-bit screen, same as the Linux job: the gpui
      # #[gpui::test] tests open real windows and render through wgpu on
      # Mesa's software Vulkan (lavapipe). The watchdog logic is the Linux
      # job's; the retry once below covers the same worker-pool flake,
      # which is a stale-tmpfs/container property too — a real regression
      # fails both passes.
      - name: Test
        timeout-minutes: 45
        shell: bash
        run: |
          run_suite() {
            xvfb-run -a -s "-screen 0 1920x1080x24" cargo test --workspace --locked &
            TEST_PID=$!
            (
              sleep 1800
              echo "::warning::test suite exceeded 1800s; dumping hung-process stacks"
              for p in $(pgrep -f 'target/debug/deps/|target/debug/oak-worker'); do
                echo "===== thread stacks of pid $p ($(readlink /proc/$p/exe 2>/dev/null)) ====="
                gdb -batch -ex 'thread apply all bt' -p "$p" || true
              done
              pkill -9 -f 'target/debug/deps/' || true
              pkill -9 -f 'target/debug/oak-worker' || true
            ) >/dev/null 2>&1 &
            WATCHDOG_PID=$!
            wait $TEST_PID
            rc=$?
            kill $WATCHDOG_PID 2>/dev/null || true
            wait $WATCHDOG_PID 2>/dev/null || true
            return $rc
          }
          if ! run_suite; then
            echo "first pass failed; retrying once for worker-pool flakes"
            run_suite
          fi