unit_tests.yml

name: Unit tests

on:
  pull_request:
    branches: [main, release/*, feature/*]
  push:
    branches: [main, release/*, feature/*]
    paths:
      - ".github/workflows/unit_tests.yml"
      - "modelopt/**"
      - "noxfile.py"
      - "pyproject.toml"
      - "tests/unit/**"
      - "tools/launcher/**"
      - "tools/mcp/**"
      - "tools/resource_monitor.py"
      - ".agents/skills/**"
      - "plugins/modelopt/skills/**"
  schedule:
    - cron: "0 0 * * *" # Nightly
  workflow_dispatch:
    # On-demand

concurrency:
  # Cancel previous runs if new commit is pushed
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  check-file-changes:
    runs-on: ubuntu-latest
    outputs:
      any_changed: ${{ steps.changed.outputs.any_modified || steps.non-pr.outputs.any_changed }}
    steps:
      - id: non-pr
        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
        run: echo "any_changed=true" >> $GITHUB_OUTPUT
      - if: github.event_name == 'pull_request'
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - if: github.event_name == 'pull_request'
        name: Check for changes in test-relevant paths
        id: changed
        uses: step-security/changed-files@v47.0.5
        with:
          files: |
            .github/workflows/unit_tests.yml
            modelopt/**
            modelopt_recipes/**
            noxfile.py
            pyproject.toml
            tests/unit/**
            tests/_test_utils/**
            tools/launcher/**
            tools/mcp/**
            tools/resource_monitor.py
            .agents/skills/**
            plugins/modelopt/skills/**
  linux:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/ubuntu-setup
      - name: Run unit tests
        env:
          COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml
          COVERAGE_FILE: ${{ github.workspace }}/.coverage
          # Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS).
          PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual' || '' }}
          # One single-threaded xdist worker per vCPU (`-n auto` counts physical cores); torch's
          # default threads per worker oversubscribe the runner and run slower than serial.
          PYTEST_ADDOPTS: -n 4 --dist loadfile
          OMP_NUM_THREADS: "1"
        run: pip install nox uv && nox -s "unit-3.12(torch_214, tf_latest)" -- $PYTEST_EXTRA
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v7
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: unit
          fail_ci_if_error: true
          # Skip GPG/SHASUM integrity check of the Codecov CLI: its key import
          # intermittently fails (codecov/codecov-action#1876), which would
          # otherwise hard-fail this required job on a transient infra blip.
          skip_validation: true
          verbose: true
  windows:
    if: needs.check-file-changes.outputs.any_changed == 'true'
    needs: [linux, check-file-changes]
    runs-on: windows-latest
    timeout-minutes: 20
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"
      - name: Record which CPU this runner got
        # The 0xc000001d crash this job used to hit tracked the host CPU, not the test, so this
        # is the first thing worth knowing if it ever recurs. See the ONEDNN_MAX_CPU_ISA comment
        # on the test step below.
        continue-on-error: true
        shell: pwsh
        run: |
          Get-CimInstance Win32_Processor | Select-Object Name, Description, NumberOfCores | Format-List
          python -c "import platform; print('machine:', platform.machine(), platform.processor())"
      - name: Run unit tests (without coverage)
        env:
          # Cap oneDNN's instruction set. bf16 linear/matmul on CPU dispatch through oneDNN,
          # which by default picks the highest ISA the host advertises -- Intel AMX on the
          # Emerald Rapids runners in the Actions fleet. On those hosts that path executes an
          # instruction that faults with #UD, which surfaces as 0xc000001d and kills the whole
          # pytest process. AMD EPYC runners have neither AMX nor AVX-512, take the AVX2 path,
          # and have never shown the crash -- so AVX2 is the ceiling with evidence behind it.
          # This is a CI mitigation for a torch/oneDNN Windows issue, not a ModelOpt bug: it
          # changes which kernel runs, not what is being tested. To check whether it is still
          # needed, drop it and run the suite repeatedly until an Intel host comes up.
          # See the PR that added this for the six crashes and the host correlation behind it.
          ONEDNN_MAX_CPU_ISA: "AVX2"
          DNNL_MAX_CPU_ISA: "AVX2"
        run: pip install nox uv && nox -s "unit-3.12(torch_214, tf_latest)"
  multi-version:
    if: needs.check-file-changes.outputs.any_changed == 'true'
    needs: [linux, check-file-changes]
    runs-on: ubuntu-latest
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        include:
          # Default torch (2.14) across the other supported Python versions
          - {nox_session: "unit-3.10(torch_214, tf_latest)", python_version: "3.10"}
          - {nox_session: "unit-3.11(torch_214, tf_latest)", python_version: "3.11"}
          - {nox_session: "unit-3.13(torch_214, tf_latest)", python_version: "3.13"}
          - {nox_session: "unit-3.14(torch_214, tf_latest)", python_version: "3.14"}
          # Older torch versions on the default Python (3.12) for back-compat.
          - {nox_session: "unit-3.12(torch_28, tf_latest)", python_version: "3.12"}
          - {nox_session: "unit-3.12(torch_29, tf_latest)", python_version: "3.12"}
          - {nox_session: "unit-3.12(torch_210, tf_latest)", python_version: "3.12"}
          - {nox_session: "unit-3.12(torch_211, tf_latest)", python_version: "3.12"}
          - {nox_session: "unit-3.12(torch_212, tf_latest)", python_version: "3.12"}
          - {nox_session: "unit-3.12(torch_213, tf_latest)", python_version: "3.12"}
          # Minimum supported transformers on the default torch.
          - {nox_session: "unit-3.12(torch_214, tf_min)", python_version: "3.12"}
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/ubuntu-setup
        with:
          python-version: ${{ matrix.python_version }}
      - name: Run unit tests
        env:
          PYTEST_ADDOPTS: -n 4 --dist loadfile
          OMP_NUM_THREADS: "1"
        run: pip install nox uv && nox -s "${{ matrix.nox_session }}"
  partial-install:
    if: needs.check-file-changes.outputs.any_changed == 'true'
    needs: [linux, check-file-changes]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        test-env: [onnx, torch]
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/ubuntu-setup
      - name: Run unit tests
        run: pip install nox uv && nox -s "partial_unit(subset='${{ matrix.test-env }}')"
  launcher:
    if: needs.check-file-changes.outputs.any_changed == 'true'
    needs: [linux, check-file-changes]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive
      - name: Run launcher tests
        working-directory: tools/launcher
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          export PATH="$HOME/.local/bin:$PATH"
          uv venv .venv
          uv pip install -e . pytest
          uv run python3 -m pytest -v
  mcp:
    if: needs.check-file-changes.outputs.any_changed == 'true'
    needs: [linux, check-file-changes]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive
      - name: Run modelopt-mcp tests
        working-directory: tools/mcp
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          export PATH="$HOME/.local/bin:$PATH"
          uv venv .venv
          # Install the sibling launcher package first; it's a runtime
          # dep declared in tools/mcp/pyproject.toml as `modelopt-launcher`
          # but uv resolves the source via [tool.uv.sources] to a local
          # editable path. -e on both packages keeps the install cheap
          # and matches the dev-mode install in tools/mcp/README.md.
          uv pip install -e ../launcher
          uv pip install -e . pytest
          uv run python3 -m pytest -v
  skills:
    if: needs.check-file-changes.outputs.any_changed == 'true'
    needs: [linux, check-file-changes]
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"
      - name: Run skill gate tests
        # Skill gate tests are stdlib-only and hermetic (no GPU/cluster/network),
        # so they run in their own lightweight job rather than the main unit lane.
        # Override addopts to drop the repo's coverage/instafail plugins (not installed here).
        run: |
          pip install pytest "tomli; python_version < '3.11'"
          python -m pytest plugins/modelopt/skills/ -o addopts="" -p no:cacheprovider -v
  unit-pr-required-check:
    # Run even if some jobs are skipped
    if: ${{ github.event_name == 'pull_request' && always() }}
    needs: [check-file-changes, linux, windows, multi-version, partial-install, launcher, mcp, skills]
    runs-on: ubuntu-latest
    steps:
      - name: Required unit tests did not succeed
        # windows is deliberately absent from this condition. It still runs and is still visible,
        # but it does not gate the merge: its torch CPU tests are flaky for host-dependent
        # reasons unrelated to the change under review. What we actually care about on that
        # platform is the ONNX surface (QA also covers Windows), so blocking every PR on
        # unrelated torch flakiness costs more than it catches.
        if: >-
          ${{ needs.linux.result != 'success' || (needs.check-file-changes.outputs.any_changed == 'true' && (
            needs.multi-version.result != 'success' ||
            needs.partial-install.result != 'success' ||
            needs.launcher.result != 'success' ||
            needs.mcp.result != 'success' ||
            needs.skills.result != 'success'
          )) }}
        run: exit 1