ignite/.github/workflows/gpu-tests.yml at master · pytorch/ignite · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
name: Run unit tests on GPUs
on:
  push:
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_gpu_tests.sh"
      - "examples/**.py"
      - "requirements-dev.txt"
      - ".github/workflows/gpu-tests.yml"
  workflow_dispatch:

concurrency:
  # <workflow_name>-<branch_name>-<true || commit_sha (if branch is protected)>
  group: gpu-tests-${{ github.ref_name }}-${{ !(github.ref_protected) || github.sha }}
  cancel-in-progress: true

jobs:
  gpu-tests:
    strategy:
      matrix:
        pytorch-channel: [pytorch, pytorch-nightly]
      fail-fast: false
    runs-on: mt-l-x86iavx512-45-172-t4-4
    container:
      image: pytorch/almalinux-builder:cuda12.8
      options: --gpus all
    timeout-minutes: 85

    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 1

      - name: Show the environment
        run: |
          set -xe

          nvidia-smi
          ls -alh

          conda --version
          python --version

      - name: Install PyTorch and dependencies
        continue-on-error: false
        run: |
          set -xe

          # Install PyTorch
          if [ "${{ matrix.pytorch-channel }}" == "pytorch" ]; then
            pip install --upgrade torch torchvision --index-url https://download.pytorch.org/whl/cu126
          else
            pip install --upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu126
          fi

          python -c "import torch; print(torch.__version__, ', CUDA is available: ', torch.cuda.is_available()); exit(not torch.cuda.is_available())"
          pip list

          # Install dependencies
          pip install -r requirements-dev.txt
          pip install -e .

          # Upgrade pyOpenSSL to avoid issue:
          # AttributeError: module 'lib' has no attribute 'X509_V_FLAG_NOTIFY_POLICY'. Did you mean: 'X509_V_FLAG_EXPLICIT_POLICY'?
          pip install -U pyOpenSSL

      - name: Run GPU Unit Tests
        continue-on-error: false
        uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
        with:
          max_attempts: 5
          timeout_minutes: 45
          shell: bash
          command: bash -xec 'bash tests/run_gpu_tests.sh 4'
          new_command_on_retry: USE_LAST_FAILED=1 bash -xec 'bash tests/run_gpu_tests.sh 4'

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
        with:
          files: coverage.xml
          flags: gpu-2
          fail_ci_if_error: false

      - name: Run examples
        continue-on-error: false
        run: |
          set -xe

          # Install additional example dependencies
          pip install fire

          # Check training on cifar10, run without backend
          ## initial run
          CI=1 python examples/cifar10/main.py run --checkpoint_every=200 --stop_iteration=500
          ## resume
          CI=1 python examples/cifar10/main.py run --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-None-1_stop-on-500/training_checkpoint_400.pt

          # Check training on cifar10, run with NCCL backend using torchrun
          ## initial run
          CI=1 torchrun --nproc_per_node=4 examples/cifar10/main.py run --backend=nccl --checkpoint_every=200 --stop_iteration=500
          ## resume
          CI=1 torchrun --nproc_per_node=4 examples/cifar10/main.py run --backend=nccl --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-nccl-4_stop-on-500/training_checkpoint_400.pt

          # Check training on cifar10, run with NCCL backend using spawn
          ## initial run
          CI=1 python -u examples/cifar10/main.py run --backend=nccl --nproc_per_node=4 --checkpoint_every=200 --stop_iteration=500
          ## resume
          CI=1 python -u examples/cifar10/main.py run --backend=nccl --nproc_per_node=4 --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-nccl-4_stop-on-500/training_checkpoint_400.pt