DEBUG: temporary ARG_MAX tracing for #70085 investigation (do not merge)

.github/workflows/build-packages.yml

@@ -24,6 +24,10 @@ on:
         type: boolean
         default: false
         description: Sign RPM Packages
+      sign-deb-packages:
+        type: boolean
+        default: false
+        description: Sign DEB Packages (via debsigs, using SIGNING_GPG_KEY)
       sign-macos-packages:
         type: boolean
         default: false
@@ -66,6 +70,7 @@ jobs:
 
   build-deb-packages:
     name: DEB
+    environment: ${{ inputs.environment }}
     if: ${{ toJSON(fromJSON(inputs.matrix)['linux']) != '[]' }}
     runs-on:
       - ${{ matrix.arch == 'x86_64' && 'ubuntu-24.04' || inputs.linux_arm_runner }}
@@ -113,6 +118,10 @@ jobs:
           apt-get install -y devscripts
           # Installing patchelf for relenv ELF binary patching
           apt-get install -y patchelf
+          # Installing debsigs for DEB signing (invoked by tools pkg
+          # build deb --key-id=<id>). Cheap install even when signing is
+          # off so the toolchain is always ready.
+          apt-get install -y debsigs
 
       - name: Download Onedir Tarball as an Artifact
         if:  inputs.source == 'onedir'
@@ -155,6 +164,30 @@ jobs:
           salt-version: "${{ inputs.salt-version }}"
           cwd: pkgs/checkout/
 
+      - name: Setup GnuPG
+        if: ${{ inputs.sign-deb-packages }}
+        env:
+          SIGNING_GPG_KEY: ${{ secrets.SIGNING_GPG_KEY }}
+          SIGNING_PASSPHRASE: ${{ secrets.SIGNING_PASSPHRASE }}
+        run: |
+          install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg
+          GNUPGHOME="$(mktemp -d -p /run/gpg)"
+          export GNUPGHOME
+          echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV"
+          cat <<EOF > "${GNUPGHOME}/gpg.conf"
+          batch
+          no-tty
+          pinentry-mode loopback
+          passphrase-file ${GNUPGHOME}/passphrase
+          EOF
+          echo "${SIGNING_PASSPHRASE}" > "${GNUPGHOME}/passphrase"
+          echo "${SIGNING_GPG_KEY}" | gpg --import -
+          # Discover the imported key's fingerprint so `tools pkg build
+          # deb --key-id=<id>` (and the debsigs call it makes) uses
+          # whatever key material was provided rather than a hardcoded id.
+          SIGN_KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '$1=="fpr" {print $10; exit}')
+          echo "SIGN_KEY_ID=${SIGN_KEY_ID}" >> "$GITHUB_ENV"
+
       - name: Configure Git
         if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
         working-directory: pkgs/checkout/
@@ -175,7 +208,7 @@ jobs:
               format('--onedir=salt-{0}-onedir-linux-{1}.tar.xz', inputs.salt-version, matrix.arch)
               ||
               format('--arch={0}', matrix.arch)
-          }}
+          }} ${{ inputs.sign-deb-packages && format('--key-id={0}', env.SIGN_KEY_ID) || '' }}
 
       - name: Cleanup
         run: |
@@ -278,6 +311,13 @@ jobs:
           EOF
           echo "${SIGNING_PASSPHRASE}" > "${GNUPGHOME}/passphrase"
           echo "${SIGNING_GPG_KEY}" | gpg --import -
+          # Discover the fingerprint of the just-imported signing key so
+          # Build RPM can pass it to rpmsign without hardcoding a specific
+          # key id. Lets each repo (saltstack/salt, saltstack/salt-nightlies)
+          # provide its own key material via SIGNING_GPG_KEY and have the
+          # workflow use whatever's in the resulting keyring.
+          SIGN_KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '$1=="fpr" {print $10; exit}')
+          echo "SIGN_KEY_ID=${SIGN_KEY_ID}" >> "$GITHUB_ENV"
 
       - name: Configure Git
         if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
@@ -296,7 +336,7 @@ jobs:
               format('--onedir=salt-{0}-onedir-linux-{1}.tar.xz', inputs.salt-version, matrix.arch)
               ||
               format('--arch={0}', matrix.arch)
-          }} ${{ inputs.sign-rpm-packages && '--key-id=64CBBC8173D76B3F' || '' }}
+          }} ${{ inputs.sign-rpm-packages && format('--key-id={0}', env.SIGN_KEY_ID) || '' }}
 
       - name: Set Artifact Name
         id: set-artifact-name

.github/workflows/ci.yml

@@ -47,6 +47,7 @@ jobs:
     name: Prepare Workflow Run
     runs-on: ubuntu-22.04
     environment: ci
+    if: vars.SKIP_CI != 'true'
     outputs:
       changed-files: ${{ steps.process-changed-files.outputs.changed-files }}
       salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
@@ -175,17 +176,8 @@ jobs:
 
       - name: Get Hash For Nox Tarball Cache
         id: nox-archive-hash
-        shell: bash
         run: |
-          HASH=$(python3 .github/scripts/hash-files.py \
-            'requirements/**/*.txt' \
-            'requirements/**/*.in' \
-            'requirements/**/*.lock' \
-            'cicd/golden-images.json' \
-            'noxfile.py' \
-            'pkg/common/env-cleanup-rules.yml' \
-            '.github/workflows/build-deps-ci-action.yml')
-          echo "nox-archive-hash=${HASH}" | tee -a "$GITHUB_OUTPUT"
+          echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'requirements/**/*.lock', 'cicd/golden-images.json', 'noxfile.py', 'pkg/common/env-cleanup-rules.yml', '.github/workflows/build-deps-ci-action.yml') }}" | tee -a "$GITHUB_OUTPUT"
 
       - name: Write Changed Files To A Local File
         run:
@@ -474,8 +466,8 @@ jobs:
     with:
       cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "0.22.18"
-      python-version: "3.14.6"
+      relenv-version: "0.22.25"
+      python-version: "3.14.7"
       ci-python-version: "3.14"
       matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
       linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }}
@@ -491,8 +483,8 @@ jobs:
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      relenv-version: "0.22.18"
-      python-version: "3.14.6"
+      relenv-version: "0.22.25"
+      python-version: "3.14.7"
       ci-python-version: "3.14"
       source: "onedir"
       matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
@@ -508,8 +500,8 @@ jobs:
     with:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      relenv-version: "0.22.18"
-      python-version: "3.14.6"
+      relenv-version: "0.22.25"
+      python-version: "3.14.7"
       ci-python-version: "3.14"
       source: "src"
       matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
@@ -524,10 +516,10 @@ jobs:
     with:
       nox-session: ci-test-onedir
       nox-version: 2022.8.7
-      python-version: "3.14.6"
+      python-version: "3.14.7"
       ci-python-version: "3.14"
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.14.6
+      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.14.7
       nox-archive-hash: "${{ needs.prepare-workflow.outputs.nox-archive-hash }}"
       matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
       linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }}
@@ -545,7 +537,7 @@ jobs:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       nox-version: 2022.8.7
       ci-python-version: "3.14"
-      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.14.6
+      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.14.7
       skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.config)['skip_code_coverage'] }}
       testing-releases: ${{ needs.prepare-workflow.outputs.testing-releases }}
       matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['pkg-test-matrix']) }}
@@ -564,7 +556,7 @@ jobs:
       ci-python-version: "3.14"
       testrun: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['testrun']) }}
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.14.6
+      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.14.7
       skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.config)['skip_code_coverage'] }}
       workflow-slug: ci
       default-timeout: 180

.github/workflows/nightly-stress-test.yml

@@ -37,11 +37,6 @@ on:
 jobs:
   stress-test:
     runs-on: ubuntu-latest
-    # ``contents: write`` lets the ``Publish panels to stress-snapshots
-    # branch`` step push the rendered PNGs to an orphan branch so the
-    # step summary can embed them via raw.githubusercontent.com URLs.
-    permissions:
-      contents: write
     steps:
       - uses: actions/checkout@v4
 
@@ -59,15 +54,6 @@ jobs:
       - name: Build and Start Environment
         run: |
           cd tests/monitoring
-          # The prometheus container runs as ``nobody`` (uid 65534) and
-          # writes its TSDB to ``/prometheus``.  Without pre-creating the
-          # bind-mount source with that ownership, Docker auto-creates
-          # it owned by root and prometheus fails to start with
-          # "permission denied" on /prometheus -- which then surfaces
-          # downstream as a bare ConnectionRefusedError when
-          # analyze_stats.py tries to query http://localhost:19090.
-          mkdir -p prometheus_data
-          sudo chown -R 65534:65534 prometheus_data
           docker compose build
           docker compose up -d
           sleep 30 # Wait for initialization
@@ -126,42 +112,8 @@ jobs:
           fi
 
       - name: Verify Connections
-        # The salt CLI returns exit 0 even when the master returns an
-        # error string (the legacy ``'str' object has no attribute
-        # 'pop'`` failure path surfaces this way), so we have to inspect
-        # the JSON output ourselves to fail the step.
         run: |
-          out=$(docker exec salt-master salt --out=json '*' test.ping)
-          echo "$out"
-          python3 - "$out" <<'PY'
-          import json, sys
-          payload = sys.argv[1].strip()
-          decoder = json.JSONDecoder()
-          idx = 0
-          bad = []
-          while idx < len(payload):
-              while idx < len(payload) and payload[idx].isspace():
-                  idx += 1
-              if idx >= len(payload):
-                  break
-              try:
-                  obj, end = decoder.raw_decode(payload, idx)
-              except ValueError as exc:
-                  bad.append(f"  non-JSON at offset {idx}: {exc}")
-                  break
-              idx = end
-              if isinstance(obj, dict):
-                  for mid, val in obj.items():
-                      if val is not True:
-                          bad.append(f"  minion {mid!r} returned {val!r}")
-              else:
-                  bad.append(f"  unexpected payload type: {type(obj).__name__}={obj!r}")
-          if bad:
-              print("Verify Connections failed:", file=sys.stderr)
-              for b in bad:
-                  print(b, file=sys.stderr)
-              sys.exit(1)
-          PY
+          docker exec salt-master salt '*' test.ping
 
       - name: Run Aggressive Stress Test
         run: |
@@ -189,19 +141,6 @@ jobs:
           sleep 30
           python3 analyze_stats.py
 
-      - name: Render Dashboard Panels
-        # Render BEFORE Snapshot Metrics stops prometheus.  We *only*
-        # produce the PNGs here; the markdown step-summary that links
-        # to them runs after the upload so it can embed a real URL.
-        # Runs even if Analyze Results failed -- the graphs are usually
-        # the most useful diagnostic for a failed run.
-        if: always()
-        run: |
-          cd tests/monitoring
-          python3 -m pip install --quiet matplotlib
-          PANELS_DIR="${GITHUB_WORKSPACE}/artifacts/panels" \
-            python3 render_panels.py
-
       - name: Snapshot Metrics
         if: always()
         run: |
@@ -216,93 +155,13 @@ jobs:
           mkdir -p artifacts
           docker logs salt-master > artifacts/salt-master.log
           docker logs salt-minion-1 > artifacts/salt-minion-1.log
-          cp tests/monitoring/event_log.txt artifacts/ || true
-          # Always grab prometheus' own log so we can tell whether the
-          # connection refused was a startup issue (bind mount permissions
-          # etc.) vs. a runtime crash.
-          docker logs prometheus > artifacts/prometheus.log 2>&1 || true
+          cp monitoring/event_log.txt artifacts/ || true
 
       - name: Upload Artifacts
-        id: upload-artifacts
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: stress-test-results
           path: |
             artifacts/
             prometheus-data.tar.gz
-
-      - name: Publish panels to stress-snapshots branch
-        # GitHub's step-summary sanitizer strips ``data:`` URIs but
-        # allows real ``https:`` image URLs, so we push the rendered
-        # PNGs to a dedicated ``stress-snapshots`` branch and let the
-        # summary embed them via ``raw.githubusercontent.com``.  The
-        # branch is orphan-style: it never merges anywhere and is
-        # auto-pruned 14 days back so it stays small.  Skipped silently
-        # when no PNGs exist (e.g. early-stage failures).
-        id: publish-snapshots
-        if: always() && hashFiles('artifacts/panels/*.png') != ''
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          set -e
-          REPO_URL="https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git"
-          BRANCH=stress-snapshots
-          RUN_DIR="runs/${{ github.run_id }}"
-
-          if git ls-remote --exit-code --heads "$REPO_URL" "$BRANCH" >/dev/null 2>&1; then
-            git clone --depth=1 --branch="$BRANCH" "$REPO_URL" snaps
-          else
-            git clone --depth=1 "$REPO_URL" snaps
-            cd snaps
-            git switch --orphan "$BRANCH"
-            git rm -rf . >/dev/null 2>&1 || true
-            cd ..
-          fi
-
-          mkdir -p "snaps/$RUN_DIR"
-          cp artifacts/panels/*.png "snaps/$RUN_DIR/"
-
-          cd snaps
-          # Drop any run directory older than 14 days so the branch
-          # never accumulates unbounded.
-          find runs -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} + 2>/dev/null || true
-
-          git config user.email "actions@github.com"
-          git config user.name  "github-actions[bot]"
-          git add -A
-          if ! git commit -m "Stress run ${{ github.run_id }}"; then
-            echo "nothing to commit"
-          else
-            # Tiny retry loop in case two runs raced.  Cron is nightly
-            # so this is essentially defensive.
-            for attempt in 1 2 3; do
-              if git push origin "$BRANCH"; then
-                break
-              fi
-              echo "push attempt $attempt failed; rebasing"
-              git fetch origin "$BRANCH"
-              git rebase "origin/$BRANCH"
-            done
-          fi
-
-          URL_PREFIX="https://raw.githubusercontent.com/${{ github.repository }}/${BRANCH}/${RUN_DIR}/"
-          echo "url-prefix=${URL_PREFIX}" >> "$GITHUB_OUTPUT"
-          echo "Published panels under ${URL_PREFIX}"
-
-      - name: Panel Summary
-        # Build the workflow step summary from the PNGs the Render
-        # step already produced, NOT by re-querying prometheus -- by
-        # this point ``Snapshot Metrics`` has stopped the prometheus
-        # container, so every range query would return no data.
-        # ``--image-url-prefix`` points at the stress-snapshots branch
-        # so each panel renders inline; without it the summary falls
-        # back to listing the artifact bundle.
-        if: always()
-        run: |
-          cd tests/monitoring
-          PANELS_DIR="${GITHUB_WORKSPACE}/artifacts/panels" \
-            python3 render_panels.py --summary --from-existing \
-              --artifact-name stress-test-results \
-              --artifact-url "${{ steps.upload-artifacts.outputs.artifact-url }}" \
-              --image-url-prefix "${{ steps.publish-snapshots.outputs.url-prefix }}"