ci: run only the integration suites a pull request can affect [APPS-37918]

.github/workflows/coverage.yml

@@ -12,6 +12,11 @@ on:
         type: string
         required: false
         default: main
+      scope_to_changes:
+        description: Run only the integration suites affected by the PR's changed files
+        type: boolean
+        required: false
+        default: false
 
 permissions:
   contents: read
@@ -54,7 +59,45 @@ jobs:
       - name: Build Project
         run: npm run build
 
+  scope:
+    # Which integration suites to run (scripts/integration-scope.mjs); anything
+    # unrecognised, or the `ci:full-integration` label, means the full run.
+    runs-on: uipath-ubuntu-latest
+    outputs:
+      run_integration: ${{ steps.resolve.outputs.run_integration }}
+      test_paths: ${{ steps.resolve.outputs.test_paths }}
+      scope: ${{ steps.resolve.outputs.scope }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
+        with:
+          node-version: '20'
+
+      - name: Resolve integration test scope
+        id: resolve
+        env:
+          SCOPE_TO_CHANGES: ${{ inputs.scope_to_changes }}
+          BASE_REF: ${{ github.base_ref }}
+          FULL_RUN_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'ci:full-integration') }}
+        run: |
+          if [ "$SCOPE_TO_CHANGES" = "true" ] && [ -n "$BASE_REF" ] && [ "$FULL_RUN_LABEL" != "true" ]; then
+            # No base ref → the resolver falls back to the full run.
+            git fetch --no-tags origin "+refs/heads/$BASE_REF:refs/remotes/origin/$BASE_REF" \
+              || echo "::warning::could not fetch $BASE_REF; running the full integration suite"
+            node scripts/integration-scope.mjs --base "origin/$BASE_REF"
+          else
+            node scripts/integration-scope.mjs --all
+          fi
+
   integration:
+    needs: scope
+    # A missing output runs everything rather than skipping.
+    if: needs.scope.outputs.run_integration != 'false'
     runs-on: uipath-ubuntu-latest
     strategy:
       fail-fast: false
@@ -248,7 +291,9 @@ jobs:
       - name: Run Integration Tests
         env:
           INTEGRATION_AUTH_MODE: ${{ matrix.auth }}
-        run: npm run test:integration:coverage -- --run --maxWorkers=2
+          # vitest path filters (empty = whole suite); built from suite folder names only.
+          TEST_PATHS: ${{ needs.scope.outputs.test_paths }}
+        run: npm run test:integration:coverage -- --run --maxWorkers=2 $TEST_PATHS
 
       # Runners are reused, so the config file — which holds the minted token —
       # must not outlive the job.
@@ -263,8 +308,14 @@ jobs:
           path: coverage-integration/lcov.info
 
   sonar:
-    needs: [unit, integration]
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false
+    needs: [scope, unit, integration]
+    # Runs on skipped integration too (unit coverage only); any failure still blocks it.
+    if: >-
+      always()
+      && needs.scope.result == 'success'
+      && needs.unit.result == 'success'
+      && (needs.integration.result == 'success' || needs.integration.result == 'skipped')
+      && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
     runs-on: uipath-ubuntu-latest
     steps:
       - name: Checkout
@@ -281,21 +332,28 @@ jobs:
         run: |
           mkdir -p coverage coverage-integration
           cp lcov/unit-coverage/lcov.info coverage/lcov.info
-          # lcov reports concatenate; Sonar merges the records per file.
-          cat lcov/integration-coverage-*/lcov.info > coverage-integration/lcov.info
+          # lcov reports concatenate; none exist when integration was skipped.
+          shopt -s nullglob
+          reports=(lcov/integration-coverage-*/lcov.info)
+          if [ ${#reports[@]} -gt 0 ]; then
+            cat "${reports[@]}" > coverage-integration/lcov.info
+          fi
 
       - name: SonarCloud Scan
         uses: SonarSource/sonarqube-scan-action@0303d6b62e310685c0e34d0b9cde218036885c4d  # v5.0.0
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
   summary:
-    needs: [unit, integration]
+    needs: [scope, unit, integration]
     if: always()
     runs-on: uipath-ubuntu-latest
     steps:
       - name: Generate Summary
-        if: needs.unit.result == 'success' && needs.integration.result == 'success'
+        if: needs.scope.result == 'success' && needs.unit.result == 'success' && (needs.integration.result == 'success' || needs.integration.result == 'skipped')
+        env:
+          SCOPE: ${{ needs.scope.outputs.scope }}
+          INTEGRATION_RESULT: ${{ needs.integration.result }}
         run: |
           echo "## ✅ Coverage Checks Passed" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
@@ -307,10 +365,14 @@ jobs:
           echo "- ✅ Typecheck passed" >> $GITHUB_STEP_SUMMARY
           echo "- ✅ SDK tests passed" >> $GITHUB_STEP_SUMMARY
           echo "- ✅ Build completed successfully" >> $GITHUB_STEP_SUMMARY
-          echo "- ✅ All integration tests passed" >> $GITHUB_STEP_SUMMARY
+          if [ "$INTEGRATION_RESULT" = "skipped" ]; then
+            echo "- ⏭️ Integration tests skipped: no changed file affects them (scope: \`$SCOPE\`)" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "- ✅ Integration tests passed (scope: \`$SCOPE\`)" >> $GITHUB_STEP_SUMMARY
+          fi
 
       - name: Generate Failure Summary
-        if: needs.unit.result != 'success' || needs.integration.result != 'success'
+        if: needs.scope.result != 'success' || needs.unit.result != 'success' || (needs.integration.result != 'success' && needs.integration.result != 'skipped')
         run: |
           echo "## ❌ Coverage Checks Failed" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY

.github/workflows/pr-checks.yml

@@ -23,6 +23,9 @@ jobs:
     secrets: inherit
     with:
       ref_label: ${{ github.base_ref }}
+      # Run only the integration suites the PR's changed files can affect;
+      # weekly-coverage.yml keeps running the full suite against main.
+      scope_to_changes: true
 
   test-and-build:
     # Gate job that reports the flat `test-and-build` status check required by

agent_docs/rules.md

@@ -43,6 +43,7 @@ Every new method must also have an integration test in `tests/integration/shared
 - **The `requirement` argument is `'both'` unless the service rejects one of the credentials.** `'both'` runs the suite once per configured credential — under the PAT *and* under the user token in CI — because the two cover different things: the PAT exercises the external-application OAuth scope model, the user token exercises the general API surface. Use `'user'` only for services that reject PAT and client-credentials tokens outright, and `'pat'` only for something specific to the external-application identity.
 - **Always `throw new Error()` when test preconditions are not met** — whether it's missing config (e.g., no `folderId`) or missing test data (e.g., no running jobs). Never use `console.warn()` + `return` to silently skip — silent skips hide unrunnable tests and make CI green when tests aren't actually exercised.
 - **When a service rejects PAT auth, run it under a user token — do not `describe.skip` it.** `insightsrtm_` endpoints (Agents, Agent Memory, Agent Traces, Governance) and the notification service return 401 for PAT and client-credentials tokens regardless of scopes. Those suites are declared with `describeIntegration(name, 'user', modes, body)`, which states the requirement once and derives the credential, the host and the collection-time skip guard from it, so they run wherever `UIPATH_USER_TOKEN` is configured and are reported as skipped where it isn't. Declaring the guard separately from the requirement lets the two disagree — use the helper. See "Authentication modes" in `tests/integration/README.md`. Do **not** use `describe.skip` for missing test data, missing config, or flakiness — those require a `beforeAll` guard or a `throw`. Equivalently, **NEVER** exclude integration test files via `vitest.integration.config.ts` using env vars or file exclusion patterns — that is functionally equivalent to `describe.skip` across an entire file and has the same problem: tests appear to pass but are never actually exercised. Guard with `beforeAll` + `throw` inside the test file instead.
+- **Scoping a pull-request run to the suites its changed files can affect (the `scope` job, `scripts/integration-scope.mjs`) is not such an exclusion**: no suite is disabled, unrecognised paths run everything, the `ci:full-integration` label forces the full run, and `weekly-coverage.yml` still runs everything. Name a new service's suite folder after its `src/services/` folder.
 - **Use snapshot+restore for integration tests that mutate shared state** — when an operation has wide-reaching, hard-to-undo side effects (e.g., marking all notifications as read, clearing a queue), read the current state before the test, perform the operation, then restore the previous state in cleanup. This prevents one test from permanently altering the shared environment and corrupting subsequent tests. **The restore value must come from the snapshotted state — never hardcode an assumed original value** (e.g., `isSubscribed: true`). If the environment's real state differed from your assumption, a hardcoded restore mutates the environment instead of restoring it, silently corrupting subsequent tests.
 - **When writing feature-gated integration tests, use universally available entity fields for the test query** (e.g., `Id` is always present on every Data Fabric entity) rather than environment-specific test data (e.g., join field names from env vars). Using env-var-gated fields as test data creates unintended coupling — environments that have the feature enabled but lack the specific fixture will fail with a misleading config error instead of a clear feature-flag error.
 - **NEVER** write redundant integration tests — each test must cover a distinct code path, error scenario, or response shape aspect.

scripts/integration-scope.mjs

@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+// Picks the integration suites a PR needs from its changed files: a change under
+// src/services/<name>, src/models/<name> or tests/integration/shared/<name> runs
+// tests/integration/shared/<name>; docs/samples/packages/unit tests run nothing;
+// anything else runs everything. Usage: --base <ref> | --files <list> | --all.
+import { execFileSync } from 'node:child_process';
+import { appendFileSync, readFileSync, readdirSync, realpathSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const SHARED = 'tests/integration/shared';
+
+/** Run whenever any suite runs. */
+export const ALWAYS_ON = Object.freeze([
+  `${SHARED}/smoke.integration.test.ts`,
+  `${SHARED}/http`,
+  'tests/integration/auth-errors.integration.test.ts',
+]);
+
+/** PR label that forces the full run. */
+export const FULL_RUN_LABEL = 'ci:full-integration';
+
+const IGNORED_PATTERNS = [
+  /^(docs|samples|packages|plugins|agent_docs|\.claude|\.agents|tests\/unit|tests\/utils\/mocks)\//,
+  /\.md$/,
+  /^(mkdocs\.yml|typedoc\.json|typedoc\.validation\.json|\.oxlintrc\.json|\.prettierrc\.docs|commitlint\.config\.js|release-metadata\.json|sonar-project\.properties|LICENSE|\.gitignore|\.npmrc|vitest\.config\.ts|rollup\.config\.js|tests\/\.env\.integration\.example)$/,
+];
+const DOMAIN_PATH = /^(?:src\/services|src\/models|tests\/integration\/shared)\/([^/]+)\//;
+
+/** Suite folders, minus the always-on ones. */
+export function suites() {
+  return readdirSync(join(ROOT, SHARED), { withFileTypes: true })
+    .filter(entry => entry.isDirectory() && !ALWAYS_ON.includes(`${SHARED}/${entry.name}`))
+    .map(entry => entry.name)
+    .sort();
+}
+
+/** One path → { kind: 'ignore' | 'always-on' | 'domain' | 'all' }. */
+export function classify(file, domains) {
+  if (IGNORED_PATTERNS.some(pattern => pattern.test(file))) return { kind: 'ignore' };
+  const domain = file.match(DOMAIN_PATH)?.[1];
+  if (domain && domains.includes(domain)) return { kind: 'domain', domain };
+  if (domain && ALWAYS_ON.includes(`${SHARED}/${domain}`)) return { kind: 'always-on' };
+  return { kind: 'all', reason: `${file} is outside the per-domain folders` };
+}
+
+/** Changed files → { run, all, domains, paths (vitest filters; empty = all), reasons }. */
+export function resolveScope(changedFiles, domains = suites()) {
+  const selected = new Set();
+  const reasons = [];
+  let alwaysOn = false;
+
+  for (const file of changedFiles) {
+    const result = classify(file, domains);
+    if (result.kind === 'all') reasons.push(result.reason);
+    else if (result.kind === 'domain') selected.add(result.domain);
+    else if (result.kind === 'always-on') alwaysOn = true;
+  }
+  if (reasons.length > 0) return { run: true, all: true, domains: [], paths: [], reasons };
+
+  const run = alwaysOn || selected.size > 0;
+  const sorted = [...selected].sort();
+  return { run, all: false, domains: sorted, paths: run ? [...ALWAYS_ON, ...sorted.map(d => `${SHARED}/${d}`)] : [], reasons };
+}
+
+export const fullScope = reason => ({ run: true, all: true, domains: [], paths: [], reasons: [reason] });
+
+/** GITHUB_OUTPUT lines for a resolved scope. */
+export function toOutputs(scope) {
+  const label = scope.all ? 'all' : scope.run ? scope.domains.join(',') || 'always-on' : 'none';
+  return [`run_integration=${scope.run}`, `test_paths=${scope.paths.join(' ')}`, `scope=${label}`];
+}
+
+// ---- CLI ----
+
+function changedFiles(argv) {
+  const at = flag => (argv.includes(flag) ? argv[argv.indexOf(flag) + 1] : undefined);
+  if (argv.includes('--all')) return { scope: fullScope('full run requested') };
+  const list = at('--files');
+  if (list) return { files: readFileSync(list, 'utf8').split('\n').map(l => l.trim()).filter(Boolean) };
+  const base = at('--base');
+  if (!base) {
+    console.error('usage: integration-scope.mjs (--base <ref> | --files <list> | --all)');
+    process.exit(2);
+  }
+  try {
+    // --no-renames: both sides of a move count as changed.
+    const diff = execFileSync('git', ['diff', '--name-only', '--no-renames', `${base}...HEAD`], { cwd: ROOT, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
+    return { files: diff.split('\n').filter(Boolean) };
+  } catch (error) {
+    // A diff problem must never skip the run.
+    return { scope: fullScope(`could not diff against ${base}: ${error.message.split('\n')[0]}`) };
+  }
+}
+
+function run() {
+  const { files, scope: forced } = changedFiles(process.argv.slice(2));
+  const scope = forced ?? resolveScope(files);
+  const outputs = toOutputs(scope);
+
+  if (files) console.log(`integration-scope: ${files.length} changed file(s)`);
+  for (const reason of scope.reasons) console.log(`integration-scope: full run — ${reason}`);
+  if (!scope.all) {
+    console.log(scope.run ? `integration-scope: running ${scope.domains.join(', ') || 'always-on suites only'}` : 'integration-scope: no integration test is affected');
+  }
+  for (const line of outputs) console.log(`  ${line}`);
+  if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${outputs.join('\n')}\n`);
+}
+
+// CLI only when run directly, not when imported by tests.
+if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  run();
+}

tests/unit/scripts/integration-scope.test.ts

@@ -0,0 +1,95 @@
+import { describe, it, expect } from 'vitest';
+// The PR integration-test scoping rules. Imported directly (the script only runs
+// its CLI when executed as main), so the pure resolver is exercised here.
+import { ALWAYS_ON, classify, resolveScope, suites, toOutputs } from '../../../scripts/integration-scope.mjs';
+
+const SHARED = 'tests/integration/shared';
+const DOMAINS = ['action-center', 'data-fabric', 'maestro', 'orchestrator'];
+
+describe('integration-scope resolveScope', () => {
+  it('runs nothing when only ignorable files change', () => {
+    const scope = resolveScope([
+      'docs/faq.md',
+      'samples/process-app/src/App.tsx',
+      'packages/coded-action-app/src/types.ts',
+      'tests/unit/services/data-fabric/entities.test.ts',
+      'tests/utils/mocks/entities.ts',
+      'README.md',
+      'rollup.config.js',
+      'tests/.env.integration.example',
+    ], DOMAINS);
+    expect(scope).toMatchObject({ run: false, all: false, domains: [], paths: [] });
+    expect(toOutputs(scope)).toEqual(['run_integration=false', 'test_paths=', 'scope=none']);
+  });
+
+  it('runs nothing for an empty change set', () => {
+    expect(resolveScope([], DOMAINS).run).toBe(false);
+  });
+
+  it('scopes service, model and suite changes to the same-named suite plus the always-on suites', () => {
+    const scope = resolveScope([
+      'src/services/data-fabric/entities.ts',
+      'src/models/maestro/cases.types.ts',
+      `${SHARED}/orchestrator/jobs.integration.test.ts`,
+      'docs/data-fabric.md',
+    ], DOMAINS);
+    expect(scope).toMatchObject({ run: true, all: false, domains: ['data-fabric', 'maestro', 'orchestrator'] });
+    expect(scope.paths).toEqual([...ALWAYS_ON, `${SHARED}/data-fabric`, `${SHARED}/maestro`, `${SHARED}/orchestrator`]);
+    expect(toOutputs(scope)).toEqual([
+      'run_integration=true',
+      `test_paths=${scope.paths.join(' ')}`,
+      'scope=data-fabric,maestro,orchestrator',
+    ]);
+  });
+
+  it('runs only the always-on suites when they are what changed', () => {
+    const scope = resolveScope([`${SHARED}/http/http-request.integration.test.ts`], DOMAINS);
+    expect(scope).toMatchObject({ run: true, all: false, domains: [], paths: [...ALWAYS_ON] });
+    expect(toOutputs(scope)[2]).toBe('scope=always-on');
+  });
+
+  it.each([
+    'src/core/http/api-client.ts',
+    'src/utils/constants/endpoints/orchestrator.ts',
+    'src/services/base.ts',
+    'src/models/common/types.ts',
+    'src/models/document-understanding/du.types.ts', // no suite of that name
+    'src/services/integration-service/connections/connections.ts', // no suite of that name
+    'src/index.ts',
+    'tests/integration/config/unified-setup.ts',
+    'tests/integration/utils/helpers.ts',
+    'tests/utils/constants/agents.ts', // imported by the agents suites
+    `${SHARED}/smoke.integration.test.ts`,
+    `${SHARED}/brand-new-domain/x.integration.test.ts`,
+    'vitest.integration.config.ts',
+    'package.json',
+    '.github/workflows/coverage.yml',
+    'scripts/integration-scope.mjs',
+    'new-top-level-dir/thing.ts',
+  ])('runs everything for anything outside the per-domain folders: %s', (file) => {
+    const scope = resolveScope(['docs/index.md', file], DOMAINS);
+    expect(scope).toMatchObject({ run: true, all: true, paths: [] });
+    expect(scope.reasons).toHaveLength(1);
+    expect(toOutputs(scope)).toEqual(['run_integration=true', 'test_paths=', 'scope=all']);
+  });
+
+  it('lets a single shared file override any number of scoped ones', () => {
+    const scope = resolveScope(['src/services/data-fabric/entities.ts', 'src/core/config.ts'], DOMAINS);
+    expect(scope).toMatchObject({ all: true, domains: [] });
+  });
+
+  it('classifies against the given domain list only', () => {
+    expect(classify('src/services/maestro/cases.ts', DOMAINS)).toEqual({ kind: 'domain', domain: 'maestro' });
+    expect(classify('src/services/maestro/cases.ts', ['data-fabric']).kind).toBe('all');
+  });
+});
+
+describe('integration-scope suites', () => {
+  it('derives the domain list from the suite folders, without the always-on ones', () => {
+    const found = suites();
+    expect(found).toContain('data-fabric');
+    expect(found).toContain('maestro');
+    expect(found).not.toContain('http');
+    expect(found).toEqual([...found].sort());
+  });
+});