Release 2026-09-24

.claude/rules/databricks.md

@@ -14,7 +14,9 @@ DB_PREFIX=yourname_            # Schema prefix (e.g., "yourname_" -> "yourname_c
 DB_CATALOG=<catalog>           # Databricks catalog name
 DB_HOST_NAME=<hostname>        # SQL Warehouse hostname
 DB_HTTP_PATH=<path>            # SQL Warehouse HTTP path
-DB_TOKEN=<token>               # Access token
+DB_TOKEN=<token>               # Personal access token — wins when set
+DB_CLIENT_ID=<client-id>       # Service principal, used when DB_TOKEN is unset
+DB_CLIENT_SECRET=<secret>      # Service principal secret; the token is renewed per job
 DB_CONNECTION=<connection>     # Databricks connection string
 DB_API_BASE_URL=<url>          # CARTO API base URL
 DB_API_ACCESS_TOKEN=<token>    # CARTO API access token

.github/workflows/bigquery.yml

@@ -192,6 +192,6 @@ jobs:
       - name: Publish package
         run: |
           cd clouds/bigquery
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-bigquery-latest.zip
-          gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/
+          gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip
+          gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-bigquery-latest.zip --content-type=application/zip
+          gcloud storage cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ --content-type=application/json

.github/workflows/databricks-ded.yml

@@ -16,6 +16,10 @@ jobs:
       (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'dedicated_databricks')) ||
       (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dedicated_databricks'))
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: write
     timeout-minutes: 20
     env:
       DB_HOST_NAME: ${{ secrets.DB_HOST_NAME_CD }}
@@ -35,34 +39,30 @@ jobs:
       - name: Authenticate with Google Cloud
         uses: 'google-github-actions/auth@v2'
         with:
-          credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }}
+          workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }}
+          service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }}
       - name: Get Databricks secrets
         id: secrets
         uses: google-github-actions/get-secretmanager-secrets@v2
         with:
           secrets: |-
             databricks-client-id:${{ secrets.DB_CLIENT_ID_NAME_CD }}
             databricks-client-secret:${{ secrets.DB_CLIENT_SECRET_NAME_CD }}
-      - name: Get Databricks OAuth token
-        run: |
-          RESPONSE=$(curl -s -X POST https://$DB_HOST_NAME/oidc/v1/token \
-            -H "Content-Type: application/x-www-form-urlencoded" \
-            -d "grant_type=client_credentials" \
-            -d "client_id=${{ steps.secrets.outputs.databricks-client-id }}" \
-            -d "client_secret=${{ steps.secrets.outputs.databricks-client-secret }}" \
-            -d "scope=all-apis")
-          TOKEN=$(echo $RESPONSE | jq -r '.access_token')
-          echo "::add-mask::$TOKEN"
-          echo "DB_TOKEN=$TOKEN" >> $GITHUB_ENV
       - name: Run deploy
         id: deploy
         if: github.event.action == 'synchronize' || github.event.action == 'labeled'
+        env:
+          DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }}
+          DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }}
         run: |
           cd clouds/databricks
           make deploy
       - name: Run remove
         id: remove
         if: github.event.action == 'unlabeled' || github.event.action == 'closed'
+        env:
+          DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }}
+          DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }}
         run: |
           cd clouds/databricks
           make remove

.github/workflows/databricks.yml

@@ -23,6 +23,9 @@ jobs:
 
   test:
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      id-token: write
     timeout-minutes: 20
     env:
       DB_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_${{ github.run_attempt }}_
@@ -43,39 +46,41 @@ jobs:
       - name: Authenticate with Google Cloud
         uses: 'google-github-actions/auth@v2'
         with:
-          credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }}
+          workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }}
+          service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }}
       - name: Get Databricks secrets
         id: secrets
         uses: google-github-actions/get-secretmanager-secrets@v2
         with:
           secrets: |-
             databricks-client-id:${{ secrets.DB_CLIENT_ID_NAME_CI }}
             databricks-client-secret:${{ secrets.DB_CLIENT_SECRET_NAME_CI }}
-      - name: Get Databricks OAuth token
-        run: |
-          RESPONSE=$(curl -s -X POST https://$DB_HOST_NAME/oidc/v1/token \
-            -H "Content-Type: application/x-www-form-urlencoded" \
-            -d "grant_type=client_credentials" \
-            -d "client_id=${{ steps.secrets.outputs.databricks-client-id }}" \
-            -d "client_secret=${{ steps.secrets.outputs.databricks-client-secret }}" \
-            -d "scope=all-apis")
-          TOKEN=$(echo $RESPONSE | jq -r '.access_token')
-          echo "::add-mask::$TOKEN"
-          echo "DB_TOKEN=$TOKEN" >> $GITHUB_ENV
       - name: Lint code
         run: make lint cloud=databricks
       - name: Deploy modules
+        env:
+          DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }}
+          DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }}
         run: make deploy cloud=databricks diff="$GIT_DIFF"
       - name: Run tests
+        env:
+          DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }}
+          DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }}
         run: make test cloud=databricks diff="$GIT_DIFF"
       - name: Cleanup resources
         if: always()
+        env:
+          DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }}
+          DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }}
         run: make remove cloud=databricks
 
   deploy-internal:
     if: github.ref_name == 'main'
     needs: test
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      id-token: write
     timeout-minutes: 10
     env:
       DB_HOST_NAME: ${{ secrets.DB_HOST_NAME_CD }}
@@ -95,26 +100,19 @@ jobs:
       - name: Authenticate with Google Cloud
         uses: 'google-github-actions/auth@v2'
         with:
-          credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }}
+          workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }}
+          service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }}
       - name: Get Databricks secrets
         id: secrets
         uses: google-github-actions/get-secretmanager-secrets@v2
         with:
           secrets: |-
             databricks-client-id:${{ secrets.DB_CLIENT_ID_NAME_CD }}
             databricks-client-secret:${{ secrets.DB_CLIENT_SECRET_NAME_CD }}
-      - name: Get Databricks OAuth token
-        run: |
-          RESPONSE=$(curl -s -X POST https://$DB_HOST_NAME/oidc/v1/token \
-            -H "Content-Type: application/x-www-form-urlencoded" \
-            -d "grant_type=client_credentials" \
-            -d "client_id=${{ steps.secrets.outputs.databricks-client-id }}" \
-            -d "client_secret=${{ steps.secrets.outputs.databricks-client-secret }}" \
-            -d "scope=all-apis")
-          TOKEN=$(echo $RESPONSE | jq -r '.access_token')
-          echo "::add-mask::$TOKEN"
-          echo "DB_TOKEN=$TOKEN" >> $GITHUB_ENV
       - name: Deploy modules
+        env:
+          DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }}
+          DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }}
         run: make deploy cloud=databricks diff="$GIT_DIFF" production=1
 
   publish:
@@ -151,6 +149,6 @@ jobs:
       - name: Publish package to storage
         run: |
           cd clouds/databricks
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-databricks-latest.zip
-          gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/
+          gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip
+          gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-databricks-latest.zip --content-type=application/zip
+          gcloud storage cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ --content-type=application/json

.github/workflows/oracle-ded.yml

@@ -17,14 +17,19 @@ jobs:
       (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'dedicated_oracle')) ||
       (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dedicated_oracle'))
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: write
     timeout-minutes: 20
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
       - name: Auth google
         uses: google-github-actions/auth@v2
         with:
-          credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }}
+          workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }}
+          service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }}
       - name: Retrieve secrets (CD)
         id: get-gcp-secrets-cd
         uses: google-github-actions/get-secretmanager-secrets@v2

.github/workflows/oracle.yml

@@ -24,6 +24,9 @@ jobs:
 
   test:
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      id-token: write
     timeout-minutes: 55
     env:
       ORA_PREFIX: CI_${{ github.run_id }}_${{ github.run_attempt }}_
@@ -35,7 +38,8 @@ jobs:
       - name: Auth google
         uses: google-github-actions/auth@v2
         with:
-          credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }}
+          workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }}
+          service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }}
       - name: Retrieve secrets
         id: get-gcp-secrets
         uses: google-github-actions/get-secretmanager-secrets@v2
@@ -98,6 +102,9 @@ jobs:
     if: github.ref_name == 'main'
     needs: test
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      id-token: write
     timeout-minutes: 20
     strategy:
       matrix:
@@ -110,7 +117,8 @@ jobs:
       - name: Auth google
         uses: google-github-actions/auth@v2
         with:
-          credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }}
+          workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }}
+          service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }}
       - name: Retrieve secrets (CI)
         if: matrix.target == 'ci'
         id: get-gcp-secrets
@@ -211,6 +219,6 @@ jobs:
       - name: Publish package
         run: |
           cd clouds/oracle
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-oracle-latest.zip
-          gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/
+          gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip
+          gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-oracle-latest.zip --content-type=application/zip
+          gcloud storage cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ --content-type=application/json

.github/workflows/postgres.yml

@@ -143,6 +143,6 @@ jobs:
       - name: Publish package
         run: |
           cd clouds/postgres
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-postgres-latest.zip
-          gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/
+          gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip
+          gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-postgres-latest.zip --content-type=application/zip
+          gcloud storage cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ --content-type=application/json

.github/workflows/redshift.yml

@@ -226,5 +226,5 @@ jobs:
           make create-package cloud=redshift production=1
       - name: Publish package
         run: |
-          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
-          gsutil -h "Content-Type:application/zip" -m cp dist/carto-at-redshift-*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-redshift-latest.zip
+          gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip
+          gcloud storage cp dist/carto-at-redshift-*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-redshift-latest.zip --content-type=application/zip

.github/workflows/snowflake-ded.yml

@@ -15,16 +15,31 @@ jobs:
       (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'dedicated_snowflake')) ||
       (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dedicated_snowflake'))
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: write
     timeout-minutes: 20
     env:
       SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_CD }}
       SF_DATABASE: ${{ secrets.SF_DATABASE_CD }}
       SF_USER: ${{ secrets.SF_USER_CD }}
-      SF_PASSWORD: ${{ secrets.SF_PASSWORD_CD }}
       SF_ROLE: ${{ secrets.SF_ROLE_CD }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
+      - name: Auth google
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }}
+          service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }}
+      - name: Retrieve RSA secrets
+        id: get-gcp-secrets
+        uses: google-github-actions/get-secretmanager-secrets@v2
+        with:
+          secrets: |-
+            SF_RSA_KEY:${{ secrets.SF_RSA_KEY_NAME_CD }}
+            SF_RSA_KEY_PASSWORD:${{ secrets.SF_RSA_KEY_PASSWORD_NAME_CD }}
       - name: Set SF_PREFIX
         run: echo "SF_PREFIX=dedicated_core_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Setup node
@@ -34,12 +49,18 @@ jobs:
       - name: Run deploy
         id: deploy
         if: github.event.action == 'synchronize' || github.event.action == 'labeled'
+        env:
+          SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY }}
+          SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD }}
         run: |
           cd clouds/snowflake
           make deploy
       - name: Run remove
         id: remove
         if: github.event.action == 'unlabeled' || github.event.action == 'closed'
+        env:
+          SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY }}
+          SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD }}
         run: |
           cd clouds/snowflake
           make remove drop-schema=1