diff --git a/.claude/rules/databricks.md b/.claude/rules/databricks.md index faeefd4bc..530002203 100644 --- a/.claude/rules/databricks.md +++ b/.claude/rules/databricks.md @@ -14,7 +14,9 @@ DB_PREFIX=yourname_ # Schema prefix (e.g., "yourname_" -> "yourname_c DB_CATALOG= # Databricks catalog name DB_HOST_NAME= # SQL Warehouse hostname DB_HTTP_PATH= # SQL Warehouse HTTP path -DB_TOKEN= # Access token +DB_TOKEN= # Personal access token — wins when set +DB_CLIENT_ID= # Service principal, used when DB_TOKEN is unset +DB_CLIENT_SECRET= # Service principal secret; the token is renewed per job DB_CONNECTION= # Databricks connection string DB_API_BASE_URL= # CARTO API base URL DB_API_ACCESS_TOKEN= # CARTO API access token diff --git a/.github/workflows/bigquery.yml b/.github/workflows/bigquery.yml index 85a93c577..b705cbd60 100644 --- a/.github/workflows/bigquery.yml +++ b/.github/workflows/bigquery.yml @@ -192,6 +192,6 @@ jobs: - name: Publish package run: | cd clouds/bigquery - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-bigquery-latest.zip - gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-bigquery-latest.zip --content-type=application/zip + gcloud storage cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ --content-type=application/json diff --git a/.github/workflows/databricks-ded.yml b/.github/workflows/databricks-ded.yml index 5bcb03e28..8daaad54c 100644 --- a/.github/workflows/databricks-ded.yml +++ b/.github/workflows/databricks-ded.yml @@ -16,6 +16,10 @@ jobs: (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'dedicated_databricks')) || (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dedicated_databricks')) runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + pull-requests: write timeout-minutes: 20 env: DB_HOST_NAME: ${{ secrets.DB_HOST_NAME_CD }} @@ -35,7 +39,8 @@ jobs: - name: Authenticate with Google Cloud uses: 'google-github-actions/auth@v2' with: - credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }} + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} - name: Get Databricks secrets id: secrets uses: google-github-actions/get-secretmanager-secrets@v2 @@ -43,26 +48,21 @@ jobs: secrets: |- databricks-client-id:${{ secrets.DB_CLIENT_ID_NAME_CD }} databricks-client-secret:${{ secrets.DB_CLIENT_SECRET_NAME_CD }} - - name: Get Databricks OAuth token - run: | - RESPONSE=$(curl -s -X POST https://$DB_HOST_NAME/oidc/v1/token \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=client_credentials" \ - -d "client_id=${{ steps.secrets.outputs.databricks-client-id }}" \ - -d "client_secret=${{ steps.secrets.outputs.databricks-client-secret }}" \ - -d "scope=all-apis") - TOKEN=$(echo $RESPONSE | jq -r '.access_token') - echo "::add-mask::$TOKEN" - echo "DB_TOKEN=$TOKEN" >> $GITHUB_ENV - name: Run deploy id: deploy if: github.event.action == 'synchronize' || github.event.action == 'labeled' + env: + DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }} + DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }} run: | cd clouds/databricks make deploy - name: Run remove id: remove if: github.event.action == 'unlabeled' || github.event.action == 'closed' + env: + DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }} + DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }} run: | cd clouds/databricks make remove diff --git a/.github/workflows/databricks.yml b/.github/workflows/databricks.yml index 1176c0e8b..c0de3a439 100644 --- a/.github/workflows/databricks.yml +++ b/.github/workflows/databricks.yml @@ -23,6 +23,9 @@ jobs: test: runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 20 env: DB_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_${{ github.run_attempt }}_ @@ -43,7 +46,8 @@ jobs: - name: Authenticate with Google Cloud uses: 'google-github-actions/auth@v2' with: - credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }} + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} - name: Get Databricks secrets id: secrets uses: google-github-actions/get-secretmanager-secrets@v2 @@ -51,31 +55,32 @@ jobs: secrets: |- databricks-client-id:${{ secrets.DB_CLIENT_ID_NAME_CI }} databricks-client-secret:${{ secrets.DB_CLIENT_SECRET_NAME_CI }} - - name: Get Databricks OAuth token - run: | - RESPONSE=$(curl -s -X POST https://$DB_HOST_NAME/oidc/v1/token \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=client_credentials" \ - -d "client_id=${{ steps.secrets.outputs.databricks-client-id }}" \ - -d "client_secret=${{ steps.secrets.outputs.databricks-client-secret }}" \ - -d "scope=all-apis") - TOKEN=$(echo $RESPONSE | jq -r '.access_token') - echo "::add-mask::$TOKEN" - echo "DB_TOKEN=$TOKEN" >> $GITHUB_ENV - name: Lint code run: make lint cloud=databricks - name: Deploy modules + env: + DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }} + DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }} run: make deploy cloud=databricks diff="$GIT_DIFF" - name: Run tests + env: + DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }} + DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }} run: make test cloud=databricks diff="$GIT_DIFF" - name: Cleanup resources if: always() + env: + DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }} + DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }} run: make remove cloud=databricks deploy-internal: if: github.ref_name == 'main' needs: test runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 10 env: DB_HOST_NAME: ${{ secrets.DB_HOST_NAME_CD }} @@ -95,7 +100,8 @@ jobs: - name: Authenticate with Google Cloud uses: 'google-github-actions/auth@v2' with: - credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }} + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} - name: Get Databricks secrets id: secrets uses: google-github-actions/get-secretmanager-secrets@v2 @@ -103,18 +109,10 @@ jobs: secrets: |- databricks-client-id:${{ secrets.DB_CLIENT_ID_NAME_CD }} databricks-client-secret:${{ secrets.DB_CLIENT_SECRET_NAME_CD }} - - name: Get Databricks OAuth token - run: | - RESPONSE=$(curl -s -X POST https://$DB_HOST_NAME/oidc/v1/token \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=client_credentials" \ - -d "client_id=${{ steps.secrets.outputs.databricks-client-id }}" \ - -d "client_secret=${{ steps.secrets.outputs.databricks-client-secret }}" \ - -d "scope=all-apis") - TOKEN=$(echo $RESPONSE | jq -r '.access_token') - echo "::add-mask::$TOKEN" - echo "DB_TOKEN=$TOKEN" >> $GITHUB_ENV - name: Deploy modules + env: + DB_CLIENT_ID: ${{ steps.secrets.outputs.databricks-client-id }} + DB_CLIENT_SECRET: ${{ steps.secrets.outputs.databricks-client-secret }} run: make deploy cloud=databricks diff="$GIT_DIFF" production=1 publish: @@ -151,6 +149,6 @@ jobs: - name: Publish package to storage run: | cd clouds/databricks - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-databricks-latest.zip - gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-databricks-latest.zip --content-type=application/zip + gcloud storage cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ --content-type=application/json diff --git a/.github/workflows/oracle-ded.yml b/.github/workflows/oracle-ded.yml index 35febfaf1..4d0002326 100644 --- a/.github/workflows/oracle-ded.yml +++ b/.github/workflows/oracle-ded.yml @@ -17,6 +17,10 @@ jobs: (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'dedicated_oracle')) || (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dedicated_oracle')) runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + pull-requests: write timeout-minutes: 20 steps: - name: Checkout repo @@ -24,7 +28,8 @@ jobs: - name: Auth google uses: google-github-actions/auth@v2 with: - credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }} + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} - name: Retrieve secrets (CD) id: get-gcp-secrets-cd uses: google-github-actions/get-secretmanager-secrets@v2 diff --git a/.github/workflows/oracle.yml b/.github/workflows/oracle.yml index 8143b84f8..2ad2bfa2d 100644 --- a/.github/workflows/oracle.yml +++ b/.github/workflows/oracle.yml @@ -24,6 +24,9 @@ jobs: test: runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 55 env: ORA_PREFIX: CI_${{ github.run_id }}_${{ github.run_attempt }}_ @@ -35,7 +38,8 @@ jobs: - name: Auth google uses: google-github-actions/auth@v2 with: - credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }} + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} - name: Retrieve secrets id: get-gcp-secrets uses: google-github-actions/get-secretmanager-secrets@v2 @@ -98,6 +102,9 @@ jobs: if: github.ref_name == 'main' needs: test runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 20 strategy: matrix: @@ -110,7 +117,8 @@ jobs: - name: Auth google uses: google-github-actions/auth@v2 with: - credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }} + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} - name: Retrieve secrets (CI) if: matrix.target == 'ci' id: get-gcp-secrets @@ -211,6 +219,6 @@ jobs: - name: Publish package run: | cd clouds/oracle - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-oracle-latest.zip - gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-oracle-latest.zip --content-type=application/zip + gcloud storage cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ --content-type=application/json diff --git a/.github/workflows/postgres.yml b/.github/workflows/postgres.yml index d3de29a82..57b6118ee 100644 --- a/.github/workflows/postgres.yml +++ b/.github/workflows/postgres.yml @@ -143,6 +143,6 @@ jobs: - name: Publish package run: | cd clouds/postgres - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-postgres-latest.zip - gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-postgres-latest.zip --content-type=application/zip + gcloud storage cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ --content-type=application/json diff --git a/.github/workflows/redshift.yml b/.github/workflows/redshift.yml index bc1d1053b..6c7b223c3 100644 --- a/.github/workflows/redshift.yml +++ b/.github/workflows/redshift.yml @@ -226,5 +226,5 @@ jobs: make create-package cloud=redshift production=1 - name: Publish package run: | - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ - gsutil -h "Content-Type:application/zip" -m cp dist/carto-at-redshift-*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-redshift-latest.zip + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip + gcloud storage cp dist/carto-at-redshift-*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-redshift-latest.zip --content-type=application/zip diff --git a/.github/workflows/snowflake-ded.yml b/.github/workflows/snowflake-ded.yml index 4d4a12258..e214ced89 100644 --- a/.github/workflows/snowflake-ded.yml +++ b/.github/workflows/snowflake-ded.yml @@ -15,16 +15,31 @@ jobs: (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'dedicated_snowflake')) || (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dedicated_snowflake')) runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + pull-requests: write timeout-minutes: 20 env: SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_CD }} SF_DATABASE: ${{ secrets.SF_DATABASE_CD }} SF_USER: ${{ secrets.SF_USER_CD }} - SF_PASSWORD: ${{ secrets.SF_PASSWORD_CD }} SF_ROLE: ${{ secrets.SF_ROLE_CD }} steps: - name: Checkout repo uses: actions/checkout@v2 + - name: Auth google + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} + - name: Retrieve RSA secrets + id: get-gcp-secrets + uses: google-github-actions/get-secretmanager-secrets@v2 + with: + secrets: |- + SF_RSA_KEY:${{ secrets.SF_RSA_KEY_NAME_CD }} + SF_RSA_KEY_PASSWORD:${{ secrets.SF_RSA_KEY_PASSWORD_NAME_CD }} - name: Set SF_PREFIX run: echo "SF_PREFIX=dedicated_core_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV - name: Setup node @@ -34,12 +49,18 @@ jobs: - name: Run deploy id: deploy if: github.event.action == 'synchronize' || github.event.action == 'labeled' + env: + SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY }} + SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD }} run: | cd clouds/snowflake make deploy - name: Run remove id: remove if: github.event.action == 'unlabeled' || github.event.action == 'closed' + env: + SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY }} + SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD }} run: | cd clouds/snowflake make remove drop-schema=1 diff --git a/.github/workflows/snowflake.yml b/.github/workflows/snowflake.yml index 68fa06886..632ef4f90 100644 --- a/.github/workflows/snowflake.yml +++ b/.github/workflows/snowflake.yml @@ -24,6 +24,9 @@ jobs: test: runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 35 env: SF_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_${{ github.run_attempt }}_ @@ -39,7 +42,8 @@ jobs: - name: Auth google uses: google-github-actions/auth@v2 with: - credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }} + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} - name: Retrieve secrets id: get-gcp-secrets uses: google-github-actions/get-secretmanager-secrets@v2 @@ -88,6 +92,9 @@ jobs: if: github.ref_name == 'main' needs: test runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 20 strategy: matrix: @@ -95,14 +102,15 @@ jobs: - account: SF_ACCOUNT_CD database: SF_DATABASE_CD user: SF_USER_CD - password: SF_PASSWORD_CD role: SF_ROLE_CD - use_rsa_key: false + rsa_key_name: SF_RSA_KEY_NAME_CD + rsa_key_password_name: SF_RSA_KEY_PASSWORD_NAME_CD - account: SF_ACCOUNT_CI database: SF_DATABASE_CI user: SF_USER_CI role: SF_ROLE_CI - use_rsa_key: true + rsa_key_name: SF_RSA_KEY_NAME_CI + rsa_key_password_name: SF_RSA_KEY_PASSWORD_NAME_CI env: SF_ACCOUNT: ${{ secrets[matrix.account] }} SF_DATABASE: ${{ secrets[matrix.database] }} @@ -113,28 +121,26 @@ jobs: uses: actions/checkout@v2 - name: Check diff uses: technote-space/get-diff-action@v6 - - name: Auth google (CI only) - if: matrix.use_rsa_key + - name: Auth google uses: google-github-actions/auth@v2 with: - credentials_json: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT_KEY }} - - name: Retrieve RSA secrets (CI only) - if: matrix.use_rsa_key + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} + - name: Retrieve RSA secrets id: get-gcp-secrets uses: google-github-actions/get-secretmanager-secrets@v2 with: secrets: |- - SF_RSA_KEY:${{ secrets.SF_RSA_KEY_NAME_CI }} - SF_RSA_KEY_PASSWORD:${{ secrets.SF_RSA_KEY_PASSWORD_NAME_CI }} + SF_RSA_KEY:${{ secrets[matrix.rsa_key_name] }} + SF_RSA_KEY_PASSWORD:${{ secrets[matrix.rsa_key_password_name] }} - name: Setup node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: ${{ env.NODE_VERSION }} - name: Run deploy env: - SF_PASSWORD: ${{ secrets[matrix.password] || '' }} - SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY || '' }} - SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD || '' }} + SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY }} + SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD }} run: | cd clouds/snowflake make deploy diff="$GIT_DIFF" production=1 @@ -143,23 +149,40 @@ jobs: if: github.ref_name == 'stable' needs: test runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 20 env: SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_CD }} SF_DATABASE: ${{ secrets.SF_DATABASE_STABLE_CD }} SF_USER: ${{ secrets.SF_USER_CD }} - SF_PASSWORD: ${{ secrets.SF_PASSWORD_CD }} SF_ROLE: ${{ secrets.SF_ROLE_CD }} steps: - name: Checkout repo uses: actions/checkout@v2 - name: Check diff uses: technote-space/get-diff-action@v6 + - name: Auth google + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} + - name: Retrieve RSA secrets + id: get-gcp-secrets + uses: google-github-actions/get-secretmanager-secrets@v2 + with: + secrets: |- + SF_RSA_KEY:${{ secrets.SF_RSA_KEY_NAME_CD }} + SF_RSA_KEY_PASSWORD:${{ secrets.SF_RSA_KEY_PASSWORD_NAME_CD }} - name: Setup node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: ${{ env.NODE_VERSION }} - name: Run deploy + env: + SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY }} + SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD }} run: | cd clouds/snowflake make deploy diff="$GIT_DIFF" production=1 @@ -168,6 +191,9 @@ jobs: if: github.ref_name == 'main' needs: test runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 30 env: APP_PACKAGE_NAME: ${{ secrets.SF_NATIVE_APP_PACKAGE_NAME_CD }} @@ -175,22 +201,39 @@ jobs: SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_NATIVE_APP }} SF_DATABASE: ${{ secrets.SF_DATABASE_NATIVE_APP }} SF_USER: ${{ secrets.SF_USER_NATIVE_APP }} - SF_PASSWORD: ${{ secrets.SF_PASSWORD_NATIVE_APP }} SF_ROLE: ${{ secrets.SF_ROLE_NATIVE_APP }} steps: - name: Checkout repo uses: actions/checkout@v2 - name: Check diff uses: technote-space/get-diff-action@v6 + - name: Auth google + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} + - name: Retrieve RSA secrets + id: get-gcp-secrets + uses: google-github-actions/get-secretmanager-secrets@v2 + with: + secrets: |- + SF_RSA_KEY:${{ secrets.SF_RSA_KEY_NAME_NATIVE_APP }} + SF_RSA_KEY_PASSWORD:${{ secrets.SF_RSA_KEY_PASSWORD_NAME_NATIVE_APP }} - name: Setup node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: ${{ env.NODE_VERSION }} - name: Deploy native app package + env: + SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY }} + SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD }} run: | cd clouds/snowflake make deploy-native-app-package production=1 - name: Deploy native app locally + env: + SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY }} + SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD }} run: | cd clouds/snowflake make deploy-native-app production=1 @@ -199,12 +242,14 @@ jobs: if: github.ref_name == 'stable' needs: test runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 20 env: SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_SHARE }} SF_DATABASE: ${{ secrets.SF_DATABASE_SHARE }} SF_USER: ${{ secrets.SF_USER_SHARE }} - SF_PASSWORD: ${{ secrets.SF_PASSWORD_SHARE }} SF_ROLE: ${{ secrets.SF_ROLE_SHARE }} SF_SHARE: ${{ secrets.SF_SHARE }} steps: @@ -212,11 +257,26 @@ jobs: uses: actions/checkout@v2 - name: Check diff uses: technote-space/get-diff-action@v6 + - name: Auth google + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.CARTO_AT_GH_ACTIONS_WIF_PROVIDER }} + service_account: ${{ secrets.CARTO_AT_GH_ACTIONS_SERVICE_ACCOUNT }} + - name: Retrieve RSA secrets + id: get-gcp-secrets + uses: google-github-actions/get-secretmanager-secrets@v2 + with: + secrets: |- + SF_RSA_KEY:${{ secrets.SF_RSA_KEY_NAME_SHARE }} + SF_RSA_KEY_PASSWORD:${{ secrets.SF_RSA_KEY_PASSWORD_NAME_SHARE }} - name: Setup node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: ${{ env.NODE_VERSION }} - name: Run deploy + env: + SF_RSA_KEY: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY }} + SF_RSA_KEY_PASSWORD: ${{ steps.get-gcp-secrets.outputs.SF_RSA_KEY_PASSWORD }} run: | cd clouds/snowflake make deploy diff="$GIT_DIFF" production=1 @@ -255,6 +315,6 @@ jobs: - name: Publish package run: | cd clouds/snowflake - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ - gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-snowflake-latest.zip - gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/ --content-type=application/zip + gcloud storage cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-snowflake-latest.zip --content-type=application/zip + gcloud storage cp dist/*.json ${{ env.PACKAGE_BUCKET }}/ --content-type=application/json diff --git a/CHANGELOG.md b/CHANGELOG.md index f48fcd80c..c05ffafe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ CARTO Analytics Toolbox Core. All notable commits to this project will be documented in this file. +## 2026-09-24 + +- chore(sf): remove the broken h3_fromlonglat JS library entry (#647) +- chore(sf): connect lazily in the Snowflake test harness (#648) +- chore(sf): ship JavaScript source maps in the native app package (#649) +- fix(sf,pg,ora,bq): insert every replacement literally when building SQL (#650) +- fix(db): renew the Databricks token instead of minting it once per job (#652) +- chore(deps): bump sqlparse from 0.5.4 to 0.6.0 (#642, #643, #644) +- refactor: migrate gsutil to gcloud storage (#651) +- chore(ci): migrate GitHub Actions auth to WIF (#641) +- chore(sf|ci): migrate the Snowflake CD jobs to key-pair auth (#654) + ## 2026-08-24 - chore(bq,sf,rs,pg,ora): pin actions/setup-node to SHA across all workflows (#628) diff --git a/clouds/bigquery/README.md b/clouds/bigquery/README.md index 138849172..3307b4743 100644 --- a/clouds/bigquery/README.md +++ b/clouds/bigquery/README.md @@ -11,7 +11,7 @@ Make sure you have installed the following tools: - `node`: (v14.18) - `yarn`: (v1.22) - `bq`: -- `gsutil`: (v5.5) +- `gcloud storage`: ## Environment variables diff --git a/clouds/bigquery/common/Makefile b/clouds/bigquery/common/Makefile index 6cd71dfe4..e9aba68ae 100644 --- a/clouds/bigquery/common/Makefile +++ b/clouds/bigquery/common/Makefile @@ -3,7 +3,7 @@ PYTHON3_VERSION = 3 VENV3_DIR ?= $(COMMON_DIR)/../venv3 VENV3_BIN = $(VENV3_DIR)/bin -GSUTIL = gsutil +GSUTIL = gcloud storage NODE_MODULES_DEV = $(COMMON_DIR)/node_modules BQ = bq --location=$(BQ_REGION) --project_id=$(BQ_PROJECT) BQ_DATASET_DEFAULT = carto diff --git a/clouds/bigquery/common/build_modules.js b/clouds/bigquery/common/build_modules.js index bcc470f28..881b37d86 100755 --- a/clouds/bigquery/common/build_modules.js +++ b/clouds/bigquery/common/build_modules.js @@ -171,7 +171,7 @@ const outputSeparator = argv.production ? '\n' : internalSeparator; let content = output.map(f => f.content).join(internalSeparator); function apply_replacements (text) { - const libraries = [... new Set(content.match(new RegExp('@@BQ_LIBRARY_[^@]*?_BUCKET@@', 'g')))]; + const libraries = [... new Set(text.match(new RegExp('@@BQ_LIBRARY_[^@]*?_BUCKET@@', 'g')))]; for (let library of libraries) { let libraryName = library.replace('@@BQ_LIBRARY_', '').replace('_BUCKET@@', '').toLowerCase(); if (makelib == libraryName) { @@ -181,7 +181,7 @@ function apply_replacements (text) { const libraryPath = path.join(libsBuildDir, libraryName); if (fs.existsSync(libraryPath)) { const libraryBucketPath = libraryBucket + '_' + libraryName; - text = text.replace(new RegExp(library, 'g'), libraryBucketPath); + text = text.replace(new RegExp(library, 'g'), () => libraryBucketPath); } else { console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`); @@ -192,7 +192,7 @@ function apply_replacements (text) { for (let replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } text = text.replace(/@@SKIP_DEP@@/g, ''); diff --git a/clouds/bigquery/common/run-script.js b/clouds/bigquery/common/run-script.js index 6f5c04e23..8ca75edc0 100755 --- a/clouds/bigquery/common/run-script.js +++ b/clouds/bigquery/common/run-script.js @@ -22,7 +22,7 @@ function apply_replacements (text) { for (let replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } } diff --git a/clouds/databricks/common/Makefile b/clouds/databricks/common/Makefile index 87e47c979..8610b4abe 100644 --- a/clouds/databricks/common/Makefile +++ b/clouds/databricks/common/Makefile @@ -35,7 +35,12 @@ ifndef DB_HTTP_PATH $(error DB_HTTP_PATH is undefined) endif ifndef DB_TOKEN - $(error DB_TOKEN is undefined) +ifndef DB_CLIENT_ID + $(error either DB_TOKEN or DB_CLIENT_ID and DB_CLIENT_SECRET are undefined) +endif +ifndef DB_CLIENT_SECRET + $(error either DB_TOKEN or DB_CLIENT_ID and DB_CLIENT_SECRET are undefined) +endif endif venv3: diff --git a/clouds/databricks/common/benchmark_utils/__init__.py b/clouds/databricks/common/benchmark_utils/__init__.py index d63ccf5b1..e49eabab4 100644 --- a/clouds/databricks/common/benchmark_utils/__init__.py +++ b/clouds/databricks/common/benchmark_utils/__init__.py @@ -8,6 +8,8 @@ from databricks import sql as dbsql +from db_token import get_access_token + __all__ = ['benchmark', 'bench', 'config_for'] # Connection-establishment time would otherwise be charged to whichever @@ -32,7 +34,7 @@ def _get_bench_conn(): _BENCH_CONN = dbsql.connect( server_hostname=os.environ['DB_HOST_NAME'], http_path=os.environ['DB_HTTP_PATH'], - access_token=os.environ['DB_TOKEN'], + access_token=get_access_token(), _disable_pandas=True, ) atexit.register(_close_bench_conn) diff --git a/clouds/databricks/common/build_modules.js b/clouds/databricks/common/build_modules.js index ffecbf972..79a024e42 100755 --- a/clouds/databricks/common/build_modules.js +++ b/clouds/databricks/common/build_modules.js @@ -164,7 +164,7 @@ function apply_replacements (text) { for (let replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } text = text.replace(/\\`/g, '`'); diff --git a/clouds/databricks/common/db_token.py b/clouds/databricks/common/db_token.py new file mode 100644 index 000000000..a7ece7ac8 --- /dev/null +++ b/clouds/databricks/common/db_token.py @@ -0,0 +1,50 @@ +import json +import os +import time +import urllib.parse +import urllib.request + + +TOKEN_PATH = '/oidc/v1/token' +REFRESH_MARGIN_SECONDS = 300 + +_token = None +_expires_at = 0.0 + + +def get_access_token(): + """Return a Databricks access token, renewing it before it expires. + + DB_TOKEN wins when set, so local runs and personal access tokens keep + working unchanged. Otherwise a token is requested with the service + principal credentials and reused until it is close to expiring. + """ + token = os.getenv('DB_TOKEN') + if token: + return token + + global _token, _expires_at + if _token is None or time.monotonic() >= _expires_at: + _token, expires_in = _request_token() + _expires_at = time.monotonic() + max(expires_in - REFRESH_MARGIN_SECONDS, 0) + return _token + + +def _request_token(): + host = os.environ['DB_HOST_NAME'] + payload = urllib.parse.urlencode( + { + 'grant_type': 'client_credentials', + 'client_id': os.environ['DB_CLIENT_ID'], + 'client_secret': os.environ['DB_CLIENT_SECRET'], + 'scope': 'all-apis', + } + ).encode() + request = urllib.request.Request( + f'https://{host}{TOKEN_PATH}', + data=payload, + headers={'Content-Type': 'application/x-www-form-urlencoded'}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + body = json.load(response) + return body['access_token'], int(body.get('expires_in', 3600)) diff --git a/clouds/databricks/common/python3_requirements.txt b/clouds/databricks/common/python3_requirements.txt index fa83829d9..8cd5911b8 100644 --- a/clouds/databricks/common/python3_requirements.txt +++ b/clouds/databricks/common/python3_requirements.txt @@ -3,7 +3,7 @@ flake8-quotes==3.2.0 flake8-docstrings==1.6.0 pep8-naming==0.12.1 brunette==0.2.0 -sqlparse==0.5.4 +sqlparse==0.6.0 sqlfluff==4.2.2 databricks-sql-connector==2.0.2 pytest==7.1.2 diff --git a/clouds/databricks/common/run_query.py b/clouds/databricks/common/run_query.py index 5225217b3..94627ea64 100644 --- a/clouds/databricks/common/run_query.py +++ b/clouds/databricks/common/run_query.py @@ -3,12 +3,14 @@ from databricks import sql +from db_token import get_access_token + def run_query(query): with sql.connect( server_hostname=os.getenv('DB_HOST_NAME'), http_path=os.getenv('DB_HTTP_PATH'), - access_token=os.getenv('DB_TOKEN'), + access_token=get_access_token(), ) as conn: with conn.cursor() as cursor: cursor.execute(query) diff --git a/clouds/databricks/common/run_script.py b/clouds/databricks/common/run_script.py index 8e3c4fff6..7cdb732c7 100644 --- a/clouds/databricks/common/run_script.py +++ b/clouds/databricks/common/run_script.py @@ -6,6 +6,8 @@ from sqlparse import split from databricks import sql +from db_token import get_access_token + function = '' @@ -26,7 +28,7 @@ def run_queries(queries): with sql.connect( server_hostname=os.getenv('DB_HOST_NAME'), http_path=os.getenv('DB_HTTP_PATH'), - access_token=os.getenv('DB_TOKEN'), + access_token=get_access_token(), ) as conn: with conn.cursor() as cursor: for i in trange(len(queries), dynamic_ncols=True, leave=True, position=0): diff --git a/clouds/databricks/common/test_utils/__init__.py b/clouds/databricks/common/test_utils/__init__.py index 7430cfc16..d91d39cd9 100644 --- a/clouds/databricks/common/test_utils/__init__.py +++ b/clouds/databricks/common/test_utils/__init__.py @@ -1,12 +1,14 @@ import os from databricks import sql +from db_token import get_access_token + def run_query(query): with sql.connect( server_hostname=os.getenv('DB_HOST_NAME'), http_path=os.getenv('DB_HTTP_PATH'), - access_token=os.getenv('DB_TOKEN'), + access_token=get_access_token(), _disable_pandas=True, ) as conn: with conn.cursor() as cursor: diff --git a/clouds/oracle/common/build_modules.js b/clouds/oracle/common/build_modules.js index 4b866328d..0f06a79ee 100755 --- a/clouds/oracle/common/build_modules.js +++ b/clouds/oracle/common/build_modules.js @@ -195,11 +195,24 @@ let content = output.map(f => f.content).join('\n'); // Inline @@ORA_LIBRARY_@@ → libraries/javascript/build/.js, then // apply env-var @@VAR@@ replacements. +// Guards against a future revert to a string replacement, where $& and $$ would expand +function insert_literally (text, placeholder, content) { + const pattern = new RegExp(placeholder, 'g'); + const count = (text.match(pattern) || []).length; + const expected = text.length + count * (content.length - placeholder.length); + const result = text.replace(pattern, () => content); + if (result.length !== expected) { + console.log(`ERROR: "${placeholder}" was not inserted literally`); + process.exit(1); + } + return result; +} + function apply_replacements (text) { const libraryDir = path.resolve( __dirname, '..', 'libraries', 'javascript', 'build' ); - const libraries = [...new Set(text.match(/@@ORA_LIBRARY_[A-Z0-9_]+@@/g) || [])]; + const libraries = [...new Set(text.match(/@@ORA_LIBRARY_[^@]+@@/g) || [])]; for (const library of libraries) { const libName = library.replace('@@ORA_LIBRARY_', '').replace('@@', ''); const file = path.join(libraryDir, libName.toLowerCase() + '.js'); @@ -210,16 +223,14 @@ function apply_replacements (text) { ); process.exit(1); } - text = text.replace( - new RegExp(library, 'g'), - fs.readFileSync(file).toString() - ); + const libraryContent = fs.readFileSync(file).toString(); + text = insert_literally(text, library, libraryContent); } const replacements = process.env.REPLACEMENTS.split(' '); for (const replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } return text; diff --git a/clouds/postgres/common/build_modules.js b/clouds/postgres/common/build_modules.js index 7ad0eb58d..430149e37 100755 --- a/clouds/postgres/common/build_modules.js +++ b/clouds/postgres/common/build_modules.js @@ -181,14 +181,27 @@ let content = anonymousBlockWrapping(postgisInstalledCheck) // Replace environment variables content += output.map(f => anonymousBlockWrapping(extensionFunctionWrapping(f.content, f.module))).join('\n'); +// Guards against a future revert to a string replacement, where $& and $$ would expand +function insert_literally (text, placeholder, content) { + const pattern = new RegExp(placeholder, 'g'); + const count = (text.match(pattern) || []).length; + const expected = text.length + count * (content.length - placeholder.length); + const result = text.replace(pattern, () => content); + if (result.length !== expected) { + console.log(`ERROR: "${placeholder}" was not inserted literally`); + process.exit(1); + } + return result; +} + function apply_replacements (text) { - const libraries = [... new Set(text.match(new RegExp('@@PG_LIBRARY_.*@@', 'g')))]; + const libraries = [... new Set(text.match(new RegExp('@@PG_LIBRARY_[^@]+@@', 'g')))]; for (let library of libraries) { const libraryName = library.replace('@@PG_LIBRARY_', '').replace('@@', '').toLowerCase() + '.js'; const libraryPath = path.join(libsBuildDir, libraryName); if (fs.existsSync(libraryPath)) { const libraryContent = fs.readFileSync(libraryPath).toString(); - text = text.replace(new RegExp(library, 'g'), libraryContent); + text = insert_literally(text, library, libraryContent); } else { console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`); @@ -199,7 +212,7 @@ function apply_replacements (text) { for (let replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } return text; diff --git a/clouds/postgres/common/list_libraries.js b/clouds/postgres/common/list_libraries.js index 8ab7ec806..385b836fc 100755 --- a/clouds/postgres/common/list_libraries.js +++ b/clouds/postgres/common/list_libraries.js @@ -129,7 +129,7 @@ function add (f, include) { functions.forEach(f => add(f)); const content = output.map(f => f.content).join('\n'); -const libraries = [... new Set(content.match(new RegExp('@@PG_LIBRARY_.*@@', 'g')))] +const libraries = [... new Set(content.match(new RegExp('@@PG_LIBRARY_[^@]+@@', 'g')))] .map(l => l.replace('@@PG_LIBRARY_', '').replace('@@', '').toLowerCase()); process.stdout.write(libraries.join(' ')); \ No newline at end of file diff --git a/clouds/postgres/common/python3_requirements.txt b/clouds/postgres/common/python3_requirements.txt index 20b1a84df..306618737 100644 --- a/clouds/postgres/common/python3_requirements.txt +++ b/clouds/postgres/common/python3_requirements.txt @@ -5,7 +5,7 @@ pep8-naming==0.12.1 brunette==0.2.0 pytest==6.2.4 psycopg2-binary==2.9.1 -sqlparse==0.5.4 +sqlparse==0.6.0 wheel==0.38.1 tqdm==4.64.0 GeoAlchemy2==0.9.3 diff --git a/clouds/redshift/common/build_modules.js b/clouds/redshift/common/build_modules.js index 228381d0b..9640fe7ff 100755 --- a/clouds/redshift/common/build_modules.js +++ b/clouds/redshift/common/build_modules.js @@ -153,7 +153,7 @@ function apply_replacements (text) { for (let replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } return text; diff --git a/clouds/redshift/common/python3_requirements.txt b/clouds/redshift/common/python3_requirements.txt index ab4b734c5..1a36717e3 100644 --- a/clouds/redshift/common/python3_requirements.txt +++ b/clouds/redshift/common/python3_requirements.txt @@ -5,7 +5,7 @@ pep8-naming==0.12.1 brunette==0.2.0 pytest==6.2.4 redshift-connector==2.1.14 -sqlparse==0.5.4 +sqlparse==0.6.0 wheel==0.38.1 tqdm==4.64.0 sqlfluff==4.2.2 diff --git a/clouds/snowflake/Makefile b/clouds/snowflake/Makefile index 35f426f7a..f5627ef41 100644 --- a/clouds/snowflake/Makefile +++ b/clouds/snowflake/Makefile @@ -13,7 +13,7 @@ include $(COMMON_DIR)/Makefile .SILENT: -.PHONY: help lint lint-fix build deploy test benchmark remove clean create-package deploy-native-app-package deploy-native-app +.PHONY: help lint lint-fix build deploy test benchmark remove clean create-package prefix-library-sourcemap-urls deploy-native-app-package deploy-native-app help: echo "Available targets: lint build deploy test remove clean create-package deploy-native-app-package deploy-native-app" @@ -79,12 +79,22 @@ build-modules: build-native-app-setup-script: rm -rf $(BUILD_DIR) - $(MAKE) build-libraries + $(MAKE) build-libraries sourcemap=1 + $(MAKE) prefix-library-sourcemap-urls mkdir -p $(BUILD_DIR) $(MAKE) -C modules build SF_SCHEMA=@@SF_SCHEMA@@ cp modules/build/modules.sql $(BUILD_DIR) $(MAKE) -C modules build-native-app-setup-script cp modules/build/setup_script.sql $(BUILD_DIR) + $(MAKE) -C modules build-source-review + cp modules/build/SOURCE_REVIEW.md $(BUILD_DIR) + +prefix-library-sourcemap-urls: + echo "Prefixing library sourceMappingURL comments with $(APP_SOURCE_MAPS_DIR)/..." + set -e; for f in libraries/javascript/build/*.js; do \ + sed -i.bak 's|//# sourceMappingURL=\($(APP_SOURCE_MAPS_DIR)/\)*|//# sourceMappingURL=$(APP_SOURCE_MAPS_DIR)/|' $$f; \ + rm -f $$f.bak; \ + done deploy: $(MAKE) build-libraries diff --git a/clouds/snowflake/common/Makefile b/clouds/snowflake/common/Makefile index 5d8023d34..0e339a49f 100644 --- a/clouds/snowflake/common/Makefile +++ b/clouds/snowflake/common/Makefile @@ -1,6 +1,10 @@ # Makefile common for Snowflake PYTHON3_VERSION = 3 + +# The sed prefix, the copy, the PUT and the index must all agree on this +APP_SOURCE_MAPS_DIR ?= sourcemaps + VENV3_DIR ?= $(COMMON_DIR)/../venv3 VENV3_BIN = $(VENV3_DIR)/bin NODE_MODULES_DEV = $(COMMON_DIR)/node_modules diff --git a/clouds/snowflake/common/build_modules.js b/clouds/snowflake/common/build_modules.js index 06dcf05da..93e1ebb6e 100755 --- a/clouds/snowflake/common/build_modules.js +++ b/clouds/snowflake/common/build_modules.js @@ -159,14 +159,36 @@ if (argv.production) { } let content = output.map(f => f.content).join(separator); +// Snowflake wraps each JS body in AS $$ ... $$, so a literal $$ in a bundle would close it early +function inline_library (text, placeholder, content) { + if (content.includes('$$')) { + console.log(`ERROR: library "${placeholder}" contains "$$", which would close the AS $$ ... $$ body`); + process.exit(1); + } + return insert_literally(text, placeholder, content); +} + +// Guards against a future revert to a string replacement, where $& and $$ would expand +function insert_literally (text, placeholder, content) { + const pattern = new RegExp(placeholder, 'g'); + const count = (text.match(pattern) || []).length; + const expected = text.length + count * (content.length - placeholder.length); + const result = text.replace(pattern, () => content); + if (result.length !== expected) { + console.log(`ERROR: "${placeholder}" was not inserted literally`); + process.exit(1); + } + return result; +} + function apply_replacements (text) { - const libraries = [... new Set(text.match(new RegExp('@@SF_LIBRARY_.*@@', 'g')))]; + const libraries = [... new Set(text.match(new RegExp('@@SF_LIBRARY_[^@]+@@', 'g')))]; for (let library of libraries) { const libraryName = library.replace('@@SF_LIBRARY_', '').replace('@@', '').toLowerCase() + '.js'; const libraryPath = path.join(libsBuildDir, libraryName); if (fs.existsSync(libraryPath)) { const libraryContent = fs.readFileSync(libraryPath).toString(); - text = text.replace(new RegExp(library, 'g'), libraryContent); + text = inline_library(text, library, libraryContent); } else { console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`); @@ -177,7 +199,7 @@ function apply_replacements (text) { for (let replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } return text; diff --git a/clouds/snowflake/common/build_native_app_setup_script.js b/clouds/snowflake/common/build_native_app_setup_script.js index cb0c0677e..320533456 100755 --- a/clouds/snowflake/common/build_native_app_setup_script.js +++ b/clouds/snowflake/common/build_native_app_setup_script.js @@ -9,7 +9,6 @@ const path = require('path'); const argv = require('minimist')(process.argv.slice(2)); const outputDir = argv.output || 'build'; -const libsBuildDir = argv.libs_build_dir || '../libraries/javascript/build'; const nativeAppDir = argv.native_app_dir || '../native_app'; // Replace environment variables @@ -22,24 +21,17 @@ if (argv.production) { let content = fs.readFileSync(path.resolve(nativeAppDir, 'SETUP_SCRIPT.sql')).toString(); function apply_replacements (text) { - const libraries = [... new Set(text.match(new RegExp('@@SF_LIBRARY_.*@@', 'g')))]; - for (let library of libraries) { - const libraryName = library.replace('@@SF_LIBRARY_', '').replace('@@', '').toLowerCase() + '.js'; - const libraryPath = path.join(libsBuildDir, libraryName); - if (fs.existsSync(libraryPath)) { - const libraryContent = fs.readFileSync(libraryPath).toString(); - text = text.replace(new RegExp(library, 'g'), libraryContent); - } - else { - console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`); - process.exit(1); - } + // The setup script does not inline libraries: modules.sql is read from the stage at install time + const libraries = text.match(new RegExp('@@SF_LIBRARY_[^@]+@@', 'g')); + if (libraries) { + console.log(`ERROR: the setup script cannot inline libraries, found ${libraries.join(', ')}`); + process.exit(1); } const replacements = process.env.REPLACEMENTS.split(' '); for (let replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } return text; diff --git a/clouds/snowflake/common/build_share.js b/clouds/snowflake/common/build_share.js index 8c410f811..91225cfe4 100755 --- a/clouds/snowflake/common/build_share.js +++ b/clouds/snowflake/common/build_share.js @@ -184,7 +184,7 @@ function apply_replacements (text) { for (let replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } return text; diff --git a/clouds/snowflake/common/build_source_review.js b/clouds/snowflake/common/build_source_review.js new file mode 100755 index 000000000..0774bae82 --- /dev/null +++ b/clouds/snowflake/common/build_source_review.js @@ -0,0 +1,136 @@ +#!/usr/bin/env node + +// Build the source review index and check every inlined library ships a usable source map + +// ./build_source_review.js modules --output=build --libs_build_dir=../libraries/javascript/build --source_maps_dir=sourcemaps + +const fs = require('fs'); +const path = require('path'); +const argv = require('minimist')(process.argv.slice(2)); + +const inputDirs = argv._[0] && argv._[0].split(','); +const outputDir = argv.output || 'build'; +const libsBuildDir = argv.libs_build_dir || '../libraries/javascript/build'; +const sourceMapsDir = argv.source_maps_dir; + +if (!sourceMapsDir) { + console.log('ERROR: --source_maps_dir is required and must match APP_SOURCE_MAPS_DIR'); + process.exit(1); +} + +// Extract functions and the libraries they inline +const functions = []; +for (let inputDir of inputDirs) { + const sqldir = path.join(inputDir, 'sql'); + const modules = fs.readdirSync(sqldir); + modules.forEach(module => { + const moduledir = path.join(sqldir, module); + if (fs.statSync(moduledir).isDirectory()) { + const files = fs.readdirSync(moduledir); + files.forEach(file => { + if (file.endsWith('.sql')) { + const name = path.parse(file).name; + // Strip comments as build_modules.js does, or a commented-out placeholder counts + const content = fs.readFileSync(path.join(moduledir, file)).toString().replace(/--.*\n/g, ''); + const libraries = [... new Set(content.match(/@@SF_LIBRARY_[A-Z0-9_]+@@/g) || [])] + .map(l => l.replace('@@SF_LIBRARY_', '').replace('@@', '').toLowerCase()); + if (libraries.length && !functions.some(f => f.name === name)) { + functions.push({ name, module, libraries }); + } + } + }); + } + }); +} + +const libraries = {}; +functions.forEach(f => { + f.libraries.forEach(library => { + libraries[library] = libraries[library] || []; + libraries[library].push(f.name); + }); +}); + +// Check the source maps +const errors = []; +const libraryNames = Object.keys(libraries).sort(); +libraryNames.forEach(library => { + const bundlePath = path.join(libsBuildDir, `${library}.js`); + const mapPath = `${bundlePath}.map`; + if (!fs.existsSync(bundlePath)) { + errors.push(`library "${library}" is inlined by ${libraries[library].join(', ')} but ${bundlePath} does not exist`); + return; + } + if (!fs.readFileSync(bundlePath).toString().includes('//# sourceMappingURL=')) { + errors.push(`bundle "${library}.js" has no sourceMappingURL comment: build the libraries with "sourcemap=1"`); + return; + } + if (!fs.existsSync(mapPath)) { + errors.push(`library "${library}" has no source map at ${mapPath}`); + return; + } + let map; + try { + map = JSON.parse(fs.readFileSync(mapPath).toString()); + } catch (e) { + errors.push(`source map "${library}.js.map" is not valid JSON: ${e.message}`); + return; + } + const sources = map.sources || []; + const contents = map.sourcesContent || []; + const missing = sources.filter((_, i) => typeof contents[i] !== 'string' || contents[i] === ''); + if (!sources.length) { + errors.push(`source map "${library}.js.map" lists no sources, so no original can be recovered from it`); + } else if (contents.length !== sources.length) { + errors.push(`source map "${library}.js.map" has ${contents.length} sourcesContent entries for ${sources.length} sources`); + } else if (missing.length) { + errors.push(`source map "${library}.js.map" is missing the content of ${missing.length} of ${sources.length} sources, starting with "${missing[0]}"`); + } + const leaked = sources.filter(s => path.isAbsolute(s) || s.startsWith('..')); + if (leaked.length) { + errors.push(`source map "${library}.js.map" leaks build paths (${leaked[0]}): check sourcemapPathTransform`); + } +}); + +if (errors.length) { + errors.forEach(e => console.log(`ERROR: ${e}`)); + process.exit(1); +} + +const functionRows = functions + .sort((a, b) => a.name.localeCompare(b.name)) + .map(f => `| \`${f.name}\` | ${f.libraries.sort().map(l => `\`${sourceMapsDir}/${l}.js.map\``).join(', ')} |`) + .join('\n'); + +const document = `# Source review + +The JavaScript inlined in \`modules.sql\` is minified. Every library it inlines ships +with its source map in \`${sourceMapsDir}/\`, and each map includes \`sourcesContent\`, so +the original un-minified source can be recovered from the map on its own, with no +other file needed. + +Snowflake JavaScript UDFs cannot import code: a function body has to be entirely +self-contained inside its \`CREATE FUNCTION\` statement. Each library is therefore +bundled and inlined into every function that uses it, which is why one map can cover +several functions. Every inlined copy ends with a \`//# sourceMappingURL=\` comment +naming its map, so each occurrence in \`modules.sql\` points at its own source. + +## Reading a source map + +A \`.map\` file is JSON. Two fields carry the original code: + +- \`sources\` — the path of each original file that went into the bundle +- \`sourcesContent\` — the full text of each of those files, at the same index + +So \`sourcesContent[i]\` is the complete, un-minified source of \`sources[i]\`. Browser +developer tools also load these maps directly and will display the original files. + +## Which source map covers which function + +| Function | Source map | +| --- | --- | +${functionRows} +`; + +fs.writeFileSync(path.join(outputDir, 'SOURCE_REVIEW.md'), document); +console.log(`Write ${outputDir}/SOURCE_REVIEW.md (${libraryNames.length} libraries, ${functions.length} functions)`); \ No newline at end of file diff --git a/clouds/snowflake/common/list_libraries.js b/clouds/snowflake/common/list_libraries.js index 6bd4e6ff6..6324d7925 100755 --- a/clouds/snowflake/common/list_libraries.js +++ b/clouds/snowflake/common/list_libraries.js @@ -130,7 +130,7 @@ function add (f, include) { functions.forEach(f => add(f)); const content = output.map(f => f.content).join('\n'); -const libraries = [... new Set(content.match(new RegExp('@@SF_LIBRARY_.*@@', 'g')))] +const libraries = [... new Set(content.match(new RegExp('@@SF_LIBRARY_[^@]+@@', 'g')))] .map(l => l.replace('@@SF_LIBRARY_', '').replace('@@', '').toLowerCase()); process.stdout.write(libraries.join(' ')); \ No newline at end of file diff --git a/clouds/snowflake/common/rollup.config.js b/clouds/snowflake/common/rollup.config.js index 3c19ed6ed..e851528c5 100644 --- a/clouds/snowflake/common/rollup.config.js +++ b/clouds/snowflake/common/rollup.config.js @@ -30,6 +30,18 @@ export default { input, output: { file: process.env.OUTPUT, + sourcemap: process.env.SOURCEMAP === '1', + sourcemapPathTransform: (relativeSourcePath, sourcemapPath) => { + // Re-root sources: the defaults leak the build layout + const absolutePath = path.resolve(path.dirname(sourcemapPath), relativeSourcePath); + const marker = `${path.sep}clouds${path.sep}`; + const index = absolutePath.lastIndexOf(marker); + if (index === -1) { + return relativeSourcePath; + } + const root = absolutePath.slice(0, index).split(path.sep).pop(); + return [root, ...absolutePath.slice(index + 1).split(path.sep)].join('/'); + }, format: process.env.UNIT_TEST ? 'umd': 'iife', name: process.env.UNIT_TEST ? name : '_' + name, banner: process.env.UNIT_TEST ? '' : 'if (typeof(' +name +') === "undefined") {', diff --git a/clouds/snowflake/common/run-script.js b/clouds/snowflake/common/run-script.js index b39ec99f5..d7379dbe8 100755 --- a/clouds/snowflake/common/run-script.js +++ b/clouds/snowflake/common/run-script.js @@ -66,7 +66,7 @@ function apply_replacements (text) { for (let replacement of replacements) { if (replacement) { const pattern = new RegExp(`@@${replacement}@@`, 'g'); - text = text.replace(pattern, process.env[replacement]); + text = text.replace(pattern, () => process.env[replacement]); } } } diff --git a/clouds/snowflake/common/test-utils.js b/clouds/snowflake/common/test-utils.js index 4a49effba..da4614f4a 100644 --- a/clouds/snowflake/common/test-utils.js +++ b/clouds/snowflake/common/test-utils.js @@ -30,20 +30,38 @@ if (process.env.SF_PASSWORD) { }); } -const connection = snowflake.createConnection(connectionOptions); - -connection.connect((err) => { - if (err) { - console.error(`Unable to connect: ${err.message}`); - } else { - // Optional: store the connection ID. - //const connection_ID = conn.getId(); +// Connect on first query rather than on import: jest imports a test file to +// discover its tests even when they are all skipped, so connecting at module +// scope left an HTTP request in flight after the suite finished and the +// environment was torn down. +// +// The promise itself is the cache, so concurrent callers share one connect and +// later callers reuse it. It is cleared on failure so a subsequent call retries +// rather than latching the error forever. +let connecting = null; + +function getConnection () { + if (!connecting) { + const conn = snowflake.createConnection(connectionOptions); + connecting = new Promise((resolve, reject) => { + conn.connect((err) => { + if (err) { + return reject(new Error(`Unable to connect: ${err.message}`)); + } + return resolve(conn); + }); + }).catch((err) => { + connecting = null; + throw err; + }); } -}); + return connecting; +} -function execAsync (query) { +async function execAsync (query) { + const conn = await getConnection(); return new Promise((resolve, reject) => { - connection.execute({ + conn.execute({ sqlText: query, complete: (err, stmt, rows) => { if (err) { diff --git a/clouds/snowflake/libraries/javascript/Makefile b/clouds/snowflake/libraries/javascript/Makefile index 5b02e24b5..5f78c0266 100644 --- a/clouds/snowflake/libraries/javascript/Makefile +++ b/clouds/snowflake/libraries/javascript/Makefile @@ -51,6 +51,7 @@ ifdef UNIT_TEST $(COMMON_DIR)/list_libraries.js $(MODULES_DIR) --all 1>/dev/null # Check errors for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIR) --all`; do \ NAME=$${f}Lib \ + SOURCEMAP=$(sourcemap) \ PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \ DIRS=$(LIBS_DIR) \ FILENAME=$${f}.js \ @@ -61,6 +62,7 @@ else $(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) 1>/dev/null # Check errors for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps)`; do \ NAME=$${f}Lib \ + SOURCEMAP=$(sourcemap) \ PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \ DIRS=$(LIBS_DIRS) \ FILENAME=$${f}.js \ diff --git a/clouds/snowflake/libraries/javascript/libs/h3_fromlonglat.js b/clouds/snowflake/libraries/javascript/libs/h3_fromlonglat.js deleted file mode 100644 index 9cd766582..000000000 --- a/clouds/snowflake/libraries/javascript/libs/h3_fromlonglat.js +++ /dev/null @@ -1,5 +0,0 @@ -import { geoToH3 } from '../src/h3/h3_fromlonglat/h3core_custom'; - -export default { - geoToH3 -}; \ No newline at end of file diff --git a/clouds/snowflake/modules/Makefile b/clouds/snowflake/modules/Makefile index fc7dba780..496bcf113 100644 --- a/clouds/snowflake/modules/Makefile +++ b/clouds/snowflake/modules/Makefile @@ -34,7 +34,7 @@ include $(COMMON_DIR)/Makefile .SILENT: -.PHONY: help lint lint-fix build build-share build-native-app-setup-script deploy deploy-share test test-fast test-slow benchmark remove remove-share clean +.PHONY: help lint lint-fix build build-share build-source-review build-native-app-setup-script deploy deploy-share test test-fast test-slow benchmark remove remove-share clean help: echo "Available targets: lint build deploy test benchmark remove clean" @@ -86,6 +86,13 @@ build-share: $(NODE_MODULES_DEV) --output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) --diff="$(diff)" \ --modules=$(modules) --functions=$(functions) --production=$(production) --nodeps=$(nodeps) --dropfirst=$(dropfirst) +build-source-review: $(NODE_MODULES_DEV) + echo "Building source review..." + mkdir -p $(BUILD_DIR) + $(COMMON_DIR)/build_source_review.js $(MODULES_DIRS) \ + --output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) \ + --source_maps_dir=$(APP_SOURCE_MAPS_DIR) + build-native-app-setup-script: $(NODE_MODULES_DEV) echo "Building native app setup script..." rm -rf $(BUILD_DIR) @@ -93,7 +100,7 @@ build-native-app-setup-script: $(NODE_MODULES_DEV) SF_APP_SCHEMA=$(SF_UNQUALIFIED_SCHEMA) APP_ROLE=app_public \ REPLACEMENTS=$(NATIVE_APP_REPLACEMENTS) \ $(COMMON_DIR)/build_native_app_setup_script.js $(MODULES_DIRS) \ - --output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) --native_app_dir=$(NATIVE_APP_DIR) \ + --output=$(BUILD_DIR) --native_app_dir=$(NATIVE_APP_DIR) \ --production=$(production) --dropfirst=1 deploy: check build diff --git a/clouds/snowflake/native_app/Makefile b/clouds/snowflake/native_app/Makefile index 2159637f1..7f8f6fc01 100644 --- a/clouds/snowflake/native_app/Makefile +++ b/clouds/snowflake/native_app/Makefile @@ -7,6 +7,7 @@ ENV_DIR ?= $(ROOT_DIR)/.. BUILD_DIR ?= $(ROOT_DIR)/../build COMMON_DIR = $(ROOT_DIR)/../common APP_DIR ?= $(ROOT_DIR) +LIBS_BUILD_DIR ?= $(APP_DIR)/../libraries/javascript/build include $(COMMON_DIR)/Makefile @@ -26,7 +27,7 @@ endif .SILENT: -.PHONY: help build deploy-app-package deploy-app drop-app-package drop-app +.PHONY: help build check-package-sourcemaps deploy-app-package deploy-app drop-app-package drop-app help: echo "Available targets: help build deploy-app-package deploy-app drop-app-package drop-app" @@ -39,11 +40,31 @@ build: sed 's/@@VERSION@@/$(APP_VERSION)/g' $(APP_DIR)/manifest.yml > $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/manifest.yml cp $(APP_DIR)/README.md $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/ cp $(APP_DIR)/get_modules_sql_from_stage.py $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/ - + cp $(BUILD_DIR)/SOURCE_REVIEW.md $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/ + mkdir -p $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/$(APP_SOURCE_MAPS_DIR) + cp $(LIBS_BUILD_DIR)/*.js.map $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/$(APP_SOURCE_MAPS_DIR)/ + $(MAKE) check-package-sourcemaps + + +check-package-sourcemaps: + set -e; pkg=$(DIST_DIR)/$(APP_PACKAGE_DIST_NAME); \ + maps=`grep -o '//# sourceMappingURL=[^ ]*' $$pkg/modules.sql | sed 's|.*=||' | sort -u`; \ + test -n "$$maps" || { echo "ERROR: modules.sql references no source map at all"; exit 1; }; \ + for map in $$maps; do \ + test -f $$pkg/$$map || { echo "ERROR: modules.sql references $$map, which is not in the package"; exit 1; }; \ + done; \ + for f in $$pkg/$(APP_SOURCE_MAPS_DIR)/*.js.map; do \ + rel=$(APP_SOURCE_MAPS_DIR)/`basename $$f`; \ + echo "$$maps" | grep -qx "$$rel" || { echo "ERROR: $$rel ships in the package but nothing references it"; exit 1; }; \ + done deploy-app-package: $(COMMON_DIR)/run-query.js "CREATE STAGE IF NOT EXISTS $(APP_STAGE_NAME);" - $(COMMON_DIR)/run-query.js "PUT file://$(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/* @$(APP_STAGE_LOCATION) overwrite=true auto_compress=false;" +# PUT never deletes and non-production builds share one stage prefix, so drop stale maps first + $(COMMON_DIR)/run-query.js "REMOVE @$(APP_STAGE_LOCATION)/$(APP_SOURCE_MAPS_DIR);" +# *.* takes files only: PUT cannot upload a directory + $(COMMON_DIR)/run-query.js "PUT file://$(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/*.* @$(APP_STAGE_LOCATION) overwrite=true auto_compress=false;" + $(COMMON_DIR)/run-query.js "PUT file://$(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/$(APP_SOURCE_MAPS_DIR)/* @$(APP_STAGE_LOCATION)/$(APP_SOURCE_MAPS_DIR) overwrite=true auto_compress=false;" result=$$(CHECK_APP_PACKAGE_EXISTENCE=1 APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) $(COMMON_DIR)/native-app-utils.js) && \ if [ $${result} -eq 0 ]; then \ diff --git a/clouds/snowflake/native_app/SETUP_SCRIPT.sql b/clouds/snowflake/native_app/SETUP_SCRIPT.sql index f42557c09..c6c046f69 100644 --- a/clouds/snowflake/native_app/SETUP_SCRIPT.sql +++ b/clouds/snowflake/native_app/SETUP_SCRIPT.sql @@ -35,7 +35,7 @@ AS $$ ] for (const [variable, value] of replacements) { - modulesSql = modulesSql.replaceAll(variable, value); + modulesSql = modulesSql.split(variable).join(value); } snowflake.execute({ sqlText: modulesSql });