fix(sf,pg,ora,bq): insert every replacement literally when building SQL [sc-572077]

clouds/bigquery/common/build_modules.js

@@ -171,7 +171,7 @@ const outputSeparator = argv.production ? '\n' : internalSeparator;
 let content = output.map(f => f.content).join(internalSeparator);
 
 function apply_replacements (text) {
-    const libraries = [... new Set(content.match(new RegExp('@@BQ_LIBRARY_[^@]*?_BUCKET@@', 'g')))];
+    const libraries = [... new Set(text.match(new RegExp('@@BQ_LIBRARY_[^@]*?_BUCKET@@', 'g')))];
     for (let library of libraries) {
         let libraryName = library.replace('@@BQ_LIBRARY_', '').replace('_BUCKET@@', '').toLowerCase();
         if (makelib == libraryName) {
@@ -181,7 +181,7 @@ function apply_replacements (text) {
         const libraryPath = path.join(libsBuildDir, libraryName);
         if (fs.existsSync(libraryPath)) {
             const libraryBucketPath = libraryBucket + '_' + libraryName;
-            text = text.replace(new RegExp(library, 'g'), libraryBucketPath);
+            text = text.replace(new RegExp(library, 'g'), () => libraryBucketPath);
         }
         else {
             console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`);
@@ -192,7 +192,7 @@ function apply_replacements (text) {
     for (let replacement of replacements) {
         if (replacement) {
             const pattern = new RegExp(`@@${replacement}@@`, 'g');
-            text = text.replace(pattern, process.env[replacement]);
+            text = text.replace(pattern, () => process.env[replacement]);
         }
     }
     text = text.replace(/@@SKIP_DEP@@/g, '');

clouds/bigquery/common/run-script.js

@@ -22,7 +22,7 @@ function apply_replacements (text) {
         for (let replacement of replacements) {
             if (replacement) {
                 const pattern = new RegExp(`@@${replacement}@@`, 'g');
-                text = text.replace(pattern, process.env[replacement]);
+                text = text.replace(pattern, () => process.env[replacement]);
             }
         }
     }

clouds/databricks/common/build_modules.js

@@ -164,7 +164,7 @@ function apply_replacements (text) {
     for (let replacement of replacements) {
         if (replacement) {
             const pattern = new RegExp(`@@${replacement}@@`, 'g');
-            text = text.replace(pattern, process.env[replacement]);
+            text = text.replace(pattern, () => process.env[replacement]);
         }
     }
     text = text.replace(/\\`/g, '`');

clouds/oracle/common/build_modules.js

@@ -195,11 +195,24 @@ let content = output.map(f => f.content).join('\n');
 
 // Inline @@ORA_LIBRARY_<NAME>@@ → libraries/javascript/build/<name>.js, then
 // apply env-var @@VAR@@ replacements.
+// Guards against a future revert to a string replacement, where $& and $$ would expand
+function insert_literally (text, placeholder, content) {
+    const pattern = new RegExp(placeholder, 'g');
+    const count = (text.match(pattern) || []).length;
+    const expected = text.length + count * (content.length - placeholder.length);
+    const result = text.replace(pattern, () => content);
+    if (result.length !== expected) {
+        console.log(`ERROR: "${placeholder}" was not inserted literally`);
+        process.exit(1);
+    }
+    return result;
+}
+
 function apply_replacements (text) {
     const libraryDir = path.resolve(
         __dirname, '..', 'libraries', 'javascript', 'build'
     );
-    const libraries = [...new Set(text.match(/@@ORA_LIBRARY_[A-Z0-9_]+@@/g) || [])];
+    const libraries = [...new Set(text.match(/@@ORA_LIBRARY_[^@]+@@/g) || [])];
     for (const library of libraries) {
         const libName = library.replace('@@ORA_LIBRARY_', '').replace('@@', '');
         const file = path.join(libraryDir, libName.toLowerCase() + '.js');
@@ -210,16 +223,14 @@ function apply_replacements (text) {
             );
             process.exit(1);
         }
-        text = text.replace(
-            new RegExp(library, 'g'),
-            fs.readFileSync(file).toString()
-        );
+        const libraryContent = fs.readFileSync(file).toString();
+        text = insert_literally(text, library, libraryContent);
     }
     const replacements = process.env.REPLACEMENTS.split(' ');
     for (const replacement of replacements) {
         if (replacement) {
             const pattern = new RegExp(`@@${replacement}@@`, 'g');
-            text = text.replace(pattern, process.env[replacement]);
+            text = text.replace(pattern, () => process.env[replacement]);
         }
     }
     return text;

clouds/postgres/common/build_modules.js

@@ -181,14 +181,27 @@ let content = anonymousBlockWrapping(postgisInstalledCheck)
 // Replace environment variables
 content += output.map(f => anonymousBlockWrapping(extensionFunctionWrapping(f.content, f.module))).join('\n');
 
+// Guards against a future revert to a string replacement, where $& and $$ would expand
+function insert_literally (text, placeholder, content) {
+    const pattern = new RegExp(placeholder, 'g');
+    const count = (text.match(pattern) || []).length;
+    const expected = text.length + count * (content.length - placeholder.length);
+    const result = text.replace(pattern, () => content);
+    if (result.length !== expected) {
+        console.log(`ERROR: "${placeholder}" was not inserted literally`);
+        process.exit(1);
+    }
+    return result;
+}
+
 function apply_replacements (text) {
-    const libraries = [... new Set(text.match(new RegExp('@@PG_LIBRARY_.*@@', 'g')))];
+    const libraries = [... new Set(text.match(new RegExp('@@PG_LIBRARY_[^@]+@@', 'g')))];
     for (let library of libraries) {
         const libraryName = library.replace('@@PG_LIBRARY_', '').replace('@@', '').toLowerCase() + '.js';
         const libraryPath = path.join(libsBuildDir, libraryName);
         if (fs.existsSync(libraryPath)) {
             const libraryContent = fs.readFileSync(libraryPath).toString();
-            text = text.replace(new RegExp(library, 'g'), libraryContent);
+            text = insert_literally(text, library, libraryContent);
         }
         else {
             console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`);
@@ -199,7 +212,7 @@ function apply_replacements (text) {
     for (let replacement of replacements) {
         if (replacement) {
             const pattern = new RegExp(`@@${replacement}@@`, 'g');
-            text = text.replace(pattern, process.env[replacement]);
+            text = text.replace(pattern, () => process.env[replacement]);
         }
     }
     return text;

clouds/postgres/common/list_libraries.js

@@ -129,7 +129,7 @@ function add (f, include) {
 functions.forEach(f => add(f));
 
 const content = output.map(f => f.content).join('\n');
-const libraries = [... new Set(content.match(new RegExp('@@PG_LIBRARY_.*@@', 'g')))]
+const libraries = [... new Set(content.match(new RegExp('@@PG_LIBRARY_[^@]+@@', 'g')))]
     .map(l => l.replace('@@PG_LIBRARY_', '').replace('@@', '').toLowerCase());
 
 process.stdout.write(libraries.join(' '));

clouds/redshift/common/build_modules.js

@@ -153,7 +153,7 @@ function apply_replacements (text) {
     for (let replacement of replacements) {
         if (replacement) {
             const pattern = new RegExp(`@@${replacement}@@`, 'g');
-            text = text.replace(pattern, process.env[replacement]);
+            text = text.replace(pattern, () => process.env[replacement]);
         }
     }
     return text;

clouds/snowflake/common/build_modules.js

@@ -159,14 +159,36 @@ if (argv.production) {
 }
 let content = output.map(f => f.content).join(separator);
 
+// Snowflake wraps each JS body in AS $$ ... $$, so a literal $$ in a bundle would close it early
+function inline_library (text, placeholder, content) {
+    if (content.includes('$$')) {
+        console.log(`ERROR: library "${placeholder}" contains "$$", which would close the AS $$ ... $$ body`);
+        process.exit(1);
+    }
+    return insert_literally(text, placeholder, content);
+}
+
+// Guards against a future revert to a string replacement, where $& and $$ would expand
+function insert_literally (text, placeholder, content) {
+    const pattern = new RegExp(placeholder, 'g');
+    const count = (text.match(pattern) || []).length;
+    const expected = text.length + count * (content.length - placeholder.length);
+    const result = text.replace(pattern, () => content);
+    if (result.length !== expected) {
+        console.log(`ERROR: "${placeholder}" was not inserted literally`);
+        process.exit(1);
+    }
+    return result;
+}
+
 function apply_replacements (text) {
-    const libraries = [... new Set(text.match(new RegExp('@@SF_LIBRARY_.*@@', 'g')))];
+    const libraries = [... new Set(text.match(new RegExp('@@SF_LIBRARY_[^@]+@@', 'g')))];
     for (let library of libraries) {
         const libraryName = library.replace('@@SF_LIBRARY_', '').replace('@@', '').toLowerCase() + '.js';
         const libraryPath = path.join(libsBuildDir, libraryName);
         if (fs.existsSync(libraryPath)) {
             const libraryContent = fs.readFileSync(libraryPath).toString();
-            text = text.replace(new RegExp(library, 'g'), libraryContent);
+            text = inline_library(text, library, libraryContent);
         }
         else {
             console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`);
@@ -177,7 +199,7 @@ function apply_replacements (text) {
     for (let replacement of replacements) {
         if (replacement) {
             const pattern = new RegExp(`@@${replacement}@@`, 'g');
-            text = text.replace(pattern, process.env[replacement]);
+            text = text.replace(pattern, () => process.env[replacement]);
         }
     }
     return text;

clouds/snowflake/common/build_native_app_setup_script.js

@@ -9,7 +9,6 @@ const path = require('path');
 const argv = require('minimist')(process.argv.slice(2));
 
 const outputDir = argv.output || 'build';
-const libsBuildDir = argv.libs_build_dir || '../libraries/javascript/build';
 const nativeAppDir = argv.native_app_dir || '../native_app';
 
 // Replace environment variables
@@ -22,24 +21,17 @@ if (argv.production) {
 let content = fs.readFileSync(path.resolve(nativeAppDir, 'SETUP_SCRIPT.sql')).toString();
 
 function apply_replacements (text) {
-    const libraries = [... new Set(text.match(new RegExp('@@SF_LIBRARY_.*@@', 'g')))];
-    for (let library of libraries) {
-        const libraryName = library.replace('@@SF_LIBRARY_', '').replace('@@', '').toLowerCase() + '.js';
-        const libraryPath = path.join(libsBuildDir, libraryName);
-        if (fs.existsSync(libraryPath)) {
-            const libraryContent = fs.readFileSync(libraryPath).toString();
-            text = text.replace(new RegExp(library, 'g'), libraryContent);
-        }
-        else {
-            console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`);
-            process.exit(1);
-        }
+    // The setup script does not inline libraries: modules.sql is read from the stage at install time
+    const libraries = text.match(new RegExp('@@SF_LIBRARY_[^@]+@@', 'g'));
+    if (libraries) {
+        console.log(`ERROR: the setup script cannot inline libraries, found ${libraries.join(', ')}`);
+        process.exit(1);
     }
     const replacements = process.env.REPLACEMENTS.split(' ');
     for (let replacement of replacements) {
         if (replacement) {
             const pattern = new RegExp(`@@${replacement}@@`, 'g');
-            text = text.replace(pattern, process.env[replacement]);
+            text = text.replace(pattern, () => process.env[replacement]);
         }
     }
     return text;

clouds/snowflake/common/build_share.js

@@ -184,7 +184,7 @@ function apply_replacements (text) {
     for (let replacement of replacements) {
         if (replacement) {
             const pattern = new RegExp(`@@${replacement}@@`, 'g');
-            text = text.replace(pattern, process.env[replacement]);
+            text = text.replace(pattern, () => process.env[replacement]);
         }
     }
     return text;

clouds/snowflake/common/list_libraries.js

@@ -130,7 +130,7 @@ function add (f, include) {
 functions.forEach(f => add(f));
 
 const content = output.map(f => f.content).join('\n');
-const libraries = [... new Set(content.match(new RegExp('@@SF_LIBRARY_.*@@', 'g')))]
+const libraries = [... new Set(content.match(new RegExp('@@SF_LIBRARY_[^@]+@@', 'g')))]
     .map(l => l.replace('@@SF_LIBRARY_', '').replace('@@', '').toLowerCase());
 
 process.stdout.write(libraries.join(' '));

clouds/snowflake/common/run-script.js

@@ -66,7 +66,7 @@ function apply_replacements (text) {
         for (let replacement of replacements) {
             if (replacement) {
                 const pattern = new RegExp(`@@${replacement}@@`, 'g');
-                text = text.replace(pattern, process.env[replacement]);
+                text = text.replace(pattern, () => process.env[replacement]);
             }
         }
     }

clouds/snowflake/modules/Makefile

@@ -93,7 +93,7 @@ build-native-app-setup-script: $(NODE_MODULES_DEV)
 	SF_APP_SCHEMA=$(SF_UNQUALIFIED_SCHEMA) APP_ROLE=app_public \
 	REPLACEMENTS=$(NATIVE_APP_REPLACEMENTS) \
 	$(COMMON_DIR)/build_native_app_setup_script.js $(MODULES_DIRS) \
-		--output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) --native_app_dir=$(NATIVE_APP_DIR) \
+		--output=$(BUILD_DIR) --native_app_dir=$(NATIVE_APP_DIR) \
 		--production=$(production) --dropfirst=1
 
 deploy: check build

clouds/snowflake/native_app/SETUP_SCRIPT.sql

@@ -35,7 +35,7 @@ AS $$
         ]
 
         for (const [variable, value] of replacements) {
-          modulesSql = modulesSql.replaceAll(variable, value);
+          modulesSql = modulesSql.split(variable).join(value);
         }
 
         snowflake.execute({ sqlText: modulesSql });