[fix](nereids) Fix SQL masking and view rewrites after supplementary characters

fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForSyncMv.java

@@ -33,6 +33,7 @@
 import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
+import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.util.PlanUtils;
@@ -109,7 +110,7 @@ public LogicalPlan visitQuery(DorisParser.QueryContext ctx) {
             for (Expression expr : outputs) {
                 aggStateFunctionFinder.find(expr, indexInSqlToString);
             }
-            querySql = Optional.of(rewriteSql(originSql, indexInSqlToString));
+            querySql = Optional.of(BaseViewInfo.rewriteSql(indexInSqlToString, originSql));
         } else {
             querySql = Optional.of(originSql);
         }
@@ -193,18 +194,4 @@ public Expression visitUnboundFunction(UnboundFunction unboundFunction,
             return unboundFunction;
         }
     }
-
-    private static String rewriteSql(String querySql,
-            Map<Pair<Integer, Integer>, String> indexStringSqlMap) {
-        StringBuilder builder = new StringBuilder();
-        int beg = 0;
-        for (Map.Entry<Pair<Integer, Integer>, String> entry : indexStringSqlMap.entrySet()) {
-            Pair<Integer, Integer> index = entry.getKey();
-            builder.append(querySql, beg, index.first);
-            builder.append(entry.getValue());
-            beg = index.second + 1;
-        }
-        builder.append(querySql, beg, querySql.length());
-        return builder.toString();
-    }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java

@@ -200,7 +200,10 @@ public static Optional<Pair<ExplainOptions, String>> tryParseExplainPlan(String
                 return Optional.empty();
             }
             ExplainOptions explainOptions = new ExplainOptions(explainLevel, showPlanProcess);
-            return Optional.of(Pair.of(explainOptions, sql.substring(explainPlanBody.getStartIndex())));
+            // The parser counts a supplementary character such as an emoji as one character, while sql
+            // counts it as two.
+            return Optional.of(Pair.of(explainOptions,
+                    sql.substring(sql.offsetByCodePoints(0, explainPlanBody.getStartIndex()))));
         } catch (Throwable t) {
             return Optional.empty();
         }
@@ -349,7 +352,7 @@ private <T> T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder,
         CommonTokenStream tokenStream = parseLeanTokens(sql);
         ParserRuleContext tree = toAst(tokenStream, parseFunction);
         LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null
-                    ? new LogicalPlanBuilder(getHintMap(sql, tokenStream, DorisParser::selectHint))
+                    ? new LogicalPlanBuilder(getHintMap(tokenStream, DorisParser::selectHint))
                     : logicalPlanBuilder;
         return (T) realLogicalPlanBuilder.visit(tree);
     }
@@ -358,7 +361,7 @@ public LogicalPlan parseForCreateView(String sql) {
         CommonTokenStream tokenStream = parseLeanTokens(sql);
         ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement);
         LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView(
-                getHintMap(sql, tokenStream, DorisParser::selectHint));
+                getHintMap(tokenStream, DorisParser::selectHint));
         return (LogicalPlan) realLogicalPlanBuilder.visit(tree);
     }
 
@@ -378,22 +381,24 @@ public Optional<String> parseForSyncMv(String sql) {
         CommonTokenStream tokenStream = parseLeanTokens(sql);
         ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement);
         LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv(
-                getHintMap(sql, tokenStream, DorisParser::selectHint));
+                getHintMap(tokenStream, DorisParser::selectHint));
         logicalPlanBuilderForSyncMv.visit(tree);
         return logicalPlanBuilderForSyncMv.getQuerySql();
     }
 
     /** get hint map */
-    public static Map<Integer, ParserRuleContext> getHintMap(String sql, CommonTokenStream hintTokenStream,
+    public static Map<Integer, ParserRuleContext> getHintMap(CommonTokenStream hintTokenStream,
                                                              Function<DorisParser, ParserRuleContext> parseFunction) {
         // parse hint first round
         Map<Integer, ParserRuleContext> selectHintMap = Maps.newHashMap();
 
         Iterator<Token> tokenIterator = hintTokenStream.getTokens().iterator();
         Token hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null;
         while (hintToken != null && hintToken.getType() != DorisLexer.EOF) {
-            if (hintToken.getChannel() == 2 && sql.charAt(hintToken.getStartIndex() + 2) == '+') {
-                String hintSql = sql.substring(hintToken.getStartIndex() + 3, hintToken.getStopIndex() + 1);
+            // Token positions count a supplementary character such as an emoji as one character, so read
+            // the hint from the token instead of cutting the SQL string.
+            if (hintToken.getChannel() == 2 && hintToken.getText().startsWith("/*+")) {
+                String hintSql = hintToken.getText().substring(3);
                 DorisLexer newHintLexer = new DorisLexer(CaseInsensitiveStream.fromString(hintSql));
                 CommonTokenStream newHintTokenStream = new CommonTokenStream(newHintLexer);
                 DorisParser hintParser = new DorisParser(newHintTokenStream);

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java

@@ -81,9 +81,9 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.RuleNode;
-import org.apache.commons.lang3.StringUtils;
 
 import java.util.List;
 import java.util.Map;
@@ -128,18 +128,25 @@ protected void analyzeAndFillRewriteSqlMap(String sql, ConnectContext ctx) throw
      * Add the full path to the field
      *
      * @param indexStringSqlMap key is the start and end position of the sql substring that needs to be replaced,
-     *         and value is the new string used for replacement.
+     *         and value is the new string used for replacement. The positions are the token positions the
+     *         parser reports for querySql.
      * @param querySql origin query sql
      * @return sql rewritten sql
      */
     public static String rewriteSql(TreeMap<Pair<Integer, Integer>, String> indexStringSqlMap, String querySql) {
         StringBuilder builder = new StringBuilder();
+        // The parser counts a supplementary character such as an emoji as one character, while querySql
+        // counts it as two. beg and begTokenPosition are the same place on both scales; the keys ascend,
+        // so each position is converted by scanning forward from the previous one.
         int beg = 0;
+        int begTokenPosition = 0;
         for (Map.Entry<Pair<Integer, Integer>, String> entry : indexStringSqlMap.entrySet()) {
             Pair<Integer, Integer> index = entry.getKey();
-            builder.append(querySql, beg, index.first);
+            int start = querySql.offsetByCodePoints(beg, index.first - begTokenPosition);
+            builder.append(querySql, beg, start);
             builder.append(entry.getValue());
-            beg = index.second + 1;
+            beg = querySql.offsetByCodePoints(start, index.second + 1 - index.first);
+            begTokenPosition = index.second + 1;
         }
         builder.append(querySql, beg, querySql.length());
         return builder.toString();
@@ -167,10 +174,11 @@ public static String rewriteProjectsToUserDefineAlias(String resSql, List<String
         List<NamedExpressionContext> namedExpressionContexts = finder.getNamedExpressionContexts();
         StringBuilder replaceWithColsBuilder = new StringBuilder();
         for (int i = 0; i < namedExpressionContexts.size(); ++i) {
-            NamedExpressionContext namedExpressionContext = namedExpressionContexts.get(i);
-            int start = namedExpressionContext.expression().start.getStartIndex();
-            int stop = namedExpressionContext.expression().stop.getStopIndex();
-            replaceWithColsBuilder.append(resSql, start, stop + 1);
+            ParserRuleContext expression = namedExpressionContexts.get(i).expression();
+            // Token positions count a supplementary character such as an emoji as one character, so read
+            // the expression from the parser's input instead of cutting resSql.
+            replaceWithColsBuilder.append(expression.start.getInputStream().getText(
+                    Interval.of(expression.start.getStartIndex(), expression.stop.getStopIndex())));
             replaceWithColsBuilder.append(" AS `");
             String escapeBacktick = finalColNames.get(i).replace("`", "``");
             replaceWithColsBuilder.append(escapeBacktick);
@@ -179,9 +187,9 @@ public static String rewriteProjectsToUserDefineAlias(String resSql, List<String
                 replaceWithColsBuilder.append(", ");
             }
         }
-        String replaceWithCols = replaceWithColsBuilder.toString();
-        return StringUtils.overlay(resSql, replaceWithCols, finder.getIndex().first,
-                finder.getIndex().second + 1);
+        TreeMap<Pair<Integer, Integer>, String> selectColumns = new TreeMap<>(new Pair.PairComparator<>());
+        selectColumns.put(finder.getIndex(), replaceWithColsBuilder.toString());
+        return rewriteSql(selectColumns, resSql);
     }
 
     protected void createFinalCols(List<Slot> outputs) throws org.apache.doris.common.AnalysisException {

fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java

@@ -268,6 +268,26 @@ public void testCreateViewWithDefinedColumnsRewritesAliases() throws Exception {
                 view.getInlineViewDef());
     }
 
+    @Test
+    public void testCreateViewRewritesAfterSupplementaryCharacters() throws Exception {
+        // The parser counts a supplementary character such as an emoji as one character, while
+        // the Java string of the view definition counts it as two.
+        ExceptionChecker.expectThrowsNoException(() -> createView(
+                "create view test.supplementary_char_view as select '😀😀' as e, k1 from test.tbl1;"));
+        ExceptionChecker.expectThrowsNoException(() -> createView(
+                "create view test.supplementary_char_alias_view(c1, c2) as select '😀😀', k1 from test.tbl1;"));
+
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test");
+        View view = (View) db.getTableOrDdlException("supplementary_char_view");
+        Assertions.assertEquals(
+                "select '😀😀' as `e`, `internal`.`test`.`tbl1`.`k1` from `internal`.`test`.`tbl1`",
+                view.getInlineViewDef());
+        View aliasView = (View) db.getTableOrDdlException("supplementary_char_alias_view");
+        Assertions.assertEquals(
+                "select '😀😀' AS `c1`, `internal`.`test`.`tbl1`.`k1` AS `c2` from `internal`.`test`.`tbl1`",
+                aliasView.getInlineViewDef());
+    }
+
     @Test
     public void testViewPreservesVarbinary() throws Exception {
         createView("create view test.vb_view as select X'AB' as vb;");

fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java

@@ -505,6 +505,98 @@ public void testAlterUserPasswordMasking() throws Exception {
         }
     }
 
+    @Test
+    public void testEncryptionAfterSupplementaryCharacters() throws Exception {
+        ctx.setDatabase("test");
+        try (MockedConstruction<StmtExecutor> ignored = Mockito.mockConstruction(StmtExecutor.class,
+                Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS),
+                (mock, context) -> {
+                    Mockito.doReturn(false).when(mock).isForwardToMaster();
+                    Profile profile = new Profile(false, 0, 0);
+                    Deencapsulation.setField(mock, "profile", profile);
+                    Mockito.doReturn(profile).when(mock).getProfile();
+                    Mockito.doReturn(ctx).when(mock).getContext();
+                    Mockito.doNothing().when(mock).execute();
+                    Deencapsulation.setField(mock, "context", ctx);
+                    if (context.arguments().size() >= 2
+                            && context.arguments().get(1) instanceof StatementBase) {
+                        Deencapsulation.setField(mock, "parsedStmt",
+                                context.arguments().get(1));
+                    }
+                    if (ctx.getStatementContext() == null) {
+                        ctx.setStatementContext(new StatementContext());
+                    }
+                })) {
+            ctx.setEnv(env);
+            ctx.setCurrentUserIdentity(UserIdentity.ROOT);
+            // The parser counts a supplementary character such as an emoji as one character, while
+            // the Java string of the SQL counts it as two.
+            String sql = "CREATE CATALOG ctl "
+                    + "PROPERTIES("
+                    + " \"type\" = \"iceberg\","
+                    + " \"iceberg.catalog.type\" = \"hadoop\","
+                    + " \"warehouse\" = \"s3://bucket/dir/key\","
+                    + " \"s3.access_key\" = \"abc\","
+                    + " \"s3.secret_key\" = \"😀😀😀😀😀😀😀secret\""
+                    + ")";
+            String res = "CREATE CATALOG ctl "
+                    + "PROPERTIES("
+                    + " \"type\" = \"iceberg\","
+                    + " \"iceberg.catalog.type\" = \"hadoop\","
+                    + " \"warehouse\" = \"s3://bucket/dir/key\","
+                    + " \"s3.access_key\" = \"*XXX\","
+                    + " \"s3.secret_key\" = \"*XXX\""
+                    + ")";
+            parseAndCheck(sql, res);
+
+            sql = "CREATE CATALOG ctl COMMENT \"😀😀😀😀😀😀😀😀\" "
+                    + "PROPERTIES("
+                    + " \"type\" = \"iceberg\","
+                    + " \"iceberg.catalog.type\" = \"hadoop\","
+                    + " \"warehouse\" = \"s3://bucket/😀/key\","
+                    + " \"s3.access_key\" = \"abc\","
+                    + " \"s3.secret_key\" = \"abc\""
+                    + ")";
+            res = "CREATE CATALOG ctl COMMENT \"😀😀😀😀😀😀😀😀\" "
+                    + "PROPERTIES("
+                    + " \"type\" = \"iceberg\","
+                    + " \"iceberg.catalog.type\" = \"hadoop\","
+                    + " \"warehouse\" = \"s3://bucket/😀/key\","
+                    + " \"s3.access_key\" = \"*XXX\","
+                    + " \"s3.secret_key\" = \"*XXX\""
+                    + ")";
+            parseAndCheck(sql, res);
+
+            sql = "CREATE EXTERNAL TABLE broker_tbl("
+                    + " k1 tinyint) "
+                    + "ENGINE=broker "
+                    + "PROPERTIES("
+                    + " \"broker_name\" = \"hdfs\","
+                    + " \"path\" = \"hdfs://abc/😀😀😀😀😀😀😀😀/a.txt\""
+                    + ") "
+                    + "BROKER PROPERTIES("
+                    + " \"username\" = \"root\","
+                    + " \"password\" = \"123\""
+                    + ")";
+            res = "CREATE EXTERNAL TABLE broker_tbl("
+                    + " k1 tinyint) "
+                    + "ENGINE=broker "
+                    + "PROPERTIES("
+                    + " \"broker_name\" = \"hdfs\","
+                    + " \"path\" = \"hdfs://abc/😀😀😀😀😀😀😀😀/a.txt\""
+                    + ") "
+                    + "BROKER PROPERTIES("
+                    + " \"username\" = \"root\","
+                    + " \"password\" = \"*XXX\""
+                    + ")";
+            parseAndCheck(sql, res);
+
+            sql = "SET PASSWORD FOR '😀😀😀😀😀😀😀😀😀' = PASSWORD('123456')";
+            res = "SET PASSWORD FOR '😀😀😀😀😀😀😀😀😀' = PASSWORD('*XXX')";
+            parseAndCheck(sql, res);
+        }
+    }
+
     private void parseAndCheck(String sql, String expected) throws Exception {
         processor.executeQuery(sql);
         AuditEvent event = auditEvents.get(auditEvents.size() - 1);

fe/fe-core/src/test/java/org/apache/doris/nereids/parser/SupplementaryCharacterOffsetTest.java

@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.parser;
+
+import org.apache.doris.nereids.properties.SelectHintOrdered;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+/**
+ * The parser counts a supplementary character such as an emoji as one character, while the Java
+ * string of the SQL counts it as two. These cases cut the SQL text after such characters.
+ */
+class SupplementaryCharacterOffsetTest extends ParserTestBase {
+    private final NereidsParser parser = new NereidsParser();
+
+    @Test
+    void recognizesHintAfterSupplementaryCharacter() {
+        LogicalPlan plan = parser.parseSingle("/* 😀 */ SELECT /*+ ORDERED */ a FROM t");
+        List<LogicalSelectHint<Plan>> hintPlans = plan.collectToList(LogicalSelectHint.class::isInstance);
+
+        Assertions.assertEquals(1, hintPlans.size());
+        Assertions.assertTrue(hintPlans.get(0).getHints().stream()
+                .anyMatch(SelectHintOrdered.class::isInstance));
+    }
+
+    @Test
+    void extractsExplainPlanBodyAfterSupplementaryCharacters() {
+        Assertions.assertEquals("SELECT 100",
+                NereidsParser.tryParseExplainPlan("EXPLAIN /* abc */ PLAN SELECT 100").orElseThrow().second);
+        Assertions.assertEquals("SELECT 100",
+                NereidsParser.tryParseExplainPlan("EXPLAIN /* 😀😀😀 */ PLAN SELECT 100").orElseThrow().second);
+    }
+
+    @Test
+    void rewritesSyncMvAggregateAfterSupplementaryCharacter() {
+        Assertions.assertEquals("SELECT /* a */ k1, avg_union(avg_state(k2)) FROM t GROUP BY k1",
+                parser.parseForSyncMv("CREATE MATERIALIZED VIEW mv AS "
+                        + "SELECT /* a */ k1, avg(k2) FROM t GROUP BY k1").orElseThrow());
+        Assertions.assertEquals("SELECT /* 😀 */ k1, avg_union(avg_state(k2)) FROM t GROUP BY k1",
+                parser.parseForSyncMv("CREATE MATERIALIZED VIEW mv AS "
+                        + "SELECT /* 😀 */ k1, avg(k2) FROM t GROUP BY k1").orElseThrow());
+    }
+}