Java Regex Refactor

core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4

@@ -28,8 +28,6 @@ CharacterEscape
  | SLASH HexEscapeSequence
  | SLASH UnicodeEscapeSequence
  | SLASH OctalEscapeSequence
- | SLASH ('p' | 'P') BRACE_open PCharacterClassEscapeLabel BRACE_close // this is only implemented in Java at the moment
-                                                        // as on JS this is allowed only while certain flags are enabled
  | SLASH ~[a-zA-Z0-9] // identity escape
  ;
 
@@ -61,10 +59,6 @@ fragment ControlLetter
 // : DecimalIntegerLiteral
 // ;
 
-DOUBLE_AMPERSAND
- : '&&'
- ;
-
 DecimalDigit
  : [0-9]
  ;
@@ -74,6 +68,9 @@ CharacterClassEscape
   // v, V, h and H are java8 exclusive, they represent vertical spaces and horizaontal spaces respectively
   // see https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html for more information
   : SLASH [dDsSwWvVhH]
+  // \p{...} escapes are included here too, since these they also represent predefined char classes
+ | SLASH ('p' | 'P') BRACE_open PCharacterClassEscapeLabel BRACE_close // this is only implemented in Java at the moment
+                                                        // as on JS this is allowed only while certain flags are enabled
  ;
 
 StartOfInputAssertion: SLASH 'A';
@@ -93,7 +90,6 @@ PLUS                       : '+';
 QUESTION                   : '?';
 PAREN_open                 : '(';
 PAREN_close                : ')';
-BRACKET_open               : '[';
 BRACKET_close              : ']';
 BRACE_open                 : '{';
 BRACE_close                : '}';
@@ -146,7 +142,7 @@ QUOTE_OPEN
  : '\\' 'Q' -> pushMode(QUOTE_MODE)
  ;
 
-
+BRACKET_open : '[' -> pushMode(CHAR_CLASS_MODE);
 
 // --------------------------------------------------------------------------------
 // QUOTE_MODE: everything after \Q and before an optional \E is literal text.
@@ -169,3 +165,23 @@ QUOTE_CLOSE
 QUOTE_CONTENT
  : ( '\\' ~[E] | ~[\\] )+
  ;
+
+// ---------------------------------------------------------------
+// CHAR_CLASS_MODE: everything not '\', ']', '[', '^', '-' is a plain
+// literal. This allows us to have a cleaner classAtomNoDash.
+// ---------------------------------------------------------------
+mode CHAR_CLASS_MODE;
+
+// metacharacters: (excluding \)
+CC_BRACKET_CLOSE        : ']' -> type(BRACKET_close), popMode;
+CC_BRACKET_OPEN         : '[' -> type(BRACKET_open), pushMode(CHAR_CLASS_MODE); // Java CC nesting, stacks mode
+CC_CARET                : '^' -> type(CARET);
+CC_MINUS                : '-' -> type(MINUS);
+CC_DOUBLE_AMPERSAND     : '&&'; // this is only a metacharacter within character classes
+
+// constructs using \
+CC_CharacterClassEscape : CharacterClassEscape  -> type(CharacterClassEscape);
+CC_CharacterEscape      : CharacterEscape       -> type(CharacterEscape);
+
+// literal characters
+CC_BaseChar : ~[\\\][^\-] -> type(BaseChar);

core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4

@@ -119,7 +119,6 @@ patternCharacter
  | BRACE_close
  | BRACKET_close
  | COLON | EQUAL | LESS_THAN
- | DOUBLE_AMPERSAND // char class intersection not supported by default in JS, only supported if "v" flag is turned on.
  ;
 
 
@@ -129,7 +128,7 @@ characterClass
     ;
 
 classContents
-    : classRanges (DOUBLE_AMPERSAND classRanges)*
+    : classRanges (CC_DOUBLE_AMPERSAND classRanges)*
     ;
 
 classRanges
@@ -160,28 +159,19 @@ classAtom
 
 classAtomNoDash
  //SourceCharacter but not one of \ or ] or - or [
- //TODO
  //: ~[-\]\\]
  : classEscape
- | BaseChar
- | DecimalDigit
- | COMMA | CARET | DOLLAR | DOT | STAR | PLUS | QUESTION
- | PAREN_open | PAREN_close | BRACE_open | BRACE_close | OR
- | COLON | EQUAL | LESS_THAN
- // should be interpreted literally:
- // As they are lexer tokens, these character sequences are captured as such. In particular these require some extra
- // steps to interpret them correctly given the context.
- // [(?iu)] -> FLAG_SCOPE_OPEN, each letter of the token should be interpreted literally.
- | FLAG_SCOPE_OPEN | FLAG_GROUP_OPEN
- | NAMED_CAPTURE_GROUP_OPEN
+ | CARET
+ | BaseChar // this is the CHAR_CLASS_MODE token (CC_BaseChar), so it includes all chars but \ or ] or - or [ or ^
  ;
 
 decimalDigits
  : DecimalDigit+
  ;
 
 classEscape
- : atomEscape
+ : CharacterClassEscape   // char class
+ | CharacterEscape        // single char
 // | SLASH 'b'
  ;
 

core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt

@@ -109,6 +109,30 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege
         return false
     }
 
+    private fun resolveCharacterEscapeString(txt: String): String = when (txt[1]) {
+        '0' -> String(Character.toChars(txt.substring(2).toInt(8)))
+        'c' -> {
+            val controlLetterValue = if (txt[2].isLowerCase()) {
+                txt[2].uppercaseChar().code.xor(0x60)
+            } else {
+                txt[2].code.xor(0x40)
+            }
+            controlLetterValue.toChar().toString()
+        }
+        in escapeMap -> escapeMap[txt[1]]!!
+        in hexEscapePrefixes -> {
+            val hexValue = if (txt[1] == 'x' && txt.length > 4 && txt[2] == '{' && txt.last() == '}') {
+                txt.substring(3, txt.length - 1).toInt(16)
+            } else {
+                txt.substring(2).toInt(16)
+            }
+            if (hexValue !in Character.MIN_CODE_POINT..Character.MAX_CODE_POINT)
+                throw IllegalArgumentException("Hexadecimal escape out of range: $txt")
+            String(Character.toChars(hexValue))
+        }
+        else -> txt.substring(1) // identity escape
+    }
+
     override fun visitPattern(ctx: RegexJavaParser.PatternContext): VisitResult {
 
         val res = ctx.disjunction().accept(this)
@@ -515,14 +539,6 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege
             if (ctx.classAtom().size == 2) throw IllegalArgumentException("Not implemented yet")
             val rec = ctx.classAtom()[0].accept(this).data as List<CharacterRange>
             list.addAll(rec)
-        } else if (
-                ctx.classAtom()[0]?.classAtomNoDash()?.FLAG_SCOPE_OPEN() != null
-                || ctx.classAtom()[0]?.classAtomNoDash()?.FLAG_GROUP_OPEN() != null
-                || ctx.classAtom()[0]?.classAtomNoDash()?.NAMED_CAPTURE_GROUP_OPEN() != null
-            ) {
-            // these should be interpreted literally within a charclass.
-            val ranges = ctx.text.map { ch -> CharacterRange(ch, ch) }
-            list.addAll(ranges)
         } else {
             val startText = ctx.classAtom()[0].text
             assert(startText.length == 1 || startText.length == 2) // single chars or \+ and \. escaped chars
@@ -615,31 +631,17 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege
 
     override fun visitClassEscape(ctx: RegexJavaParser.ClassEscapeContext): VisitResult {
 
-        val res = VisitResult()
-        res.data = if(ctx.atomEscape() != null &&
-            (ctx.atomEscape().BackReference() != null || ctx.atomEscape().NamedBackReference() != null)
-            ) {
-            // In Java using backrefs or named backrefs is illegal within char classes. (i.e.: [\1\k<name>])
-            throw IllegalArgumentException("Illegal/unsupported escape sequence")
-        } else if (ctx.atomEscape() != null) {
-            when (val rec = ctx.atomEscape().accept(this).genes[0]) {
-                is CharacterClassEscapeRxGene -> {
-                    rec.multiCharRange.ranges
-                }
-
-                is PatternCharacterBlockGene -> {
-                    if (rec.stringBlock.length > 1) {
-                        throw IllegalArgumentException("CharClass element cannot be strings")
-                    }
-                    else listOf(CharacterRange(rec.stringBlock[0], rec.stringBlock[0]))
-                }
-
-                else -> throw IllegalArgumentException("Unexpected CharClass content")
+        val txt = ctx.text
+        val ranges = when {
+            ctx.CharacterClassEscape() != null ->
+                CharacterClassEscapeRxGene(txt.substring(1), currentFlags).multiCharRange.ranges
+            else -> { // character escape
+                val s = resolveCharacterEscapeString(txt)
+                if (s.length > 1) throw IllegalArgumentException("CharClass element cannot be strings")
+                listOf(CharacterRange(s[0], s[0]))
             }
-        } else {
-            throw IllegalArgumentException("Not implemented yet")
         }
-        return res
+        return VisitResult(data = ranges)
     }
 
     override fun visitAtomEscape(ctx: RegexJavaParser.AtomEscapeContext): VisitResult {
@@ -686,44 +688,11 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege
             return VisitResult(BackReferenceRxGene(groupIndex, group))
         }
 
-        return VisitResult(when (txt[1]) {
-            '0' -> {
-                val octalValue = txt.substring(2).toInt(8)
-                PatternCharacterBlockGene(
-                        txt,
-                        String(Character.toChars(octalValue)),
-                        currentFlags
-                )
-            }
-            'c' -> {
-                val controlLetterValue = if (txt[2].isLowerCase()){
-                    txt[2].uppercaseChar().code.xor(0x60)
-                } else {
-                    txt[2].code.xor(0x40)
-                }
-                PatternCharacterBlockGene(txt, controlLetterValue.toChar().toString(), currentFlags)
-            }
-            in escapeMap -> {
-                val escape = escapeMap[txt[1]]!!
-                PatternCharacterBlockGene(txt, escape, currentFlags)
-            }
-            in hexEscapePrefixes -> {
-                val hexValue = if (txt[1] == 'x' && txt.length > 4 && txt[2] == '{' && txt[txt.length - 1] == '}') {
-                    txt.substring(3, txt.length - 1).toInt(16)
-                } else {
-                    txt.substring(2).toInt(16)
-                }
-                if(hexValue !in Character.MIN_CODE_POINT..Character.MAX_CODE_POINT){
-                    throw IllegalArgumentException("Hexadecimal escape out of range: ${ctx.text}")
-                }
-                PatternCharacterBlockGene(
-                        txt,
-                        String(Character.toChars(hexValue)),
-                        currentFlags
-                )
-            }
-            !in notIdentityEscapes -> PatternCharacterBlockGene(txt, txt.substring(1), currentFlags)
-            else -> CharacterClassEscapeRxGene(txt.substring(1), currentFlags)
-        })
+        if (ctx.CharacterClassEscape() != null) {
+            return VisitResult(CharacterClassEscapeRxGene(ctx.CharacterClassEscape().text.substring(1), currentFlags))
+        }
+
+        // character escape
+        return VisitResult(PatternCharacterBlockGene(txt, resolveCharacterEscapeString(txt), currentFlags))
     }
 }

core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt

@@ -264,7 +264,11 @@ object RegexHandler {
      */
     private class ThrowingErrorListener : BaseErrorListener() {
 
-        override fun syntaxError(recognizer: Recognizer<*, *>, offendingSymbol: Any, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException) {
+        // offendingSymbol is nullable as otherwise lexer errors throw NPE instead of ParseCancellationException
+        // for example, on our Java grammar [\1] is a lexer error since \1 has no token within that context
+        // org.antlr.v4.runtime.BaseErrorListener.syntaxError declares this parameter as Object, which is nullable
+        // org.antlr.v4.runtime.Lexer.notifyListeners calls this method with offendingSymbol as null
+        override fun syntaxError(recognizer: Recognizer<*, *>, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException) {
             throw ParseCancellationException("line $line:$charPositionInLine $msg")
         }
     }

core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt

@@ -1,5 +1,6 @@
 package org.evomaster.core.parser
 
+import org.antlr.v4.runtime.misc.ParseCancellationException
 import org.evomaster.core.search.gene.regex.RegexGene
 import org.evomaster.core.utils.RegexFlags
 import org.junit.jupiter.api.Test
@@ -134,6 +135,22 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() {
         checkSameAsJava("""Pe""")
     }
 
+    @Test
+    fun testCharacterClass(){
+        checkSameAsJava("^[\\p{Lower}a-z]$")
+        checkSameAsJava("^[a\\p{Lower}]$")
+        checkSameAsJava("^[a[b-c]&&[^x]]$")
+        checkSameAsJava("^a]b$")
+        checkSameAsJava("^a&&b$")
+        checkSameAsJava("^[(?i)-a]$")
+        checkSameAsJava("^[(?i:abc)-a]$")
+        checkCanSample("^[a&b]$", "&", 100)
+        // these are rejected as lexer has no token for \1 or \k<name> here, as they are not available on CHAR_CLASS_MODE
+        // on Java these throw on Pattern.compile since backreferences are not allowed within character classes
+        assertThrows<ParseCancellationException> { checkSameAsJava("^[\\1]$") }
+        assertThrows<ParseCancellationException> { checkSameAsJava("^[\\k<name>]$") }
+    }
+
     @Test
     override fun testPredefinedCharClassInsideCharClass(){
         checkSameAsJava("""[\V\p{Lower}\p{Upper}\W\d]""")