diff --git a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 index a4450c58a9..c26bd66cae 100644 --- a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 +++ b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 @@ -28,8 +28,6 @@ CharacterEscape | SLASH HexEscapeSequence | SLASH UnicodeEscapeSequence | SLASH OctalEscapeSequence - | SLASH ('p' | 'P') BRACE_open PCharacterClassEscapeLabel BRACE_close // this is only implemented in Java at the moment - // as on JS this is allowed only while certain flags are enabled | SLASH ~[a-zA-Z0-9] // identity escape ; @@ -61,10 +59,6 @@ fragment ControlLetter // : DecimalIntegerLiteral // ; -DOUBLE_AMPERSAND - : '&&' - ; - DecimalDigit : [0-9] ; @@ -74,6 +68,9 @@ CharacterClassEscape // v, V, h and H are java8 exclusive, they represent vertical spaces and horizaontal spaces respectively // see https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html for more information : SLASH [dDsSwWvVhH] + // \p{...} escapes are included here too, since these they also represent predefined char classes + | SLASH ('p' | 'P') BRACE_open PCharacterClassEscapeLabel BRACE_close // this is only implemented in Java at the moment + // as on JS this is allowed only while certain flags are enabled ; StartOfInputAssertion: SLASH 'A'; @@ -93,7 +90,6 @@ PLUS : '+'; QUESTION : '?'; PAREN_open : '('; PAREN_close : ')'; -BRACKET_open : '['; BRACKET_close : ']'; BRACE_open : '{'; BRACE_close : '}'; @@ -146,7 +142,7 @@ QUOTE_OPEN : '\\' 'Q' -> pushMode(QUOTE_MODE) ; - +BRACKET_open : '[' -> pushMode(CHAR_CLASS_MODE); // -------------------------------------------------------------------------------- // QUOTE_MODE: everything after \Q and before an optional \E is literal text. @@ -169,3 +165,23 @@ QUOTE_CLOSE QUOTE_CONTENT : ( '\\' ~[E] | ~[\\] )+ ; + +// --------------------------------------------------------------- +// CHAR_CLASS_MODE: everything not '\', ']', '[', '^', '-' is a plain +// literal. This allows us to have a cleaner classAtomNoDash. +// --------------------------------------------------------------- +mode CHAR_CLASS_MODE; + +// metacharacters: (excluding \) +CC_BRACKET_CLOSE : ']' -> type(BRACKET_close), popMode; +CC_BRACKET_OPEN : '[' -> type(BRACKET_open), pushMode(CHAR_CLASS_MODE); // Java CC nesting, stacks mode +CC_CARET : '^' -> type(CARET); +CC_MINUS : '-' -> type(MINUS); +CC_DOUBLE_AMPERSAND : '&&'; // this is only a metacharacter within character classes + +// constructs using \ +CC_CharacterClassEscape : CharacterClassEscape -> type(CharacterClassEscape); +CC_CharacterEscape : CharacterEscape -> type(CharacterEscape); + +// literal characters +CC_BaseChar : ~[\\\][^\-] -> type(BaseChar); \ No newline at end of file diff --git a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 index 77622aca0b..0b0f7d2834 100644 --- a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 +++ b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 @@ -119,7 +119,6 @@ patternCharacter | BRACE_close | BRACKET_close | COLON | EQUAL | LESS_THAN - | DOUBLE_AMPERSAND // char class intersection not supported by default in JS, only supported if "v" flag is turned on. ; @@ -129,7 +128,7 @@ characterClass ; classContents - : classRanges (DOUBLE_AMPERSAND classRanges)* + : classRanges (CC_DOUBLE_AMPERSAND classRanges)* ; classRanges @@ -160,20 +159,10 @@ classAtom classAtomNoDash //SourceCharacter but not one of \ or ] or - or [ - //TODO //: ~[-\]\\] : classEscape - | BaseChar - | DecimalDigit - | COMMA | CARET | DOLLAR | DOT | STAR | PLUS | QUESTION - | PAREN_open | PAREN_close | BRACE_open | BRACE_close | OR - | COLON | EQUAL | LESS_THAN - // should be interpreted literally: - // As they are lexer tokens, these character sequences are captured as such. In particular these require some extra - // steps to interpret them correctly given the context. - // [(?iu)] -> FLAG_SCOPE_OPEN, each letter of the token should be interpreted literally. - | FLAG_SCOPE_OPEN | FLAG_GROUP_OPEN - | NAMED_CAPTURE_GROUP_OPEN + | CARET + | BaseChar // this is the CHAR_CLASS_MODE token (CC_BaseChar), so it includes all chars but \ or ] or - or [ or ^ ; decimalDigits @@ -181,7 +170,8 @@ decimalDigits ; classEscape - : atomEscape + : CharacterClassEscape // char class + | CharacterEscape // single char // | SLASH 'b' ; diff --git a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt index 80ff61317b..bbd5ef7cbe 100644 --- a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt @@ -109,6 +109,30 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege return false } + private fun resolveCharacterEscapeString(txt: String): String = when (txt[1]) { + '0' -> String(Character.toChars(txt.substring(2).toInt(8))) + 'c' -> { + val controlLetterValue = if (txt[2].isLowerCase()) { + txt[2].uppercaseChar().code.xor(0x60) + } else { + txt[2].code.xor(0x40) + } + controlLetterValue.toChar().toString() + } + in escapeMap -> escapeMap[txt[1]]!! + in hexEscapePrefixes -> { + val hexValue = if (txt[1] == 'x' && txt.length > 4 && txt[2] == '{' && txt.last() == '}') { + txt.substring(3, txt.length - 1).toInt(16) + } else { + txt.substring(2).toInt(16) + } + if (hexValue !in Character.MIN_CODE_POINT..Character.MAX_CODE_POINT) + throw IllegalArgumentException("Hexadecimal escape out of range: $txt") + String(Character.toChars(hexValue)) + } + else -> txt.substring(1) // identity escape + } + override fun visitPattern(ctx: RegexJavaParser.PatternContext): VisitResult { val res = ctx.disjunction().accept(this) @@ -515,14 +539,6 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege if (ctx.classAtom().size == 2) throw IllegalArgumentException("Not implemented yet") val rec = ctx.classAtom()[0].accept(this).data as List list.addAll(rec) - } else if ( - ctx.classAtom()[0]?.classAtomNoDash()?.FLAG_SCOPE_OPEN() != null - || ctx.classAtom()[0]?.classAtomNoDash()?.FLAG_GROUP_OPEN() != null - || ctx.classAtom()[0]?.classAtomNoDash()?.NAMED_CAPTURE_GROUP_OPEN() != null - ) { - // these should be interpreted literally within a charclass. - val ranges = ctx.text.map { ch -> CharacterRange(ch, ch) } - list.addAll(ranges) } else { val startText = ctx.classAtom()[0].text assert(startText.length == 1 || startText.length == 2) // single chars or \+ and \. escaped chars @@ -615,31 +631,17 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege override fun visitClassEscape(ctx: RegexJavaParser.ClassEscapeContext): VisitResult { - val res = VisitResult() - res.data = if(ctx.atomEscape() != null && - (ctx.atomEscape().BackReference() != null || ctx.atomEscape().NamedBackReference() != null) - ) { - // In Java using backrefs or named backrefs is illegal within char classes. (i.e.: [\1\k]) - throw IllegalArgumentException("Illegal/unsupported escape sequence") - } else if (ctx.atomEscape() != null) { - when (val rec = ctx.atomEscape().accept(this).genes[0]) { - is CharacterClassEscapeRxGene -> { - rec.multiCharRange.ranges - } - - is PatternCharacterBlockGene -> { - if (rec.stringBlock.length > 1) { - throw IllegalArgumentException("CharClass element cannot be strings") - } - else listOf(CharacterRange(rec.stringBlock[0], rec.stringBlock[0])) - } - - else -> throw IllegalArgumentException("Unexpected CharClass content") + val txt = ctx.text + val ranges = when { + ctx.CharacterClassEscape() != null -> + CharacterClassEscapeRxGene(txt.substring(1), currentFlags).multiCharRange.ranges + else -> { // character escape + val s = resolveCharacterEscapeString(txt) + if (s.length > 1) throw IllegalArgumentException("CharClass element cannot be strings") + listOf(CharacterRange(s[0], s[0])) } - } else { - throw IllegalArgumentException("Not implemented yet") } - return res + return VisitResult(data = ranges) } override fun visitAtomEscape(ctx: RegexJavaParser.AtomEscapeContext): VisitResult { @@ -686,44 +688,11 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege return VisitResult(BackReferenceRxGene(groupIndex, group)) } - return VisitResult(when (txt[1]) { - '0' -> { - val octalValue = txt.substring(2).toInt(8) - PatternCharacterBlockGene( - txt, - String(Character.toChars(octalValue)), - currentFlags - ) - } - 'c' -> { - val controlLetterValue = if (txt[2].isLowerCase()){ - txt[2].uppercaseChar().code.xor(0x60) - } else { - txt[2].code.xor(0x40) - } - PatternCharacterBlockGene(txt, controlLetterValue.toChar().toString(), currentFlags) - } - in escapeMap -> { - val escape = escapeMap[txt[1]]!! - PatternCharacterBlockGene(txt, escape, currentFlags) - } - in hexEscapePrefixes -> { - val hexValue = if (txt[1] == 'x' && txt.length > 4 && txt[2] == '{' && txt[txt.length - 1] == '}') { - txt.substring(3, txt.length - 1).toInt(16) - } else { - txt.substring(2).toInt(16) - } - if(hexValue !in Character.MIN_CODE_POINT..Character.MAX_CODE_POINT){ - throw IllegalArgumentException("Hexadecimal escape out of range: ${ctx.text}") - } - PatternCharacterBlockGene( - txt, - String(Character.toChars(hexValue)), - currentFlags - ) - } - !in notIdentityEscapes -> PatternCharacterBlockGene(txt, txt.substring(1), currentFlags) - else -> CharacterClassEscapeRxGene(txt.substring(1), currentFlags) - }) + if (ctx.CharacterClassEscape() != null) { + return VisitResult(CharacterClassEscapeRxGene(ctx.CharacterClassEscape().text.substring(1), currentFlags)) + } + + // character escape + return VisitResult(PatternCharacterBlockGene(txt, resolveCharacterEscapeString(txt), currentFlags)) } } diff --git a/core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt b/core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt index a1a522ec0a..0fdfc09b9c 100644 --- a/core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt +++ b/core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt @@ -264,7 +264,11 @@ object RegexHandler { */ private class ThrowingErrorListener : BaseErrorListener() { - override fun syntaxError(recognizer: Recognizer<*, *>, offendingSymbol: Any, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException) { + // offendingSymbol is nullable as otherwise lexer errors throw NPE instead of ParseCancellationException + // for example, on our Java grammar [\1] is a lexer error since \1 has no token within that context + // org.antlr.v4.runtime.BaseErrorListener.syntaxError declares this parameter as Object, which is nullable + // org.antlr.v4.runtime.Lexer.notifyListeners calls this method with offendingSymbol as null + override fun syntaxError(recognizer: Recognizer<*, *>, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException) { throw ParseCancellationException("line $line:$charPositionInLine $msg") } } diff --git a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt index 5a10072b75..f81cead7c1 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt @@ -1,5 +1,6 @@ package org.evomaster.core.parser +import org.antlr.v4.runtime.misc.ParseCancellationException import org.evomaster.core.search.gene.regex.RegexGene import org.evomaster.core.utils.RegexFlags import org.junit.jupiter.api.Test @@ -134,6 +135,22 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("""Pe""") } + @Test + fun testCharacterClass(){ + checkSameAsJava("^[\\p{Lower}a-z]$") + checkSameAsJava("^[a\\p{Lower}]$") + checkSameAsJava("^[a[b-c]&&[^x]]$") + checkSameAsJava("^a]b$") + checkSameAsJava("^a&&b$") + checkSameAsJava("^[(?i)-a]$") + checkSameAsJava("^[(?i:abc)-a]$") + checkCanSample("^[a&b]$", "&", 100) + // these are rejected as lexer has no token for \1 or \k here, as they are not available on CHAR_CLASS_MODE + // on Java these throw on Pattern.compile since backreferences are not allowed within character classes + assertThrows { checkSameAsJava("^[\\1]$") } + assertThrows { checkSameAsJava("^[\\k]$") } + } + @Test override fun testPredefinedCharClassInsideCharClass(){ checkSameAsJava("""[\V\p{Lower}\p{Upper}\W\d]""")