Skip to content

Commit 725fdc1

Browse files
dmitriplotnikovcopybara-github
dmitriplotnikov
authored and
copybara-github
committed
Optimize CEL PrattParser and Lexer performance
Improve parsing throughput and reduce memory allocations across CEL expressions. PiperOrigin-RevId: 979511160
1 parent 6a8a056 commit 725fdc1

3 files changed

Lines changed: 233 additions & 243 deletions

File tree

‎parser/src/main/java/dev/cel/parser/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ java_library(
118118
"//common:source_location",
119119
"//common/ast",
120120
"//common/internal",
121+
"@maven//:com_google_errorprone_error_prone_annotations",
121122
"@maven//:com_google_guava_guava",
122123
"@maven//:org_jspecify_jspecify",
123124
],

‎parser/src/main/java/dev/cel/parser/Lexer.java‎

Lines changed: 46 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,17 @@ static final class Token {
9898
final TokenType type;
9999
final int start;
100100
final int end;
101+
final @Nullable String text;
101102

102103
Token(TokenType type, int start, int end) {
104+
this(type, start, end, null);
105+
}
106+
107+
Token(TokenType type, int start, int end, @Nullable String text) {
103108
this.type = type;
104109
this.start = start;
105110
this.end = end;
111+
this.text = text;
106112
}
107113

108114
@Override
@@ -149,35 +155,28 @@ static final class LexerError {
149155
.buildOrThrow();
150156

151157
private final CelCodePointArray content;
158+
private final int size;
152159
private int position;
153160
private LexerError error;
154161

155162
Lexer(CelCodePointArray content) {
156163
this.content = content;
164+
this.size = content.size();
157165
this.position = 0;
158166
this.error = null;
159167
}
160168

161169
Token lex() {
170+
consumeWhitespaceAndComments();
162171
int start = position;
163-
if (position >= content.size()) {
172+
if (position >= size) {
164173
return makeToken(TokenType.END, start, start);
165174
}
166175
int c = content.get(position);
167176
switch (c) {
168-
case '\f':
169-
case '\n':
170-
case ' ':
171-
case '\r':
172-
case 0x0B: // \v (vertical tab)
173-
case '\t':
174-
{
175-
consumeWhitespace();
176-
return makeToken(TokenType.WHITESPACE, start, position);
177-
}
178177
case '.':
179178
{
180-
if (position + 1 < content.size() && isDigit(content.get(position + 1))) {
179+
if (position + 1 < size && isDigit(content.get(position + 1))) {
181180
return consumeNumericLiteral();
182181
}
183182
advance(1);
@@ -283,10 +282,6 @@ Token lex() {
283282
case '/':
284283
{
285284
advance(1);
286-
if (consume('/')) {
287-
consumeLine();
288-
return makeToken(TokenType.COMMENT, start, position);
289-
}
290285
return makeToken(TokenType.SLASH, start, position);
291286
}
292287
case '&':
@@ -391,7 +386,7 @@ private void advance(int n) {
391386
}
392387

393388
private boolean match(int c) {
394-
return position < content.size() && content.get(position) == c;
389+
return position < size && content.get(position) == c;
395390
}
396391

397392
private boolean consume(int c) {
@@ -403,7 +398,7 @@ private boolean consume(int c) {
403398
}
404399

405400
private boolean consumeIf(IntPredicate predicate) {
406-
if (position < content.size()) {
401+
if (position < size) {
407402
int cp = content.get(position);
408403
if (predicate.test(cp)) {
409404
advance(1);
@@ -414,7 +409,7 @@ private boolean consumeIf(IntPredicate predicate) {
414409
}
415410

416411
private void consumeLine() {
417-
while (position < content.size()) {
412+
while (position < size) {
418413
if (content.get(position) == '\n') {
419414
advance(1);
420415
return;
@@ -423,8 +418,8 @@ private void consumeLine() {
423418
}
424419
}
425420

426-
private void consumeWhitespace() {
427-
while (position < content.size()) {
421+
private void consumeWhitespaceAndComments() {
422+
while (position < size) {
428423
int c = content.get(position);
429424
switch (c) {
430425
case '\f':
@@ -433,38 +428,35 @@ private void consumeWhitespace() {
433428
case '\r':
434429
case 11: // \v
435430
case '\t':
436-
advance(1);
431+
position++;
437432
break;
433+
case '/':
434+
if (position + 1 < size && content.get(position + 1) == '/') {
435+
consumeLine();
436+
break;
437+
} else {
438+
return;
439+
}
438440
default:
439441
return;
440442
}
441443
}
442444
}
443445

444446
private boolean consumeDigits() {
445-
boolean advanced = false;
446-
while (position < content.size()) {
447-
int c = content.get(position);
448-
if (!isDigit(c)) {
449-
break;
450-
}
451-
advance(1);
452-
advanced = true;
447+
int start = position;
448+
while (position < size && isDigit(content.get(position))) {
449+
position++;
453450
}
454-
return advanced;
451+
return position > start;
455452
}
456453

457454
private boolean consumeHexDigits() {
458-
boolean advanced = false;
459-
while (position < content.size()) {
460-
int c = content.get(position);
461-
if (!isHexDigit(c)) {
462-
break;
463-
}
464-
advance(1);
465-
advanced = true;
455+
int start = position;
456+
while (position < size && isHexDigit(content.get(position))) {
457+
position++;
466458
}
467-
return advanced;
459+
return position > start;
468460
}
469461

470462
private TokenType consumeIntegralSuffix() {
@@ -486,7 +478,7 @@ private Token consumeQuotedIdent() {
486478
private boolean consumeUntilAfter(int c, boolean isRaw) {
487479
int pos = position;
488480
boolean escaped = false;
489-
while (pos < content.size()) {
481+
while (pos < size) {
490482
int cc = content.get(pos);
491483
if (cc == '\n' || cc == '\r') {
492484
position = pos;
@@ -503,20 +495,20 @@ private boolean consumeUntilAfter(int c, boolean isRaw) {
503495
}
504496
pos++;
505497
}
506-
position = content.size();
498+
position = size;
507499
return false;
508500
}
509501

510502
private boolean consumeUntilAfterTripleQuote(int quote, boolean isRaw) {
511503
int pos = position;
512504
boolean escaped = false;
513-
while (pos < content.size()) {
505+
while (pos < size) {
514506
int cc = content.get(pos);
515507
if (!isRaw && cc == '\\') {
516508
escaped = !escaped;
517509
} else {
518510
if ((isRaw || !escaped)
519-
&& pos + 2 < content.size()
511+
&& pos + 2 < size
520512
&& cc == quote
521513
&& content.get(pos + 1) == quote
522514
&& content.get(pos + 2) == quote) {
@@ -527,16 +519,14 @@ private boolean consumeUntilAfterTripleQuote(int quote, boolean isRaw) {
527519
}
528520
pos++;
529521
}
530-
position = content.size();
522+
position = size;
531523
return false;
532524
}
533525

534526
private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolean isRaw) {
535527
advance(1);
536528
boolean isTripleQuote =
537-
position + 1 < content.size()
538-
&& content.get(position) == quote
539-
&& content.get(position + 1) == quote;
529+
position + 1 < size && content.get(position) == quote && content.get(position + 1) == quote;
540530
if (isTripleQuote) {
541531
advance(2);
542532
if (!consumeUntilAfterTripleQuote(quote, isRaw)) {
@@ -556,7 +546,7 @@ private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolea
556546

557547
private @Nullable Token consumePrefixedStringLiteral() {
558548
int start = position;
559-
if (position >= content.size()) {
549+
if (position >= size) {
560550
return null;
561551
}
562552
int c = content.get(position);
@@ -566,15 +556,15 @@ private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolea
566556
return null;
567557
}
568558
int lookahead = 1;
569-
if (position + 1 < content.size()) {
559+
if (position + 1 < size) {
570560
int c2 = content.get(position + 1);
571561
if (isBytes ? (c2 == 'r' || c2 == 'R') : (c2 == 'b' || c2 == 'B')) {
572562
isBytes = true;
573563
isRaw = true;
574564
lookahead = 2;
575565
}
576566
}
577-
if (position + lookahead < content.size()) {
567+
if (position + lookahead < size) {
578568
int quote = content.get(position + lookahead);
579569
if (quote == '"' || quote == '\'') {
580570
advance(lookahead);
@@ -612,9 +602,9 @@ private Token consumeNumericLiteral() {
612602
return makeToken(tokenType, start, position);
613603
}
614604
consumeDigits();
615-
if (position < content.size()
605+
if (position < size
616606
&& content.get(position) == '.'
617-
&& position + 1 < content.size()
607+
&& position + 1 < size
618608
&& isDigit(content.get(position + 1))) {
619609
floatingPoint = true;
620610
advance(1);
@@ -639,19 +629,15 @@ && isDigit(content.get(position + 1))) {
639629

640630
private Token consumeIdent() {
641631
int start = position;
642-
while (position < content.size()) {
643-
int c = content.get(position);
644-
if (!isIdentTrailing(c)) {
645-
break;
646-
}
647-
advance(1);
632+
while (position < size && isIdentTrailing(content.get(position))) {
633+
position++;
648634
}
649635
int end = position;
650636
String word = content.slice(start, end).toString();
651637
TokenType keywordType = KEYWORDS.get(word);
652638
if (keywordType != null) {
653639
return makeToken(keywordType, start, end);
654640
}
655-
return makeToken(TokenType.IDENT, start, end);
641+
return new Token(TokenType.IDENT, start, end, word);
656642
}
657643
}

0 commit comments

Comments
 (0)