Fix tokenization of escaped backslashes in SQL string literals

sqlparse/keywords.py

@@ -59,9 +59,9 @@
     (r'(?![_A-ZÀ-Ü])-?(\d+(\.\d*)|\.\d+)(?![_A-ZÀ-Ü])',
      tokens.Number.Float),
     (r'(?![_A-ZÀ-Ü])-?\d+(?![_A-ZÀ-Ü])', tokens.Number.Integer),
-    (r"'(''|\\'|[^'])*'", tokens.String.Single),
+    (r"'(''|\\'|\\\\|[^'])*'", tokens.String.Single),
     # not a real string literal in ANSI SQL:
-    (r'"(""|\\"|[^"])*"', tokens.String.Symbol),
+    (r'"(""|\\"|\\\\|[^"])*"', tokens.String.Symbol),
     (r'(""|".*?[^\\]")', tokens.String.Symbol),
     # sqlite names can be escaped with [square brackets]. left bracket
     # cannot be preceded by word character or a right bracket --

tests/test_issue_814.py

@@ -0,0 +1,28 @@
+from sqlparse import lexer
+from sqlparse import tokens as T
+
+
+def test_escaped_backslashes_single_quoted_token_stream():
+    tokens = list(lexer.tokenize(r"SELECT '\\', '\\'"))
+
+    assert tokens == [
+        (T.Keyword.DML, "SELECT"),
+        (T.Whitespace, " "),
+        (T.String.Single, r"'\\'"),
+        (T.Punctuation, ","),
+        (T.Whitespace, " "),
+        (T.String.Single, r"'\\'"),
+    ]
+
+
+def test_escaped_backslashes_double_quoted_token_stream():
+    tokens = list(lexer.tokenize(r'SELECT "\\", "\\"'))
+
+    assert tokens == [
+        (T.Keyword.DML, "SELECT"),
+        (T.Whitespace, " "),
+        (T.String.Symbol, r'"\\"'),
+        (T.Punctuation, ","),
+        (T.Whitespace, " "),
+        (T.String.Symbol, r'"\\"'),
+    ]

tests/test_tokenize.py

@@ -99,6 +99,21 @@ def test_single_quotes():
     assert repr(p.tokens[0])[:len(tst)] == tst
 
 
+def test_single_quotes_escaped_backslash():
+    # issue 814 - Incorrect Tokenization of Escaped Backslashes
+    # A string containing an escaped backslash (\\) should be tokenized
+    # as a single string literal, not split incorrectly.
+    sql = r"SELECT '\\', '\\'"
+    tokens = list(lexer.tokenize(sql))
+    # Should be: SELECT, ws, '\\', ,, ws, '\\'
+    assert tokens[0] == (T.Keyword.DML, 'SELECT')
+    assert tokens[1] == (T.Whitespace, ' ')
+    assert tokens[2] == (T.String.Single, "'\\\\'")
+    assert tokens[3] == (T.Punctuation, ',')
+    assert tokens[4] == (T.Whitespace, ' ')
+    assert tokens[5] == (T.String.Single, "'\\\\'")
+
+
 def test_tokenlist_first():
     p = sqlparse.parse(' select foo')[0]
     first = p.token_first()