Skip to content

Commit 503c16c

Browse files
fix(dbt): quote exported YAML scalars so values keep their type (#372)
export_dbt_tests hand-writes schema.yml. _scalar left a string unquoted when it matched ^[A-Za-z0-9_][A-Za-z0-9_.-]*$ and float() rejected it, but YAML 1.1 loaders (PyYAML, used by dbt) still re-type such text: 2024-01-01 loads as a date, 0x1F as 31, 0b101 as 5. The `$` anchor also matched before a trailing newline, so "a\n" was written bare, and the double-quoted branch left newlines raw, so they folded to spaces. Strings are now emitted plain only when they fully match a letter or underscore followed by [A-Za-z0-9_.-] and are not a bool/null/nan/inf word; everything else is double-quoted. Inside quotes, backslash, quote, \n, \t, \r, other C0/C1 controls, DEL, U+2028/U+2029, lone surrogates and U+FFFE/U+FFFF are escaped. Python floats are written so PyYAML reads floats back: .nan/.inf/-.inf, and 1e+16 as 1.0e+16. Genuine ints, floats and bools stay unquoted YAML numbers/bools. Closes #342
1 parent 6d88427 commit 503c16c

2 files changed

Lines changed: 301 additions & 18 deletions

File tree

‎src/freshdata/integrations/dbt/tests_exporter.py‎

Lines changed: 45 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
from __future__ import annotations
2525

26+
import math
2627
import re
2728
from pathlib import Path
2829
from typing import Any
@@ -34,35 +35,61 @@
3435
#: Canonical severity -> dbt test ``config.severity`` (dbt only knows warn/error).
3536
_DEFAULT_DBT_SEVERITY = {"info": "warn", "warning": "warn", "error": "error"}
3637

37-
_YAML_SAFE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.\-]*$")
38-
_YAML_RESERVED = {"true", "false", "null", "yes", "no", "on", "off", "none", "~"}
39-
40-
41-
def _is_number_like(text: str) -> bool:
42-
try:
43-
float(text)
44-
except ValueError:
45-
return False
46-
return True
38+
# A string may be written as a plain (unquoted) scalar only when it starts with a
39+
# letter or underscore. YAML 1.1 loaders (PyYAML, which dbt uses) resolve ints, floats,
40+
# dates/timestamps, sexagesimals and 0x/0o/0b literals only from text starting with a
41+
# digit, sign or dot, so such text is always quoted. Of letter-initial words, only the
42+
# bool/null spellings below re-type; ``nan``/``inf`` are quoted too, as a precaution.
43+
# Matched with ``fullmatch`` so a trailing newline can never slip through unquoted.
44+
_YAML_SAFE = re.compile(r"[A-Za-z_][A-Za-z0-9_.\-]*")
45+
_YAML_RESERVED = frozenset(
46+
{"true", "false", "null", "yes", "no", "on", "off", "y", "n", "none", "nan", "inf", "infinity"}
47+
)
48+
49+
# Characters that must be escaped inside a double-quoted scalar: backslash and quote,
50+
# C0 controls, DEL, C1 controls (incl. NEL), Unicode line/paragraph separators, lone
51+
# surrogates and the non-characters U+FFFE/U+FFFF.
52+
_NEEDS_ESCAPE = re.compile(r'[\\"\x00-\x1f\x7f-\x9f\u2028\u2029\ud800-\udfff\ufffe\uffff]')
53+
_NAMED_ESCAPES = {"\\": "\\\\", '"': '\\"', "\n": "\\n", "\t": "\\t", "\r": "\\r"}
54+
55+
56+
def _escape_char(match: re.Match[str]) -> str:
57+
char = match.group(0)
58+
named = _NAMED_ESCAPES.get(char)
59+
if named is not None:
60+
return named
61+
code = ord(char)
62+
return f"\\x{code:02x}" if code < 0x100 else f"\\u{code:04x}"
63+
64+
65+
def _float_scalar(value: float) -> str:
66+
"""Render a float so YAML 1.1 loaders read it back as the same float."""
67+
if math.isnan(value):
68+
return ".nan"
69+
if math.isinf(value):
70+
return ".inf" if value > 0 else "-.inf"
71+
text = repr(value)
72+
# PyYAML's float resolver requires a '.' in the mantissa: '1e+16' would load as str.
73+
mantissa, sep, exponent = text.partition("e")
74+
if sep and "." not in mantissa:
75+
text = f"{mantissa}.0e{exponent}"
76+
return text
4777

4878

4979
def _scalar(value: Any) -> str:
50-
"""Render one YAML scalar, quoting only when needed to preserve meaning."""
80+
"""Render one YAML scalar, quoting strings unless they cannot change type."""
5181
if value is None:
5282
return "null"
5383
if isinstance(value, bool):
5484
return "true" if value else "false"
5585
if isinstance(value, int):
56-
return str(value)
86+
return str(int(value))
5787
if isinstance(value, float):
58-
return repr(value)
88+
return _float_scalar(float(value))
5989
text = str(value)
60-
if not text:
61-
return '""'
62-
if (_YAML_SAFE.match(text) and not _is_number_like(text)
63-
and text.lower() not in _YAML_RESERVED):
90+
if _YAML_SAFE.fullmatch(text) and text.lower() not in _YAML_RESERVED:
6491
return text
65-
return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"'
92+
return '"' + _NEEDS_ESCAPE.sub(_escape_char, text) + '"'
6693

6794

6895
def _emit(obj: Any, indent: int = 0) -> list[str]:
Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
"""YAML round-trip tests for the dbt tests exporter's scalar writer (issue #342).
2+
3+
Every value ``export_dbt_tests`` writes must load back (via PyYAML, which dbt uses)
4+
as the same Python value and type: strings stay strings, numbers stay numbers.
5+
"""
6+
7+
from __future__ import annotations
8+
9+
import itertools
10+
import math
11+
12+
import pytest
13+
14+
import freshdata as fd
15+
from freshdata import QualityFinding
16+
from freshdata.integrations.dbt.tests_exporter import _scalar
17+
18+
yaml = pytest.importorskip("yaml")
19+
20+
TRICKY_STRINGS = [
21+
# YAML 1.1 dates, timestamps and sexagesimals
22+
"2024-01-01",
23+
"2024-01-01T10:00:00",
24+
"2024-01-01 10:00:00",
25+
"12:30",
26+
"1:30:00",
27+
# int / float literal spellings
28+
"0x1F",
29+
"0b101",
30+
"0o17",
31+
"017",
32+
"012",
33+
"1e3",
34+
"1_000",
35+
".5",
36+
"+1",
37+
"-1",
38+
"1.0",
39+
"0",
40+
"1",
41+
"-0",
42+
".inf",
43+
"-.inf",
44+
".NaN",
45+
"nan",
46+
"inf",
47+
"Infinity",
48+
# bool / null words
49+
"yes",
50+
"No",
51+
"on",
52+
"OFF",
53+
"true",
54+
"False",
55+
"null",
56+
"Null",
57+
"NULL",
58+
"~",
59+
"y",
60+
"n",
61+
"Y",
62+
"N",
63+
"none",
64+
# indicators and flow characters
65+
"a: b",
66+
"# x",
67+
"a #b",
68+
"a:b",
69+
"- x",
70+
"-",
71+
"?",
72+
"? x",
73+
"|",
74+
">",
75+
"!tag",
76+
"&anchor",
77+
"*alias",
78+
"%dir",
79+
"@at",
80+
"`tick`",
81+
"[a]",
82+
"{a: 1}",
83+
"a, b",
84+
"<<",
85+
"=",
86+
"'single'",
87+
'"double"',
88+
# whitespace and escaping
89+
"",
90+
" ",
91+
" lead",
92+
"trail ",
93+
'with "quote"',
94+
"back\\slash",
95+
"tab\tchar",
96+
"new\nline",
97+
"cr\rchar",
98+
"a\n",
99+
"\nb",
100+
# control and special Unicode characters
101+
"nul\x00",
102+
"bell\x07",
103+
"esc\x1b",
104+
"del\x7f",
105+
"nel\x85",
106+
"c1\x9f",
107+
"ls\u2028",
108+
"ps\u2029",
109+
"bom\ufeff",
110+
"nonchar\ufffe",
111+
"nonchar\uffff",
112+
"café",
113+
"中文",
114+
"emoji\U0001f600",
115+
]
116+
117+
PLAIN_STRINGS = [
118+
"open",
119+
"closed",
120+
"not_null",
121+
"freshdata_expectation",
122+
"warn",
123+
"orders",
124+
"_private",
125+
"a-b.c",
126+
"Col1",
127+
]
128+
129+
NON_STRING_VALUES = [
130+
None,
131+
True,
132+
False,
133+
0,
134+
1,
135+
-5,
136+
10**20,
137+
-(10**20),
138+
1.5,
139+
-0.0,
140+
0.1,
141+
1e16,
142+
1e-5,
143+
1e300,
144+
-2.5e-300,
145+
float("inf"),
146+
float("-inf"),
147+
]
148+
149+
150+
def _load_in_contexts(rendered: str) -> list[object]:
151+
"""Load *rendered* as a mapping value and as a sequence item."""
152+
as_value = yaml.safe_load(f"key: {rendered}\n")["key"]
153+
as_item = yaml.safe_load(f"items:\n - {rendered}\n")["items"][0]
154+
return [as_value, as_item]
155+
156+
157+
def _assert_same(loaded: object, expected: object) -> None:
158+
assert type(loaded) is type(expected), (loaded, expected)
159+
assert loaded == expected
160+
161+
162+
@pytest.mark.parametrize("text", TRICKY_STRINGS + PLAIN_STRINGS)
163+
def test_strings_round_trip_as_same_string(text: str) -> None:
164+
for loaded in _load_in_contexts(_scalar(text)):
165+
_assert_same(loaded, text)
166+
167+
168+
@pytest.mark.parametrize("value", NON_STRING_VALUES)
169+
def test_numbers_bools_and_none_keep_their_type(value: object) -> None:
170+
rendered = _scalar(value)
171+
assert not rendered.startswith('"')
172+
for loaded in _load_in_contexts(rendered):
173+
_assert_same(loaded, value)
174+
175+
176+
def test_nan_stays_a_float() -> None:
177+
rendered = _scalar(float("nan"))
178+
assert rendered == ".nan"
179+
for loaded in _load_in_contexts(rendered):
180+
assert isinstance(loaded, float) and math.isnan(loaded)
181+
182+
183+
def test_numpy_float_stays_a_float() -> None:
184+
np = pytest.importorskip("numpy")
185+
for loaded in _load_in_contexts(_scalar(np.float64(1e16))):
186+
_assert_same(loaded, 1e16)
187+
188+
189+
@pytest.mark.parametrize("text", PLAIN_STRINGS)
190+
def test_ordinary_identifiers_stay_unquoted(text: str) -> None:
191+
assert _scalar(text) == text
192+
193+
194+
def test_lone_surrogate_is_escaped_to_ascii() -> None:
195+
rendered = _scalar("x\ud800")
196+
assert rendered == '"x\\ud800"'
197+
for loaded in _load_in_contexts(rendered):
198+
_assert_same(loaded, "x\ud800")
199+
200+
201+
def test_all_short_strings_over_tricky_alphabet_round_trip() -> None:
202+
alphabet = "aZ_09 .-+:#'\"\\\n\t~ynx!&*,[]{}%@`|>?=<e"
203+
for length in (1, 2):
204+
for chars in itertools.product(alphabet, repeat=length):
205+
text = "".join(chars)
206+
for loaded in _load_in_contexts(_scalar(text)):
207+
_assert_same(loaded, text)
208+
209+
210+
def test_issue_342_repro_end_to_end(tmp_path) -> None:
211+
values = ["2024-01-01", "0x1F", "0b101", "017"]
212+
names = ["2024-01-01", "a\nb"]
213+
findings = [
214+
QualityFinding.create(
215+
severity="error",
216+
step="s",
217+
column=name,
218+
rule_name="allowed_values",
219+
message="m",
220+
extra={"value_set": values},
221+
)
222+
for name in names
223+
]
224+
path = tmp_path / "schema.yml"
225+
text = fd.export_dbt_tests(findings, "orders", str(path))
226+
227+
assert path.read_text(encoding="utf-8") == text
228+
doc = yaml.safe_load(text)
229+
cols = doc["models"][0]["columns"]
230+
assert [c["name"] for c in cols] == ["2024-01-01", "a\nb"]
231+
for col in cols:
232+
accepted = col["tests"][0]["accepted_values"]
233+
assert accepted["values"] == values
234+
assert all(type(v) is str for v in accepted["values"])
235+
236+
237+
def test_export_round_trips_mixed_value_sets(tmp_path) -> None:
238+
values: list[object] = [*TRICKY_STRINGS, *PLAIN_STRINGS, 1, 2.5, True, None]
239+
finding = QualityFinding.create(
240+
severity="warning",
241+
step="s",
242+
column="yes",
243+
rule_name="allowed_values",
244+
message="on: off",
245+
extra={"value_set": values},
246+
)
247+
text = fd.export_dbt_tests([finding], "2024-01-01", str(tmp_path / "s.yml"))
248+
doc = yaml.safe_load(text)
249+
model = doc["models"][0]
250+
assert model["name"] == "2024-01-01"
251+
col = model["columns"][0]
252+
assert col["name"] == "yes"
253+
loaded = col["tests"][0]["accepted_values"]["values"]
254+
assert len(loaded) == len(values)
255+
for got, expected in zip(loaded, values):
256+
_assert_same(got, expected)

0 commit comments

Comments
 (0)