Skip to content

Commit be5d0d3

Browse files
fix(er): strict pandas blocking parser, NaT-safe comparisons, quoted link keys (#356)
- resolve_entities/link_entities(backend="duckdb") no longer reject valid DuckDB blocking SQL (e.g. jaro_winkler_similarity(...) > 0.8). The pandas parser still runs to attribute blocking_rule_ids, but its EntityResolutionError is now caught alongside ValueError/KeyError, so rules it cannot read are skipped for attribution as documented. - The pandas blocking parser is now strict. Predicates are split on AND and on "="/"==" outside quoted identifiers and string literals. Each side must be a column reference (l.col, r."quoted col", col) or lower/upper/trim/left/right/substr over one. OR, <, <=, >, >=, !=, <>, literals and arithmetic now raise EntityResolutionError instead of resolving to missing keys and silently producing zero candidate pairs. Quoted identifiers unescape "" and may contain spaces, "=" or "and". A rule that references a column absent from the frame logs a warning. - _is_missing treats every scalar pandas missing value (NaN, NaT, pd.NA, numpy NaT) as missing, not only None and float NaN. Two NaT values no longer score as full agreement and merge records, and NaT/pd.NA blocking keys no longer form candidate pairs. - fd.link builds its default blocking SQL with double-quoted identifiers (embedded quotes escaped as ""), so keys such as "first name" or "e-mail" work on the DuckDB backend. - link_entities records match_threshold, clerical_review_threshold and scoring in runtime_metadata, as resolve_entities does. The external fd.link report records its thresholds too. build_review_queue therefore sorts linkage queues around the configured midpoint instead of 0.75. Closes #236 Closes #237 Closes #238 Closes #266 Closes #271
1 parent 9c02f94 commit be5d0d3

3 files changed

Lines changed: 402 additions & 22 deletions

File tree

‎src/freshdata/enterprise/entity_resolution.py‎

Lines changed: 100 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
from __future__ import annotations
3434

3535
import json
36+
import logging
3637
import re
3738
from collections.abc import Iterable, Sequence
3839
from dataclasses import dataclass, field, replace
@@ -50,6 +51,8 @@
5051
EntityResolutionConfig,
5152
)
5253

54+
logger = logging.getLogger("freshdata.enterprise.entity_resolution")
55+
5356
_PAIRS_SAMPLE = 50
5457
_CLUSTERS_SAMPLE = 50
5558
_PREVIEW_LEN = 24
@@ -546,7 +549,12 @@ def __str__(self) -> str:
546549

547550

548551
def _is_missing(v: Any) -> bool:
549-
return v is None or (isinstance(v, float) and pd.isna(v))
552+
"""True for ``None`` and every scalar pandas missing value (NaN, NaT, pd.NA)."""
553+
if v is None:
554+
return True
555+
if isinstance(v, (str, bytes)):
556+
return False
557+
return bool(pd.api.types.is_scalar(v) and pd.isna(v))
550558

551559

552560
def _compare(cmp: ComparisonLevel, a: Any, b: Any) -> float | None:
@@ -714,7 +722,7 @@ def _parse_blocking_rules(
714722
for idx, rule in enumerate(config.blocking_rules):
715723
try:
716724
left_key, right_key = _parse_blocking(rule.sql)
717-
except (ValueError, KeyError):
725+
except (ValueError, KeyError, EntityResolutionError):
718726
continue
719727
parsed.append((blocking_rule_id(idx), left_key, right_key))
720728
return parsed
@@ -797,16 +805,42 @@ def _candidates_duckdb(
797805
# =====================================================================
798806

799807
_FUNC_RE = re.compile(r"^(\w+)\s*\((.*)\)$", re.DOTALL)
808+
# An SQL identifier: unquoted (``name``) or double-quoted with ``""`` escapes.
809+
_IDENT = r'(?:[^\W\d][\w$]*|"(?:[^"]|"")*")'
810+
# A bare column reference, optionally table-prefixed (``l.col`` / ``r."a b"``).
811+
_COLUMN_REF_RE = re.compile(rf"^(?:{_IDENT}\s*\.\s*)?({_IDENT})$")
812+
# Quoted spans (identifiers or string literals) — masked before splitting.
813+
_QUOTED_RE = re.compile(r'"(?:[^"]|"")*"|\'(?:[^\']|\'\')*\'')
814+
_AND_RE = re.compile(r"\band\b", re.IGNORECASE)
815+
_EQ_RE = re.compile(r"==?")
816+
817+
818+
def _quote_identifier(name: object) -> str:
819+
"""Double-quote *name* as an SQL identifier, escaping embedded quotes."""
820+
return '"' + str(name).replace('"', '""') + '"'
821+
822+
823+
def _unquote_identifier(ident: str) -> str:
824+
if len(ident) >= 2 and ident[0] == ident[-1] == '"':
825+
return ident[1:-1].replace('""', '"')
826+
return ident
827+
800828

829+
def _make_expr(expr: str, refs: list[str] | None = None) -> Callable[[dict[str, Any]], Any]:
830+
"""Compile a tiny SQL expression subset to a record→value function.
801831
802-
def _make_expr(expr: str) -> Callable[[dict[str, Any]], Any]:
803-
"""Compile a tiny SQL expression subset to a record→value function."""
832+
Supported: column references (``l.col``, ``r."quoted col"``, ``col``) and
833+
``lower``/``upper``/``trim``/``left``/``right``/``substr`` over them. Any
834+
other expression (operators, literals, other functions) raises
835+
:class:`EntityResolutionError`. Referenced column names are appended to
836+
*refs* when given.
837+
"""
804838
expr = expr.strip()
805839
m = _FUNC_RE.match(expr)
806840
if m:
807841
func = m.group(1).lower()
808842
args = _split_args(m.group(2))
809-
inner = _make_expr(args[0])
843+
inner = _make_expr(args[0] if args else "", refs)
810844
if func == "lower":
811845
return lambda rec: _safe_str(inner(rec)).lower()
812846
if func == "upper":
@@ -826,9 +860,16 @@ def _make_expr(expr: str) -> Callable[[dict[str, Any]], Any]:
826860
raise EntityResolutionError(
827861
f"unsupported SQL function {func!r} in pandas blocking; use the duckdb backend"
828862
)
829-
# bare column reference, possibly prefixed (l.col / r.col)
830-
col = expr.split(".", 1)[1] if "." in expr else expr
831-
col = col.strip().strip('"')
863+
# bare column reference, possibly prefixed (l.col / r."col")
864+
ref = _COLUMN_REF_RE.match(expr)
865+
if ref is None:
866+
raise EntityResolutionError(
867+
f"unsupported expression {expr!r} in pandas blocking; only column references "
868+
"and lower/upper/trim/left/right/substr are supported; use the duckdb backend"
869+
)
870+
col = _unquote_identifier(ref.group(1))
871+
if refs is not None:
872+
refs.append(col)
832873
return lambda rec: rec.get(col)
833874

834875

@@ -863,22 +904,38 @@ def _split_args(text: str) -> list[str]:
863904
return [a.strip() for a in args]
864905

865906

907+
def _split_outside_quotes(text: str, pattern: re.Pattern[str]) -> list[str]:
908+
"""Split *text* on *pattern*, ignoring matches inside quoted spans."""
909+
masked = _QUOTED_RE.sub(lambda m: "\0" * len(m.group()), text)
910+
parts: list[str] = []
911+
start = 0
912+
for m in pattern.finditer(masked):
913+
parts.append(text[start : m.start()])
914+
start = m.end()
915+
parts.append(text[start:])
916+
return parts
917+
918+
866919
def _parse_blocking(
867-
sql: str,
920+
sql: str, refs: list[str] | None = None
868921
) -> tuple[Callable[[dict[str, Any]], Any], Callable[[dict[str, Any]], Any]]:
869-
"""Parse ``a = b [and c = d ...]`` into (left_key_fn, right_key_fn)."""
870-
predicates = re.split(r"\band\b", sql, flags=re.IGNORECASE)
922+
"""Parse ``a = b [and c = d ...]`` into (left_key_fn, right_key_fn).
923+
924+
Each predicate must be exactly ``<expr> = <expr>`` (see :func:`_make_expr`);
925+
``OR``, ``<``/``<=``/``>``/``>=``/``!=``/``<>``, literals and other SQL raise
926+
:class:`EntityResolutionError` instead of silently matching nothing.
927+
"""
871928
left_fns: list[Callable[[dict[str, Any]], Any]] = []
872929
right_fns: list[Callable[[dict[str, Any]], Any]] = []
873-
for pred in predicates:
874-
if "=" not in pred:
930+
for pred in _split_outside_quotes(sql, _AND_RE):
931+
sides = _split_outside_quotes(pred, _EQ_RE)
932+
if len(sides) != 2 or sides[0].rstrip()[-1:] in ("<", ">", "!"):
875933
raise EntityResolutionError(
876-
f"pandas blocking only supports equality predicates, got {pred!r}; "
934+
f"pandas blocking only supports equality predicates, got {pred.strip()!r}; "
877935
"use the duckdb backend for richer SQL"
878936
)
879-
lhs, rhs = pred.split("=", 1)
880-
left_fns.append(_make_expr(lhs))
881-
right_fns.append(_make_expr(rhs))
937+
left_fns.append(_make_expr(sides[0], refs))
938+
right_fns.append(_make_expr(sides[1], refs))
882939

883940
def left_key(rec: dict[str, Any]) -> tuple[Any, ...] | None:
884941
vals = tuple(fn(rec) for fn in left_fns)
@@ -900,7 +957,16 @@ def _candidates_pandas(
900957
n = len(records)
901958
pairs: set[tuple[int, int]] = set()
902959
for rule in config.blocking_rules:
903-
left_key, right_key = _parse_blocking(rule.sql)
960+
refs: list[str] = []
961+
left_key, right_key = _parse_blocking(rule.sql, refs)
962+
missing = sorted({c for c in refs if c not in frame.columns})
963+
if missing:
964+
logger.warning(
965+
"blocking rule %r references column(s) %s not in the frame; "
966+
"it produces no candidate pairs",
967+
rule.sql,
968+
missing,
969+
)
904970
buckets: dict[Any, list[int]] = defaultdict(list)
905971
right_keys = [right_key(records[j]) for j in range(n)]
906972
for j, rk in enumerate(right_keys):
@@ -1140,7 +1206,12 @@ def link_entities(
11401206
backend=backend,
11411207
pairs=pairs,
11421208
clusters=[c for c in clusters if c.size > 1],
1143-
runtime_metadata={"link_type": config.link_type},
1209+
runtime_metadata={
1210+
"link_type": config.link_type,
1211+
"match_threshold": config.match_threshold,
1212+
"clerical_review_threshold": config.clerical_review_threshold,
1213+
"scoring": "rule_weighted_probabilistic_linkage",
1214+
},
11441215
)
11451216
out = from_pandas(resolved, left_df)
11461217
return (out, report) if return_report else out
@@ -1177,10 +1248,12 @@ def _link_config(
11771248
"""Build an EntityResolutionConfig from keys + strategy for exact/fuzzy linking."""
11781249
rules = _coerce_blocking(blocking)
11791250
if not rules:
1251+
# Quote identifiers so keys like "first name" / "e-mail" stay valid SQL.
1252+
quoted = [_quote_identifier(k) for k in keys]
11801253
if strategy == "exact":
1181-
sql = " AND ".join(f"l.{k} = r.{k}" for k in keys)
1254+
sql = " AND ".join(f"l.{q} = r.{q}" for q in quoted)
11821255
else: # fuzzy: block on the first key to bound the candidate space
1183-
sql = f"l.{keys[0]} = r.{keys[0]}"
1256+
sql = f"l.{quoted[0]} = r.{quoted[0]}"
11841257
rules = (BlockingRule(sql=sql, description=f"{strategy} block on {list(keys)}"),)
11851258

11861259
comparisons: list[ComparisonLevel] = []
@@ -1269,7 +1342,12 @@ def _external_report(
12691342
backend="external",
12701343
pairs=pairs,
12711344
clusters=[],
1272-
runtime_metadata={"link_type": "link_only", "strategy": "external"},
1345+
runtime_metadata={
1346+
"link_type": "link_only",
1347+
"strategy": "external",
1348+
"match_threshold": match_threshold,
1349+
"clerical_review_threshold": review_threshold,
1350+
},
12731351
)
12741352

12751353

0 commit comments

Comments
 (0)