3333from __future__ import annotations
3434
3535import json
36+ import logging
3637import re
3738from collections .abc import Iterable , Sequence
3839from dataclasses import dataclass , field , replace
5051 EntityResolutionConfig ,
5152)
5253
54+ logger = logging .getLogger ("freshdata.enterprise.entity_resolution" )
55+
5356_PAIRS_SAMPLE = 50
5457_CLUSTERS_SAMPLE = 50
5558_PREVIEW_LEN = 24
@@ -546,7 +549,12 @@ def __str__(self) -> str:
546549
547550
548551def _is_missing (v : Any ) -> bool :
549- return v is None or (isinstance (v , float ) and pd .isna (v ))
552+ """True for ``None`` and every scalar pandas missing value (NaN, NaT, pd.NA)."""
553+ if v is None :
554+ return True
555+ if isinstance (v , (str , bytes )):
556+ return False
557+ return bool (pd .api .types .is_scalar (v ) and pd .isna (v ))
550558
551559
552560def _compare (cmp : ComparisonLevel , a : Any , b : Any ) -> float | None :
@@ -714,7 +722,7 @@ def _parse_blocking_rules(
714722 for idx , rule in enumerate (config .blocking_rules ):
715723 try :
716724 left_key , right_key = _parse_blocking (rule .sql )
717- except (ValueError , KeyError ):
725+ except (ValueError , KeyError , EntityResolutionError ):
718726 continue
719727 parsed .append ((blocking_rule_id (idx ), left_key , right_key ))
720728 return parsed
@@ -797,16 +805,42 @@ def _candidates_duckdb(
797805# =====================================================================
798806
799807_FUNC_RE = re .compile (r"^(\w+)\s*\((.*)\)$" , re .DOTALL )
808+ # An SQL identifier: unquoted (``name``) or double-quoted with ``""`` escapes.
809+ _IDENT = r'(?:[^\W\d][\w$]*|"(?:[^"]|"")*")'
810+ # A bare column reference, optionally table-prefixed (``l.col`` / ``r."a b"``).
811+ _COLUMN_REF_RE = re .compile (rf"^(?:{ _IDENT } \s*\.\s*)?({ _IDENT } )$" )
812+ # Quoted spans (identifiers or string literals) — masked before splitting.
813+ _QUOTED_RE = re .compile (r'"(?:[^"]|"")*"|\'(?:[^\']|\'\')*\'' )
814+ _AND_RE = re .compile (r"\band\b" , re .IGNORECASE )
815+ _EQ_RE = re .compile (r"==?" )
816+
817+
818+ def _quote_identifier (name : object ) -> str :
819+ """Double-quote *name* as an SQL identifier, escaping embedded quotes."""
820+ return '"' + str (name ).replace ('"' , '""' ) + '"'
821+
822+
823+ def _unquote_identifier (ident : str ) -> str :
824+ if len (ident ) >= 2 and ident [0 ] == ident [- 1 ] == '"' :
825+ return ident [1 :- 1 ].replace ('""' , '"' )
826+ return ident
827+
800828
829+ def _make_expr (expr : str , refs : list [str ] | None = None ) -> Callable [[dict [str , Any ]], Any ]:
830+ """Compile a tiny SQL expression subset to a record→value function.
801831
802- def _make_expr (expr : str ) -> Callable [[dict [str , Any ]], Any ]:
803- """Compile a tiny SQL expression subset to a record→value function."""
832+ Supported: column references (``l.col``, ``r."quoted col"``, ``col``) and
833+ ``lower``/``upper``/``trim``/``left``/``right``/``substr`` over them. Any
834+ other expression (operators, literals, other functions) raises
835+ :class:`EntityResolutionError`. Referenced column names are appended to
836+ *refs* when given.
837+ """
804838 expr = expr .strip ()
805839 m = _FUNC_RE .match (expr )
806840 if m :
807841 func = m .group (1 ).lower ()
808842 args = _split_args (m .group (2 ))
809- inner = _make_expr (args [0 ])
843+ inner = _make_expr (args [0 ] if args else "" , refs )
810844 if func == "lower" :
811845 return lambda rec : _safe_str (inner (rec )).lower ()
812846 if func == "upper" :
@@ -826,9 +860,16 @@ def _make_expr(expr: str) -> Callable[[dict[str, Any]], Any]:
826860 raise EntityResolutionError (
827861 f"unsupported SQL function { func !r} in pandas blocking; use the duckdb backend"
828862 )
829- # bare column reference, possibly prefixed (l.col / r.col)
830- col = expr .split ("." , 1 )[1 ] if "." in expr else expr
831- col = col .strip ().strip ('"' )
863+ # bare column reference, possibly prefixed (l.col / r."col")
864+ ref = _COLUMN_REF_RE .match (expr )
865+ if ref is None :
866+ raise EntityResolutionError (
867+ f"unsupported expression { expr !r} in pandas blocking; only column references "
868+ "and lower/upper/trim/left/right/substr are supported; use the duckdb backend"
869+ )
870+ col = _unquote_identifier (ref .group (1 ))
871+ if refs is not None :
872+ refs .append (col )
832873 return lambda rec : rec .get (col )
833874
834875
@@ -863,22 +904,38 @@ def _split_args(text: str) -> list[str]:
863904 return [a .strip () for a in args ]
864905
865906
907+ def _split_outside_quotes (text : str , pattern : re .Pattern [str ]) -> list [str ]:
908+ """Split *text* on *pattern*, ignoring matches inside quoted spans."""
909+ masked = _QUOTED_RE .sub (lambda m : "\0 " * len (m .group ()), text )
910+ parts : list [str ] = []
911+ start = 0
912+ for m in pattern .finditer (masked ):
913+ parts .append (text [start : m .start ()])
914+ start = m .end ()
915+ parts .append (text [start :])
916+ return parts
917+
918+
866919def _parse_blocking (
867- sql : str ,
920+ sql : str , refs : list [ str ] | None = None
868921) -> tuple [Callable [[dict [str , Any ]], Any ], Callable [[dict [str , Any ]], Any ]]:
869- """Parse ``a = b [and c = d ...]`` into (left_key_fn, right_key_fn)."""
870- predicates = re .split (r"\band\b" , sql , flags = re .IGNORECASE )
922+ """Parse ``a = b [and c = d ...]`` into (left_key_fn, right_key_fn).
923+
924+ Each predicate must be exactly ``<expr> = <expr>`` (see :func:`_make_expr`);
925+ ``OR``, ``<``/``<=``/``>``/``>=``/``!=``/``<>``, literals and other SQL raise
926+ :class:`EntityResolutionError` instead of silently matching nothing.
927+ """
871928 left_fns : list [Callable [[dict [str , Any ]], Any ]] = []
872929 right_fns : list [Callable [[dict [str , Any ]], Any ]] = []
873- for pred in predicates :
874- if "=" not in pred :
930+ for pred in _split_outside_quotes (sql , _AND_RE ):
931+ sides = _split_outside_quotes (pred , _EQ_RE )
932+ if len (sides ) != 2 or sides [0 ].rstrip ()[- 1 :] in ("<" , ">" , "!" ):
875933 raise EntityResolutionError (
876- f"pandas blocking only supports equality predicates, got { pred !r} ; "
934+ f"pandas blocking only supports equality predicates, got { pred . strip () !r} ; "
877935 "use the duckdb backend for richer SQL"
878936 )
879- lhs , rhs = pred .split ("=" , 1 )
880- left_fns .append (_make_expr (lhs ))
881- right_fns .append (_make_expr (rhs ))
937+ left_fns .append (_make_expr (sides [0 ], refs ))
938+ right_fns .append (_make_expr (sides [1 ], refs ))
882939
883940 def left_key (rec : dict [str , Any ]) -> tuple [Any , ...] | None :
884941 vals = tuple (fn (rec ) for fn in left_fns )
@@ -900,7 +957,16 @@ def _candidates_pandas(
900957 n = len (records )
901958 pairs : set [tuple [int , int ]] = set ()
902959 for rule in config .blocking_rules :
903- left_key , right_key = _parse_blocking (rule .sql )
960+ refs : list [str ] = []
961+ left_key , right_key = _parse_blocking (rule .sql , refs )
962+ missing = sorted ({c for c in refs if c not in frame .columns })
963+ if missing :
964+ logger .warning (
965+ "blocking rule %r references column(s) %s not in the frame; "
966+ "it produces no candidate pairs" ,
967+ rule .sql ,
968+ missing ,
969+ )
904970 buckets : dict [Any , list [int ]] = defaultdict (list )
905971 right_keys = [right_key (records [j ]) for j in range (n )]
906972 for j , rk in enumerate (right_keys ):
@@ -1140,7 +1206,12 @@ def link_entities(
11401206 backend = backend ,
11411207 pairs = pairs ,
11421208 clusters = [c for c in clusters if c .size > 1 ],
1143- runtime_metadata = {"link_type" : config .link_type },
1209+ runtime_metadata = {
1210+ "link_type" : config .link_type ,
1211+ "match_threshold" : config .match_threshold ,
1212+ "clerical_review_threshold" : config .clerical_review_threshold ,
1213+ "scoring" : "rule_weighted_probabilistic_linkage" ,
1214+ },
11441215 )
11451216 out = from_pandas (resolved , left_df )
11461217 return (out , report ) if return_report else out
@@ -1177,10 +1248,12 @@ def _link_config(
11771248 """Build an EntityResolutionConfig from keys + strategy for exact/fuzzy linking."""
11781249 rules = _coerce_blocking (blocking )
11791250 if not rules :
1251+ # Quote identifiers so keys like "first name" / "e-mail" stay valid SQL.
1252+ quoted = [_quote_identifier (k ) for k in keys ]
11801253 if strategy == "exact" :
1181- sql = " AND " .join (f"l.{ k } = r.{ k } " for k in keys )
1254+ sql = " AND " .join (f"l.{ q } = r.{ q } " for q in quoted )
11821255 else : # fuzzy: block on the first key to bound the candidate space
1183- sql = f"l.{ keys [0 ]} = r.{ keys [0 ]} "
1256+ sql = f"l.{ quoted [0 ]} = r.{ quoted [0 ]} "
11841257 rules = (BlockingRule (sql = sql , description = f"{ strategy } block on { list (keys )} " ),)
11851258
11861259 comparisons : list [ComparisonLevel ] = []
@@ -1269,7 +1342,12 @@ def _external_report(
12691342 backend = "external" ,
12701343 pairs = pairs ,
12711344 clusters = [],
1272- runtime_metadata = {"link_type" : "link_only" , "strategy" : "external" },
1345+ runtime_metadata = {
1346+ "link_type" : "link_only" ,
1347+ "strategy" : "external" ,
1348+ "match_threshold" : match_threshold ,
1349+ "clerical_review_threshold" : review_threshold ,
1350+ },
12731351 )
12741352
12751353
0 commit comments