Skip to content

Commit 9989011

Browse files
wsp1911wsp1911
committed
fix(tool-call-jsonrepair): sync fork with upstream 0.2.5
Refresh the local parser and non-CLI regression coverage to the upstream 0.2.5 baseline while preserving OpenBitFun's tool-call behavior: - keep comment markers as argument content in the tool-call profile - include parser fixes for truncated URLs, invalid numbers, Unicode, control characters, recursion, and separator rollback - omit the upstream schema-guided correction API Update the README and add dedicated profile regression coverage.
1 parent dacba0e commit 9989011

13 files changed

Lines changed: 263 additions & 63 deletions

File tree

‎src/crates/execution/tool-call-jsonrepair/README.md‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Tool-call JSON repair
22

33
This crate is a local fork of
4-
[`jsonrepair-rs` 0.2.1](https://github.com/majiayu000/jsonrepair-rs), licensed
4+
[`jsonrepair-rs` 0.2.5](https://github.com/majiayu000/jsonrepair-rs), licensed
55
under MIT. The upstream license is retained in [`LICENSE`](LICENSE).
66

77
It retains the upstream generic repair API for compatibility, and adds
@@ -14,9 +14,15 @@ The profile still supports bounded syntax recovery needed for malformed model
1414
tool arguments, including missing string quotes, commas, and closing
1515
delimiters. The caller must parse and schema-validate the result before use.
1616

17+
The fork keeps the upstream 0.2.5 parser fixes and regression coverage, while
18+
retaining the separate comment-free tool-call profile. Schema-guided correction
19+
from upstream 0.2.5 is intentionally not included; tool arguments are parsed
20+
and validated by the owning tool pipeline.
21+
1722
## Upstream regression coverage
1823

19-
The non-CLI regression tests and parity fixture from `jsonrepair-rs` 0.2.1 are
20-
vendored under `tests/`. They differ only in the local crate import path. The
21-
upstream CLI tests are intentionally excluded because this internal library
22-
sets `autobins = false` and does not ship the upstream command-line program.
24+
The non-CLI parser, streaming, and parity regression tests from
25+
`jsonrepair-rs` 0.2.5 are vendored under `tests/`, with the local crate import
26+
path adjusted. Schema-specific tests and the upstream CLI tests are omitted:
27+
this internal library does not expose the schema helper, sets `autobins = false`,
28+
and does not ship the upstream command-line program.

‎src/crates/execution/tool-call-jsonrepair/src/lib.rs‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
//! # OpenBitFun tool-call JSON repair
22
//!
3-
//! A local MIT-licensed fork of `jsonrepair-rs` 0.2.1 for streamed tool
3+
//! A local MIT-licensed fork of `jsonrepair-rs` 0.2.5 for streamed tool
44
//! arguments. The tool-call profile preserves Markdown-like `#` content rather
55
//! than treating it as a configuration-file comment.
66
//!
@@ -304,6 +304,21 @@ where
304304
serde_json::from_str(&repaired).map_err(JsonRepairParseError::from)
305305
}
306306

307+
#[cfg(test)]
308+
mod tests {
309+
use super::repair_tool_call_json;
310+
311+
#[test]
312+
fn tool_call_profile_preserves_hash_prefixed_unquoted_string_values() {
313+
let repaired = repair_tool_call_json(r##"{"plan": # Markdown heading"}"##)
314+
.expect("repair should succeed");
315+
let value: serde_json::Value =
316+
serde_json::from_str(&repaired).expect("repaired output should be JSON");
317+
318+
assert_eq!(value["plan"], "# Markdown heading");
319+
}
320+
}
321+
307322
fn reject_if_changed(input: &str, repaired: &str) -> Result<(), JsonRepairError> {
308323
if input == repaired {
309324
return Ok(());
@@ -347,18 +362,3 @@ fn line_column(input: &str, position: usize) -> (usize, usize) {
347362

348363
(line, column)
349364
}
350-
351-
#[cfg(test)]
352-
mod tests {
353-
use super::repair_tool_call_json;
354-
355-
#[test]
356-
fn tool_call_profile_preserves_hash_prefixed_unquoted_string_values() {
357-
let repaired = repair_tool_call_json(r##"{"plan": # Markdown heading"}"##)
358-
.expect("repair should succeed");
359-
let value: serde_json::Value =
360-
serde_json::from_str(&repaired).expect("repaired output should be JSON");
361-
362-
assert_eq!(value["plan"], "# Markdown heading");
363-
}
364-
}

‎src/crates/execution/tool-call-jsonrepair/src/parser/array.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ impl JsonRepairer {
1111

1212
self.enter_container()?;
1313
self.output.push('[');
14+
let frame_start = self.output.len();
1415
self.pos += 1;
1516
self.parse_whitespace_and_comments();
1617

@@ -41,7 +42,7 @@ impl JsonRepairer {
4142
let processed_value = self.parse_value()?;
4243
if !processed_value {
4344
// Trailing comma or truncated input.
44-
self.strip_trailing_comma();
45+
self.strip_trailing_comma(frame_start);
4546
break;
4647
}
4748
}

‎src/crates/execution/tool-call-jsonrepair/src/parser/mod.rs‎

Lines changed: 7 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -284,39 +284,23 @@ impl JsonRepairer {
284284
true
285285
}
286286

287-
/// Remove last occurrence of `c` from output.
288-
pub(super) fn strip_last_occurrence(&mut self, c: char) {
289-
if let Some(idx) = self.output.rfind(c) {
290-
self.output.remove(idx);
291-
}
292-
}
293-
294287
/// Fast path for the common trailing-comma rollback case.
295-
/// Removes a comma only when it's the last non-whitespace output char.
296-
pub(super) fn strip_trailing_comma(&mut self) {
288+
/// Never remove a separator before the current container's output.
289+
pub(super) fn strip_trailing_comma(&mut self, min_index: usize) {
297290
let bytes = self.output.as_bytes();
298-
if let Some(&last) = bytes.last() {
299-
if last == b',' {
300-
self.output.pop();
301-
return;
302-
}
303-
if !matches!(last, b' ' | b'\n' | b'\r' | b'\t') {
304-
self.strip_last_occurrence(',');
305-
return;
306-
}
307-
}
308-
309291
let mut idx = bytes.len();
310-
while idx > 0 && matches!(bytes[idx - 1], b' ' | b'\n' | b'\r' | b'\t') {
292+
while idx > min_index && matches!(bytes[idx - 1], b' ' | b'\n' | b'\r' | b'\t') {
311293
idx -= 1;
312294
}
313295

314-
if idx > 0 && bytes[idx - 1] == b',' {
296+
if idx > min_index && bytes[idx - 1] == b',' {
315297
self.output.remove(idx - 1);
316298
return;
317299
}
318300

319-
self.strip_last_occurrence(',');
301+
if let Some(relative_index) = self.output[min_index..].rfind(',') {
302+
self.output.remove(min_index + relative_index);
303+
}
320304
}
321305

322306
/// Insert `text` before any trailing whitespace in the output buffer.

‎src/crates/execution/tool-call-jsonrepair/src/parser/number.rs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ impl JsonRepairer {
3030
let mut append_trailing_zero = false;
3131
let mut has_leading_dot = false;
3232
let mut has_invalid_leading_zero = false;
33+
// Exponent (`e`/`E`) is only valid after at least one integer or fraction
34+
// digit. Without this guard, plus-prefixed forms like `+e` / `+e1` (after
35+
// `parse_plus_number` strips `+`) would emit invalid JSON such as `e0`.
36+
let mut has_digit = false;
3337

3438
if self.peek() == Some('-') {
3539
self.pos += 1;
@@ -55,6 +59,7 @@ impl JsonRepairer {
5559
return Ok(false);
5660
}
5761

62+
has_digit = true;
5863
while self.pos < len && chars::is_digit(self.chars[self.pos]) {
5964
self.pos += 1;
6065
}
@@ -75,6 +80,9 @@ impl JsonRepairer {
7580
integer_digits += 1;
7681
self.pos += 1;
7782
}
83+
if integer_digits > 0 {
84+
has_digit = true;
85+
}
7886

7987
if self.pos < len && self.chars[self.pos] == '.' {
8088
self.pos += 1;
@@ -84,14 +92,18 @@ impl JsonRepairer {
8492
self.pos = start;
8593
return Ok(false);
8694
} else {
95+
has_digit = true;
8796
while self.pos < len && chars::is_digit(self.chars[self.pos]) {
8897
self.pos += 1;
8998
}
9099
}
91100
}
92101
}
93102

94-
if !append_trailing_zero && self.pos < len && matches!(self.chars[self.pos], 'e' | 'E')
103+
if !append_trailing_zero
104+
&& has_digit
105+
&& self.pos < len
106+
&& matches!(self.chars[self.pos], 'e' | 'E')
95107
{
96108
self.pos += 1;
97109
if self.pos < len && matches!(self.chars[self.pos], '-' | '+') {

‎src/crates/execution/tool-call-jsonrepair/src/parser/object.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ impl JsonRepairer {
1212

1313
self.enter_container()?;
1414
self.output.push('{');
15+
let frame_start = self.output.len();
1516
self.pos += 1;
1617
self.parse_whitespace_and_comments();
1718

@@ -43,7 +44,7 @@ impl JsonRepairer {
4344
|| matches!(self.peek(), Some('}') | Some('{') | Some(']') | Some('['));
4445
if near_end {
4546
// Trailing comma.
46-
self.strip_trailing_comma();
47+
self.strip_trailing_comma(frame_start);
4748
} else {
4849
return Err(self.error_kind(
4950
"Object key expected",

‎src/crates/execution/tool-call-jsonrepair/src/parser/string.rs‎

Lines changed: 76 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ impl JsonRepairer {
9696
.and_then(|idx| self.prev_non_whitespace_index(idx));
9797
let prev_char = prev_non_ws.and_then(|idx| self.peek_at(idx));
9898

99-
if prev_char == Some(',') {
99+
if prev_char == Some(',') && stop_at_index != prev_non_ws {
100100
// {"a":"b,c,"d":"e"} -> stop at comma before quote.
101101
self.pos = input_start;
102102
self.output.truncate(output_start);
@@ -116,9 +116,12 @@ impl JsonRepairer {
116116
self.pos = quote_pos + 1;
117117
} else if stop_at_delimiter && chars::is_unquoted_string_delimiter(c) {
118118
// URL like "https://..." should not stop at '/'.
119-
if self.pos > input_start + 1
120-
&& self.peek_at(self.pos.saturating_sub(1)) == Some(':')
121-
&& self.looks_like_url_start(input_start + 1, self.pos)
119+
if c == '/'
120+
&& self.pos > input_start + 1
121+
&& self.peek_at(self.pos - 1) == Some(':')
122+
&& (self.looks_like_url_start(input_start + 1, self.pos)
123+
|| (self.pos + 1 == self.chars.len()
124+
&& self.ends_with_url_scheme(input_start + 1, self.pos)))
122125
{
123126
while self.peek().is_some_and(chars::is_url_char) {
124127
self.output.push(self.chars[self.pos]);
@@ -173,6 +176,47 @@ impl JsonRepairer {
173176
}
174177

175178
if digits == 4 {
179+
let code_unit = self.hex_quad(self.pos + 1).ok_or_else(|| {
180+
self.error_at_kind(
181+
"Invalid unicode escape",
182+
backslash_pos,
183+
JsonRepairErrorKind::InvalidUnicode,
184+
)
185+
})?;
186+
if (0xD800..=0xDBFF).contains(&code_unit) {
187+
let next = self.pos + 5;
188+
let low = if self.peek_at(next) == Some('\\')
189+
&& self.peek_at(next + 1) == Some('u')
190+
{
191+
self.hex_quad(next + 2)
192+
} else {
193+
None
194+
};
195+
if !low.is_some_and(|unit| (0xDC00..=0xDFFF).contains(&unit)) {
196+
return Err(self.error_at_kind(
197+
"Invalid unicode surrogate pair",
198+
backslash_pos,
199+
JsonRepairErrorKind::InvalidUnicode,
200+
));
201+
}
202+
self.output.push_str("\\u");
203+
for i in 0..4 {
204+
self.output.push(self.chars[self.pos + 1 + i]);
205+
}
206+
self.output.push_str("\\u");
207+
for i in 0..4 {
208+
self.output.push(self.chars[next + 2 + i]);
209+
}
210+
self.pos += 11;
211+
return Ok(());
212+
}
213+
if (0xDC00..=0xDFFF).contains(&code_unit) {
214+
return Err(self.error_at_kind(
215+
"Invalid unicode surrogate pair",
216+
backslash_pos,
217+
JsonRepairErrorKind::InvalidUnicode,
218+
));
219+
}
176220
self.output.push_str("\\u");
177221
for i in 0..4 {
178222
self.output.push(self.chars[self.pos + 1 + i]);
@@ -202,13 +246,22 @@ impl JsonRepairer {
202246
}
203247
_ => {
204248
// Invalid escape: drop '\' and keep char.
205-
self.output.push(esc);
249+
self.push_string_char(esc);
206250
self.pos += 1;
207251
}
208252
}
209253
Ok(())
210254
}
211255

256+
fn hex_quad(&self, start: usize) -> Option<u16> {
257+
let mut value = 0u16;
258+
for offset in 0..4 {
259+
let digit = self.peek_at(start + offset)?.to_digit(16)? as u16;
260+
value = value * 16 + digit;
261+
}
262+
Some(value)
263+
}
264+
212265
fn parse_string_char(&mut self, c: char) -> Result<()> {
213266
if c >= '\u{0020}' && c != '"' && c != '\\' {
214267
self.output.push(c);
@@ -324,6 +377,13 @@ impl JsonRepairer {
324377
self.pos -= 1;
325378
}
326379

380+
// A whitespace-only token has no value. Form feed is trimmed here but
381+
// is not consumed by parse_whitespace_and_comments. Returning false
382+
// prevents the array parser from looping at an unchanged position.
383+
if self.pos == start {
384+
return Ok(false);
385+
}
386+
327387
// Compare directly on char slice — no String allocation.
328388
if !is_key && self.slice_eq(start, self.pos, "undefined") {
329389
self.output.push_str("null");
@@ -395,6 +455,17 @@ impl JsonRepairer {
395455
|| self.matches_at(start, "irc://")
396456
}
397457

458+
fn ends_with_url_scheme(&self, start: usize, slash_idx: usize) -> bool {
459+
[
460+
"http:", "https:", "ftp:", "mailto:", "file:", "data:", "irc:",
461+
]
462+
.iter()
463+
.any(|scheme| {
464+
slash_idx >= start + scheme.len()
465+
&& self.slice_eq(slash_idx - scheme.len(), slash_idx, scheme)
466+
})
467+
}
468+
398469
fn is_known_wrapper_function(&self, start: usize, end: usize) -> bool {
399470
self.slice_starts_with_ignore_ascii_case(start, end, "callback")
400471
|| self.slice_eq_ignore_ascii_case(start, end, "cb")

‎src/crates/execution/tool-call-jsonrepair/src/parser/toplevel.rs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ impl JsonRepairer {
3939
self.parse_ndjson()?;
4040
} else if processed_comma {
4141
// Remove trailing comma after a single root value.
42-
self.strip_trailing_comma();
42+
self.strip_trailing_comma(0);
4343
}
4444

4545
// Repair redundant closing brackets at the root level.
@@ -67,12 +67,16 @@ impl JsonRepairer {
6767
fn parse_ndjson(&mut self) -> Result<()> {
6868
let mut initial = true;
6969
let mut processed_value = true;
70+
// repair() has already inserted or consumed the separator before the
71+
// first additional value.
72+
let mut pending_comma = self.output.rfind(',');
7073
while processed_value {
7174
if !initial {
7275
let processed_comma = self.parse_char(',');
7376
if !processed_comma {
7477
self.insert_before_last_whitespace(",");
7578
}
79+
pending_comma = self.output.rfind(',');
7680
} else {
7781
initial = false;
7882
}
@@ -81,7 +85,9 @@ impl JsonRepairer {
8185
}
8286

8387
if !processed_value {
84-
self.strip_trailing_comma();
88+
if let Some(index) = pending_comma {
89+
self.output.remove(index);
90+
}
8591
}
8692

8793
self.output.insert_str(0, "[\n");

‎src/crates/execution/tool-call-jsonrepair/tests/number_regression_tests.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,13 @@ fn plus_prefixed_leading_dot_numbers_remain_parseable() {
2828
assert_repaired_json("[+.5]");
2929
assert_repaired_json("{value:+.5}");
3030
}
31+
32+
#[test]
33+
fn plus_prefixed_exponent_only_tokens_are_rejected() {
34+
for input in ["+e", "+e1", "+E", "+e+1", "[+e]", "{v:+e1}"] {
35+
assert!(
36+
jsonrepair(input).is_err(),
37+
"plus-prefixed exponent-only token {input:?} must not repair to invalid JSON"
38+
);
39+
}
40+
}

0 commit comments

Comments
 (0)