From a8d16968f109847d287cf0b955c56a216fa5d223 Mon Sep 17 00:00:00 2001 From: Elon-Jiang <191222665+Elon-Jiang@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:06:45 +0800 Subject: [PATCH] fix(paper2code): preserve fenced code in paper sections --- CHANGELOG.md | 6 +++ CHANGELOG_ZH.md | 5 +++ tests/test_file_processor_fences.py | 67 +++++++++++++++++++++++++++++ utils/file_processor.py | 21 +++++++++ 4 files changed, 99 insertions(+) create mode 100644 tests/test_file_processor_fences.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 27740df1e..17c526ec6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ Released versions are named in the entry title; the version on PyPI is [deepcode-hku](https://pypi.org/project/deepcode-hku/). 中文版见 [CHANGELOG_ZH.md](CHANGELOG_ZH.md). +## 2026-10-31 · Preserve code examples in paper sections + +- **Keep fenced code in its real paper section.** Python comments and other + hash-prefixed lines inside backtick or tilde code blocks no longer create + false headings or break the paper context passed to the workflow. + ## 2026-10-29 · Lighter compaction, checked Goal completion, and folder browsing on the Web - **Clear old tool output first.** With `agents.defaults.microcompact` on, diff --git a/CHANGELOG_ZH.md b/CHANGELOG_ZH.md index 88b980e32..8ecc58d34 100644 --- a/CHANGELOG_ZH.md +++ b/CHANGELOG_ZH.md @@ -4,6 +4,11 @@ DeepCode 的主要变化，最新的在前，每天一条。正式版本号写 PyPI 上的包名是 [deepcode-hku](https://pypi.org/project/deepcode-hku/)。 English: [CHANGELOG.md](CHANGELOG.md). +## 2026-10-31 · 保留论文章节中的代码示例 + +- **代码注释不再变成论文章节。** 反引号或波浪线代码块中的 Python 注释等 + 井号行保持为代码，不再生成错误标题或拆坏传给工作流的论文上下文。 + ## 2026-10-29 · 更轻的上下文压缩、目标完成前先检查、Web 端可浏览文件夹 - **先清掉旧的工具输出。** 打开 `agents.defaults.microcompact` 后，长会话接近 diff --git a/tests/test_file_processor_fences.py b/tests/test_file_processor_fences.py new file mode 100644 index 000000000..399f89d43 --- /dev/null +++ b/tests/test_file_processor_fences.py @@ -0,0 +1,67 @@ +"""Paper headings must not be inferred from fenced code examples.""" + +import asyncio + +import pytest + +from utils.file_processor import FileProcessor + + +@pytest.mark.parametrize("fence", ["```python", "~~~python", " ```python"]) +def test_fenced_comments_stay_in_their_paper_section(fence): + marker = "~~~" if "~" in fence else "```" + code = f"{fence}\n# initialize the model\nx = 1\n{marker}" + sections = FileProcessor.parse_markdown_sections( + f"# Paper\n\n{code}\n\n## Results\nAccurate." + ) + assert len(sections) == 1 + assert sections[0]["title"] == "Paper" + assert sections[0]["content"] == code.strip() + assert [s["title"] for s in sections[0]["subsections"]] == ["Results"] + + +@pytest.mark.parametrize( + "non_closer", ["```", "~~~~", "```` trailing text", " ````"] +) +def test_only_a_matching_long_enough_fence_closes_code(non_closer): + code = f"````python\n{non_closer}\n# still a comment\n`````" + sections = FileProcessor.parse_markdown_sections( + f"# Paper\n{code}\n## Results\nDone." + ) + assert len(sections) == 1 + assert sections[0]["content"] == code.strip() + assert sections[0]["subsections"][0]["title"] == "Results" + + +def test_unclosed_fence_keeps_later_hash_lines_as_code(): + sections = FileProcessor.parse_markdown_sections( + "# Paper\n```python\n# code comment" + ) + assert len(sections) == 1 + assert sections[0]["content"] == "```python\n# code comment" + + +def test_backticks_in_info_string_do_not_open_a_fence(): + sections = FileProcessor.parse_markdown_sections( + "# Paper\n```not`a-fence\n## Results" + ) + assert sections[0]["subsections"][0]["title"] == "Results" + + +def test_process_file_input_preserves_code_before_standardization(tmp_path): + paper = tmp_path / "paper.md" + code = "```python\n# initialize the model\nx = 1\n```" + paper.write_text(f"# Paper\n{code}\n## Results\nDone.", encoding="utf-8") + result = asyncio.run(FileProcessor.process_file_input(str(paper))) + assert len(result["sections"]) == 1 + assert code in result["standardized_text"] + assert result["sections"][0]["subsections"][0]["title"] == "Results" + + +def test_crlf_fences_close_before_the_next_heading(): + sections = FileProcessor.parse_markdown_sections( + "# Paper\r\n```python\r\n# code comment\r\n```\r\n## Results\r\nDone." + ) + assert len(sections) == 1 + assert sections[0]["subsections"][0]["title"] == "Results" + assert "# code comment" in sections[0]["content"] diff --git a/utils/file_processor.py b/utils/file_processor.py index 0605026d9..cab465263 100644 --- a/utils/file_processor.py +++ b/utils/file_processor.py @@ -109,8 +109,29 @@ def parse_markdown_sections(content: str) -> List[Dict[str, Union[str, int, List sections = [] current_section = None current_content = [] + fence = None for line in lines: + fence_match = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line) + if fence is not None: + if ( + fence_match + and fence_match.group(1)[0] == fence[0] + and len(fence_match.group(1)) >= len(fence) + and not fence_match.group(2).strip(" \t\r") + ): + fence = None + if current_section is not None: + current_content.append(line) + continue + if fence_match and not ( + fence_match.group(1)[0] == "`" and "`" in fence_match.group(2) + ): + fence = fence_match.group(1) + if current_section is not None: + current_content.append(line) + continue + # Check if line is a header header_match = re.match(r"^(#{1,6})\s+(.+)$", line)