Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ Released versions are named in the entry title; the version on PyPI is
[deepcode-hku](https://pypi.org/project/deepcode-hku/). 中文版见
[CHANGELOG_ZH.md](CHANGELOG_ZH.md).

## 2026-10-31 · Preserve code examples in paper sections

- **Keep fenced code in its real paper section.** Python comments and other
hash-prefixed lines inside backtick or tilde code blocks no longer create
false headings or break the paper context passed to the workflow.

## 2026-10-29 · Lighter compaction, checked Goal completion, and folder browsing on the Web

- **Clear old tool output first.** With `agents.defaults.microcompact` on,
Expand Down
5 changes: 5 additions & 0 deletions CHANGELOG_ZH.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ DeepCode 的主要变化,最新的在前,每天一条。正式版本号写
PyPI 上的包名是 [deepcode-hku](https://pypi.org/project/deepcode-hku/)。
English: [CHANGELOG.md](CHANGELOG.md).

## 2026-10-31 · 保留论文章节中的代码示例

- **代码注释不再变成论文章节。** 反引号或波浪线代码块中的 Python 注释等
井号行保持为代码,不再生成错误标题或拆坏传给工作流的论文上下文。

## 2026-10-29 · 更轻的上下文压缩、目标完成前先检查、Web 端可浏览文件夹

- **先清掉旧的工具输出。** 打开 `agents.defaults.microcompact` 后,长会话接近
Expand Down
67 changes: 67 additions & 0 deletions tests/test_file_processor_fences.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
"""Paper headings must not be inferred from fenced code examples."""

import asyncio

import pytest

from utils.file_processor import FileProcessor


@pytest.mark.parametrize("fence", ["```python", "~~~python", " ```python"])
def test_fenced_comments_stay_in_their_paper_section(fence):
marker = "~~~" if "~" in fence else "```"
code = f"{fence}\n# initialize the model\nx = 1\n{marker}"
sections = FileProcessor.parse_markdown_sections(
f"# Paper\n\n{code}\n\n## Results\nAccurate."
)
assert len(sections) == 1
assert sections[0]["title"] == "Paper"
assert sections[0]["content"] == code.strip()
assert [s["title"] for s in sections[0]["subsections"]] == ["Results"]


@pytest.mark.parametrize(
"non_closer", ["```", "~~~~", "```` trailing text", " ````"]
)
def test_only_a_matching_long_enough_fence_closes_code(non_closer):
code = f"````python\n{non_closer}\n# still a comment\n`````"
sections = FileProcessor.parse_markdown_sections(
f"# Paper\n{code}\n## Results\nDone."
)
assert len(sections) == 1
assert sections[0]["content"] == code.strip()
assert sections[0]["subsections"][0]["title"] == "Results"


def test_unclosed_fence_keeps_later_hash_lines_as_code():
sections = FileProcessor.parse_markdown_sections(
"# Paper\n```python\n# code comment"
)
assert len(sections) == 1
assert sections[0]["content"] == "```python\n# code comment"


def test_backticks_in_info_string_do_not_open_a_fence():
sections = FileProcessor.parse_markdown_sections(
"# Paper\n```not`a-fence\n## Results"
)
assert sections[0]["subsections"][0]["title"] == "Results"


def test_process_file_input_preserves_code_before_standardization(tmp_path):
paper = tmp_path / "paper.md"
code = "```python\n# initialize the model\nx = 1\n```"
paper.write_text(f"# Paper\n{code}\n## Results\nDone.", encoding="utf-8")
result = asyncio.run(FileProcessor.process_file_input(str(paper)))
assert len(result["sections"]) == 1
assert code in result["standardized_text"]
assert result["sections"][0]["subsections"][0]["title"] == "Results"


def test_crlf_fences_close_before_the_next_heading():
sections = FileProcessor.parse_markdown_sections(
"# Paper\r\n```python\r\n# code comment\r\n```\r\n## Results\r\nDone."
)
assert len(sections) == 1
assert sections[0]["subsections"][0]["title"] == "Results"
assert "# code comment" in sections[0]["content"]
21 changes: 21 additions & 0 deletions utils/file_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,29 @@ def parse_markdown_sections(content: str) -> List[Dict[str, Union[str, int, List
sections = []
current_section = None
current_content = []
fence = None

for line in lines:
fence_match = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)
if fence is not None:
if (
fence_match
and fence_match.group(1)[0] == fence[0]
and len(fence_match.group(1)) >= len(fence)
and not fence_match.group(2).strip(" \t\r")
):
fence = None
if current_section is not None:
current_content.append(line)
continue
if fence_match and not (
fence_match.group(1)[0] == "`" and "`" in fence_match.group(2)
):
fence = fence_match.group(1)
if current_section is not None:
current_content.append(line)
continue

# Check if line is a header
header_match = re.match(r"^(#{1,6})\s+(.+)$", line)

Expand Down