From c0ef287bfa45dd3d915a2d3b6ab550ad22fda6d9 Mon Sep 17 00:00:00 2001 From: Greta Date: Mon, 19 Oct 2026 11:45:03 +0800 Subject: [PATCH] docs: define contribution workflow and clarify setup guides --- .github/ISSUE_TEMPLATE/bug_report.yml | 30 +++++++ .github/PULL_REQUEST_TEMPLATE.md | 21 +++++ CONTRIBUTING.md | 125 ++++++++++++++++++++++++++ README.md | 42 +++++---- README_zh.md | 40 +++++---- docs/README.md | 1 + docs/README_zh.md | 1 + docs/agent_memory/README.md | 15 ++-- docs/agent_memory/README_zh.md | 8 +- docs/user_memory/README.md | 21 ++--- docs/user_memory/README_zh.md | 19 ++-- docs/user_memory/results.md | 6 ++ env_examples/PARAMETERS.md | 4 +- scripts/client_factory/registry.py | 8 +- 14 files changed, 261 insertions(+), 80 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 CONTRIBUTING.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..71d86e9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,30 @@ +name: Bug report +description: Report a reproducible code or documentation problem. +body: + - type: textarea + id: problem + attributes: + label: Expected and actual behavior + description: Explain the difference. Include file/section links for documentation issues. + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: Include the command and a minimal input/configuration with credentials removed. + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: Repository commit, OS, Python version, benchmark/backend, and relevant dependency versions. + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant output + description: Remove API keys, private endpoints, and private conversation data before posting. + render: text diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..55ad931 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +## Problem and change + + + +## Validation + + + +- [ ] The change is focused, and relevant tests/documentation are updated. +- [ ] Credentials, private endpoints, and generated benchmark data are excluded. + +## Adapter or benchmark results (if applicable) + + + +- Contribution type: adapter / Published Reference Results / Reproduced Results +- Backend, harness, and dataset versions: +- Run author, evaluated scope, and failure/skip accounting: +- Commands, configuration, scoring, and aggregation: +- Stable public result/artifact links: +- Backend integration validation and any limitations: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d7c4e92 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,125 @@ +# Contributing to OmniMemEval + +Contributions to adapters, evaluation pipelines, documentation, and public +benchmark results are welcome. Use an issue to report a reproducible problem +or discuss a substantial change, and a pull request to propose a focused change. + +## Development workflow + +Create a branch from the latest `main` (use a fork if you do not have write +access). Names such as `fix/search-status`, `chore/test-ci`, or +`docs/quickstart` describe the purpose; no specific prefix is required. +Keep one PR focused on one goal. All changes to `main` go through a PR. + +For offline development, use Python 3.11 or 3.12 in a virtual or conda environment: + +```bash +python -m pip install -r requirements-dev.txt +python -m pytest -q +python scripts/check_doc_links.py +git diff --check +``` + +See the [test guide](scripts/tests/README.md) for targeted suites and repository +checks. Install [User Memory](docs/user_memory/README.md) or +[Agent Memory](docs/agent_memory/README.md) runtime dependencies separately when +running real evaluations. Record the dependency versions used for a run; +the requirements files are dependency lists, not locked reproduction environments. + +Match the surrounding code style, avoid unrelated formatting changes, and add +a regression test for a behavior fix. Documentation-only changes should pass +the link checks; include a minimal before/after example when explaining a bug. + +## Adding a User Memory adapter + +The public entry points use `add(messages, user_id, ...)` and +`search(query, user_id, top_k, ...)`, plus a user-scoped cleanup method. +Inspect an existing adapter with similar behavior and its tests; product-specific +results are normalized by the search dispatch helpers. + +1. Add the client under `scripts/client_factory/` and register its class in + [registry.py](scripts/client_factory/registry.py). Keep optional SDK imports + lazy so other backends remain usable without that SDK. +2. Register its search behavior in + [search_helpers.py](scripts/utils/search_helpers.py) and the LoCoMo-specific + dispatch in [locomo_search.py](scripts/locomo/locomo_search.py). +3. Check timestamp and session handling in + [ingest_helpers.py](scripts/utils/ingest_helpers.py), along with any + benchmark-specific ingestion behavior. Registering the client class alone + does not define these behaviors. +4. Expose cleanup through a method supported by + [delete_user_data](scripts/utils/streaming.py), such as `delete(user_id)`. + Verify that deleting one evaluation user leaves all other users intact. +5. Add a credential-free template under `env_examples/`, dependency/setup + instructions, and entries in the template index and both backend guide tables. + Document the backend version, deployment mode, required settings, and + non-default settings used by any submitted results. +6. Add offline tests for configuration, registration, add/search/cleanup, + user isolation, and failures. Document a small real-backend smoke run and + its version/configuration when integration has been tested. + +For Agent Memory extensions, follow the separate +[architecture and extension contracts](docs/agent_memory/architecture.md). + +## Submitting benchmark results + +The result tables distinguish **Published Reference Results** from +**Reproduced Results**. Adding an adapter does not automatically add its scores. +Only submit benchmarks whose supporting materials are ready; other rows can +follow in another PR. Missing or unrun benchmarks use `-`. + +For every proposed result, provide a stable public report/artifact link and: + +- Run author and date; backend/deployment version; OmniMemEval or other harness + commit; dataset revision or checksum, split, answer-key version, and any corrections. +- Evaluated question counts and coverage; failed, skipped, timed-out, and excluded + items; the denominator used for the reported metric. +- Answer/judge model identifiers, prompts or prompt versions, retrieval settings, + exact commands, and credential-free configuration. +- Scoring and aggregation rules, repetition counts, and how multiple runs are + combined. Explain checkpoint reuse and any re-evaluation; do not combine retained + high scores with score-selected reruns into an unexplained aggregate. +- Per-question records sufficient to check coverage and recompute the aggregate, + along with summary files. Provide answer prompt-token totals and call counts + when submitting `Context Tokens`; otherwise use `-`. + +Link large artifacts in a public repository or release rather than adding them +to this repository's ignored `results/` directory. Share the records needed for +verification with credentials and private endpoints removed. Respect the +upstream dataset licenses listed in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). + +### Published Reference Results + +Add rows to the relevant sections of +[docs/user_memory/results.md](docs/user_memory/results.md), with a direct source +link. These are externally reported results and may use different data, +models, prompts, or budgets. State such differences next to the result or in +the linked methodology. For the LoCoMo four-category table, identify the +non-adversarial subset; report any full-set score separately. + +An adapter contribution and a new shared-baseline run are not prerequisites +for submitting published reference results. + +### Reproduced Results + +Use an available/reviewable OmniMemEval adapter and the shared pipeline, +dataset scope, prompts, metrics, and baseline answer/judge settings documented +in [Evaluation Setup](docs/user_memory/results.md#evaluation-setup). +PersonaMem v2 uses option matching rather than an LLM judge. Backend-specific +settings must be disclosed, including modes that generate answers directly. + +Contributor-run results are eligible for review when these conditions and the +artifact requirements are met. Identify who ran them. The review should record +whether verification covered artifact arithmetic, replay of answer/judge stages, +or a complete rerun; these establish different levels of evidence. + +## Review and merge + +Describe the problem, change, and validation in the PR. If a command was not +run, say so. Separate blocking issues from optional suggestions and reply to +review comments when updating the PR. New commits may require another review. + +CI checks offline tests, Python/Shell syntax, and relative Markdown file links. +It does not establish benchmark validity or run live integrations. Maintainers +review evidence and test results before merging; squash merging is the normal +choice for a focused PR. Follow-up fixes should use another branch and PR. diff --git a/README.md b/README.md index 314900f..7fafaa0 100644 --- a/README.md +++ b/README.md @@ -9,24 +9,6 @@ OmniMemEval is an evaluation framework for memory systems and memory-augmented a | User Memory Evaluation | Memory backend APIs exposed through `add()` and `search()` | LoCoMo, LongMemEval, BEAM, PersonaMem v2, HaluMem | [docs/user_memory/README.md](./docs/user_memory/README.md) | | Agent Memory Evaluation | Agent runtimes equipped with memory plugins | AgentBench domains: reasoning, information retrieval, knowledge work, code implementation, software engineering | [docs/agent_memory/README.md](./docs/agent_memory/README.md) | -## Evaluation Tracks - -### User Memory Evaluation - -User Memory Evaluation measures the capability of memory backend systems through a standardized API adapter layer. The same benchmark pipeline can be run against mainstream memory products, self-hosted memory frameworks, or custom adapters by selecting a backend with `--lib`. - -The pipeline covers ingestion, retrieval, answer generation, LLM-as-Judge evaluation, metric aggregation, and report generation. Results are written under `results//-/`. - -Start here: [docs/user_memory/README.md](./docs/user_memory/README.md) - -### Agent Memory Evaluation - -Agent Memory Evaluation measures the task performance of an agent runtime after a memory plugin is installed. The current implementation is based on AgentBench and evaluates OpenClaw across five task domains. The memory-plugin protocol includes memory cleanup, training, memory settling, backup, restore, and test execution. - -Results are written under `results/agentbench/`. - -Start here: [docs/agent_memory/README.md](./docs/agent_memory/README.md) - ## Installation User Memory Evaluation and Agent Memory Evaluation use separate environments. For User Memory Evaluation, create the base Python environment from the repository root: @@ -59,6 +41,13 @@ Each evaluation track may require additional dependencies: ```bash cp env_examples/.env.memos .env.memos +``` + +Fill in the memory backend credentials and `ANSWER_*` / `EVAL_*` model settings +in `.env.memos` before continuing. Then prepare data and run the full LoCoMo +evaluation (this calls the configured backend and model APIs): + +```bash python data/locomo/prepare_locomo.py ./scripts/run_locomo_eval.sh --lib memos --env .env.memos ``` @@ -67,14 +56,22 @@ python data/locomo/prepare_locomo.py ```bash cp env_examples/.env.agent .env.agent +``` + +Complete the OpenClaw setup and credentials in the +[Agent Memory guide](./docs/agent_memory/README.md), then run one reasoning +task as a smoke check: + +```bash mkdir -p data/agentbench -huggingface-cli download EverMind-AI/EvoAgentBench \ +hf download EverMind-AI/EvoAgentBench \ --repo-type dataset \ --local-dir ./data/agentbench ./scripts/run_agent_eval.sh \ --agent openclaw \ --domain reasoning \ --protocol test_only \ + --test-split 1 \ --version smoke_agentbench \ --trials 1 \ --parallel 1 @@ -109,8 +106,15 @@ scripts/ utils/ # Shared utilities requirements_user_memory.txt # User Memory Evaluation dependencies requirements_agentbench.txt # Agent Memory Evaluation dependencies +requirements-dev.txt # Offline tests and development checks ``` +## Contributing + +See [CONTRIBUTING.md](./CONTRIBUTING.md) for the branch/PR workflow, adapter +integration, and benchmark-result submission requirements. The +[test guide](./scripts/tests/README.md) documents offline checks and CI. + ## License See [LICENSE](./LICENSE). Third-party benchmark data keeps its upstream license; the OmniMemEval code license does not relicense external datasets. See [THIRD_PARTY_NOTICES.md](./THIRD_PARTY_NOTICES.md). diff --git a/README_zh.md b/README_zh.md index 07739c8..9433f27 100644 --- a/README_zh.md +++ b/README_zh.md @@ -9,24 +9,6 @@ OmniMemEval 是面向记忆系统和记忆增强 Agent 的评测框架，当前 | User Memory Evaluation | 通过 `add()` 和 `search()` 暴露能力的 memory backend API | LoCoMo、LongMemEval、BEAM、PersonaMem v2、HaluMem | [docs/user_memory/README_zh.md](./docs/user_memory/README_zh.md) | | Agent Memory Evaluation | 安装记忆插件后的 Agent Runtime | AgentBench domains：reasoning、information retrieval、knowledge work、code implementation、software engineering | [docs/agent_memory/README_zh.md](./docs/agent_memory/README_zh.md) | -## 评测线 - -### User Memory Evaluation - -User Memory Evaluation 通过统一 adapter 层评估 memory backend 系统能力。使用 `--lib` 可以在同一套 benchmark pipeline 下切换不同 memory 产品、自托管 memory framework 或自定义 adapter。 - -该流程覆盖写入、检索、答案生成、LLM-as-Judge 评估、指标聚合和报告生成。结果写入 `results//-/`。 - -入口文档：[docs/user_memory/README_zh.md](./docs/user_memory/README_zh.md) - -### Agent Memory Evaluation - -Agent Memory Evaluation 评估安装记忆插件后的 Agent Runtime 在任务域中的实际表现。当前实现基于 AgentBench，评估 OpenClaw 在五个任务域上的能力。记忆插件协议包含记忆清理、训练、沉淀、备份、恢复和测试执行。 - -结果写入 `results/agentbench/`。 - -入口文档：[docs/agent_memory/README_zh.md](./docs/agent_memory/README_zh.md) - ## 安装 User Memory Evaluation 和 Agent Memory Evaluation 使用两套独立环境。User Memory Evaluation 可在仓库根目录创建基础 Python 环境： @@ -59,6 +41,12 @@ OpenClaw、系统包和各 domain 的额外安装步骤详见 [docs/agent_memory ```bash cp env_examples/.env.memos .env.memos +``` + +先在 `.env.memos` 中填写后端凭证和 `ANSWER_*` / `EVAL_*` 模型配置，再准备数据并 +运行完整 LoCoMo 评测。以下命令会调用配置的后端和模型 API： + +```bash python data/locomo/prepare_locomo.py ./scripts/run_locomo_eval.sh --lib memos --env .env.memos ``` @@ -67,14 +55,21 @@ python data/locomo/prepare_locomo.py ```bash cp env_examples/.env.agent .env.agent +``` + +按 [Agent Memory 指南](./docs/agent_memory/README_zh.md) 完成 OpenClaw 安装和凭证 +配置后，使用一个 reasoning 任务进行 smoke 检查： + +```bash mkdir -p data/agentbench -huggingface-cli download EverMind-AI/EvoAgentBench \ +hf download EverMind-AI/EvoAgentBench \ --repo-type dataset \ --local-dir ./data/agentbench ./scripts/run_agent_eval.sh \ --agent openclaw \ --domain reasoning \ --protocol test_only \ + --test-split 1 \ --version smoke_agentbench \ --trials 1 \ --parallel 1 @@ -109,8 +104,15 @@ scripts/ utils/ # 通用工具 requirements_user_memory.txt # User Memory Evaluation 依赖 requirements_agentbench.txt # Agent Memory Evaluation 依赖 +requirements-dev.txt # 离线测试和开发检查依赖 ``` +## 参与贡献 + +分支和 PR 流程、adapter 接入及评测结果提交要求见 +[贡献指南（英文）](./CONTRIBUTING.md)。离线检查和 CI 用法见 +[测试指南（英文）](./scripts/tests/README.md)。 + ## License 见 [LICENSE](./LICENSE)。第三方 benchmark 数据保留其上游许可证；OmniMemEval 代码许可证不重新授权外部数据集。详见 [THIRD_PARTY_NOTICES.md](./THIRD_PARTY_NOTICES.md)。 diff --git a/docs/README.md b/docs/README.md index 2e4452e..caef676 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,4 +20,5 @@ Related documentation remains next to the files it describes: [env_examples/README.md](../env_examples/README.md) and [env_examples/PARAMETERS.md](../env_examples/PARAMETERS.md) - Test commands: [scripts/tests/README.md](../scripts/tests/README.md) +- Contribution and result submission rules: [CONTRIBUTING.md](../CONTRIBUTING.md) - Third-party attribution: [THIRD_PARTY_NOTICES.md](../THIRD_PARTY_NOTICES.md) diff --git a/docs/README_zh.md b/docs/README_zh.md index bdcdf0c..4ea4a0d 100644 --- a/docs/README_zh.md +++ b/docs/README_zh.md @@ -19,4 +19,5 @@ OmniMemEval 包含两条互补的评测线。请根据待评测对象选择对 [env_examples/README.md](../env_examples/README.md) 和 [env_examples/PARAMETERS.md](../env_examples/PARAMETERS.md) - 测试命令：[scripts/tests/README.md](../scripts/tests/README.md) +- 开发与结果提交规范：[CONTRIBUTING.md（英文）](../CONTRIBUTING.md) - 第三方归属：[THIRD_PARTY_NOTICES.md](../THIRD_PARTY_NOTICES.md) diff --git a/docs/agent_memory/README.md b/docs/agent_memory/README.md index d3eacb5..3b4a142 100644 --- a/docs/agent_memory/README.md +++ b/docs/agent_memory/README.md @@ -119,7 +119,7 @@ Run the following commands from the repository root: ```bash mkdir -p data/agentbench -huggingface-cli download EverMind-AI/EvoAgentBench \ +hf download EverMind-AI/EvoAgentBench \ --repo-type dataset \ --local-dir ./data/agentbench ``` @@ -135,6 +135,8 @@ data/agentbench/ swebench/ ``` +The `hf` CLI is provided by `huggingface_hub`; see the +[upstream CLI guide](https://huggingface.co/docs/huggingface_hub/en/guides/cli). Domain data paths are configured under `configs/agentbench/domains/*.yaml`. If a custom data location is used, update the relevant domain yaml fields. LiveCodeBench requires its official verifier package: @@ -164,19 +166,18 @@ systemctl start docker ## Quick Start -Reasoning smoke run: +After completing environment, credentials, and data preparation above, run one +reasoning task. `--test-split 1` selects the first test question for this domain; +omit it to evaluate the complete test split. `--parallel` controls concurrency, +not the number of selected tasks. ```bash -cp env_examples/.env.agent .env.agent -mkdir -p data/agentbench -huggingface-cli download EverMind-AI/EvoAgentBench \ - --repo-type dataset \ - --local-dir ./data/agentbench ./scripts/run_agent_eval.sh \ --agent openclaw \ --domain reasoning \ --protocol test_only \ --version smoke_agentbench \ + --test-split 1 \ --trials 1 \ --parallel 1 ``` diff --git a/docs/agent_memory/README_zh.md b/docs/agent_memory/README_zh.md index 9228156..09d7b70 100644 --- a/docs/agent_memory/README_zh.md +++ b/docs/agent_memory/README_zh.md @@ -38,7 +38,7 @@ AgentBench 使用 EverMind（Hugging Face/GitHub namespace: `EverMind-AI`）发 ```bash mkdir -p data/agentbench -huggingface-cli download EverMind-AI/EvoAgentBench \ +hf download EverMind-AI/EvoAgentBench \ --repo-type dataset \ --local-dir ./data/agentbench ``` @@ -54,6 +54,8 @@ data/agentbench/ swebench/ ``` +`hf` CLI 随 `huggingface_hub` 提供，详见 +[上游 CLI 指南](https://huggingface.co/docs/huggingface_hub/en/guides/cli)。 各域数据路径由 `configs/agentbench/domains/*.yaml` 管理。若采用自定义数据目录，请同步更新相应 domain yaml 中的 `data_path`、`train_file`、`test_file`、`repo_root` 或同类字段。 LiveCodeBench 域依赖官方 verifier 包： @@ -206,7 +208,9 @@ results/agentbench/ ### `test_only` -执行 test split，适用于 baseline smoke test 或单域功能验证。 +以下命令执行完整 test split。需要单任务 reasoning smoke 检查时，追加 +`--test-split 1` 以选择第一道测试题；`--parallel` 只控制并发，不限制题数。 +运行前完成上文的环境、凭证和数据准备。 ```bash ./scripts/run_agent_eval.sh \ diff --git a/docs/user_memory/README.md b/docs/user_memory/README.md index 3715838..c502fec 100644 --- a/docs/user_memory/README.md +++ b/docs/user_memory/README.md @@ -142,19 +142,9 @@ files for upstream dataset licenses and redistribution notes. ./scripts/run_halumem_eval.sh --lib memos --env .env.memos ``` -Useful shared options: - -| Option | Purpose | -|--------|---------| -| `--version ` | Result directory suffix. Defaults to `omnimemeval_`. | -| `--from-step N` / `--to-step N` | Run a subset of pipeline steps. | -| `--replay ` | Recompute later stages from an existing result directory. | -| `--top-k N` | Search result count. Overrides `TOPK` from the env file. | -| `--llm-workers N` | Concurrent answer/eval LLM workers. | -| `--allow-empty-search 1` | Allow successful runs with no raw memory returned. | -| `--skip-failed-search 1` | Mark failed search items as skipped instead of failing the step. | -| `--skip-failed-answer 1` | Mark failed answer items as skipped instead of failing the step. | -| `--skip-failed-judge 1` | Mark failed judge items as skipped instead of failing the step. | +Use `--help` on a runner for its accepted options. Shared defaults and +benchmark-specific applicability are maintained in the +[parameter reference](../../env_examples/PARAMETERS.md). Streaming mode is available for LongMemEval, BEAM, PersonaMem v2, and HaluMem. In streaming mode, OmniMemEval runs add, search, save, and delete for @@ -171,7 +161,8 @@ corresponding runner: Streaming runs support `--start-idx`, `--end-idx`, `--restart-unit`, `--no-resume`, and `--skip-failed-streaming`. -Minimal smoke commands: +Ingestion/search checks (these call the backend; only the LongMemEval example +below limits the dataset to one unit): ```bash # LoCoMo: run ingestion and search only @@ -235,6 +226,8 @@ without changing the benchmark stages, prompt flow, or metric calculation. | `backboard` | Backboard.io | | `mem9` | mem9 | +To contribute a backend, follow the [adapter checklist](../../CONTRIBUTING.md#adding-a-user-memory-adapter). + ## Benchmarks diff --git a/docs/user_memory/README_zh.md b/docs/user_memory/README_zh.md index 9bb5ca3..c6042c5 100644 --- a/docs/user_memory/README_zh.md +++ b/docs/user_memory/README_zh.md @@ -133,19 +133,8 @@ Benchmark 数据按需下载，不提交到仓库。上游数据许可证和再 ./scripts/run_halumem_eval.sh --lib memos --env .env.memos ``` -常用参数： - -| 参数 | 用途 | -|------|------| -| `--version ` | 结果目录后缀，默认 `omnimemeval_`。 | -| `--from-step N` / `--to-step N` | 只运行部分阶段。 | -| `--replay ` | 从已有结果目录重算后续阶段。 | -| `--top-k N` | 检索数量，覆盖 env 中的 `TOPK`。 | -| `--llm-workers N` | Answer/Eval LLM 并发数。 | -| `--allow-empty-search 1` | 允许 raw memory 为空的 search 结果通过。 | -| `--skip-failed-search 1` | search 失败时标记 skipped，而不是失败退出。 | -| `--skip-failed-answer 1` | answer 失败时标记 skipped，而不是失败退出。 | -| `--skip-failed-judge 1` | judge 失败时标记 skipped，而不是失败退出。 | +使用 runner 的 `--help` 查看接受的选项。共享参数默认值和各 benchmark 的适用范围 +统一维护在 [参数说明](../../env_examples/PARAMETERS.md) 中。 LongMemEval、BEAM、PersonaMem v2 和 HaluMem 支持 streaming 模式。在 streaming 模式下，OmniMemEval 会对每个 benchmark unit 依次执行 add、search、 @@ -161,7 +150,7 @@ streaming 模式下，OmniMemEval 会对每个 benchmark unit 依次执行 add Streaming 模式可配合 `--start-idx`、`--end-idx`、`--restart-unit`、 `--no-resume` 和 `--skip-failed-streaming` 使用。 -最小 smoke 命令： +Ingestion/search 检查（会调用后端；以下只有 LongMemEval 示例限制为一个数据单元）： ```bash # LoCoMo：只跑 ingestion 和 search @@ -223,6 +212,8 @@ memory 方案运行同一套评测。 | `backboard` | Backboard.io | | `mem9` | mem9 | +新增后端时，请遵循 [adapter 接入清单（英文）](../../CONTRIBUTING.md#adding-a-user-memory-adapter)。 + ## Benchmarks diff --git a/docs/user_memory/results.md b/docs/user_memory/results.md index 0c710c4..02870e0 100644 --- a/docs/user_memory/results.md +++ b/docs/user_memory/results.md @@ -307,3 +307,9 @@ results/halumem/{LIB}-{VERSION}/ Benchmark datasets are downloaded on demand and are not committed to this repository. See [THIRD_PARTY_NOTICES.md](../../THIRD_PARTY_NOTICES.md) for dataset license information. + +## Contributing Results + +See the [result submission requirements](../../CONTRIBUTING.md#submitting-benchmark-results) +for required evidence, aggregation details, and the distinction between +Published Reference Results and Reproduced Results. diff --git a/env_examples/PARAMETERS.md b/env_examples/PARAMETERS.md index 7062c0c..2b80c19 100644 --- a/env_examples/PARAMETERS.md +++ b/env_examples/PARAMETERS.md @@ -20,7 +20,7 @@ Unless a product-specific table is present below, the corresponding | `--workers` | `2` | ingestion/search | Memory API worker count. | | `--top-k` | `TOPK` or `20` | search | Number of search results requested from the memory backend. | | `--llm-workers` | `LLM_WORKERS` or `10` | answer/eval | Concurrent LLM workers. | -| `--num-runs` | `1` | BEAM, PersonaMem v2, HaluMem | Number of answer or judge repetitions, depending on benchmark. | +| `--num-runs` | `1` | all five User Memory benchmarks | Number of answer repetitions for PersonaMem v2, or judge repetitions for LoCoMo, LongMemEval, BEAM, and HaluMem. | | `--save-model-input` | `0` | answer | Save response-stage model input for inspection. | | `--allow-empty-search` | `1` | search | Allow successful records when no raw memories are returned. | | `--skip-failed-search` | `0` | search | Mark failed search records as skipped instead of failing the step. | @@ -92,7 +92,7 @@ inspect target ids. | Benchmark | Data path | Notes | |-----------|-----------|-------| -| LoCoMo | `data/locomo/locomo10.json` | Included with upstream license notes. | +| LoCoMo | `data/locomo/locomo10.json` | Download with `python data/locomo/prepare_locomo.py`; see its dataset README for license notes. | | LongMemEval | `data/longmemeval/longmemeval_s_cleaned.json` | Download with `python data/longmemeval/prepare_longmemeval.py`. | | BEAM | `data/beam/beam_100k.json` | Default 100K scale; prepare script can download additional scales. | | PersonaMem v2 | `data/personamem_v2/benchmark/text/benchmark.csv` and `data/personamem_v2/data/chat_history_32k/` | Download with `python data/personamem_v2/prepare_personamem.py`. | diff --git a/scripts/client_factory/registry.py b/scripts/client_factory/registry.py index c8b9f16..08c3864 100644 --- a/scripts/client_factory/registry.py +++ b/scripts/client_factory/registry.py @@ -1,11 +1,13 @@ """Central registry of supported memory framework libraries. -Add new libraries here — all evaluation scripts import SUPPORTED_LIBS -from this module, so a single change propagates everywhere. +All evaluation scripts import SUPPORTED_LIBS from this module. Registration +exposes the client key; search, ingestion, and cleanup integration must also +match the backend's behavior. When adding a new lib: 1. Add a Client class in scripts/client_factory/_client.py 2. Register it in _LIB_CLIENT_REGISTRY below (module_name, class_name) + 3. Follow CONTRIBUTING.md for dispatch, configuration, docs, and tests """ from importlib import import_module @@ -62,4 +64,4 @@ def create_client(lib_name: str): don't need their own if/elif chains for client creation. """ cls = _load_client_class(lib_name) - return cls() \ No newline at end of file + return cls()