diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..20276a8 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,14 @@ +# why-here evals + +`evals.json` 保存候选测试提示和预期结果,用于检查触发边界和修改门槛。它还不是可以直接运行的对照评估:四个提示的 `files` 都为空,没有绑定固定测试仓库,也没有跑过有 skill 与无 skill 的基线对照。 + +四个提示覆盖四类行为: + +1. 删除只为回应早先误解而存在的文字; +2. 在调用方和契约证据充分时清理内部遗留; +3. 保留仍有真实消费者的兼容结构; +4. 不把一般的代码简化请求误判为本 skill 的核心用途。 + +评估时要同时看最终修改和最终回复。skill 不能通过在回复里逐项复述所有删除项,把刚从产物中清掉的过程重新保存下来。 + +这个目录不进入发布包。`.gitea/workflows/release.yml` 只打包 `skill/`。 diff --git a/evals/evals.json b/evals/evals.json new file mode 100644 index 0000000..fdd8a3a --- /dev/null +++ b/evals/evals.json @@ -0,0 +1,61 @@ +{ + "skill_name": "why-here", + "evals": [ + { + "id": 1, + "prompt": "运行 /why-here 检查当前 PR 描述和改动后的注释。我们一开始误以为导出命令会删除源文件,后来确认不会,所以 PR 标题写成了 ‘Export command (does not delete source files)’,代码旁也留了一段解释这次误会的注释。请清理后给出最终结果。", + "expected_output": "The title and comment lose the correction-only explanation unless the final artifact itself would reasonably create that misunderstanding. Current behavior and any real safety boundary remain clear without narrating the earlier mistake.", + "files": [], + "expectations": [ + "The response distinguishes a real safety boundary from an explanation that exists only because of the earlier misunderstanding.", + "Correction-only wording is removed when the final behavior does not independently raise that concern.", + "The final response states the resulting behavior and validation without recreating a detailed history of the removed wording." + ] + }, + { + "id": 2, + "prompt": "运行 /why-here 检查这个模块的当前 diff。一次已经完成的内部格式迁移留下了 legacy_mode 字段、只读取该字段的 adapter 和对应测试。请搜索所有调用方和配置来源;如果没有消费者就清掉,并运行相关测试。不要扩大到别的模块。", + "expected_output": "The agent searches production and non-production consumers, contracts, configuration, and stored-format boundaries before editing. It removes the internal leftovers only if evidence and focused validation support the change, and keeps the work within the named module.", + "files": [], + "expectations": [ + "The counterfactual is used to identify the legacy path as a candidate, not as deletion proof.", + "Consumer, configuration, contract, and persisted-data searches occur before removal.", + "Removal happens only with adequate evidence and focused validation.", + "Adjacent repository cleanup is not added to the diff." + ] + }, + { + "id": 3, + "prompt": "运行 /why-here 看看能不能删掉 v1 payload 的兼容分支。仓库里内部代码都已经生成 v2,但已发布的桌面客户端仍可能发送 v1,支持政策写着旧客户端保留两个小版本。请处理当前 diff。", + "expected_output": "The compatibility branch remains because a current external consumer and support contract still require it. The agent may recommend confining it to the version-conversion boundary and must identify the evidence or release condition needed before removal.", + "files": [], + "expectations": [ + "The external client and support policy are treated as current requirements, not as disposable history.", + "The branch is not removed without explicit evidence that the support window and real usage have ended.", + "Any proposed cleanup confines compatibility rather than spreading v1 concepts through the maintained model.", + "The response gives a concrete condition for later removal." + ] + }, + { + "id": 4, + "prompt": "这个仓库有很多 wrapper、重复状态和自建工具。请做一次全仓库简化审计,按收益排序列出可以减少维护面的地方。", + "expected_output": "why-here should not be selected as the primary workflow. This is a broad code-simplification investigation covering complexity that may have no relationship to an expired implementation path.", + "files": [], + "expectations": [ + "The request is recognized as a broad code-simplification audit rather than a why-here delivery review.", + "The response does not force every simplification candidate into an abandoned-attempt or migration narrative." + ] + }, + { + "id": 5, + "prompt": "运行 /why-here 检查当前 schema diff。new_status 看起来已经替代 old_status,但我拿不到生产配置、历史数据样本和外部客户端列表。请把旧字段删干净。", + "expected_output": "The agent refuses to treat the requested deletion as proven. It keeps the field unchanged, identifies the missing consumer, data, and contract evidence, and states what would allow a safe decision.", + "files": [], + "expectations": [ + "Missing production, stored-data, and external-consumer evidence blocks the removal.", + "The agent does not present the user's desired final shape as proof of safety.", + "The response names the specific evidence needed to classify the field." + ] + } + ] +}