第 7 周：同一批 Bug 报告，Inspect 和 promptfoo 打的分一样吗？

def check_schema(output: str) -> tuple[bool, str]:
    report = parse_report(output)
    if report is None:
        return False, "输出不是 JSON 对象"
    try:
        jsonschema.validate(report, SCHEMA)        # data/report_schema.json
    except jsonschema.ValidationError as e:
        where = "/".join(str(p) for p in e.absolute_path) or "(根)"
        return False, f"schema 不符：{where}: {e.message}"
    return True, "格式合格"


def check_hit(output: str, target: str, keywords: list[str]) -> tuple[bool, str]:
    report = parse_report(output)
    if report is None:
        return False, "输出不是 JSON 对象，无法判定"
    verdict = report.get("verdict")
    if target == NONE:
        if verdict == "no_bug":
            return True, "该模块没有注入 Bug，Agent 也没报"
        return False, f"误报：该模块没有注入 Bug，Agent 报了「{report.get('title', '')}」"
    if verdict != "bug":
        return False, f"漏报：应发现 {target}，Agent 报告 no_bug"
    steps = report.get("steps") or []
    text = " ".join([str(report.get("title", "")), *map(str, steps), str(report.get("actual", ""))])
    missing = [k for k in keywords if k not in text]
    if missing:
        return False, f"没对上 {target}：缺关键词 {missing}"
    return True, f"命中 {target}"

PROMPT = ("任务 {id}：你是测试 Agent。请探索被测应用的 {module} 模块，按 JSON 提交一份 Bug 报告；"
          "没发现 Bug 就提交 {{\"verdict\": \"no_bug\"}}。")
TASK_ID = re.compile(r"任务 (s\d+)：")


def bughunt_dataset() -> list[Sample]:
    return [
        Sample(id=s["id"], input=PROMPT.format(**s), target=s["target"],
               metadata={"module": s["module"], "keywords": s["keywords"]})
        for s in grading.load_samples()
    ]


def replay_model(recording: str):
    """mockllm 的 custom_outputs 可以是一个函数：每次 generate 都用 (input, tools, tool_choice, config) 调它。"""
    reports = grading.load_recording(recording)

    def respond(input, tools, tool_choice, config) -> ModelOutput:
        sample_id = TASK_ID.search(input[-1].text).group(1)
        return ModelOutput.from_content(model=recording, content=reports[sample_id])

    return get_model(f"mockllm/{recording}", custom_outputs=respond)


@scorer(metrics=[accuracy(), stderr()])
def report_schema():
    async def score(state: TaskState, target: Target) -> Score:
        ok, why = grading.check_schema(state.output.completion)
        return Score(value=CORRECT if ok else INCORRECT, explanation=why)
    return score


@scorer(metrics=[accuracy(), stderr()])
def hits_injected_bug():
    async def score(state: TaskState, target: Target) -> Score:
        ok, why = grading.check_hit(state.output.completion, target.text, state.metadata["keywords"])
        return Score(value=CORRECT if ok else INCORRECT, explanation=why)
    return score


@task
def bughunt_reports():
    return Task(
        dataset=bughunt_dataset(),
        solver=generate(),
        scorer=[report_schema(), hits_injected_bug()],
    )

== mockllm/agent_v1  status=success  samples=10
   report_schema      accuracy=0.800  stderr=0.1333
   hits_injected_bug  accuracy=0.600  stderr=0.1633
   s01  schema=C  hit=C  命中 BUG-001
   s02  schema=C  hit=I  没对上 BUG-002：缺关键词 ['叠加']
   s03  schema=I  hit=C  命中 BUG-003
   s04  schema=C  hit=C  命中 BUG-004
   s05  schema=C  hit=I  漏报：应发现 BUG-005，Agent 报告 no_bug
   s06  schema=C  hit=C  命中 BUG-006
   s07  schema=I  hit=I  输出不是 JSON 对象，无法判定
   s08  schema=C  hit=C  命中 BUG-008
   s09  schema=C  hit=I  误报：该模块没有注入 Bug，Agent 报了「帮助页加载较慢」
   s10  schema=C  hit=C  该模块没有注入 Bug，Agent 也没报
== mockllm/agent_v2  status=success  samples=10
   report_schema      accuracy=0.900  stderr=0.1000
   hits_injected_bug  accuracy=0.800  stderr=0.1333
   s01  schema=C  hit=C  命中 BUG-001
   s02  schema=C  hit=C  命中 BUG-002
   s03  schema=C  hit=C  命中 BUG-003
   s04  schema=C  hit=C  命中 BUG-004
   s05  schema=C  hit=C  命中 BUG-005
   s06  schema=I  hit=C  命中 BUG-006
   s07  schema=C  hit=C  命中 BUG-007
   s08  schema=C  hit=I  漏报：应发现 BUG-008，Agent 报告 no_bug
   s09  schema=C  hit=C  该模块没有注入 Bug，Agent 也没报
   s10  schema=C  hit=I  误报：该模块没有注入 Bug，Agent 报了「关于页版本号与发布说明不一致」
汇总写到 logs/inspect_summary.json

$ inspect eval inspect_bughunt.py@bughunt_reports --model mockllm/model --display plain
...
bughunt_reports (10 samples): mockllm/model
total time:                    0:00:00
mockllm/model                  810 tokens [I: 480, O: 330]
report_schema         hits_injected_bug
accuracy       0.000  accuracy           0.000
stderr         0.000  stderr             0.000

prompts:
  - '任务 {{id}}：你是测试 Agent。请探索被测应用的 {{module}} 模块，按 JSON 提交一份 Bug 报告；没发现 Bug 就提交 {"verdict": "no_bug"}。'

providers:
  - id: file://promptfoo_provider.py
    label: agent_v1
    config:
      recording: agent_v1
  - id: file://promptfoo_provider.py
    label: agent_v2
    config:
      recording: agent_v2

defaultTest:
  options:
    # keywords 是数组。不关掉的话，promptfoo 会把数组型 vars 展开成多条用例（每个关键词一条）
    disableVarExpansion: true
  assert:
    - type: is-json
      value: file://data/report_schema.json
      metric: report_schema
    - type: python
      value: file://promptfoo_assert.py
      metric: hits_injected_bug

tests: file://promptfoo_tests.py:generate_tests

# promptfoo_provider.py：每个 (prompt, provider, test) 组合调一次
def call_api(prompt, options, context):
    recording = options["config"]["recording"]
    sample_id = context["vars"]["id"]
    reports = grading.load_recording(recording)
    if sample_id not in reports:
        return {"error": f"{recording} 里没有任务 {sample_id} 的录制"}
    return {"output": reports[sample_id]}


# promptfoo_assert.py：和 Inspect 的 hits_injected_bug 用同一条规则
def get_assert(output, context):
    v = context["vars"]
    ok, why = grading.check_hit(output, v["target"], v["keywords"])
    return {"pass": ok, "score": 1.0 if ok else 0.0, "reason": why}


# promptfoo_tests.py：从 data/samples.jsonl 读，和 Inspect 同一份数据
def generate_tests(config=None):
    return [
        {"description": f"{s['id']} {s['module']} → {s['target']}",
         "vars": {"id": s["id"], "module": s["module"], "target": s["target"], "keywords": s["keywords"]}}
        for s in grading.load_samples()
    ]

Cache is disabled.
Starting evaluation eval-X07-2026-10-01T11:13:40
Running 20 test cases (up to 4 at a time)...
✓ Eval complete (ID: eval-X07-2026-10-01T11:13:40)

» View results: promptfoo view
» Share with your team: https://promptfoo.app
» Feedback: https://promptfoo.dev/feedback


Results:
  ✓ 12 passed (60.00%)
  ✗ 8 failed (40.00%)
  0 errors (0%)
Duration: 1s (concurrency: 4)

Writing output to logs/promptfoo_results.json

Results:
  ✓ 23 passed (63.89%)
  ✗ 13 failed (36.11%)
  0 errors (0%)

逐条判定：Inspect 40 个，promptfoo 40 个，不一致 0 个

口径对照（Inspect | promptfoo）：
  agent_v1  0.800 | 0.800  report_schema 通过率
  agent_v1  0.600 | 0.600  hits_injected_bug 通过率
  agent_v1  0.500 | 0.500  两条都过（promptfoo 的 pass）
  agent_v1    —   | 7.000  score 合计（promptfoo，按断言均分）
  agent_v2  0.900 | 0.900  report_schema 通过率
  agent_v2  0.800 | 0.800  hits_injected_bug 通过率
  agent_v2  0.700 | 0.700  两条都过（promptfoo 的 pass）
  agent_v2    —   | 8.500  score 合计（promptfoo，按断言均分）

hit_weight0             agent_v1  用例 10  pass  8  score 8.0000
hit_weight0             agent_v2  用例 10  pass  9  score 9.0000
threshold05             agent_v1  用例 10  pass  9  score 7.0000
threshold05             agent_v2  用例 10  pass 10  score 8.5000
hit_weight2_threshold05 agent_v1  用例 10  pass  6  score 6.6667
hit_weight2_threshold05 agent_v2  用例 10  pass  8  score 8.3333
var_expansion_on        agent_v1  用例 18  pass 10  score 13.0000
var_expansion_on        agent_v2  用例 18  pass 13  score 15.5000

.......                                                                  [100%]
7 passed in 2.29s