Leroy
d3e9bd45bd
- SKILL.md 双工作流（创建/迭代）+ 两道门（验料/通用化）+ 三档位 - 7 篇方法论 references + 3 个纯标准库脚本（scaffold/validate/trigger_eval） - 自身触发用例 16 条 + 先例研究/创建/迭代证据报告 - 方法论吸收自 qiaomu-meta-skill 与 luban-skill，取舍见 reports/prior-art.md
238 lines
10 KiB
Python
238 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""LeMetaSkill 触发冒烟测试：词面检查 description 是否覆盖用户会说的原话。
|
||
|
||
这是**词面信号检查**，不是行为判断——它快速暴露"description 里没写用户
|
||
会用的词"这类硬伤；真正的触发行为要靠真实对话实测（见 references/trigger-eval.md）。
|
||
|
||
做法：
|
||
1. 解析 SKILL.md 的 description，与用例文件 positive_concepts 的概念族词表合并；
|
||
2. 对每条测试原话做**概念族匹配**：负触发词命中直接判不触发；否则需命中全部
|
||
required_families 且总命中族数 >= families_required（无概念族配置时退化为覆盖率阈值）；
|
||
3. 与期望比对，报告不一致条目；结果连同词面覆盖率写入 reports/trigger-eval.json。
|
||
|
||
用法: python3 trigger_eval.py <skill目录> [--cases 路径] [--output 路径] [--threshold 0.3]
|
||
退出码: 0=全部一致 1=有不一致或 FAIL 2=缺少用例文件
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
from datetime import date
|
||
from pathlib import Path
|
||
|
||
CJK = r"\u4e00-\u9fff"
|
||
LATIN_WORD = re.compile(r"[a-z0-9]+")
|
||
CJK_RUN = re.compile(rf"[{CJK}]+")
|
||
|
||
# 中文虚词/口语助词：不承重的触发信号，剔除后再匹配
|
||
STOP_BIGRAMS = {
|
||
"我们", "你们", "帮我", "帮忙", "一下", "这个", "那个", "一个", "一些",
|
||
"然后", "可能", "可以", "应该", "就是", "还是", "什么", "怎么", "如果",
|
||
"但是", "所以", "觉得", "感觉", "一下", "每次", "先不", "不用", "不要",
|
||
"没有", "已经", "正在", "出来", "上去", "过去", "东西", "事情", "麻烦",
|
||
}
|
||
STOP_SINGLE = set("的了吧呢啊呀嘛么是有在和与我你他她它就把被这那也给从对上下里中要去会来能多很太都还再又就才只")
|
||
|
||
NEG_HINT = re.compile(r"不要|不该|不用于|排除|exclude|not for|do not use|don'?t use", re.I)
|
||
|
||
|
||
def normalize(text: str) -> str:
|
||
return re.sub(r"\s+", " ", text.lower().strip())
|
||
|
||
|
||
def tokenize(text: str) -> set[str]:
|
||
"""中文 → 滑动二元组（剔除虚词）；英文/数字 → 单词。"""
|
||
tokens: set[str] = set()
|
||
for run in CJK_RUN.findall(text):
|
||
if len(run) == 1:
|
||
if run not in STOP_SINGLE:
|
||
tokens.add(run)
|
||
continue
|
||
for i in range(len(run) - 1):
|
||
bg = run[i : i + 2]
|
||
if bg not in STOP_BIGRAMS:
|
||
tokens.add(bg)
|
||
for w in LATIN_WORD.findall(text):
|
||
if len(w) > 1:
|
||
tokens.add(w)
|
||
return tokens
|
||
|
||
|
||
def parse_description(skill_md: Path) -> tuple[str, dict[str, str]]:
|
||
text = skill_md.read_text(encoding="utf-8")
|
||
lines = text.splitlines()
|
||
fields: dict[str, str] = {}
|
||
if lines and lines[0].strip() == "---":
|
||
key = ""
|
||
for line in lines[1:60]:
|
||
if line.strip() == "---":
|
||
break
|
||
m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
|
||
if m:
|
||
key = m.group(1)
|
||
fields[key] = m.group(2).strip().strip("'\"")
|
||
elif line.startswith((" ", "\t")) and key:
|
||
fields[key] = (fields[key] + " " + line.strip()).strip()
|
||
return fields.get("description", ""), fields
|
||
|
||
|
||
def coverage(case_text: str, desc_tokens: set[str]) -> tuple[float, list[str]]:
|
||
case_tokens = tokenize(normalize(case_text))
|
||
if not case_tokens:
|
||
return 0.0, []
|
||
hits = sorted(t for t in case_tokens if t in desc_tokens)
|
||
return len(hits) / len(case_tokens), hits
|
||
|
||
|
||
def phrase_in(phrase_norm: str, text_norm: str) -> bool:
|
||
"""短语命中：含中文按子串，纯拉丁按整词边界。"""
|
||
if re.search(rf"[{CJK}]", phrase_norm):
|
||
return phrase_norm in text_norm
|
||
return re.search(rf"\b{re.escape(phrase_norm)}\b", text_norm) is not None
|
||
|
||
|
||
def family_hits(text_norm: str, concepts: dict[str, list[str]]) -> dict[str, str]:
|
||
"""返回 {命中的族: 命中的短语}。"""
|
||
hits: dict[str, str] = {}
|
||
for family, phrases in concepts.items():
|
||
for p in phrases:
|
||
pn = normalize(p)
|
||
if pn and phrase_in(pn, text_norm):
|
||
hits[family] = p
|
||
break
|
||
return hits
|
||
|
||
|
||
def load_cases(path: Path) -> dict:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def as_items(raw) -> list[dict]:
|
||
items = []
|
||
for entry in raw or []:
|
||
if isinstance(entry, str):
|
||
items.append({"text": entry, "family": "default"})
|
||
elif isinstance(entry, dict):
|
||
it = dict(entry)
|
||
it.setdefault("family", "default")
|
||
items.append(it)
|
||
return items
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser(description="触发冒烟测试")
|
||
ap.add_argument("skill_dir", type=Path)
|
||
ap.add_argument("--cases", type=Path, default=None)
|
||
ap.add_argument("--output", type=Path, default=None)
|
||
ap.add_argument("--threshold", type=float, default=None)
|
||
args = ap.parse_args()
|
||
|
||
root = args.skill_dir.resolve()
|
||
skill_md = root / "SKILL.md"
|
||
if not skill_md.is_file():
|
||
print(f"FAIL SKILL.md 不存在: {skill_md}")
|
||
return 1
|
||
|
||
cases_path = (args.cases or root / "evals" / "trigger_cases.json").resolve()
|
||
if not cases_path.is_file():
|
||
cases_path.parent.mkdir(parents=True, exist_ok=True)
|
||
starter = {
|
||
"_说明": "填入用户真实会说的话：应触发/不应触发/边界各若干条。family 标注场景族，便于回归分析。",
|
||
"recommended_threshold": 0.3,
|
||
"negative_patterns": ["不要做成 skill", "只要总结", "翻译一下"],
|
||
"should_trigger": [
|
||
{"text": "【TODO：用户原话】", "family": "cn_workflow"},
|
||
],
|
||
"should_not_trigger": [
|
||
{"text": "【TODO：相邻但不该触发的原话】", "family": "near_task"},
|
||
],
|
||
"near_neighbor": [],
|
||
}
|
||
cases_path.write_text(json.dumps(starter, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print(f"未找到用例文件，已生成模板: {cases_path}")
|
||
print("请填入真实用例后重跑。")
|
||
return 2
|
||
|
||
cases = load_cases(cases_path)
|
||
desc, _fields = parse_description(skill_md)
|
||
if not desc:
|
||
print("FAIL description 为空，触发信号不存在")
|
||
return 1
|
||
if not NEG_HINT.search(desc):
|
||
print("WARN description 缺负触发表述，不应触发的判定仅靠用例负词兜底")
|
||
|
||
threshold = args.threshold if args.threshold is not None else float(cases.get("recommended_threshold", 0.3))
|
||
desc_tokens = tokenize(normalize(desc))
|
||
# positive_concepts: {族: [词]}，视作 description 的延伸词面
|
||
concepts = cases.get("positive_concepts")
|
||
if isinstance(concepts, dict):
|
||
for words in concepts.values():
|
||
for w in words:
|
||
desc_tokens |= tokenize(normalize(w))
|
||
|
||
neg_patterns = [normalize(p) for p in cases.get("negative_patterns", [])]
|
||
concepts = cases.get("positive_concepts") if isinstance(cases.get("positive_concepts"), dict) else {}
|
||
required_families = set(cases.get("required_families", []))
|
||
families_required = int(cases.get("families_required", 2))
|
||
use_families = bool(concepts)
|
||
report: dict = {
|
||
"skill": root.name,
|
||
"date": str(date.today()),
|
||
"threshold": threshold,
|
||
"prediction_mode": "concept_families" if use_families else "coverage_fallback",
|
||
"note": "词面冒烟测试；行为结论需真实对话实测（live_turns 字段手工补记）",
|
||
"cases": [],
|
||
}
|
||
|
||
n_mismatch = n_total = 0
|
||
for bucket, expected in (("should_trigger", True), ("should_not_trigger", False), ("near_neighbor", None)):
|
||
for item in as_items(cases.get(bucket)):
|
||
text = normalize(item["text"])
|
||
hit_neg = any(p and p in text for p in neg_patterns)
|
||
cov, cov_hits = coverage(item["text"], desc_tokens)
|
||
fams = family_hits(text, concepts)
|
||
if use_families:
|
||
predicted = (not hit_neg) and required_families <= set(fams) and len(fams) >= families_required
|
||
basis = f"fams={','.join(sorted(fams)) or '-'}"
|
||
else:
|
||
predicted = (not hit_neg) and cov >= threshold
|
||
basis = f"cov={cov:.2f}"
|
||
if expected is None:
|
||
verdict = "boundary"
|
||
entry_verdict = "边界条目（不计分）"
|
||
else:
|
||
ok_ = predicted == expected
|
||
n_total += 1
|
||
n_mismatch += 0 if ok_ else 1
|
||
verdict = "pass" if ok_ else "MISMATCH"
|
||
entry_verdict = ("一致" if ok_ else "不一致") + f"（预测={'触发' if predicted else '不触发'}，期望={'触发' if expected else '不触发'}）"
|
||
flag = "" if verdict in ("pass", "boundary") else " <-- 需修 description"
|
||
print(f"[{verdict:>9}] {item['text'][:48]} {basis} 负词命中={hit_neg} {entry_verdict}{flag}")
|
||
report["cases"].append({
|
||
"text": item["text"], "family": item["family"], "bucket": bucket,
|
||
"coverage": round(cov, 3), "coverage_hits": cov_hits,
|
||
"families_hit": sorted(fams),
|
||
"negative_hit": hit_neg, "predicted_trigger": predicted,
|
||
"expected_trigger": expected, "verdict": verdict,
|
||
})
|
||
|
||
report["summary"] = {"scored": n_total, "mismatch": n_mismatch,
|
||
"pass_rate": round((n_total - n_mismatch) / n_total, 3) if n_total else None}
|
||
out = args.output or (root / "reports" / "trigger-eval.json")
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
|
||
print(f"\n计分条目 {n_total}，不一致 {n_mismatch}；报告已写入 {out}")
|
||
if n_mismatch:
|
||
print("结论：description 与用户原话存在词面缺口，先补触发短语/负触发再扩正文。")
|
||
return 1
|
||
print("结论：冒烟全绿。仍需真实对话实测后方可声称[已验证]。")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|