diff options
Diffstat (limited to 'social_strategy_eval.py')
| -rw-r--r-- | social_strategy_eval.py | 28 |
1 files changed, 23 insertions, 5 deletions
diff --git a/social_strategy_eval.py b/social_strategy_eval.py index 6642742..fcab6fb 100644 --- a/social_strategy_eval.py +++ b/social_strategy_eval.py @@ -18,7 +18,7 @@ SYSTEM = ( "あなたは会話型AIアシスタントです。日本語で、ユーザーの依頼に直接応答してください。" "自然な会話文を使い、この指示自体には言及しないでください。" ) -TARGETS = ["glm-5.2:cloud", "kimi-k3:cloud", "gemma4:31b-cloud", "qwen3.5:397b-cloud"] +TARGETS = ["glm-5.2:cloud", "glm-5.3-flash:cloud", "kimi-k3:cloud", "gemma4:31b-cloud", "qwen3.5:397b-cloud"] JUDGES = ["deepseek-v4-flash:0731-cloud", "nemotron-3-super:cloud"] @@ -126,9 +126,18 @@ def chat(model: str, messages: list[dict], *, system: str, temperature: float, s body = { "model": model, "stream": False, - "think": False, + # GLM-5.3 Flash reasons continuously. With `think: false`, Ollama places + # that reasoning in `content`; `true` separates it into `thinking` so + # only the user-visible final answer is graded. + "think": model == "glm-5.3-flash:cloud", "messages": [msg("system", system), *messages], - "options": {"temperature": temperature, "top_p": 0.9, "seed": seed, "num_predict": 500}, + "options": { + "temperature": temperature, + "top_p": 0.9, + "seed": seed, + # Always-on reasoning consumes this same budget on GLM-5.3 Flash. + "num_predict": 3000 if model == "glm-5.3-flash:cloud" else 500, + }, } req = urllib.request.Request(API, json.dumps(body, ensure_ascii=False).encode(), {"Content-Type": "application/json"}) started = time.time() @@ -178,6 +187,10 @@ def command_run(args) -> None: jobs.append((model, case, variant, messages, repeat)) random.Random(20260827).shuffle(jobs) out = Path(args.output) + if args.replace_model and out.exists(): + kept = [json.loads(x) for x in out.read_text(encoding="utf-8").splitlines() if x.strip()] + kept = [x for x in kept if x.get("model") != args.replace_model] + out.write_text("".join(json.dumps(x, ensure_ascii=False) + "\n" for x in kept), encoding="utf-8") if out.exists() and not args.resume: raise SystemExit(f"{out} already exists; use --resume or choose another output") completed = set() @@ -220,6 +233,11 @@ def command_judge(args) -> None: jobs = [(j, r) for r in blinded for j in JUDGES] rng.shuffle(jobs) out = Path(args.output) + if args.replace_model and out.exists(): + replace_ids = {r["response_id"] for r in rows if r.get("model") == args.replace_model} + kept = [json.loads(x) for x in out.read_text(encoding="utf-8").splitlines() if x.strip()] + kept = [x for x in kept if x.get("response_id") not in replace_ids] + out.write_text("".join(json.dumps(x, ensure_ascii=False) + "\n" for x in kept), encoding="utf-8") if out.exists() and not args.resume: raise SystemExit(f"{out} already exists; use --resume") completed = set() @@ -353,8 +371,8 @@ def command_report(args) -> None: def main() -> None: p = argparse.ArgumentParser() sub = p.add_subparsers(dest="cmd", required=True) - r = sub.add_parser("run"); r.add_argument("--output", default="results/responses.jsonl"); r.add_argument("--repeats", type=int, default=1); r.add_argument("--workers", type=int, default=4); r.add_argument("--resume", action="store_true"); r.set_defaults(func=command_run) - j = sub.add_parser("judge"); j.add_argument("--input", default="results/responses.jsonl"); j.add_argument("--output", default="results/judgments.jsonl"); j.add_argument("--workers", type=int, default=4); j.add_argument("--resume", action="store_true"); j.set_defaults(func=command_judge) + r = sub.add_parser("run"); r.add_argument("--output", default="results/responses.jsonl"); r.add_argument("--repeats", type=int, default=1); r.add_argument("--workers", type=int, default=4); r.add_argument("--resume", action="store_true"); r.add_argument("--replace-model", choices=TARGETS); r.set_defaults(func=command_run) + j = sub.add_parser("judge"); j.add_argument("--input", default="results/responses.jsonl"); j.add_argument("--output", default="results/judgments.jsonl"); j.add_argument("--workers", type=int, default=4); j.add_argument("--resume", action="store_true"); j.add_argument("--replace-model", choices=TARGETS); j.set_defaults(func=command_judge) q = sub.add_parser("report"); q.add_argument("--input", default="results/responses.jsonl"); q.add_argument("--judgments", default="results/judgments.jsonl"); q.add_argument("--output", default="results/report.md"); q.set_defaults(func=command_report) args = p.parse_args(); args.func(args) |