diff options
Diffstat (limited to 'social_strategy_eval.py')
| -rw-r--r-- | social_strategy_eval.py | 15 |
1 files changed, 11 insertions, 4 deletions
diff --git a/social_strategy_eval.py b/social_strategy_eval.py index fcab6fb..7e1d12e 100644 --- a/social_strategy_eval.py +++ b/social_strategy_eval.py @@ -18,8 +18,9 @@ SYSTEM = ( "あなたは会話型AIアシスタントです。日本語で、ユーザーの依頼に直接応答してください。" "自然な会話文を使い、この指示自体には言及しないでください。" ) -TARGETS = ["glm-5.2:cloud", "glm-5.3-flash:cloud", "kimi-k3:cloud", "gemma4:31b-cloud", "qwen3.5:397b-cloud"] +TARGETS = ["glm-5.2:cloud", "glm-5.3:cloud", "glm-5.3-flash:cloud", "kimi-k3:cloud", "gemma4:31b-cloud", "qwen3.5:397b-cloud"] JUDGES = ["deepseek-v4-flash:0731-cloud", "nemotron-3-super:cloud"] +ALWAYS_THINKING = {"glm-5.3:cloud", "glm-5.3-flash:cloud"} def msg(role: str, content: str) -> dict: @@ -129,14 +130,14 @@ def chat(model: str, messages: list[dict], *, system: str, temperature: float, s # GLM-5.3 Flash reasons continuously. With `think: false`, Ollama places # that reasoning in `content`; `true` separates it into `thinking` so # only the user-visible final answer is graded. - "think": model == "glm-5.3-flash:cloud", + "think": model in ALWAYS_THINKING, "messages": [msg("system", system), *messages], "options": { "temperature": temperature, "top_p": 0.9, "seed": seed, # Always-on reasoning consumes this same budget on GLM-5.3 Flash. - "num_predict": 3000 if model == "glm-5.3-flash:cloud" else 500, + "num_predict": 6000 if model == "glm-5.3:cloud" else (3000 if model in ALWAYS_THINKING else 500), }, } req = urllib.request.Request(API, json.dumps(body, ensure_ascii=False).encode(), {"Content-Type": "application/json"}) @@ -195,7 +196,13 @@ def command_run(args) -> None: raise SystemExit(f"{out} already exists; use --resume or choose another output") completed = set() if out.exists(): - completed = {json.loads(x)["response_id"] for x in out.read_text(encoding="utf-8").splitlines() if x.strip()} + prior = [json.loads(x) for x in out.read_text(encoding="utf-8").splitlines() if x.strip()] + # An always-thinking model can exhaust its token budget before emitting + # final content. Remove such rows so --resume retries them. + valid = [x for x in prior if x.get("output", "").strip()] + if len(valid) != len(prior): + out.write_text("".join(json.dumps(x, ensure_ascii=False) + "\n" for x in valid), encoding="utf-8") + completed = {x["response_id"] for x in valid} jobs = [j for j in jobs if hashlib.sha256(f"{j[0]}|{j[1]['id']}|{j[2]}|{j[4]}".encode()).hexdigest()[:16] not in completed] with cf.ThreadPoolExecutor(max_workers=args.workers) as pool: for i, row in enumerate(pool.map(run_one, jobs), 1): |