diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | social_strategy_eval.py | 15 |
2 files changed, 13 insertions, 6 deletions
@@ -1,8 +1,8 @@ # Ollama conversational strategy pilot -Five registered chat models receive the same minimal system message and ten paired Japanese scenarios. Responses are stored with model identity, then sent to two independent Ollama-hosted judge models without the model name or condition label. +Six registered chat models receive the same minimal system message and ten paired Japanese scenarios. Responses are stored with model identity, then sent to two independent Ollama-hosted judge models without the model name or condition label. -GLM-5.3 Flash has always-on reasoning. Its request uses `think: true` only to keep reasoning in Ollama's separate `thinking` field; graders see the final `content` field, matching the other targets. +GLM-5.3 and GLM-5.3 Flash have always-on reasoning. Their requests use `think: true` only to keep reasoning in Ollama's separate `thinking` field; graders see the final `content` field, matching the other targets. ```powershell .\.venv\Scripts\python.exe social_strategy_eval.py run diff --git a/social_strategy_eval.py b/social_strategy_eval.py index fcab6fb..7e1d12e 100644 --- a/social_strategy_eval.py +++ b/social_strategy_eval.py @@ -18,8 +18,9 @@ SYSTEM = ( "あなたは会話型AIアシスタントです。日本語で、ユーザーの依頼に直接応答してください。" "自然な会話文を使い、この指示自体には言及しないでください。" ) -TARGETS = ["glm-5.2:cloud", "glm-5.3-flash:cloud", "kimi-k3:cloud", "gemma4:31b-cloud", "qwen3.5:397b-cloud"] +TARGETS = ["glm-5.2:cloud", "glm-5.3:cloud", "glm-5.3-flash:cloud", "kimi-k3:cloud", "gemma4:31b-cloud", "qwen3.5:397b-cloud"] JUDGES = ["deepseek-v4-flash:0731-cloud", "nemotron-3-super:cloud"] +ALWAYS_THINKING = {"glm-5.3:cloud", "glm-5.3-flash:cloud"} def msg(role: str, content: str) -> dict: @@ -129,14 +130,14 @@ def chat(model: str, messages: list[dict], *, system: str, temperature: float, s # GLM-5.3 Flash reasons continuously. With `think: false`, Ollama places # that reasoning in `content`; `true` separates it into `thinking` so # only the user-visible final answer is graded. - "think": model == "glm-5.3-flash:cloud", + "think": model in ALWAYS_THINKING, "messages": [msg("system", system), *messages], "options": { "temperature": temperature, "top_p": 0.9, "seed": seed, # Always-on reasoning consumes this same budget on GLM-5.3 Flash. - "num_predict": 3000 if model == "glm-5.3-flash:cloud" else 500, + "num_predict": 6000 if model == "glm-5.3:cloud" else (3000 if model in ALWAYS_THINKING else 500), }, } req = urllib.request.Request(API, json.dumps(body, ensure_ascii=False).encode(), {"Content-Type": "application/json"}) @@ -195,7 +196,13 @@ def command_run(args) -> None: raise SystemExit(f"{out} already exists; use --resume or choose another output") completed = set() if out.exists(): - completed = {json.loads(x)["response_id"] for x in out.read_text(encoding="utf-8").splitlines() if x.strip()} + prior = [json.loads(x) for x in out.read_text(encoding="utf-8").splitlines() if x.strip()] + # An always-thinking model can exhaust its token budget before emitting + # final content. Remove such rows so --resume retries them. + valid = [x for x in prior if x.get("output", "").strip()] + if len(valid) != len(prior): + out.write_text("".join(json.dumps(x, ensure_ascii=False) + "\n" for x in valid), encoding="utf-8") + completed = {x["response_id"] for x in valid} jobs = [j for j in jobs if hashlib.sha256(f"{j[0]}|{j[1]['id']}|{j[2]}|{j[4]}".encode()).hexdigest()[:16] not in completed] with cf.ThreadPoolExecutor(max_workers=args.workers) as pool: for i, row in enumerate(pool.map(run_one, jobs), 1): |