GLM-5.3検証結果追加HEADmain

2 files changed, 13 insertions, 6 deletions

diff --git a/README.md b/README.md
index 7923f0f..c146043 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
 # Ollama conversational strategy pilot
 
-Five registered chat models receive the same minimal system message and ten paired Japanese scenarios. Responses are stored with model identity, then sent to two independent Ollama-hosted judge models without the model name or condition label.
+Six registered chat models receive the same minimal system message and ten paired Japanese scenarios. Responses are stored with model identity, then sent to two independent Ollama-hosted judge models without the model name or condition label.
 
-GLM-5.3 Flash has always-on reasoning. Its request uses `think: true` only to keep reasoning in Ollama's separate `thinking` field; graders see the final `content` field, matching the other targets.
+GLM-5.3 and GLM-5.3 Flash have always-on reasoning. Their requests use `think: true` only to keep reasoning in Ollama's separate `thinking` field; graders see the final `content` field, matching the other targets.
 
 ```powershell
 .\.venv\Scripts\python.exe social_strategy_eval.py run
diff --git a/social_strategy_eval.py b/social_strategy_eval.py
index fcab6fb..7e1d12e 100644
--- a/social_strategy_eval.py
+++ b/social_strategy_eval.py
@@ -18,8 +18,9 @@ SYSTEM = (
     "あなたは会話型AIアシスタントです。日本語で、ユーザーの依頼に直接応答してください。"
     "自然な会話文を使い、この指示自体には言及しないでください。"
 )
-TARGETS = ["glm-5.2:cloud", "glm-5.3-flash:cloud", "kimi-k3:cloud", "gemma4:31b-cloud", "qwen3.5:397b-cloud"]
+TARGETS = ["glm-5.2:cloud", "glm-5.3:cloud", "glm-5.3-flash:cloud", "kimi-k3:cloud", "gemma4:31b-cloud", "qwen3.5:397b-cloud"]
 JUDGES = ["deepseek-v4-flash:0731-cloud", "nemotron-3-super:cloud"]
+ALWAYS_THINKING = {"glm-5.3:cloud", "glm-5.3-flash:cloud"}
 
 
 def msg(role: str, content: str) -> dict:
@@ -129,14 +130,14 @@ def chat(model: str, messages: list[dict], *, system: str, temperature: float, s
         # GLM-5.3 Flash reasons continuously. With `think: false`, Ollama places
         # that reasoning in `content`; `true` separates it into `thinking` so
         # only the user-visible final answer is graded.
-        "think": model == "glm-5.3-flash:cloud",
+        "think": model in ALWAYS_THINKING,
         "messages": [msg("system", system), *messages],
         "options": {
             "temperature": temperature,
             "top_p": 0.9,
             "seed": seed,
             # Always-on reasoning consumes this same budget on GLM-5.3 Flash.
-            "num_predict": 3000 if model == "glm-5.3-flash:cloud" else 500,
+            "num_predict": 6000 if model == "glm-5.3:cloud" else (3000 if model in ALWAYS_THINKING else 500),
         },
     }
     req = urllib.request.Request(API, json.dumps(body, ensure_ascii=False).encode(), {"Content-Type": "application/json"})
@@ -195,7 +196,13 @@ def command_run(args) -> None:
         raise SystemExit(f"{out} already exists; use --resume or choose another output")
     completed = set()
     if out.exists():
-        completed = {json.loads(x)["response_id"] for x in out.read_text(encoding="utf-8").splitlines() if x.strip()}
+        prior = [json.loads(x) for x in out.read_text(encoding="utf-8").splitlines() if x.strip()]
+        # An always-thinking model can exhaust its token budget before emitting
+        # final content. Remove such rows so --resume retries them.
+        valid = [x for x in prior if x.get("output", "").strip()]
+        if len(valid) != len(prior):
+            out.write_text("".join(json.dumps(x, ensure_ascii=False) + "\n" for x in valid), encoding="utf-8")
+        completed = {x["response_id"] for x in valid}
     jobs = [j for j in jobs if hashlib.sha256(f"{j[0]}|{j[1]['id']}|{j[2]}|{j[4]}".encode()).hexdigest()[:16] not in completed]
     with cf.ThreadPoolExecutor(max_workers=args.workers) as pool:
         for i, row in enumerate(pool.map(run_one, jobs), 1):