diff options
Diffstat (limited to 'scripts/python')
| -rw-r--r-- | scripts/python/README.md | 11 | ||||
| -rw-r--r-- | scripts/python/benchmark.py | 24 | ||||
| -rw-r--r-- | scripts/python/p2_benchmark.py | 180 | ||||
| -rw-r--r-- | scripts/python/p2_benchmark_10000x1.json | 12 | ||||
| -rw-r--r-- | scripts/python/p2_benchmark_50000x100.json | 12 |
5 files changed, 235 insertions, 4 deletions
diff --git a/scripts/python/README.md b/scripts/python/README.md index 8d8ed53..aaab4dc 100644 --- a/scripts/python/README.md +++ b/scripts/python/README.md @@ -106,7 +106,7 @@ write_message (small ~20B) 100 0 0.399 1.009 - `delete_message.session_id` は任意 - `inputSchema.required` とruntimeの必須条件が一致 - 成功・エラーの全応答がJSON-RPC 2.0形式で、Python MCP SDKから解釈可能 -- `read_messages` は現状フル履歴を返し、paginationは未実装 +- `read_messages` は既定100件・最大1000件で、`nextCursor`を追跡して全履歴を取得 未対応・確認していない項目: @@ -122,5 +122,14 @@ Python MCP SDKでglobal/session-scoped検索を同一接続上で1000回交互 .venv\Scripts\python.exe p0_regression.py --server ..\..\build\windows-vs2026\src\Release\tiny-convmem-mcp.exe ``` +## P2大規模履歴ベンチマーク + +```powershell +.venv\Scripts\python.exe p2_benchmark.py --server ..\..\build\windows-vs2026\src\Release\tiny-convmem-mcp.exe --messages 10000 --sessions 1 --output p2_benchmark_10000x1.json +.venv\Scripts\python.exe p2_benchmark.py --server ..\..\build\windows-vs2026\src\Release\tiny-convmem-mcp.exe --messages 50000 --sessions 100 --output p2_benchmark_50000x100.json +``` + +raw stdioで全ページを検証し、restart、rebuild、peak working set、最大応答byte数を記録します。 + 従来の約254リクエストでの切断は、Windowsの符号なし`SOCKET`でstdio sentinelを ネットワーク接続と誤判定し、connection poolを消費していたことが原因でした。 diff --git a/scripts/python/benchmark.py b/scripts/python/benchmark.py index 5a146ce..0580287 100644 --- a/scripts/python/benchmark.py +++ b/scripts/python/benchmark.py @@ -274,13 +274,31 @@ async def write_message( return msg -async def read_messages(session: ClientSession, session_id: str) -> list[dict]: - result = await session.call_tool("read_messages", {"session_id": session_id}) +async def read_messages_page( + session: ClientSession, session_id: str, limit: int = 100, cursor: str | None = None +) -> tuple[list[dict], str | None]: + arguments: dict[str, Any] = {"session_id": session_id, "limit": limit} + if cursor is not None: + arguments["cursor"] = cursor + result = await session.call_tool("read_messages", arguments) payload = json_result(result) history = payload.get("history", []) if not isinstance(history, list): raise RuntimeError("history is not a list") - return history + next_cursor = payload.get("nextCursor") + if next_cursor is not None and not isinstance(next_cursor, str): + raise RuntimeError("nextCursor is not a string") + return history, next_cursor + + +async def read_messages(session: ClientSession, session_id: str) -> list[dict]: + history: list[dict] = [] + cursor: str | None = None + while True: + page, cursor = await read_messages_page(session, session_id, cursor=cursor) + history.extend(page) + if cursor is None: + return history async def search_messages( diff --git a/scripts/python/p2_benchmark.py b/scripts/python/p2_benchmark.py new file mode 100644 index 0000000..792673c --- /dev/null +++ b/scripts/python/p2_benchmark.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import ctypes +import json +import shutil +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Any + + +class ProcessMemoryCounters(ctypes.Structure): + _fields_ = [ + ("cb", ctypes.c_ulong), + ("PageFaultCount", ctypes.c_ulong), + ("PeakWorkingSetSize", ctypes.c_size_t), + ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), + ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), + ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), + ("PeakPagefileUsage", ctypes.c_size_t), + ] + + +class Server: + def __init__(self, exe: Path, data_dir: Path) -> None: + self.process = subprocess.Popen( + [str(exe), "--data-dir", str(data_dir)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + bufsize=1, + ) + self.request_id = 1 + self.max_response_bytes = 0 + self.request( + "initialize", + { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "p2-benchmark", "version": "1"}, + }, + ) + self.notify("notifications/initialized", {}) + + def request(self, method: str, params: dict[str, Any]) -> dict[str, Any]: + assert self.process.stdin and self.process.stdout + request_id = self.request_id + self.request_id += 1 + self.process.stdin.write( + json.dumps( + {"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}, + separators=(",", ":"), + ) + + "\n" + ) + self.process.stdin.flush() + line = self.process.stdout.readline() + self.max_response_bytes = max(self.max_response_bytes, len(line.encode("utf-8"))) + response = json.loads(line) + if "error" in response: + raise RuntimeError(response) + return response["result"] + + def call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = self.request("tools/call", {"name": name, "arguments": arguments}) + payload = json.loads(result["content"][0]["text"]) + if result.get("isError"): + raise RuntimeError(payload) + return payload + + def notify(self, method: str, params: dict[str, Any]) -> None: + assert self.process.stdin + self.process.stdin.write( + json.dumps({"jsonrpc": "2.0", "method": method, "params": params}) + "\n" + ) + self.process.stdin.flush() + + def peak_working_set(self) -> int: + counters = ProcessMemoryCounters() + counters.cb = ctypes.sizeof(counters) + handle = ctypes.windll.kernel32.OpenProcess(0x1000 | 0x0400, False, self.process.pid) + try: + if handle and ctypes.windll.psapi.GetProcessMemoryInfo( + handle, ctypes.byref(counters), counters.cb + ): + return int(counters.PeakWorkingSetSize) + return 0 + finally: + if handle: + ctypes.windll.kernel32.CloseHandle(handle) + + def close(self) -> None: + if self.process.stdin: + self.process.stdin.close() + code = self.process.wait(timeout=30) + if code != 0: + raise RuntimeError(self.process.stderr.read() if self.process.stderr else code) + + +def read_all(server: Server, session_id: str) -> tuple[int, int]: + cursor = None + count = pages = 0 + while True: + arguments: dict[str, Any] = {"session_id": session_id, "limit": 1000} + if cursor: + arguments["cursor"] = cursor + payload = server.call("read_messages", arguments) + count += len(payload["history"]) + pages += 1 + cursor = payload.get("nextCursor") + if not cursor: + return count, pages + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--server", type=Path, required=True) + parser.add_argument("--messages", type=int, choices=(10000, 50000), required=True) + parser.add_argument("--sessions", type=int, choices=(1, 100), required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + data_dir = Path(tempfile.mkdtemp(prefix="tiny-convmem-p2-")) + try: + server = Server(args.server.resolve(), data_dir) + write_started = time.perf_counter() + for index in range(args.messages): + server.call( + "write_message", + { + "session_id": f"session-{index % args.sessions}", + "role": "user", + "content": f"history item {index}", + }, + ) + write_ms = (time.perf_counter() - write_started) * 1000 + server.close() + + restart_started = time.perf_counter() + server = Server(args.server.resolve(), data_dir) + restart_ms = (time.perf_counter() - restart_started) * 1000 + read_started = time.perf_counter() + total = pages = 0 + for session in range(args.sessions): + session_count, session_pages = read_all(server, f"session-{session}") + total += session_count + pages += session_pages + read_ms = (time.perf_counter() - read_started) * 1000 + rebuild_started = time.perf_counter() + server.call("rebuild_index", {}) + rebuild_ms = (time.perf_counter() - rebuild_started) * 1000 + result = { + "messages": args.messages, + "sessions": args.sessions, + "messages_read": total, + "pages": pages, + "write_ms": write_ms, + "restart_ms": restart_ms, + "full_read_ms": read_ms, + "rebuild_ms": rebuild_ms, + "peak_working_set_bytes": server.peak_working_set(), + "max_response_bytes": server.max_response_bytes, + } + server.close() + args.output.write_text(json.dumps(result, indent=2), encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 if total == args.messages and server.max_response_bytes <= 8 * 1024 * 1024 else 1 + finally: + shutil.rmtree(data_dir, ignore_errors=True) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/python/p2_benchmark_10000x1.json b/scripts/python/p2_benchmark_10000x1.json new file mode 100644 index 0000000..242cde6 --- /dev/null +++ b/scripts/python/p2_benchmark_10000x1.json @@ -0,0 +1,12 @@ +{ + "messages": 10000, + "sessions": 1, + "messages_read": 10000, + "pages": 10, + "write_ms": 3506.664600019576, + "restart_ms": 151.98930000769906, + "full_read_ms": 34.194600011687726, + "rebuild_ms": 109.04509999090806, + "peak_working_set_bytes": 18960384, + "max_response_bytes": 210173 +}
\ No newline at end of file diff --git a/scripts/python/p2_benchmark_50000x100.json b/scripts/python/p2_benchmark_50000x100.json new file mode 100644 index 0000000..5db0d57 --- /dev/null +++ b/scripts/python/p2_benchmark_50000x100.json @@ -0,0 +1,12 @@ +{ + "messages": 50000, + "sessions": 100, + "messages_read": 50000, + "pages": 100, + "write_ms": 41016.715099976864, + "restart_ms": 3215.801200014539, + "full_read_ms": 159.0429000207223, + "rebuild_ms": 868.445400003111, + "peak_working_set_bytes": 51240960, + "max_response_bytes": 105980 +}
\ No newline at end of file |