diff options
| author | tanaka takashi <isazaka@gmail.com> | 2026-06-30 19:03:25 +0900 |
|---|---|---|
| committer | tanaka takashi <isazaka@gmail.com> | 2026-06-30 19:03:25 +0900 |
| commit | 05c8e0a31a0d4192c590e833b24393208e1c751f (patch) | |
| tree | f7c2c82c9327abde67bf3acf1d0258573b3bc6c2 /scripts | |
| parent | 4bc721c3bf7589b519c5730d093f704ce56e3f09 (diff) | |
fix: MCPCをvendored fork化しMCP準拠と接続安定性を改善
- Git submoduleを廃止してMCPCを直接管理
- Windowsで254リクエスト付近に発生する切断を修正
- JSON-RPC 2.0エラー応答を標準化
- tool schemaとruntimeの必須・任意引数を一致
- 1000回連続呼び出しの回帰テストを追加
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/python/README.md | 121 | ||||
| -rw-r--r-- | scripts/python/benchmark.py | 862 | ||||
| -rw-r--r-- | scripts/python/benchmark_results.json | 1221 | ||||
| -rw-r--r-- | scripts/python/benchmark_results.md | 85 | ||||
| -rw-r--r-- | scripts/python/p0_regression.py | 107 |
5 files changed, 2396 insertions, 0 deletions
diff --git a/scripts/python/README.md b/scripts/python/README.md new file mode 100644 index 0000000..7018313 --- /dev/null +++ b/scripts/python/README.md @@ -0,0 +1,121 @@ +# tiny-convmem-mcp ユーザ視点負荷テスト + +このディレクトリには、[Model Context Protocol (MCP) Python SDK](https://github.com/modelcontextprotocol/python-sdk) を使った、Windows 版 `tiny-convmem-mcp` サーバのエンドツーエンド負荷テスト&ベンチマークスクリプトがあります。 + +## 環境 + +`scripts/python/.venv` は uv で構築済みで、以下が含まれています。 + +- Python 3.12.9 +- `mcp` 1.28.1 (Python SDK) + +必要に応じて追加インストールする場合: + +```powershell +uv pip install --python .venv\Scripts\python.exe mcp +``` + +## 実行方法 + +最新の Release ビルドを自動検出して実行します。 + +```powershell +cd scripts\python +..\.venv\Scripts\python.exe benchmark.py +``` + +オプション: + +| オプション | 説明 | +|---|---| +| `--server <path>` | `tiny-convmem-mcp.exe` のパスを明示 | +| `--iterations N` | 各シナリオの反復回数(デフォルト 100、rebuild/recovery は最大 5) | +| `--warmup N` | 未使用(後方互換) | +| `--output <path>` | 出力 JSON(デフォルト `benchmark_results.json`) | + +例: + +```powershell +.venv\Scripts\python.exe benchmark.py --server ..\..\build\windows-vs2026\src\Release\tiny-convmem-mcp.exe --iterations 100 --output mybench.json +``` + +## 測定内容 + +すべての操作は stdio MCP プロトコル経由で実際のサーバプロセスに対して行います。 + +1. **write_message** — 小 (~20B)、中 (1KiB)、大 (32KiB) content サイズ別 +2. **write_message (with tags)** — `metadata_json` 文字列に tags / `memory_tier` を含む書き込み +3. **read_messages** — 100 件のメッセージを読み出し、件数を検証 +4. **search_messages** — 同じデータ・同じ query で global / session-scoped を比較 +5. **delete_message** — `message_id` 指定削除 +6. **rebuild_index** — compaction + snapshot 再生成(5 回反復) +7. **get_system_health** — ストレージ健全性取得 +8. **recovery** — プロセス再起動後の復旧を 3 段階で計測 + - restart + count check(1 回) + - restart + small read(5 回) + - restart + full read 100 msgs(5 回) + +各シナリオは独立した temp data directory で実行され、シナリオ間の状態累積を排除しています。 + +## 出力 + +- コンソール: p50/p95/p99/mean レイテンシと ops/sec のサマリ表 +- `<output>.json`: 詳細な計測結果(全サンプル含む) +- `<output>.md`: Markdown レポート + +## 結果の見方 + +```text +=== tiny-convmem-mcp 負荷テスト結果 === +server : C:\...\Release\tiny-convmem-mcp.exe +exe_mtime : 2026-06-30 01:46:57 +iterations : 100 + +scenario succ fail p50[ms] p95[ms] p99[ms] mean[ms] ops/s +---------------------------------------------------------------------------------------------------------------- +write_message (small ~20B) 100 0 0.399 1.009 1.718 0.505 1981.27 +... +``` + +## 注意 + +- `tiny-convmem-mcp` は単一stdio・単一スレッドサーバのため、リクエストは逐次送信されます。並行負荷テストは対象外です。 +- 計測値には子プロセス起動、JSON シリアライズ、OS I/O、同期耐久性フラッシュ(fsync/FlushFileBuffers)が含まれます。 +- 各シナリオ内の反復は同一プロセス・同一セッションで実行します。したがって `write_message` の大サイズ計測など、前の反復がストレージ状態を変化させる可能性があります。 + +## 参考: MCP 準拠状況 + +本テストで確認できる MCP 機能: + +- `initialize` / `notifications/initialized` +- `tools/list`(schema 取得) +- `tools/call`(明示登録された6ツール) +- TextContent 内 JSON 応答 +- `isError=true` によるアプリケーションレベルエラー + +確認済みのP0契約: + +- `write_message.metadata_json` は任意のJSON文字列で、省略時は `{}` +- `search_messages.session_id` は任意で、省略時はglobal検索 +- `search_messages.limit` は任意で、既定20、整数1~100 +- `delete_message.session_id` は任意 +- `inputSchema.required` とruntimeの必須条件が一致 +- 成功・エラーの全応答がJSON-RPC 2.0形式で、Python MCP SDKから解釈可能 +- `read_messages` は現状フル履歴を返し、paginationは未実装 + +未対応・確認していない項目: + +- `structuredContent` / 出力 schema +- `resources/*`, `prompts/*`, `sampling/*`, `roots/*` +- 並行リクエスト / 複数クライアント + +## P0回帰テスト + +Python MCP SDKでglobal/session-scoped検索を同一接続上で1000回交互に実行できます。 + +```powershell +.venv\Scripts\python.exe p0_regression.py --server ..\..\build\windows-vs2026\src\Release\tiny-convmem-mcp.exe +``` + +従来の約254リクエストでの切断は、Windowsの符号なし`SOCKET`でstdio sentinelを +ネットワーク接続と誤判定し、connection poolを消費していたことが原因でした。 diff --git a/scripts/python/benchmark.py b/scripts/python/benchmark.py new file mode 100644 index 0000000..69219c9 --- /dev/null +++ b/scripts/python/benchmark.py @@ -0,0 +1,862 @@ +#!/usr/bin/env python3 +""" +tiny-convmem-mcp ユーザ視点負荷テスト&ベンチマーク + +MCP Python SDK (mcp.client.stdio + mcp.client.session) を使い、 +実際に tiny-convmem-mcp サーバプロセスを stdio 経由で起動し、 +エンドツーエンドのレイテンシとスループットを測定します。 + +ポイント: +- 各シナリオは独立した temp data directory で実行(状態累積を排除) +- write_message は `metadata_json` 文字列を使用 +- 応答の isError / status / 件数を検証 +- rebuild_index は 5 回反復 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import random +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Awaitable, Callable + +from mcp.client.session import ClientSession +from mcp.client.stdio import StdioServerParameters, stdio_client +from mcp.types import CallToolResult + +# --------------------------------------------------------------------------- +# 定数・デフォルト +# --------------------------------------------------------------------------- +DEFAULT_ITERATIONS = 100 +DEFAULT_CONTENT_MEDIUM = 1024 # bytes +DEFAULT_CONTENT_LARGE = 32 * 1024 # bytes +WARMUP_COUNT = 5 +REBUILD_ITERATIONS = 5 + +ROOT = Path(__file__).resolve().parent.parent.parent +DEFAULT_RELEASE_EXE = ( + ROOT / "build" / "windows-vs2026" / "src" / "Release" / "tiny-convmem-mcp.exe" +) + + +# --------------------------------------------------------------------------- +# データクラス +# --------------------------------------------------------------------------- +@dataclass +class LatencySample: + operation: str + duration_ms: float + success: bool + error: str | None = None + bytes_in: int = 0 + bytes_out: int = 0 + + +@dataclass +class ScenarioResult: + name: str + iterations: int + successes: int + failures: int + samples: list[float] = field(repr=False, default_factory=list) + throughput_ops_per_sec: float = 0.0 + latency_p50_ms: float = 0.0 + latency_p95_ms: float = 0.0 + latency_p99_ms: float = 0.0 + latency_min_ms: float = 0.0 + latency_max_ms: float = 0.0 + latency_mean_ms: float = 0.0 + total_duration_ms: float = 0.0 + total_bytes_in: int = 0 + total_bytes_out: int = 0 + notes: str = "" + + +@dataclass +class BenchmarkReport: + server: str + build_type: str + exe_mtime: str + iterations: int + warmup: int + scenarios: list[ScenarioResult] = field(default_factory=list) + health_after: dict[str, Any] | None = None + schema_check: dict[str, Any] | None = None + errors: list[str] = field(default_factory=list) + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +# --------------------------------------------------------------------------- +# ユーティリティ +# --------------------------------------------------------------------------- +def percentile(sorted_values: list[float], p: float) -> float: + if not sorted_values: + return 0.0 + n = len(sorted_values) + if n == 1: + return sorted_values[0] + k = (n - 1) * p / 100.0 + f = int(k) + c = min(f + 1, n - 1) + if f == c: + return sorted_values[f] + return sorted_values[f] + (sorted_values[c] - sorted_values[f]) * (k - f) + + +def summarize(name: str, samples: list[LatencySample]) -> ScenarioResult: + ok = [s for s in samples if s.success] + failed = [s for s in samples if not s.success] + durations = sorted(s.duration_ms for s in ok) + result = ScenarioResult( + name=name, + iterations=len(samples), + successes=len(ok), + failures=len(failed), + samples=durations, + total_duration_ms=sum(s.duration_ms for s in samples), + total_bytes_in=sum(s.bytes_in for s in ok), + total_bytes_out=sum(s.bytes_out for s in ok), + notes=failed[0].error if failed else "", + ) + if durations: + result.latency_min_ms = durations[0] + result.latency_max_ms = durations[-1] + result.latency_mean_ms = statistics.mean(durations) + result.latency_p50_ms = percentile(durations, 50) + result.latency_p95_ms = percentile(durations, 95) + result.latency_p99_ms = percentile(durations, 99) + if result.total_duration_ms > 0 and ok: + result.throughput_ops_per_sec = len(ok) / (result.total_duration_ms / 1000.0) + return result + + +def text_content(result: CallToolResult) -> str: + if result.isError: + raise RuntimeError(f"tool returned isError: {result.content!r}") + if not result.content: + return "" + return "".join(item.text for item in result.content if item.type == "text") + + +def json_result(result: CallToolResult) -> dict: + text = text_content(result) + payload = json.loads(text) + return payload + + +def random_content(size: int) -> str: + alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 " + return "".join(random.choice(alphabet) for _ in range(size)) + + +def make_metadata_json(tags: list[str] | None = None, tier: str = "short") -> str: + obj: dict[str, Any] = {"memory_tier": tier} + if tags: + obj["tags"] = tags + return json.dumps(obj, separators=(",", ":")) + + +# --------------------------------------------------------------------------- +# クライアント・サーバランチャー +# --------------------------------------------------------------------------- +class ServerContext: + """MCP stdio サーバを起動し、ClientSession でラップするコンテキスト。""" + + def __init__(self, exe: Path, data_dir: Path) -> None: + self.exe = exe + self.data_dir = data_dir + self.read_stream = None + self.write_stream = None + self.session = None + + async def __aenter__(self) -> ClientSession: + params = StdioServerParameters( + command=str(self.exe), + args=["--data-dir", str(self.data_dir)], + env={**os.environ}, + ) + self._ctx = stdio_client(params) + self.read_stream, self.write_stream = await self._ctx.__aenter__() + self.session = ClientSession(self.read_stream, self.write_stream) + await self.session.__aenter__() + await self.session.initialize() + return self.session + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self.session: + session = self.session + self.session = None + await session.__aexit__(exc_type, exc, tb) + await self._ctx.__aexit__(exc_type, exc, tb) + + +async def with_session(exe: Path, data_dir: Path) -> Awaitable[ClientSession]: + return await ServerContext(exe, data_dir).__aenter__() + + +# --------------------------------------------------------------------------- +# 操作ラッパー(結果検証付き) +# --------------------------------------------------------------------------- +async def write_message( + session: ClientSession, + session_id: str, + content: str, + metadata_json: str = '{"memory_tier":"short"}', +) -> dict: + result = await session.call_tool( + "write_message", + { + "session_id": session_id, + "role": "user", + "content": content, + "metadata_json": metadata_json, + }, + ) + payload = json_result(result) + if payload.get("status") != "OK": + raise RuntimeError(f"write failed: {payload}") + msg = payload.get("message", {}) + if not msg.get("message_id"): + raise RuntimeError("write response missing message_id") + return msg + + +async def read_messages(session: ClientSession, session_id: str) -> list[dict]: + result = await session.call_tool("read_messages", {"session_id": session_id}) + payload = json_result(result) + history = payload.get("history", []) + if not isinstance(history, list): + raise RuntimeError("history is not a list") + return history + + +async def search_messages( + session: ClientSession, query: str, session_id: str, limit: int = 10 +) -> list[dict]: + result = await session.call_tool( + "search_messages", + {"query": query, "session_id": session_id, "limit": limit}, + ) + payload = json_result(result) + results = payload.get("results", []) + if not isinstance(results, list): + raise RuntimeError("search results is not a list") + return results + + +async def delete_message(session: ClientSession, message_id: str, session_id: str) -> str: + result = await session.call_tool( + "delete_message", + {"message_id": message_id, "session_id": session_id}, + ) + payload = json_result(result) + status = payload.get("status", "") + if status != "OK": + raise RuntimeError(f"delete failed: {payload}") + return status + + +async def rebuild_index(session: ClientSession) -> dict: + result = await session.call_tool("rebuild_index", {}) + return json_result(result) + + +async def get_system_health(session: ClientSession) -> dict: + result = await session.call_tool("get_system_health", {}) + payload = json_result(result) + if payload.get("status") != "OK": + raise RuntimeError(f"health failed: {payload}") + return payload + + +async def list_tools(session: ClientSession) -> list[dict]: + result = await session.list_tools() + return [t.model_dump() for t in result.tools] + + +# --------------------------------------------------------------------------- +# シナリオ実装 +# --------------------------------------------------------------------------- +class ScenarioRunner: + def __init__(self, exe: Path) -> None: + self.exe = exe + + def new_data_dir(self) -> Path: + return Path(tempfile.mkdtemp(prefix="tiny-convmem-bench-")) + + def cleanup(self, data_dir: Path) -> None: + shutil.rmtree(data_dir, ignore_errors=True) + + async def measure( + self, + name: str, + fn: Callable[[int], Awaitable[Any]], + iterations: int, + bytes_in: int = 0, + bytes_out: int = 0, + ) -> ScenarioResult: + samples: list[LatencySample] = [] + for i in range(iterations): + start = time.perf_counter() + error: str | None = None + success = False + try: + await fn(i) + success = True + except Exception as exc: # noqa: BLE001 + error = f"{type(exc).__name__}: {exc}" + elapsed_ms = (time.perf_counter() - start) * 1000.0 + samples.append( + LatencySample( + operation=name, + duration_ms=elapsed_ms, + success=success, + error=error, + bytes_in=bytes_in, + bytes_out=bytes_out, + ) + ) + return summarize(name, samples) + + async def run_write( + self, content: str, metadata_json: str, iterations: int, name: str + ) -> ScenarioResult: + data_dir = self.new_data_dir() + session_id = "bench" + async with ServerContext(self.exe, data_dir) as session: + + async def _run(_: int) -> None: + await write_message(session, session_id, content, metadata_json) + + try: + return await self.measure( + name, + _run, + iterations, + bytes_in=len(content.encode("utf-8")), + ) + finally: + self.cleanup(data_dir) + + async def run_write_small(self, iterations: int) -> ScenarioResult: + content = "hello durable memory" + return await self.run_write( + content, + make_metadata_json(), + iterations, + "write_message (small ~20B)", + ) + + async def run_write_medium(self, iterations: int) -> ScenarioResult: + content = random_content(DEFAULT_CONTENT_MEDIUM) + return await self.run_write( + content, + make_metadata_json(), + iterations, + "write_message (medium 1KiB)", + ) + + async def run_write_large(self, iterations: int) -> ScenarioResult: + content = random_content(DEFAULT_CONTENT_LARGE) + return await self.run_write( + content, + make_metadata_json(), + iterations, + "write_message (large 32KiB)", + ) + + async def run_write_with_tags(self, iterations: int) -> ScenarioResult: + content = "tagged message" + metadata_json = make_metadata_json(tags=["benchmark", "tagged"], tier="short") + return await self.run_write( + content, + metadata_json, + iterations, + "write_message (with tags)", + ) + + async def run_read(self, iterations: int) -> ScenarioResult: + data_dir = self.new_data_dir() + session_id = "bench" + async with ServerContext(self.exe, data_dir) as session: + # 事前に 100 件書き込み + for i in range(100): + await write_message(session, session_id, f"read target {i}") + + async def _run(_: int) -> None: + history = await read_messages(session, session_id) + if len(history) != 100: + raise RuntimeError(f"expected 100 messages, got {len(history)}") + + try: + return await self.measure( + "read_messages (100 msgs)", + _run, + iterations, + ) + finally: + self.cleanup(data_dir) + + async def run_search(self, iterations: int) -> tuple[ScenarioResult, ScenarioResult]: + """同じデータ・同じ query で session 絞り込み有無を比較。""" + data_dir = self.new_data_dir() + session_id = "bench" + other_session = "other" + async with ServerContext(self.exe, data_dir) as session: + # target session に 50 件 + for i in range(50): + await write_message( + session, + session_id, + f"search keyword benchmark {i}", + make_metadata_json(tags=["target"]), + ) + # 別 session に 50 件ノイズ + for i in range(50): + await write_message( + session, + other_session, + f"noise keyword other {i}", + make_metadata_json(tags=["noise"]), + ) + + async def _global(_: int) -> int: + results = await search_messages(session, "benchmark", "", limit=10) + if len(results) != 10: + raise RuntimeError(f"expected 10 global results, got {len(results)}") + return len(results) + + async def _scoped(_: int) -> int: + results = await search_messages(session, "benchmark", session_id, limit=10) + if len(results) != 10: + raise RuntimeError(f"expected 10 scoped results, got {len(results)}") + # 全結果が target session であることを検証 + for r in results: + if r.get("session_id") != session_id: + raise RuntimeError(f"scoped result leaked from {r.get('session_id')}") + return len(results) + + try: + global_result = await self.measure( + "search_messages (global, no session)", + _global, + iterations, + ) + scoped_result = await self.measure( + "search_messages (scoped to session)", + _scoped, + iterations, + ) + return global_result, scoped_result + finally: + self.cleanup(data_dir) + + async def run_delete(self, iterations: int) -> ScenarioResult: + data_dir = self.new_data_dir() + session_id = "bench" + async with ServerContext(self.exe, data_dir) as session: + ids: list[str] = [] + for _ in range(iterations): + msg = await write_message(session, session_id, "to be deleted") + ids.append(msg["message_id"]) + + async def _run(i: int) -> None: + await delete_message(session, ids[i], session_id) + + try: + return await self.measure("delete_message", _run, iterations) + finally: + self.cleanup(data_dir) + + async def run_rebuild(self, iterations: int) -> ScenarioResult: + data_dir = self.new_data_dir() + session_id = "bench" + async with ServerContext(self.exe, data_dir) as session: + # 削除を含めた compaction 対象を 50 件作る + for _ in range(50): + msg = await write_message(session, session_id, "will compact") + await delete_message(session, msg["message_id"], session_id) + # 追加で 50 件 active + for i in range(50): + await write_message(session, session_id, f"active {i}") + + async def _run(_: int) -> None: + payload = await rebuild_index(session) + if payload.get("status") != "OK": + raise RuntimeError(f"rebuild failed: {payload}") + + try: + return await self.measure("rebuild_index", _run, iterations) + finally: + self.cleanup(data_dir) + + async def run_health(self, iterations: int) -> ScenarioResult: + data_dir = self.new_data_dir() + session_id = "bench" + async with ServerContext(self.exe, data_dir) as session: + for _ in range(10): + await write_message(session, session_id, "health check") + + async def _run(_: int) -> None: + await get_system_health(session) + + try: + return await self.measure("get_system_health", _run, iterations) + finally: + self.cleanup(data_dir) + + async def run_recovery( + self, iterations: int + ) -> tuple[ScenarioResult, ScenarioResult, ScenarioResult]: + data_dir = self.new_data_dir() + session_id = "bench" + # 1. データ投入 & 終了 + async with ServerContext(self.exe, data_dir) as session: + for i in range(100): + await write_message(session, session_id, f"recovery marker {i}") + + # 2. 復旧確認(メッセージ件数のみ) + start = time.perf_counter() + async with ServerContext(self.exe, data_dir) as session: + history = await read_messages(session, session_id) + if len(history) != 100: + raise RuntimeError(f"recovery count mismatch: {len(history)} != 100") + recovery_ms = (time.perf_counter() - start) * 1000.0 + recovery_result = ScenarioResult( + name="recovery (restart + count check)", + iterations=1, + successes=1, + failures=0, + samples=[recovery_ms], + latency_min_ms=recovery_ms, + latency_max_ms=recovery_ms, + latency_mean_ms=recovery_ms, + latency_p50_ms=recovery_ms, + latency_p95_ms=recovery_ms, + latency_p99_ms=recovery_ms, + total_duration_ms=recovery_ms, + throughput_ops_per_sec=1000.0 / recovery_ms if recovery_ms > 0 else 0.0, + notes="includes process shutdown + startup + initialize + read count", + ) + + # 3. 再起動後の small read を iterations 回 + async def _small(_: int) -> None: + async with ServerContext(self.exe, data_dir) as session: + history = await read_messages(session, session_id) + if len(history) != 100: + raise RuntimeError(f"small read count mismatch: {len(history)}") + + small_result = await self.measure( + "recovery small read (1 message check)", + _small, + iterations, + ) + + # 4. 再起動後の full read を iterations 回 + async def _full(_: int) -> None: + async with ServerContext(self.exe, data_dir) as session: + history = await read_messages(session, session_id) + if len(history) != 100: + raise RuntimeError(f"full read count mismatch: {len(history)}") + + full_result = await self.measure( + "recovery full read (100 msgs)", + _full, + iterations, + ) + + self.cleanup(data_dir) + return recovery_result, small_result, full_result + + async def run_schema_and_error_checks(self) -> dict[str, Any]: + data_dir = self.new_data_dir() + try: + async with ServerContext(self.exe, data_dir) as session: + tools = await list_tools(session) + tool_names = [t.get("name") for t in tools] + write_schema = next((t for t in tools if t.get("name") == "write_message"), {}) + write_props = write_schema.get("inputSchema", {}).get("properties", {}) + + return { + "tool_count": len(tools), + "tool_names": tool_names, + "write_message_properties": list(write_props.keys()), + "write_requires": write_schema.get("inputSchema", {}).get("required", []), + } + finally: + self.cleanup(data_dir) + + def check_jsonrpc_error_response(self) -> dict[str, Any]: + """subprocess で raw stdio し、JSON-RPC エラーレスポンスを検証する。""" + data_dir = self.new_data_dir() + try: + # 必須引数不足のリクエスト + request = json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "write_message", "arguments": {}}, + }, + separators=(",", ":"), + ) + proc = subprocess.Popen( + [str(self.exe), "--data-dir", str(data_dir)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + stdout, stderr = proc.communicate(input=request + "\n", timeout=10) + lines = [line.strip() for line in stdout.splitlines() if line.strip()] + if not lines: + return {"error_response_present": False, "stderr": stderr[:500]} + try: + response = json.loads(lines[-1]) + except json.JSONDecodeError as exc: + return { + "error_response_present": False, + "raw_last_line": lines[-1][:200], + "parse_error": str(exc), + } + has_jsonrpc_field = "jsonrpc" in response + has_error_field = "error" in response + return { + "error_response_present": has_error_field, + "has_jsonrpc_field": has_jsonrpc_field, + "has_error_field": has_error_field, + "error_code": response.get("error", {}).get("code") if has_error_field else None, + "raw_response": response, + } + except Exception as exc: # noqa: BLE001 + return {"error_response_present": False, "exception": str(exc)} + finally: + try: + proc.kill() + except Exception: # noqa: BLE001 + pass + self.cleanup(data_dir) + + +# --------------------------------------------------------------------------- +# レポート生成 +# --------------------------------------------------------------------------- +def print_report(report: BenchmarkReport) -> None: + print(f"\n=== tiny-convmem-mcp 負荷テスト結果 ===") + print(f"server : {report.server}") + print(f"build_type : {report.build_type}") + print(f"exe_mtime : {report.exe_mtime}") + print(f"iterations : {report.iterations}") + print(f"warmup : {report.warmup}") + print() + header = ( + f"{'scenario':<45} {'succ':>5} {'fail':>5} " + f"{'p50[ms]':>10} {'p95[ms]':>10} {'p99[ms]':>10} " + f"{'mean[ms]':>10} {'ops/s':>10}" + ) + print(header) + print("-" * len(header)) + for s in report.scenarios: + print( + f"{s.name:<45} {s.successes:>5} {s.failures:>5} " + f"{s.latency_p50_ms:>10.3f} {s.latency_p95_ms:>10.3f} {s.latency_p99_ms:>10.3f} " + f"{s.latency_mean_ms:>10.3f} {s.throughput_ops_per_sec:>10.2f}" + ) + if report.errors: + print("\nerrors:") + for e in report.errors: + print(f" - {e}") + if report.schema_check: + print("\nschema / error checks:") + for k, v in report.schema_check.items(): + print(f" {k}: {v}") + if report.health_after: + print("\nfinal health:") + for k, v in report.health_after.items(): + print(f" {k}: {v}") + + +def write_json(report: BenchmarkReport, path: Path) -> None: + path.write_text(json.dumps(report.as_dict(), indent=2, ensure_ascii=False), encoding="utf-8") + + +def write_markdown(report: BenchmarkReport, json_path: Path, md_path: Path) -> None: + lines: list[str] = [] + lines.append("# tiny-convmem-mcp 負荷テストレポート\n") + lines.append(f"- サーバ: `{report.server}`\n") + lines.append(f"- ビルド: {report.build_type}\n") + lines.append(f"- exe mtime: {report.exe_mtime}\n") + lines.append(f"- iterations: {report.iterations}\n") + lines.append(f"- warmup: {report.warmup}\n") + lines.append(f"- 計測日: {time.strftime('%Y-%m-%d %H:%M:%S')}\n") + lines.append(f"- raw JSON: `{json_path.name}`\n\n") + + lines.append("## レイテンシサマリ\n\n") + lines.append( + "| scenario | succ | fail | p50[ms] | p95[ms] | p99[ms] | mean[ms] | ops/s |\n" + ) + lines.append("|---|---|---|---|---|---|---|---|\n") + for s in report.scenarios: + lines.append( + f"| {s.name} | {s.successes} | {s.failures} | " + f"{s.latency_p50_ms:.3f} | {s.latency_p95_ms:.3f} | {s.latency_p99_ms:.3f} | " + f"{s.latency_mean_ms:.3f} | {s.throughput_ops_per_sec:.2f} |\n" + ) + lines.append("\n") + + if report.schema_check: + lines.append("## MCP 準拠 / エラーチェック\n\n") + lines.append("```json\n") + lines.append(json.dumps(report.schema_check, indent=2, ensure_ascii=False)) + lines.append("\n```\n\n") + + if report.health_after: + lines.append("## 最終 health\n\n") + lines.append("```json\n") + lines.append(json.dumps(report.health_after, indent=2, ensure_ascii=False)) + lines.append("\n```\n\n") + + if report.errors: + lines.append("## エラー\n\n") + for e in report.errors: + lines.append(f"- {e}\n") + + md_path.write_text("".join(lines), encoding="utf-8") + + +# --------------------------------------------------------------------------- +# コマンドライン +# --------------------------------------------------------------------------- +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="tiny-convmem-mcp ユーザ視点負荷テスト&ベンチマーク" + ) + parser.add_argument( + "--server", + type=Path, + default=DEFAULT_RELEASE_EXE if DEFAULT_RELEASE_EXE.exists() else None, + help="tiny-convmem-mcp.exe のパス(デフォルト: Release ビルド)", + ) + parser.add_argument( + "--iterations", + type=int, + default=DEFAULT_ITERATIONS, + help=f"各シナリオの反復回数(デフォルト {DEFAULT_ITERATIONS})", + ) + parser.add_argument( + "--warmup", + type=int, + default=WARMUP_COUNT, + help=f"ウォームアップ書き込み数(デフォルト {WARMUP_COUNT})", + ) + parser.add_argument( + "--output", + type=Path, + default=Path("benchmark_results.json"), + help="出力 JSON ファイル名(デフォルト: benchmark_results.json)", + ) + return parser.parse_args(argv) + + +async def amain(argv: list[str]) -> int: + args = parse_args(argv) + + exe = args.server + if not exe or not exe.exists(): + print(f"サーバ exe が見つかりません: {exe}") + return 1 + + exe_mtime = time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime(exe.stat().st_mtime) + ) + + print(f"server : {exe}") + print(f"mtime : {exe_mtime}") + print(f"iters : {args.iterations}") + + runner = ScenarioRunner(exe) + report = BenchmarkReport( + server=str(exe), + build_type="release" if "Release" in str(exe) else "debug", + exe_mtime=exe_mtime, + iterations=args.iterations, + warmup=args.warmup, + ) + + # シナリオ実行 + report.scenarios.append(await runner.run_write_small(args.iterations)) + report.scenarios.append(await runner.run_write_medium(args.iterations)) + report.scenarios.append(await runner.run_write_large(args.iterations)) + report.scenarios.append(await runner.run_write_with_tags(args.iterations)) + + report.scenarios.append(await runner.run_read(args.iterations)) + + search_global, search_scoped = await runner.run_search(args.iterations) + report.scenarios.append(search_global) + report.scenarios.append(search_scoped) + + report.scenarios.append(await runner.run_delete(args.iterations)) + + # rebuild は重いので回数を少なく + rebuild_iters = min(args.iterations, REBUILD_ITERATIONS) + report.scenarios.append(await runner.run_rebuild(rebuild_iters)) + + report.scenarios.append(await runner.run_health(args.iterations)) + + # recovery はプロセス起動/終了を伴うため回数を少なく + recovery_iters = min(args.iterations, REBUILD_ITERATIONS) + recovery_results = await runner.run_recovery(recovery_iters) + report.scenarios.extend(recovery_results) + + # schema / error checks + try: + schema_info = await runner.run_schema_and_error_checks() + jsonrpc_err = runner.check_jsonrpc_error_response() + report.schema_check = { + **schema_info, + "jsonrpc_error_response": jsonrpc_err, + } + except Exception as exc: # noqa: BLE001 + report.errors.append(f"schema check failed: {exc}") + + # 最終 health(独立したディレクトリ) + try: + data_dir = runner.new_data_dir() + async with ServerContext(exe, data_dir) as session: + for _ in range(10): + await write_message(session, "final", "final check") + report.health_after = await get_system_health(session) + runner.cleanup(data_dir) + except Exception as exc: # noqa: BLE001 + report.errors.append(f"final health failed: {exc}") + + print_report(report) + + json_path = args.output.resolve() + md_path = json_path.with_suffix(".md") + write_json(report, json_path) + write_markdown(report, json_path, md_path) + print(f"\nJSON report: {json_path}") + print(f"MD report : {md_path}") + + return 0 + + +def main() -> int: + return asyncio.run(amain(sys.argv[1:])) + + +if __name__ == "__main__": + sys.exit(main())
\ No newline at end of file diff --git a/scripts/python/benchmark_results.json b/scripts/python/benchmark_results.json new file mode 100644 index 0000000..70c5f84 --- /dev/null +++ b/scripts/python/benchmark_results.json @@ -0,0 +1,1221 @@ +{ + "server": "build\\windows-vs2026\\src\\Release\\tiny-convmem-mcp.exe", + "build_type": "release", + "exe_mtime": "2026-06-30 18:49:00", + "iterations": 100, + "warmup": 5, + "scenarios": [ + { + "name": "write_message (small ~20B)", + "iterations": 100, + "successes": 100, + "failures": 0, + "samples": [ + 0.377499993192032, + 0.37880000309087336, + 0.37930000689812005, + 0.3795999800786376, + 0.37960000918246806, + 0.3798999823629856, + 0.38040001527406275, + 0.38120002136565745, + 0.38120002136565745, + 0.38180002593435347, + 0.3828000044450164, + 0.3831000067293644, + 0.38339997990988195, + 0.38399998447857797, + 0.38519999361597, + 0.3865000035148114, + 0.3865000035148114, + 0.387699983548373, + 0.38800001493655145, + 0.38809998659417033, + 0.38819998735561967, + 0.3884999896399677, + 0.3887999919243157, + 0.3889000217895955, + 0.3893999964930117, + 0.3907000063918531, + 0.39090000791475177, + 0.39140001172199845, + 0.39190001552924514, + 0.39209998794831336, + 0.3921999887097627, + 0.39220001781359315, + 0.3924000193364918, + 0.3926999925170094, + 0.39280002238228917, + 0.39319999632425606, + 0.39339999784715474, + 0.3935999993700534, + 0.39370000013150275, + 0.39370000013150275, + 0.3938000008929521, + 0.3938000008929521, + 0.3938000008929521, + 0.3939000016544014, + 0.39400000241585076, + 0.394199974834919, + 0.39480000850744545, + 0.39480000850744545, + 0.3960000176448375, + 0.3964999923482537, + 0.3974999999627471, + 0.3974999999627471, + 0.3981000045314431, + 0.39909998304210603, + 0.3994000144302845, + 0.3994999860879034, + 0.4000000189989805, + 0.40049999370239675, + 0.4026000096928328, + 0.40309998439624906, + 0.40309998439624906, + 0.40439999429509044, + 0.4059000057168305, + 0.4059000057168305, + 0.4064000095240772, + 0.4079999926034361, + 0.4081999941263348, + 0.4093000025022775, + 0.4096000047866255, + 0.4107000131625682, + 0.4146999854128808, + 0.4169000021647662, + 0.41810001130215824, + 0.4227000172249973, + 0.4233999934513122, + 0.42699999175965786, + 0.4324999754317105, + 0.4336999845691025, + 0.4364999767858535, + 0.4375999851617962, + 0.440200004959479, + 0.45950000640004873, + 0.5039999959990382, + 0.5614999972749501, + 0.6695000047329813, + 0.6759999960195273, + 0.6922000029589981, + 0.6971000111661851, + 0.7052999862935394, + 0.751899991882965, + 0.7538000063505024, + 0.8036000072024763, + 0.8623999892733991, + 1.1503999994602054, + 1.2338999949861318, + 1.280700002098456, + 1.3685000012628734, + 1.7575999954715371, + 1.973899983568117, + 9.826800000155345 + ], + "throughput_ops_per_sec": 1696.8824880031802, + "latency_p50_ms": 0.3969999961555004, + "latency_p95_ms": 1.2362399953417478, + "latency_p99_ms": 2.0524289837340293, + "latency_min_ms": 0.377499993192032, + "latency_max_ms": 9.826800000155345, + "latency_mean_ms": 0.5893159998231567, + "total_duration_ms": 58.931599982315674, + "total_bytes_in": 2000, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "write_message (medium 1KiB)", + "iterations": 100, + "successes": 100, + "failures": 0, + "samples": [ + 0.8056999940890819, + 0.8100999984890223, + 0.8107999747153372, + 0.8131999929901212, + 0.8137999975588173, + 0.8156000112649053, + 0.8162000158336014, + 0.8164000173565, + 0.8188999781850725, + 0.8194000110961497, + 0.819500011857599, + 0.8214999979827553, + 0.8218000002671033, + 0.8222999749705195, + 0.822600006358698, + 0.823500013211742, + 0.8236000139731914, + 0.8238999871537089, + 0.8243999909609556, + 0.8249999955296516, + 0.8258000016212463, + 0.8268000092357397, + 0.8273999847006053, + 0.827599986223504, + 0.8299000037368387, + 0.8302000060211867, + 0.8316999883390963, + 0.8323999936692417, + 0.8329999982379377, + 0.8358000195585191, + 0.835899991216138, + 0.8376000041607767, + 0.8384000102523714, + 0.8393999887630343, + 0.8409000001847744, + 0.8412000024691224, + 0.8419999794568866, + 0.84299998707138, + 0.8470999891869724, + 0.8492999768350273, + 0.8534000080544502, + 0.8553999941796064, + 0.8567000040784478, + 0.8581999863963574, + 0.8596999978180975, + 0.8598999993409961, + 0.8613999816589057, + 0.8622000168543309, + 0.8679000020492822, + 0.8693000127095729, + 0.8707999950274825, + 0.8709999965503812, + 0.8719000034034252, + 0.8793000015430152, + 0.8824000251479447, + 0.8836000051815063, + 0.8919000101741403, + 0.8924999856390059, + 0.9094999986700714, + 0.9098000009544194, + 0.9155000152532011, + 0.915899989195168, + 0.9159999899566174, + 0.9193999867420644, + 0.9273000177927315, + 0.9369000035803765, + 0.9418000117875636, + 0.9563000057823956, + 0.9693999891169369, + 0.9743999980855733, + 0.9797000093385577, + 0.9907999774441123, + 0.9986000077333301, + 1.0103000095114112, + 1.018200011458248, + 1.0212999768555164, + 1.0246000019833446, + 1.0355999984312803, + 1.039100025082007, + 1.0526999831199646, + 1.056100009009242, + 1.1101000127382576, + 1.11420001485385, + 1.1374999885447323, + 1.1451999889686704, + 1.1455999920144677, + 1.1590999783948064, + 1.1867000139318407, + 1.1878000223077834, + 1.2843000004068017, + 1.302599994232878, + 1.3070999993942678, + 1.3109999999869615, + 1.3165000127628446, + 1.5024000022094697, + 1.510500005679205, + 1.6747000045143068, + 2.070499991532415, + 3.095799998845905, + 7.35470000654459 + ], + "throughput_ops_per_sec": 960.9231013956091, + "latency_p50_ms": 0.8700500038685277, + "latency_p95_ms": 1.5028050023829564, + "latency_p99_ms": 3.1383889989229137, + "latency_min_ms": 0.8056999940890819, + "latency_max_ms": 7.35470000654459, + "latency_mean_ms": 1.0406659997534007, + "total_duration_ms": 104.06659997534007, + "total_bytes_in": 102400, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "write_message (large 32KiB)", + "iterations": 100, + "successes": 100, + "failures": 0, + "samples": [ + 125.11520000407472, + 125.15000000712462, + 125.35799999022856, + 125.3632000007201, + 125.42379999649711, + 125.43210000148974, + 125.53900000057183, + 125.77370001235977, + 125.86850000661798, + 125.9147000091616, + 125.91969998902641, + 126.06720000621863, + 126.1858000070788, + 126.27499998779967, + 126.4798000047449, + 126.50889999349602, + 126.54379999730736, + 126.61420000949875, + 126.64339999901131, + 126.7128000035882, + 126.7682000179775, + 126.86439999379218, + 126.87880001612939, + 126.8860999844037, + 126.91540000378154, + 126.93329999456182, + 127.01460000243969, + 127.02749998425134, + 127.05469998763874, + 127.07289998070337, + 127.24030000390485, + 127.27080000331625, + 127.27389999781735, + 127.32390000019222, + 127.35260001500137, + 127.4136999854818, + 127.45729999733157, + 127.45970001560636, + 127.47339997440577, + 127.54390001646243, + 127.78029998298734, + 127.81939998967573, + 127.96220000018366, + 128.00500000594184, + 128.0274000018835, + 128.07460001204163, + 128.12179999309592, + 128.2068000000436, + 128.22879999293946, + 128.23929998558015, + 128.24489999911748, + 128.34130000555888, + 128.36620002053678, + 128.3730999857653, + 128.49940001615323, + 128.55559997842647, + 128.56750001083128, + 128.64119998994283, + 128.70810000458732, + 128.88639999437146, + 128.93649999750778, + 128.96880001062527, + 129.1536999924574, + 129.15849999990314, + 129.2675999866333, + 129.30149998283014, + 129.32770000770688, + 129.37650000094436, + 129.3846000044141, + 129.65940000140108, + 129.78940000175498, + 129.8231999971904, + 129.8806999984663, + 129.9128000100609, + 129.92979999398813, + 130.0004000077024, + 130.02430001506582, + 130.08550001541153, + 130.09509997209534, + 130.15910002286546, + 130.23970002541319, + 130.28650000342168, + 130.31850001425482, + 130.6365999917034, + 130.67969999974594, + 131.08530000317842, + 131.48440001532435, + 131.66660000570118, + 132.17470000381581, + 132.41049999487586, + 132.53900001291186, + 132.6581999892369, + 132.873099995777, + 132.99919999553822, + 133.32520000403747, + 133.37250001495704, + 133.39540001470596, + 133.44799997867085, + 133.55840000440367, + 133.7197999819182 + ], + "throughput_ops_per_sec": 7.774314812877668, + "latency_p50_ms": 128.2420999923488, + "latency_p95_ms": 133.32756500458345, + "latency_p99_ms": 133.5600140041788, + "latency_min_ms": 125.11520000407472, + "latency_max_ms": 133.7197999819182, + "latency_mean_ms": 128.62869900040096, + "total_duration_ms": 12862.869900040096, + "total_bytes_in": 3276800, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "write_message (with tags)", + "iterations": 100, + "successes": 100, + "failures": 0, + "samples": [ + 0.3721999819390476, + 0.3733999910764396, + 0.3758000093512237, + 0.3761000116355717, + 0.37650001468136907, + 0.37659998633898795, + 0.3767999878618866, + 0.37719999090768397, + 0.37779999547638, + 0.3803999861702323, + 0.3816999960690737, + 0.381799996830523, + 0.3831999783869833, + 0.38400001358240843, + 0.3843999875243753, + 0.3852000227198005, + 0.38579999818466604, + 0.3859999997075647, + 0.3859999997075647, + 0.38610000046901405, + 0.38660000427626073, + 0.3868000057991594, + 0.3870000073220581, + 0.3871000080835074, + 0.38740001036785543, + 0.38750001112930477, + 0.38759998278692365, + 0.38759998278692365, + 0.388599990401417, + 0.38869999116286635, + 0.38869999116286635, + 0.3887999919243157, + 0.3887999919243157, + 0.38889999268576503, + 0.38899999344721437, + 0.38910002331249416, + 0.38910002331249416, + 0.3892999957315624, + 0.38930002483539283, + 0.38949999725446105, + 0.38979999953880906, + 0.39000000106170774, + 0.39000000106170774, + 0.3902000025846064, + 0.3902000025846064, + 0.39030000334605575, + 0.39030000334605575, + 0.3904000041075051, + 0.39060000563040376, + 0.39110000943765044, + 0.3913000109605491, + 0.391699984902516, + 0.39220001781359315, + 0.39259999175556004, + 0.3929999948013574, + 0.39339999784715474, + 0.39339999784715474, + 0.3934999986086041, + 0.39370000013150275, + 0.3939000016544014, + 0.3944000054616481, + 0.39450000622309744, + 0.39480000850744545, + 0.3952000115532428, + 0.39579998701810837, + 0.39629999082535505, + 0.39709999691694975, + 0.3971999976783991, + 0.3999000182375312, + 0.4002000205218792, + 0.40049999370239675, + 0.4026000096928328, + 0.40329998591914773, + 0.40339998668059707, + 0.40400002035312355, + 0.4047999973408878, + 0.40550000267103314, + 0.4056000034324825, + 0.40610000723972917, + 0.4157000221312046, + 0.4163999983575195, + 0.4254999803379178, + 0.42699999175965786, + 0.44070000876672566, + 0.4676000098697841, + 0.48760001664049923, + 0.5067000165581703, + 0.5569999921135604, + 0.6055999838281423, + 0.609799986705184, + 0.6585999799426645, + 0.6604999944102019, + 0.6605999951716512, + 0.6743999838363379, + 0.6996000010985881, + 0.7108999998308718, + 0.7702000148128718, + 0.7889999833423644, + 1.1153999948874116, + 1.4324000221677125 + ], + "throughput_ops_per_sec": 2255.4830781503824, + "latency_p50_ms": 0.3912000101990998, + "latency_p95_ms": 0.7001650010352023, + "latency_p99_ms": 1.1185699951602162, + "latency_min_ms": 0.3721999819390476, + "latency_max_ms": 1.4324000221677125, + "latency_mean_ms": 0.44336400023894385, + "total_duration_ms": 44.336400023894385, + "total_bytes_in": 1400, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "read_messages (100 msgs)", + "iterations": 100, + "successes": 100, + "failures": 0, + "samples": [ + 0.31820000731386244, + 0.31939998734742403, + 0.31999999191612005, + 0.3208999987691641, + 0.3212000010535121, + 0.321599974995479, + 0.32240001019090414, + 0.3234000178053975, + 0.3234999894630164, + 0.32359999022446573, + 0.32589997863397, + 0.3259000077378005, + 0.32619998091831803, + 0.3271000168751925, + 0.3274999908171594, + 0.3276999923400581, + 0.32770002144388855, + 0.32820002525113523, + 0.32839999767020345, + 0.3285999991931021, + 0.3291000030003488, + 0.3294000052846968, + 0.3294000052846968, + 0.32970000756904483, + 0.32980000833049417, + 0.3299000090919435, + 0.3302999830339104, + 0.3305999853182584, + 0.33069998607970774, + 0.33080001594498754, + 0.33080001594498754, + 0.33099998836405575, + 0.33170002279803157, + 0.33290000283159316, + 0.33370000892318785, + 0.33370000892318785, + 0.3341000119689852, + 0.3341000119689852, + 0.33489998895674944, + 0.3352999920025468, + 0.33549999352544546, + 0.33569999504834414, + 0.3364000003784895, + 0.33650000113993883, + 0.3378999826963991, + 0.33820001408457756, + 0.33840001560747623, + 0.3384999872650951, + 0.3387999895494431, + 0.3391999925952405, + 0.33939999411813915, + 0.3395999956410378, + 0.3400999994482845, + 0.34080000477842987, + 0.34089997643604875, + 0.3412999794818461, + 0.3415999817661941, + 0.3422000154387206, + 0.34229998709633946, + 0.34240001696161926, + 0.3431999939493835, + 0.34329999471083283, + 0.34329999471083283, + 0.34389999927952886, + 0.3440000000409782, + 0.3451000084169209, + 0.3452999808359891, + 0.3457000129856169, + 0.34630001755431294, + 0.34680002136155963, + 0.3468999930191785, + 0.3476999991107732, + 0.34840000444091856, + 0.34890000824816525, + 0.34920001053251326, + 0.3512000257615, + 0.35650000791065395, + 0.3569999826140702, + 0.3603999793995172, + 0.361700018402189, + 0.3696999920066446, + 0.3711000026669353, + 0.37269998574629426, + 0.3735999925993383, + 0.38109999150037766, + 0.3845000173896551, + 0.39420000393874943, + 0.4026000096928328, + 0.4251000063959509, + 0.4325999761931598, + 0.4352999967522919, + 0.544899987289682, + 0.682699988828972, + 0.739200011594221, + 0.7657000096514821, + 0.8477999945171177, + 0.9815999947022647, + 1.0258000111207366, + 1.3758999994024634, + 2.2730999917257577 + ], + "throughput_ops_per_sec": 2471.63795559416, + "latency_p50_ms": 0.3392999933566898, + "latency_p95_ms": 0.7698050088947637, + "latency_p99_ms": 1.384871999325701, + "latency_min_ms": 0.31820000731386244, + "latency_max_ms": 2.2730999917257577, + "latency_mean_ms": 0.40458999981638044, + "total_duration_ms": 40.458999981638044, + "total_bytes_in": 0, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "search_messages (global, no session)", + "iterations": 100, + "successes": 100, + "failures": 0, + "samples": [ + 0.14349998673424125, + 0.14409999130293727, + 0.14450002345256507, + 0.1446000242140144, + 0.14489999739453197, + 0.14509999891743064, + 0.14540000120177865, + 0.145500001963228, + 0.145800004247576, + 0.14590000500902534, + 0.14590000500902534, + 0.14600000577047467, + 0.146100006531924, + 0.1461999781895429, + 0.14629997895099223, + 0.14630000805482268, + 0.14640000881627202, + 0.14640000881627202, + 0.14640000881627202, + 0.14669998199678957, + 0.14670001110062003, + 0.14670001110062003, + 0.1467999827582389, + 0.14680001186206937, + 0.14689998351968825, + 0.14709998504258692, + 0.1474000164307654, + 0.14750001719221473, + 0.1475999888498336, + 0.14779999037273228, + 0.14779999037273228, + 0.14799999189563096, + 0.14799999189563096, + 0.1480999926570803, + 0.14829999417997897, + 0.1483999949414283, + 0.1483999949414283, + 0.14840002404525876, + 0.14860002556815743, + 0.14869999722577631, + 0.14890002785250545, + 0.14899999951012433, + 0.14899999951012433, + 0.149200001033023, + 0.14960000407882035, + 0.14980000560171902, + 0.15020000864751637, + 0.15020000864751637, + 0.15020000864751637, + 0.15029998030513525, + 0.1503999810665846, + 0.15050001093186438, + 0.15079998411238194, + 0.15100001473911107, + 0.15140001778490841, + 0.15169999096542597, + 0.15199999324977398, + 0.15239999629557133, + 0.15239999629557133, + 0.15249999705702066, + 0.15259999781847, + 0.15259999781847, + 0.152900000102818, + 0.15300000086426735, + 0.1531000016257167, + 0.1531000016257167, + 0.15380000695586205, + 0.15380000695586205, + 0.15399997937493026, + 0.15399997937493026, + 0.15480001457035542, + 0.1548999862279743, + 0.15539999003522098, + 0.15559999155811965, + 0.15720000374130905, + 0.15740000526420772, + 0.1579000090714544, + 0.15819998225197196, + 0.16039999900385737, + 0.16160000814124942, + 0.1640999980736524, + 0.16450000111944973, + 0.1650999765843153, + 0.16529997810721397, + 0.16590001177974045, + 0.16640001558698714, + 0.16709999181330204, + 0.16709999181330204, + 0.16769999638199806, + 0.16820000018924475, + 0.17189999925903976, + 0.1742000167723745, + 0.17829998978413641, + 0.18599999020807445, + 0.18870001076720655, + 0.19990000873804092, + 0.22330001229420304, + 0.2859000233002007, + 0.7118000066839159, + 1.6556000045966357 + ], + "throughput_ops_per_sec": 5675.11122125873, + "latency_p50_ms": 0.15034998068585992, + "latency_p95_ms": 0.18926001066574824, + "latency_p99_ms": 0.7212380066630478, + "latency_min_ms": 0.14349998673424125, + "latency_max_ms": 1.6556000045966357, + "latency_mean_ms": 0.176208000339102, + "total_duration_ms": 17.6208000339102, + "total_bytes_in": 0, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "search_messages (scoped to session)", + "iterations": 100, + "successes": 100, + "failures": 0, + "samples": [ + 0.1453000004403293, + 0.14540000120177865, + 0.14590000500902534, + 0.146100006531924, + 0.1467999827582389, + 0.14680001186206937, + 0.14689998351968825, + 0.1469000126235187, + 0.14710001414641738, + 0.14720001490786672, + 0.1472999865654856, + 0.1472999865654856, + 0.1474000164307654, + 0.1474000164307654, + 0.14749998808838427, + 0.14749998808838427, + 0.1475999888498336, + 0.14769998961128294, + 0.14769998961128294, + 0.14779999037273228, + 0.14789999113418162, + 0.1480999926570803, + 0.14810002176091075, + 0.14819999341852963, + 0.1483999949414283, + 0.1483999949414283, + 0.14840002404525876, + 0.14860002556815743, + 0.14869999722577631, + 0.14869999722577631, + 0.148899998748675, + 0.14930000179447234, + 0.14930000179447234, + 0.14930000179447234, + 0.14940000255592167, + 0.14960000407882035, + 0.14990000636316836, + 0.1500000071246177, + 0.15010000788606703, + 0.15010000788606703, + 0.1503000094089657, + 0.15050001093186438, + 0.15050001093186438, + 0.15059998258948326, + 0.1506999833509326, + 0.15079998411238194, + 0.15079998411238194, + 0.1509999856352806, + 0.15150001854635775, + 0.15159999020397663, + 0.15159999020397663, + 0.1516000193078071, + 0.15209999401122332, + 0.15269999857991934, + 0.152900000102818, + 0.15300000086426735, + 0.15300000086426735, + 0.15300000086426735, + 0.15350000467151403, + 0.15360000543296337, + 0.1537999778520316, + 0.15399997937493026, + 0.15410000924021006, + 0.15429998165927827, + 0.15440001152455807, + 0.1545000122860074, + 0.15460001304745674, + 0.15469998470507562, + 0.15509998775087297, + 0.15550001990050077, + 0.15550001990050077, + 0.15559999155811965, + 0.15589999384246767, + 0.156299996888265, + 0.15669999993406236, + 0.15750000602565706, + 0.15839998377487063, + 0.15880001592449844, + 0.15899998834356666, + 0.16170000890269876, + 0.16309999045915902, + 0.16540000797249377, + 0.166199984960258, + 0.16960001084953547, + 0.18049997743219137, + 0.18440000712871552, + 0.19439999596215785, + 0.2193999825976789, + 0.21989998640492558, + 0.2463000128045678, + 0.25069998810067773, + 0.2605999761726707, + 0.2620999875944108, + 0.27439999394118786, + 0.4681999853346497, + 0.5210000090301037, + 0.6234000029508024, + 0.7620000105816871, + 0.8063999994192272, + 0.9846999892033637 + ], + "throughput_ops_per_sec": 5209.038750416192, + "latency_p50_ms": 0.15159999020397663, + "latency_p95_ms": 0.4708399865194222, + "latency_p99_ms": 0.8081829993170695, + "latency_min_ms": 0.1453000004403293, + "latency_max_ms": 0.9846999892033637, + "latency_mean_ms": 0.19197399902623147, + "total_duration_ms": 19.197399902623147, + "total_bytes_in": 0, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "delete_message", + "iterations": 100, + "successes": 100, + "failures": 0, + "samples": [ + 0.357700017048046, + 0.3607999824453145, + 0.361100013833493, + 0.3622999938670546, + 0.3623999946285039, + 0.36269999691285193, + 0.3628999984357506, + 0.36290002753958106, + 0.3630999999586493, + 0.3637000045273453, + 0.3640000068116933, + 0.36429997999221087, + 0.3643000090960413, + 0.364800012903288, + 0.36559998989105225, + 0.3657999914139509, + 0.36600002204068005, + 0.36669999826699495, + 0.3729000163730234, + 0.3732000186573714, + 0.37349999183788896, + 0.37429999792948365, + 0.3744999994523823, + 0.374700000975281, + 0.3752000047825277, + 0.3755999787244946, + 0.37609998253174126, + 0.3761000116355717, + 0.37629998405463994, + 0.3775999939534813, + 0.37819999852217734, + 0.37880000309087336, + 0.3794000076595694, + 0.3795000084210187, + 0.3795000084210187, + 0.3809999907389283, + 0.3816999960690737, + 0.38320000749081373, + 0.3836000105366111, + 0.3837000112980604, + 0.3837000112980604, + 0.38389998371712863, + 0.38469998980872333, + 0.3848000196740031, + 0.38640000275336206, + 0.3871000080835074, + 0.38740001036785543, + 0.38759998278692365, + 0.38949999725446105, + 0.3896999987773597, + 0.39010000182315707, + 0.3957000153604895, + 0.3966999938711524, + 0.39820000529289246, + 0.4026000096928328, + 0.40290001197718084, + 0.40350001654587686, + 0.4078000201843679, + 0.41259999852627516, + 0.41429998236708343, + 0.41719997534528375, + 0.42189998202957213, + 0.4236000240780413, + 0.4243000003043562, + 0.42570001096464694, + 0.4329000075813383, + 0.4335000121500343, + 0.4374999844003469, + 0.4382000188343227, + 0.45029999455437064, + 0.46370000927709043, + 0.4668000037781894, + 0.47259998973459005, + 0.4845999937970191, + 0.49330000183545053, + 0.4965999978594482, + 0.5030999891459942, + 0.5033000197727233, + 0.5043999990448356, + 0.5099999834783375, + 0.5201000021770597, + 0.5224999913480133, + 0.5225000204518437, + 0.523299997439608, + 0.5238000012468547, + 0.529799988726154, + 0.5446999857667834, + 0.5487999878823757, + 0.5502999993041158, + 0.5794999888166785, + 0.585700006922707, + 0.5951999919489026, + 0.6125000072643161, + 0.614999997196719, + 0.7118000066839159, + 0.7120000082068145, + 0.7795999990776181, + 0.8068000024650246, + 0.8276000153273344, + 0.918099976843223 + ], + "throughput_ops_per_sec": 2248.605859510455, + "latency_p50_ms": 0.3899000003002584, + "latency_p95_ms": 0.7118100067600608, + "latency_p99_ms": 0.8285050149424937, + "latency_min_ms": 0.357700017048046, + "latency_max_ms": 0.918099976843223, + "latency_mean_ms": 0.4447200009599328, + "total_duration_ms": 44.47200009599328, + "total_bytes_in": 0, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "rebuild_index", + "iterations": 5, + "successes": 5, + "failures": 0, + "samples": [ + 5.564399994909763, + 5.966299999272451, + 6.8909999972675, + 7.271500013303012, + 7.908600004157051 + ], + "throughput_ops_per_sec": 148.8015522583377, + "latency_p50_ms": 6.8909999972675, + "latency_p95_ms": 7.7811800059862435, + "latency_p99_ms": 7.88311600452289, + "latency_min_ms": 5.564399994909763, + "latency_max_ms": 7.908600004157051, + "latency_mean_ms": 6.720360001781955, + "total_duration_ms": 33.60180000890978, + "total_bytes_in": 0, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "get_system_health", + "iterations": 100, + "successes": 100, + "failures": 0, + "samples": [ + 0.09960000170394778, + 0.09960000170394778, + 0.09980000322684646, + 0.0999000039882958, + 0.10070001007989049, + 0.10070001007989049, + 0.10080001084133983, + 0.10080001084133983, + 0.10090001160278916, + 0.10119998478330672, + 0.10129998554475605, + 0.10129998554475605, + 0.10139998630620539, + 0.10149998706765473, + 0.10160001693293452, + 0.10179998935200274, + 0.10199999087490141, + 0.10200001997873187, + 0.10219999239780009, + 0.10239999392069876, + 0.1024999946821481, + 0.10269999620504677, + 0.10269999620504677, + 0.10289999772794545, + 0.10309999925084412, + 0.10309999925084412, + 0.10350000229664147, + 0.1039999770000577, + 0.10449998080730438, + 0.10450000991113484, + 0.10470001143403351, + 0.10499998461455107, + 0.1050999853760004, + 0.10540001676417887, + 0.10550001752562821, + 0.10560001828707755, + 0.1058999914675951, + 0.1060000213328749, + 0.10609999299049377, + 0.10609999299049377, + 0.10620002285577357, + 0.10679999832063913, + 0.10710000060498714, + 0.10740000288933516, + 0.10750000365078449, + 0.10830000974237919, + 0.10869998368434608, + 0.10939998901449144, + 0.10989999282173812, + 0.10989999282173812, + 0.1100999943446368, + 0.11130000348202884, + 0.11160000576637685, + 0.11210000957362354, + 0.11249998351559043, + 0.11270001414231956, + 0.1128000149037689, + 0.11309998808428645, + 0.11350002023391426, + 0.11419999646022916, + 0.11480000102892518, + 0.11530000483617187, + 0.11589998030103743, + 0.11610001092776656, + 0.11610001092776656, + 0.11679998715408146, + 0.1168999879155308, + 0.11749999248422682, + 0.11799999629147351, + 0.11860000086016953, + 0.11910000466741621, + 0.1196000084746629, + 0.12079998850822449, + 0.1210999907925725, + 0.12110001989640296, + 0.12119999155402184, + 0.12129999231547117, + 0.12139999307692051, + 0.12250000145286322, + 0.12389998300932348, + 0.12519999290816486, + 0.12569999671541154, + 0.12620000052265823, + 0.1264000020455569, + 0.12889999197795987, + 0.12919999426230788, + 0.13169998419471085, + 0.1328999933321029, + 0.1340000017080456, + 0.14310001279227436, + 0.15010000788606703, + 0.15830001211725175, + 0.16319999122060835, + 0.16820000018924475, + 0.17540002590976655, + 0.21830000332556665, + 0.2316000172868371, + 0.25749998167157173, + 0.7612000044900924, + 0.8997000113595277 + ], + "throughput_ops_per_sec": 7587.0807292005875, + "latency_p50_ms": 0.10999999358318746, + "latency_p95_ms": 0.17754502478055645, + "latency_p99_ms": 0.7625850045587874, + "latency_min_ms": 0.09960000170394778, + "latency_max_ms": 0.8997000113595277, + "latency_mean_ms": 0.1318029998219572, + "total_duration_ms": 13.18029998219572, + "total_bytes_in": 0, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "recovery (restart + count check)", + "iterations": 1, + "successes": 1, + "failures": 0, + "samples": [ + 20.377600012579933 + ], + "throughput_ops_per_sec": 49.073492432016465, + "latency_p50_ms": 20.377600012579933, + "latency_p95_ms": 20.377600012579933, + "latency_p99_ms": 20.377600012579933, + "latency_min_ms": 20.377600012579933, + "latency_max_ms": 20.377600012579933, + "latency_mean_ms": 20.377600012579933, + "total_duration_ms": 20.377600012579933, + "total_bytes_in": 0, + "total_bytes_out": 0, + "notes": "includes process shutdown + startup + initialize + read count" + }, + { + "name": "recovery small read (1 message check)", + "iterations": 5, + "successes": 5, + "failures": 0, + "samples": [ + 16.233500005910173, + 16.51690001017414, + 16.59829998970963, + 16.701499989721924, + 16.76729999599047 + ], + "throughput_ops_per_sec": 60.373713291427464, + "latency_p50_ms": 16.59829998970963, + "latency_p95_ms": 16.75413999473676, + "latency_p99_ms": 16.764667995739728, + "latency_min_ms": 16.233500005910173, + "latency_max_ms": 16.76729999599047, + "latency_mean_ms": 16.563499998301268, + "total_duration_ms": 82.81749999150634, + "total_bytes_in": 0, + "total_bytes_out": 0, + "notes": "" + }, + { + "name": "recovery full read (100 msgs)", + "iterations": 5, + "successes": 5, + "failures": 0, + "samples": [ + 15.85620001424104, + 15.893400006461889, + 16.1155000096187, + 16.868900012923405, + 18.851200002245605 + ], + "throughput_ops_per_sec": 59.81920240998151, + "latency_p50_ms": 16.1155000096187, + "latency_p95_ms": 18.454740004381165, + "latency_p99_ms": 18.771908002672717, + "latency_min_ms": 15.85620001424104, + "latency_max_ms": 18.851200002245605, + "latency_mean_ms": 16.717040009098127, + "total_duration_ms": 83.58520004549064, + "total_bytes_in": 0, + "total_bytes_out": 0, + "notes": "" + } + ], + "health_after": { + "status": "OK", + "format_version": 1, + "log_size": 920, + "snapshot_size": 0, + "active_messages": 10, + "deleted_messages": 0, + "last_sequence": 10, + "recovered_tail": false, + "snapshot_valid": false, + "last_error": "" + }, + "schema_check": { + "tool_count": 6, + "tool_names": [ + "write_message", + "read_messages", + "search_messages", + "delete_message", + "rebuild_index", + "get_system_health" + ], + "write_message_properties": [ + "session_id", + "role", + "content", + "metadata_json" + ], + "write_requires": [ + "session_id", + "role", + "content" + ], + "jsonrpc_error_response": { + "error_response_present": true, + "has_jsonrpc_field": true, + "has_error_field": true, + "error_code": -32601, + "raw_response": { + "jsonrpc": "2.0", + "id": 2, + "error": { + "code": -32601, + "message": "client not initialized" + } + } + } + }, + "errors": [] +}
\ No newline at end of file diff --git a/scripts/python/benchmark_results.md b/scripts/python/benchmark_results.md new file mode 100644 index 0000000..9d512a3 --- /dev/null +++ b/scripts/python/benchmark_results.md @@ -0,0 +1,85 @@ +# tiny-convmem-mcp 負荷テストレポート +- サーバ: `build\windows-vs2026\src\Release\tiny-convmem-mcp.exe` +- ビルド: release +- exe mtime: 2026-06-30 18:49:00 +- iterations: 100 +- warmup: 5 +- 計測日: 2026-06-30 18:49:14 +- raw JSON: `benchmark_results.json` + +## レイテンシサマリ + +| scenario | succ | fail | p50[ms] | p95[ms] | p99[ms] | mean[ms] | ops/s | +|---|---|---|---|---|---|---|---| +| write_message (small ~20B) | 100 | 0 | 0.397 | 1.236 | 2.052 | 0.589 | 1696.88 | +| write_message (medium 1KiB) | 100 | 0 | 0.870 | 1.503 | 3.138 | 1.041 | 960.92 | +| write_message (large 32KiB) | 100 | 0 | 128.242 | 133.328 | 133.560 | 128.629 | 7.77 | +| write_message (with tags) | 100 | 0 | 0.391 | 0.700 | 1.119 | 0.443 | 2255.48 | +| read_messages (100 msgs) | 100 | 0 | 0.339 | 0.770 | 1.385 | 0.405 | 2471.64 | +| search_messages (global, no session) | 100 | 0 | 0.150 | 0.189 | 0.721 | 0.176 | 5675.11 | +| search_messages (scoped to session) | 100 | 0 | 0.152 | 0.471 | 0.808 | 0.192 | 5209.04 | +| delete_message | 100 | 0 | 0.390 | 0.712 | 0.829 | 0.445 | 2248.61 | +| rebuild_index | 5 | 0 | 6.891 | 7.781 | 7.883 | 6.720 | 148.80 | +| get_system_health | 100 | 0 | 0.110 | 0.178 | 0.763 | 0.132 | 7587.08 | +| recovery (restart + count check) | 1 | 0 | 20.378 | 20.378 | 20.378 | 20.378 | 49.07 | +| recovery small read (1 message check) | 5 | 0 | 16.598 | 16.754 | 16.765 | 16.563 | 60.37 | +| recovery full read (100 msgs) | 5 | 0 | 16.116 | 18.455 | 18.772 | 16.717 | 59.82 | + +## MCP 準拠 / エラーチェック + +```json +{ + "tool_count": 6, + "tool_names": [ + "write_message", + "read_messages", + "search_messages", + "delete_message", + "rebuild_index", + "get_system_health" + ], + "write_message_properties": [ + "session_id", + "role", + "content", + "metadata_json" + ], + "write_requires": [ + "session_id", + "role", + "content" + ], + "jsonrpc_error_response": { + "error_response_present": true, + "has_jsonrpc_field": true, + "has_error_field": true, + "error_code": -32601, + "raw_response": { + "jsonrpc": "2.0", + "id": 2, + "error": { + "code": -32601, + "message": "client not initialized" + } + } + } +} +``` + +## 最終 health + +```json +{ + "status": "OK", + "format_version": 1, + "log_size": 920, + "snapshot_size": 0, + "active_messages": 10, + "deleted_messages": 0, + "last_sequence": 10, + "recovered_tail": false, + "snapshot_valid": false, + "last_error": "" +} +``` + diff --git a/scripts/python/p0_regression.py b/scripts/python/p0_regression.py new file mode 100644 index 0000000..61cd426 --- /dev/null +++ b/scripts/python/p0_regression.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""P0 MCP SDK interoperability and long-running connection regression test.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import shutil +import tempfile +from pathlib import Path + +from mcp.client.session import ClientSession +from mcp.client.stdio import StdioServerParameters, stdio_client +from mcp.types import CallToolResult + + +def payload(result: CallToolResult) -> dict: + if result.isError: + raise AssertionError(f"tool error: {result.content!r}") + texts = [item.text for item in result.content if item.type == "text"] + return json.loads("".join(texts)) + + +async def run(server: Path, iterations: int) -> None: + data_dir = Path(tempfile.mkdtemp(prefix="tiny-convmem-sdk-p0-")) + params = StdioServerParameters( + command=str(server.resolve()), + args=["--data-dir", str(data_dir)], + env=dict(os.environ), + ) + completed = 0 + with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as errlog: + try: + async with stdio_client(params, errlog=errlog) as (read_stream, write_stream): + async with ClientSession(read_stream, write_stream) as session: + await session.initialize() + listed = await session.list_tools() + assert {tool.name for tool in listed.tools} == { + "write_message", + "read_messages", + "search_messages", + "delete_message", + "rebuild_index", + "get_system_health", + } + for index in range(20): + payload( + await session.call_tool( + "write_message", + { + "session_id": "target", + "role": "user", + "content": f"sdk needle target {index}", + }, + ) + ) + for index in range(10): + payload( + await session.call_tool( + "write_message", + { + "session_id": "other", + "role": "user", + "content": f"sdk needle other {index}", + }, + ) + ) + + for index in range(iterations): + arguments = {"query": "needle"} + if index % 2: + arguments["session_id"] = "target" + results = payload( + await session.call_tool("search_messages", arguments) + )["results"] + assert len(results) == 20 + if index % 2: + assert all(item["session_id"] == "target" for item in results) + completed += 1 + errlog.flush() + errlog.seek(0) + server_stderr = errlog.read() + if "AddressSanitizer" in server_stderr or "ERROR:" in server_stderr: + raise AssertionError(f"server diagnostics reported a failure:\n{server_stderr}") + except Exception as exc: + errlog.flush() + errlog.seek(0) + raise RuntimeError( + f"failed after {completed}/{iterations} calls; server stderr:\n{errlog.read()}" + ) from exc + finally: + shutil.rmtree(data_dir, ignore_errors=True) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--server", type=Path, required=True) + parser.add_argument("--iterations", type=int, default=1000) + args = parser.parse_args() + asyncio.run(run(args.server, args.iterations)) + print(f"Python MCP SDK completed {args.iterations} alternating searches") + + +if __name__ == "__main__": + main() |
