From 5b063df8868ef8cdc2a6aab7a670f911bc0ea9a4 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Sat, 29 Aug 2026 09:15:24 +0900 Subject: feat: read_messagesに本文範囲取得viewを追加 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 大きなメッセージをMCPホストのtool result上限に触れず取得できるよう、read_messagesへcontent viewを追加する。UTF-8 byte単位のcontent_offset/content_limitとnext_content_offsetにより、文字境界を壊さず反復取得できる。 各chunkにrevisionとtotal_bytesを返し、expected_revisionで取得途中の更新をCONFLICTとして検出する。未知または削除済みmessageはNOT_FOUND、不正な範囲はVALIDATION_ERRORとする。読み取りには既存memory indexを利用し、永続data、revision、I/O counterを変更しない。 新規MCP toolは増やさず、従来の7 tool構成とhistory/context/sessionsの契約を維持する。storage recordとsnapshot形式も変更しないためmigrationとrebuildは不要。 README、設計書、MCPリファレンスを更新し、多言語UTF-8本文の完全再構成、境界・limit・revision・削除済みmessage、read-only I/O、既存view回帰をprotocol testで検証する。 Tests: ctest --test-dir build/range-read-test -C Release --output-on-failure (20/20 passed) --- README.md | 2 +- docs/Design.md | 12 +++ docs/mcp.md | 41 +++++++- src/mcp/tools/read_messages.c | 152 +++++++++++++++++++++++++++- tests/function-coverage.md | 2 +- tests/mcp_protocol_test.py | 228 +++++++++++++++++++++++++++++++++++++++++- 6 files changed, 428 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e86c8f9..20310b6 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ MCPクライアントのstdio server設定へ、実行ファイルと保存先 | Tool | 用途 | |---|---| | `write_message` | session、role、contentと任意metadataを新しいメッセージとして保存します。 | -| `read_messages` | session履歴、session一覧、対象メッセージ前後の文脈を取得します。 | +| `read_messages` | session履歴、本文の範囲、session一覧、対象メッセージ前後の文脈を取得します。 | | `search_messages` | content、summary、tagを検索し、score内訳や任意の前後文脈を返します。 | | `update_message` | IDや作成日時を維持したまま、本文またはmetadataを更新します。 | | `delete_message` | メッセージをread/search結果から除外します。物理除去はrebuild後です。 | diff --git a/docs/Design.md b/docs/Design.md index f370b74..b9a05c7 100644 --- a/docs/Design.md +++ b/docs/Design.md @@ -96,12 +96,24 @@ content または metadata を部分更新する。`expected_revision` が一致 次の view を持つ。 - `history`: session の時系列ページ。安定 cursor を返す +- `content`: message contentをUTF-8 byte offsetとbounded limitで分割取得する - `context`: target の前後メッセージと incoming/outgoing relation を返す。relation depth は 0～2 - `sessions`: session ごとの件数、kind 集計、代表 summary、最終時刻を返す - `export`: versioned JSONL をページ出力する互換 view レスポンスは 8 MiB budget を超えない位置でページを区切る。 +`content` viewは既定4096 bytes、最大65536 bytesを返す。offsetは0、末尾、またはUTF-8 +code point境界だけを受理し、limit位置がcode point途中なら直前の境界まで戻す。limitが +1 code pointより小さい場合だけ、そのcode pointを完全に返して進捗を保証する。各responseの +`revision`と`total_bytes`を返し、2回目以降の`expected_revision`で途中更新を`CONFLICT`として +検出する。このviewは永続data、access履歴、revisionを変更しない。message検索に必要な既存の +in-memory indexは、他のread viewと同様に必要時だけ構築する。 + +このview追加は既存toolへのadditive schema changeだけである。既存viewのinput、response、 +cursor semanticsと、保存recordおよびsnapshotのformat versionは変更しないため、data migration +とrebuildは不要であり、旧binaryと新binaryは同じdataを読み込める。 + ### `search_messages` content、summary、tags を対象に検索する。session、kind、tier、tag、minimum importance で絞り込める。matched fields、score と field 別 score 内訳、任意の前後 context を返す。 diff --git a/docs/mcp.md b/docs/mcp.md index 788fc84..e7b959d 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -143,13 +143,13 @@ messageのidentity、session、role、作成時刻を維持し、contentまた ## `read_messages` -`view`によりhistory、context、sessionsの3種類を返す。`view`省略時は`history`。 +`view`によりhistory、content、context、sessionsの4種類を返す。`view`省略時は`history`。 ### 共通Input | Parameter | 必須 | 型 | 説明 | |---|---:|---|---| -| `view` | No | string | `history` / `context` / `sessions` | +| `view` | No | string | `history` / `content` / `context` / `sessions` | | `limit` | No | integer | 1～1000、default 100 | | `cursor` | No | string | 前回responseの`nextCursor`。内容はopaque | @@ -174,6 +174,43 @@ session内のlive messageを作成順に返す。 続きを読む場合は同じ`session_id`と`nextCursor`を渡す。 +### `content` view + +live message 1件のcontentを、保存状態を変更せずにUTF-8 byte rangeで取得する。MCP hostの +tool-result上限を避けるため、`next_content_offset`を使って末尾まで反復する。 + +追加parameter: + +| Parameter | 必須 | 型 | Default | 説明 | +|---|---:|---|---:|---| +| `message_id` | Yes | string | — | 対象の26文字ULID | +| `content_offset` | No | integer | 0 | UTF-8 byte offset。0、末尾、またはcode point境界 | +| `content_limit` | No | integer | 4096 | 目標chunk bytes、1～65536 | +| `expected_revision` | No | integer | — | 指定時は現在revisionとの一致を要求 | + +```json +{ + "content_range": { + "message_id": "01ARZ3NDEKTSV4RRFFQ69G5FAV", + "revision": 1, + "total_bytes": 14420, + "content_offset": 4096, + "next_content_offset": 8191, + "complete": false, + "content": "..." + } +} +``` + +`complete`がfalseなら、同じ`message_id`、返された`next_content_offset`、最初のresponseで得た +`revision`を`expected_revision`として次を取得する。limit位置がUTF-8 code point途中なら、 +通常は直前の境界まで短くする。limitが1 code pointより小さい場合だけ、その1文字を完全に +返すため最大3 bytes超えることがある。 + +messageが存在しないか削除済みなら`NOT_FOUND`、offsetが範囲外またはUTF-8境界でなければ +`VALIDATION_ERROR`、`expected_revision`不一致なら`CONFLICT`を返す。このviewはlogical I/O +counterを含む永続状態を変更しない。 + ### `context` view 指定message、同一sessionの前後message、relation graphを返す。 diff --git a/src/mcp/tools/read_messages.c b/src/mcp/tools/read_messages.c index 131c513..e217aa2 100644 --- a/src/mcp/tools/read_messages.c +++ b/src/mcp/tools/read_messages.c @@ -15,6 +15,8 @@ #define READ_DEFAULT_LIMIT 100 #define READ_MAX_LIMIT 1000 #define READ_RESPONSE_BUDGET (8U * 1024U * 1024U - 64U * 1024U) +#define CONTENT_RANGE_DEFAULT_LIMIT 4096 +#define CONTENT_RANGE_MAX_LIMIT 65536 #define HISTORY_CURSOR_BYTES 47U #define HISTORY_CURSOR_TEXT_MAX 64U #define SESSION_CURSOR_BYTES_MAX (1U + 8U + 4U + TINY_SESSION_ID_MAX + 4U) @@ -26,6 +28,120 @@ typedef struct history_cursor char message_id[27]; } history_cursor_t; +static bool +utf8_boundary(const char *content, size_t length, size_t offset) +{ + return offset == 0 || offset == length || ((unsigned char)content[offset] & 0xc0U) != 0x80U; +} + +static void +call_content_view(tiny_mcp_tools_context_t *context, const mcpc_tool_t *tool, mcpc_ucbr_t *result) +{ + char *message_id = tiny_mcp_tool_get_string(tool, "message_id", TINY_MESSAGE_ID_LENGTH); + int32_t requested_offset = 0, requested_limit = CONTENT_RANGE_DEFAULT_LIMIT; + int32_t expected_revision = 0; + mcpc_errcode_t offset_status + = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"content_offset", &requested_offset); + mcpc_errcode_t limit_status + = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"content_limit", &requested_limit); + mcpc_errcode_t revision_status + = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"expected_revision", &expected_revision); + const tiny_store_entry_t *entry = NULL; + size_t entry_index = 0, content_length = 0, offset = 0, end = 0, chunk_length = 0; + char *chunk = NULL; + tiny_json_buffer_t json; + bool valid = context && message_id && tiny_message_id_valid(message_id) + && (offset_status == MCPC_EC_0 || offset_status == MCPC_EC_ANYDATA_NOTFOUND) + && requested_offset >= 0 + && (limit_status == MCPC_EC_0 || limit_status == MCPC_EC_ANYDATA_NOTFOUND) + && requested_limit >= 1 && requested_limit <= CONTENT_RANGE_MAX_LIMIT + && (revision_status == MCPC_EC_0 || revision_status == MCPC_EC_ANYDATA_NOTFOUND) + && (revision_status != MCPC_EC_0 || expected_revision >= 1); + if (!valid) + { + tiny_mcp_tool_add_error(result, + "VALIDATION_ERROR", + "message_id, content_offset, content_limit or expected_revision is " + "invalid"); + } + else if (!tiny_mcp_tools_ensure_memory_index(context)) + { + tiny_mcp_tool_add_error(result, "INTERNAL_ERROR", "message index allocation failed"); + } + else if (!tiny_memory_index_find(context->memory_index, message_id, &entry_index) + || !(entry = tiny_store_entry(context->store, entry_index)) || entry->deleted) + { + tiny_mcp_tool_add_error(result, "NOT_FOUND", "message was not found"); + } + else if (revision_status == MCPC_EC_0 && entry->message.revision != (uint64_t)expected_revision) + { + tiny_mcp_tool_add_error(result, "CONFLICT", "message revision does not match"); + } + else + { + content_length = strlen(entry->message.content); + offset = (size_t)requested_offset; + if (offset > content_length || !utf8_boundary(entry->message.content, content_length, offset)) + { + tiny_mcp_tool_add_error(result, + "VALIDATION_ERROR", + "content_offset is outside content or not a UTF-8 boundary"); + } + else + { + end = offset + (size_t)requested_limit; + if (end > content_length) + { + end = content_length; + } + while (end > offset && !utf8_boundary(entry->message.content, content_length, end)) + { + --end; + } + if (end == offset && end < content_length) + { + do + { + ++end; + } + while (end < content_length + && !utf8_boundary(entry->message.content, content_length, end)); + } + chunk_length = end - offset; + chunk = malloc(chunk_length + 1); + if (!chunk) + { + tiny_mcp_tool_add_error(result, "INTERNAL_ERROR", "out of memory"); + } + else + { + memcpy(chunk, entry->message.content + offset, chunk_length); + chunk[chunk_length] = '\0'; + tiny_json_init(&json); + tiny_json_raw(&json, "{\"content_range\":{\"message_id\":"); + tiny_json_string(&json, entry->message.message_id); + tiny_json_raw(&json, ",\"revision\":"); + tiny_json_u64(&json, entry->message.revision); + tiny_json_raw(&json, ",\"total_bytes\":"); + tiny_json_u64(&json, content_length); + tiny_json_raw(&json, ",\"content_offset\":"); + tiny_json_u64(&json, offset); + tiny_json_raw(&json, ",\"next_content_offset\":"); + tiny_json_u64(&json, end); + tiny_json_raw(&json, ",\"complete\":"); + tiny_json_raw(&json, end == content_length ? "true" : "false"); + tiny_json_raw(&json, ",\"content\":"); + tiny_json_string(&json, chunk); + tiny_json_raw(&json, "}}"); + tiny_mcp_tool_add_json(result, &json); + tiny_json_free(&json); + } + } + } + free(chunk); + free(message_id); +} + static uint64_t session_hash(const char *value) { @@ -800,6 +916,14 @@ call_read_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) const tiny_message_t *last = NULL; tiny_json_buffer_t json; + if (view && strcmp(view, "content") == 0) + { + call_content_view(context, tool, result); + free(view); + free(session); + free(cursor_text); + return; + } if (view && strcmp(view, "context") == 0) { call_context_view(context, tool, result); @@ -950,14 +1074,38 @@ tiny_mcp_register_read_messages(mcpc_server_t *server) call_read_messages); return tool && tiny_mcp_tool_add_property( - tool, "view", "Read view: history (default), context, or sessions.", MCPC_U8STR, false) + tool, + "view", + "Read view: history (default), content, context, or sessions.", + MCPC_U8STR, + false) && tiny_mcp_tool_add_property(tool, "session_id", "Conversation session whose chronological history to read.", MCPC_U8STR, false) + && tiny_mcp_tool_add_property(tool, + "message_id", + "Message ULID required by the content and context views.", + MCPC_U8STR, + false) + && tiny_mcp_tool_add_property(tool, + "content_offset", + "UTF-8 byte offset for the content view; defaults to 0.", + MCPC_I32, + false) && tiny_mcp_tool_add_property( - tool, "message_id", "Message ULID required by the context view.", MCPC_U8STR, false) + tool, + "content_limit", + "Target content chunk bytes from 1 to 65536; defaults to 4096.", + MCPC_I32, + false) + && tiny_mcp_tool_add_property( + tool, + "expected_revision", + "Optional positive revision for a consistent multi-call content read.", + MCPC_I32, + false) && tiny_mcp_tool_add_property( tool, "before", "Context messages before the target, from 0 to 10.", MCPC_I32, false) && tiny_mcp_tool_add_property( diff --git a/tests/function-coverage.md b/tests/function-coverage.md index 8976580..9f5808d 100644 --- a/tests/function-coverage.md +++ b/tests/function-coverage.md @@ -15,5 +15,5 @@ byte/encoding logic is exposed through internal headers and tested directly. | `domain/search.c` | `decode_codepoints`, `gram_hash`, `find_bucket`, `add_gram`, `index_text`, all `tiny_search_*` | `domain`: ASCII case folding, Unicode grams/tags, candidates, scoring and ownership | | `storage/store.c` | error/I/O helpers, ULID, entry growth/search, record codec, replay/snapshot/recovery, all `tiny_store_*` | `storage`, `lifecycle`: lock, append/reopen, corruption, rebuild rollback, Unicode path | | `mcp/json.c` | `reserve`, all `tiny_json_*` | `boundaries`: empty append, growth, control escaping, signed/unsigned integer limits, OOM | -| `mcp/server.c` | line/JSON/parser/result/search helpers, `handle_call`, `tiny_convmem_server_run` | `smoke`, `lifecycle`: initialize, list/call, malformed/EOF behavior, persistence | +| `mcp/server.c`, `mcp/tools/read_messages.c` | line/JSON/parser/result/search helpers, range read, `handle_call`, `tiny_convmem_server_run` | `smoke`, `lifecycle`, `protocol`: initialize, list/call, malformed/EOF behavior, persistence, UTF-8 range boundaries, revision conflict, read-only I/O | | `app/main.c` | `usage`, `run`, `main` | `requires-data-dir`, `smoke`: missing CLI argument, normal startup and EOF | diff --git a/tests/mcp_protocol_test.py b/tests/mcp_protocol_test.py index c206515..3efbca0 100644 --- a/tests/mcp_protocol_test.py +++ b/tests/mcp_protocol_test.py @@ -34,7 +34,11 @@ class Server: def send(self, message: Mapping[str, Any] | str) -> dict[str, Any]: assert self.process.stdin and self.process.stdout - line = message if isinstance(message, str) else json.dumps(message, separators=(",", ":")) + line = ( + message + if isinstance(message, str) + else json.dumps(message, separators=(",", ":"), ensure_ascii=False) + ) self.process.stdin.write(line + "\n") self.process.stdin.flush() response_line = self.process.stdout.readline() @@ -81,6 +85,7 @@ def tool_call(server: Server, request_id: int, name: str, arguments: Mapping[str def tool_payload(response: Mapping[str, Any], expect_error: bool = False) -> dict[str, Any]: + assert "result" in response, response result = response["result"] assert bool(result.get("isError", False)) is expect_error, response return json.loads(result["content"][0]["text"]) @@ -157,9 +162,12 @@ def main() -> None: "metadata_json": "Optional JSON object string containing tool_call_id, tags, memory_tier, memory_kind, importance, summary, relations, and/or source_message_ids; defaults to {}.", }, "read_messages": { - "view": "Read view: history (default), context, or sessions.", + "view": "Read view: history (default), content, context, or sessions.", "session_id": "Conversation session whose chronological history to read.", - "message_id": "Message ULID required by the context view.", + "message_id": "Message ULID required by the content and context views.", + "content_offset": "UTF-8 byte offset for the content view; defaults to 0.", + "content_limit": "Target content chunk bytes from 1 to 65536; defaults to 4096.", + "expected_revision": "Optional positive revision for a consistent multi-call content read.", "before": "Context messages before the target, from 0 to 10.", "after": "Context messages after the target, from 0 to 10.", "relation_depth": "Relation traversal depth from 0 to 2; defaults to 1.", @@ -195,6 +203,8 @@ def main() -> None: "content", ] assert tools["read_messages"]["inputSchema"].get("required", []) == [] + assert tools["read_messages"]["inputSchema"]["properties"]["content_offset"]["type"] == "integer" + assert tools["read_messages"]["inputSchema"]["properties"]["content_limit"]["type"] == "integer" assert tools["read_messages"]["inputSchema"]["properties"]["limit"]["type"] == "integer" assert tools["read_messages"]["inputSchema"]["properties"]["cursor"]["type"] == "string" assert tools["search_messages"]["inputSchema"]["required"] == ["query"] @@ -205,6 +215,218 @@ def main() -> None: next_id += 1 assert empty_health["session_count"] == 0 + range_content = ("Aあ𐍈B\n" * 1000) + "range-end" + range_message = tool_payload( + tool_call( + server, + next_id, + "write_message", + {"session_id": "range-read", "role": "assistant", "content": range_content}, + ) + )["message"] + next_id += 1 + range_health_before = tool_payload( + tool_call(server, next_id, "get_system_health", {}) + )["io"] + next_id += 1 + single_codepoint = tool_payload( + tool_call( + server, + next_id, + "read_messages", + { + "view": "content", + "message_id": range_message["message_id"], + "content_offset": 1, + "content_limit": 1, + }, + ) + )["content_range"] + next_id += 1 + assert single_codepoint["content"] == "あ" + assert single_codepoint["content_offset"] == 1 + assert single_codepoint["next_content_offset"] == 4 + assert single_codepoint["complete"] is False + reconstructed: list[str] = [] + range_offset = 0 + while True: + chunk = tool_payload( + tool_call( + server, + next_id, + "read_messages", + { + "view": "content", + "message_id": range_message["message_id"], + "content_offset": range_offset, + "content_limit": 4097, + "expected_revision": range_message["revision"], + }, + ) + )["content_range"] + next_id += 1 + assert chunk["content_offset"] == range_offset + assert chunk["revision"] == range_message["revision"] + assert chunk["total_bytes"] == len(range_content.encode("utf-8")) + assert chunk["next_content_offset"] > range_offset + reconstructed.append(chunk["content"]) + range_offset = chunk["next_content_offset"] + if chunk["complete"]: + break + assert "".join(reconstructed) == range_content + end_chunk = tool_payload( + tool_call( + server, + next_id, + "read_messages", + { + "view": "content", + "message_id": range_message["message_id"], + "content_offset": range_offset, + }, + ) + )["content_range"] + next_id += 1 + assert end_chunk["content"] == "" + assert end_chunk["complete"] is True + maximum_chunk = tool_payload( + tool_call( + server, + next_id, + "read_messages", + { + "view": "content", + "message_id": range_message["message_id"], + "content_limit": 65536, + }, + ) + )["content_range"] + next_id += 1 + assert maximum_chunk["content"] == range_content + assert maximum_chunk["complete"] is True + for arguments, expected_code in [ + ( + { + "view": "content", + "message_id": range_message["message_id"], + "content_offset": 2, + }, + "VALIDATION_ERROR", + ), + ( + { + "view": "content", + "message_id": range_message["message_id"], + "content_offset": len(range_content.encode("utf-8")) + 1, + }, + "VALIDATION_ERROR", + ), + ( + { + "view": "content", + "message_id": range_message["message_id"], + "content_offset": -1, + }, + "VALIDATION_ERROR", + ), + ( + { + "view": "content", + "message_id": range_message["message_id"], + "content_limit": 0, + }, + "VALIDATION_ERROR", + ), + ( + { + "view": "content", + "message_id": range_message["message_id"], + "expected_revision": 0, + }, + "VALIDATION_ERROR", + ), + ( + { + "view": "content", + "message_id": range_message["message_id"], + "content_limit": 65537, + }, + "VALIDATION_ERROR", + ), + ( + { + "view": "content", + "message_id": range_message["message_id"], + "expected_revision": range_message["revision"] + 1, + }, + "CONFLICT", + ), + ( + { + "view": "content", + "message_id": "01ARZ3NDEKTSV4RRFFQ69G5FAW", + }, + "NOT_FOUND", + ), + ]: + error = tool_payload( + tool_call(server, next_id, "read_messages", arguments), True + ) + next_id += 1 + assert error["error"]["code"] == expected_code + range_health_after = tool_payload( + tool_call(server, next_id, "get_system_health", {}) + )["io"] + next_id += 1 + assert range_health_after == range_health_before + range_history = tool_payload( + tool_call( + server, + next_id, + "read_messages", + {"view": "history", "session_id": "range-read"}, + ) + )["history"] + next_id += 1 + persisted_range_message = next( + item for item in range_history if item["message_id"] == range_message["message_id"] + ) + assert persisted_range_message["revision"] == range_message["revision"] + assert persisted_range_message["content"] == range_content + + deleted_range_message = tool_payload( + tool_call( + server, + next_id, + "write_message", + {"session_id": "range-read", "role": "user", "content": "delete me"}, + ) + )["message"] + next_id += 1 + assert tool_payload( + tool_call( + server, + next_id, + "delete_message", + {"message_id": deleted_range_message["message_id"]}, + ) + )["status"] == "OK" + next_id += 1 + deleted_range_error = tool_payload( + tool_call( + server, + next_id, + "read_messages", + { + "view": "content", + "message_id": deleted_range_message["message_id"], + }, + ), + True, + ) + next_id += 1 + assert deleted_range_error["error"]["code"] == "NOT_FOUND" + message_ids: list[str] = [] for index in range(20): payload = tool_payload( -- cgit v1.2.3