diff options
| author | tanaka takashi <isazaka@gmail.com> | 2026-06-29 02:36:40 +0900 |
|---|---|---|
| committer | tanaka takashi <isazaka@gmail.com> | 2026-06-29 02:36:40 +0900 |
| commit | 4c6a831aa2a57e2bcc20aa3fc2f1b7172f5be285 (patch) | |
| tree | 074021381abf57cad434430f25027115b681b8a9 | |
| parent | 8d38b06f11c713e547d70468340242f23322152c (diff) | |
feat: 会話メモリのプロトタイプを実装
- 6種類のMCPツールと入力検証を追加
- バイナリログ、スナップショット、復旧、圧縮処理を実装
- Unicode n-gram検索とドメインモデルを追加
- 単体テストと統合テストを追加
- アーキテクチャと実装状況を文書化
- Cコードの記述スタイルを統一
| -rw-r--r-- | .clang-format | 21 | ||||
| -rw-r--r-- | README.md | 96 | ||||
| -rw-r--r-- | docs/architecture.md | 363 | ||||
| -rw-r--r-- | docs/implementation-status.md | 227 | ||||
| -rw-r--r-- | src/CMakeLists.txt | 13 | ||||
| -rw-r--r-- | src/app/main.c | 48 | ||||
| -rw-r--r-- | src/domain/README.md | 11 | ||||
| -rw-r--r-- | src/domain/message.c | 255 | ||||
| -rw-r--r-- | src/domain/message.h | 60 | ||||
| -rw-r--r-- | src/domain/search.c | 350 | ||||
| -rw-r--r-- | src/domain/search.h | 22 | ||||
| -rw-r--r-- | src/mcp/README.md | 14 | ||||
| -rw-r--r-- | src/mcp/json.c | 137 | ||||
| -rw-r--r-- | src/mcp/json.h | 23 | ||||
| -rw-r--r-- | src/mcp/server.c | 763 | ||||
| -rw-r--r-- | src/mcp/server.h | 4 | ||||
| -rw-r--r-- | src/mcp/tools/health.c | 29 | ||||
| -rw-r--r-- | src/mcp/tools/health.h | 9 | ||||
| -rw-r--r-- | src/storage/README.md | 9 | ||||
| -rw-r--r-- | src/storage/store.c | 1111 | ||||
| -rw-r--r-- | src/storage/store.h | 64 | ||||
| -rw-r--r-- | tests/CMakeLists.txt | 39 | ||||
| -rw-r--r-- | tests/domain_test.c | 71 | ||||
| -rw-r--r-- | tests/fixtures/health.jsonl | 3 | ||||
| -rw-r--r-- | tests/fixtures/smoke-input.jsonl | 8 | ||||
| -rw-r--r-- | tests/fixtures/write-once.jsonl | 2 | ||||
| -rw-r--r-- | tests/lifecycle.cmake | 51 | ||||
| -rw-r--r-- | tests/smoke.cmake | 21 | ||||
| -rw-r--r-- | tests/storage_test.c | 97 |
29 files changed, 3785 insertions, 136 deletions
diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..e7e5aa3 --- /dev/null +++ b/.clang-format @@ -0,0 +1,21 @@ +--- +Language: Cpp +BasedOnStyle: GNU +ColumnLimit: 100 +IndentWidth: 2 +InsertBraces: true +ContinuationIndentWidth: 4 +UseTab: Never +BreakBeforeBraces: GNU +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortEnumsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: None +AllowShortLoopsOnASingleLine: false +BinPackArguments: false +BinPackParameters: false +SortIncludes: Never +SpaceBeforeParens: ControlStatements +... @@ -1,7 +1,8 @@ # tiny-convmem-mcp -LLMとの会話履歴を扱うための、小さなC言語製MCPサーバです。現在は設計前の -基盤段階で、stdioによるMCP初期化と`health` toolだけを実装しています。 +LLMの会話メッセージをローカルの追記ログへ保存する、Windows向けC17製MCP +サーバです。書込みと削除はディスクへの同期完了後に成功を返し、プロセス再起動時は +ログまたはsnapshotから状態を復旧します。 ## 必要環境 @@ -9,64 +10,81 @@ LLMとの会話履歴を扱うための、小さなC言語製MCPサーバですã - Visual Studio Community 2026(Desktop development with C++) - CMake 4.3以降 - Git -- vcpkg(PATHから`vcpkg`を実行できること) +- vcpkg(PATHから実行可能であること) -vcpkgはコンパイラを導入するためではなく、今後SQLite等のライブラリを管理する -ためにmanifest modeで使用します。現時点の`vcpkg.json`に外部依存はありません。 - -## 初回構築 - -PowerShellで次を実行します。スクリプトは不足ツールを自動インストールしません。 +## ビルドとテスト ```powershell git submodule update --init --recursive .\scripts\bootstrap.ps1 -Configuration Debug ``` -Releaseも確認する場合は次を実行します。 - -```powershell -.\scripts\bootstrap.ps1 -Configuration Release -``` - -## 手動ビルドとテスト - -presetを直接使う場合は、先にvcpkgルートを設定します。 +手動実行する場合は次のとおりです。 ```powershell $config = vcpkg z-print-config | ConvertFrom-Json $env:VCPKG_ROOT = $config.'vcpkg-root' - cmake --preset windows-vs2026 cmake --build --preset windows-vs2026-debug ctest --preset windows-vs2026-debug ``` -成果物は`build/windows-vs2026/`以下に生成されます。サーバはstdoutをMCP -プロトコル専用、stderrを診断メッセージ専用として使用します。 - -## mcpcの管理 +## 起動 -mcpcは`third_party/mcpc`のGit submoduleとしてコミット -`4025fddfedcfa7573803a47236b147733fcb3c07`へ固定しています。更新時は新しい -コミットを明示的にcheckoutし、Debug/Releaseのsmoke testを両方実行してください。 +保存先は必ず明示します。相対パスも使用できます。同じdata directoryを使用する +プロセスの二重起動は拒否されます。 ```powershell -git -C third_party/mcpc fetch origin -git -C third_party/mcpc checkout <reviewed-commit> -git add third_party/mcpc +.\build\windows-vs2026\src\Debug\tiny-convmem-mcp.exe --data-dir .\data ``` -固定コミットの上流CMakeには、欠落fixtureの参照、Makefileに対するソース列挙漏れ、 -MSVC互換マクロ名の不整合があります。そのため上流を変更せず、 -`cmake/mcpc.cmake`でMakefileと同じ13ソースを静的ライブラリ化しています。 +stdoutはMCP JSON-RPC専用、stderrは診断専用です。 + +## MCP tools + +- `write_message`: サーバ生成ULID・UTC timestamp付きメッセージを同期永続化 +- `read_messages`: sessionの有効メッセージを時系列順に取得 +- `search_messages`: contentとtagsをUTF-8コードポイントの2/3-gramで検索 +- `delete_message`: DELレコードを同期永続化して即時不可視化 +- `rebuild_index`: 有効PUTだけにcompactionしsnapshotを再生成 +- `get_system_health`: ログ、snapshot、復旧状態、件数を報告 -また、上流は同じ階層に`LICENSE`ファイルと`license/`ディレクトリを保持しており、 -Windowsでは名前が衝突します。`bootstrap.ps1`はライセンス本文を含む`license/`を -維持し、checkoutできないトップレベル`LICENSE`だけをローカルsubmodule indexで -`skip-worktree`に設定します。上流の追跡ファイルやコミットは変更しません。 +ツール結果はMCP TextContent内のJSON文字列です。入力schema、サイズ制限、エラーコード +および物理フォーマットは[architecture](docs/architecture.md)を参照してください。 +実装済み・未実装・テスト状況の追跡表は +[implementation status](docs/implementation-status.md)にまとめています。 -## 現在の境界 +## 永続化と復旧 -会話・メッセージのデータモデル、保持期限、検索方式、ストレージAPI、SQLite採用は -未決定です。責務の境界は[docs/architecture.md](docs/architecture.md)を参照してください。 +`active.log` はversion付きlittle-endianレコード、payload CRC32、単調sequenceを持ちます。 +PUT/DELは `FlushFileBuffers` が成功するまで `OK` を返しません。末尾で中断したレコード +だけは正常境界まで自動切詰めし、途中のCRC不一致はデータを保存したまま起動を拒否します。 + +`rebuild_index` は新しいgeneration markerを持つログを作成後、同じgenerationとログoffsetを +記録した `snapshot.bin` を置換します。起動時に両者が一致するとsnapshotをロードし、後続 +レコードだけを再生します。不一致または不正なsnapshotは変更せず、ログ全体を再生します。 + +削除は呼出時には論理削除です。元本文が `active.log` から除去されるのはcompaction成功時で、 +SSD、バックアップ等からの安全消去は保証しません。`memory_tier` は初期版では属性であり、 +自動要約・昇格・破棄を行いません。 + +## mcpcの管理 + +mcpc submoduleはコミット `4025fddfedcfa7573803a47236b147733fcb3c07` に固定しています。 +固定版tool APIではoptional/object/array schemaとcallback contextを表現できないため、 +プロジェクト側MCP互換境界がmcpc同梱のmjson parserを利用してraw argumentsとschemaを扱います。 +submoduleのソースは変更しません。詳細は[src/mcp/README.md](src/mcp/README.md)を参照してください。 + +## コーディングスタイル + +プロジェクト所有のCソースとヘッダーは、リポジトリ直下の +[`.clang-format`](.clang-format)に従います。制御文には常に波括弧を付け、1行には +1つのstatementだけを記述します。`third_party/` は整形対象外です。 + +Visual Studio同梱のclang-formatを使う場合は、PowerShellで次のように整形できます。 + +```powershell +$formatter = 'C:\Program Files\Microsoft Visual Studio\18\Community\VC\Tools\Llvm\bin\clang-format.exe' +$files = rg --files src tests -g '*.c' -g '*.h' +& $formatter -i --style=file $files +``` diff --git a/docs/architecture.md b/docs/architecture.md index f23f06a..edf2dfa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,9 +1,14 @@ # Architecture -`tiny-convmem-mcp` currently proves only the MCP integration boundary. The -executable composes an mcpc stdio server and registers a single `health` tool. +設計項目ごとの実装・テスト状況は +[設計・実装・テスト状況一覧](implementation-status.md)を参照してください。 -Future conversation-memory work is split into these dependencies: +`tiny-convmem-mcp` is a single-process Windows C17 MCP server backed by a +durable append-only conversation log. The executable owns the stdio dispatch +boundary and uses mcpc's bundled mjson parser through a local compatibility +layer. + +Conversation-memory work is split into these dependencies: ```text MCP transport and tools -> domain contracts <- storage adapters @@ -11,11 +16,353 @@ MCP transport and tools -> domain contracts <- storage adapters - `src/mcp` translates MCP requests and responses. It may depend on mcpc and domain contracts, but it must not contain persistence logic. -- `src/domain` will own conversation and message concepts without depending on +- `src/domain` owns conversation and message concepts without depending on mcpc or a database. -- `src/storage` will implement domain persistence contracts using a storage - engine selected in the next design phase. +- `src/storage` implements the versioned binary log, recovery, snapshot, and + compaction using Win32 file APIs. + +## Implemented prototype contract (v1) + +The six public tools are `write_message`, `read_messages`, `search_messages`, +`delete_message`, `rebuild_index`, and `get_system_health`. The detailed +"ProtoType Architecture v0.0.1" below is retained as the original proposal; +where it conflicts with this section, the implemented contract is authoritative. + +- `write_message` creates the ULID and UTC millisecond timestamp. A PUT or DEL + succeeds only after `FlushFileBuffers` succeeds. +- Deletion is immediately invisible but remains a tombstone until synchronous + compaction. `metadata.deleted` is not part of immutable Message data. +- Metadata is limited to optional `tool_call_id`, `tags`, and `memory_tier`. + Tiers are labels only; no automatic lifecycle operation is performed. +- Search uses ASCII-case-folded Unicode codepoint 2/3-grams, verifies hash + matches against the original codepoints, and ranks exact substrings first. +- A log record contains magic, format version, type, payload length, sequence, + and payload CRC32. PUT, DEL, and generation marker records are supported. +- A valid snapshot contains immutable messages and a log generation/offset. + Startup loads it only when the active log generation matches, then replays the + tail. Otherwise startup performs a full log replay. +- An incomplete final record is truncated. Unknown versions/types, invalid + lengths, non-monotonic sequences, or CRC errors within a complete record stop + startup without silently skipping data. +- `--data-dir <path>` is mandatory and guarded by a process-exclusive lock. + +The prototype remains Windows x64/MSVC and single-process. Retention, +encryption, pagination, multiple writers, automated summarization, and secure +physical erasure are outside v1. + +# ProtoType Architecture v0.0.1 +Tiny-ConvMem-MCP: FINAL TOOL INTERFACE SPECIFICATION (v1.0) + +この定義は、システムと外部(LLMプロセス)との間の「契約書」となります。これで実装に着手できますね! +No. Tool Name カテゴリ 機能概要 入力引数 (Arguments) 出力値 (Response) +1 write_message SET/WRITE 新しいメッセージイベントをログファイルへ追記し、インデックスを更新する。 session_id, role, content, timestamp status: Enum(OK, ERROR) +2 read_messages GET/READ 特定セッションIDの全メッセージを時系列順に読み出す。(フル履歴取得) session_id: 文字列 history: 配列(メッセージオブジェクト) +3 search_messages GET/READ クエリに基づいてログファイルから関連するメッセージを特定し、返す。(インデックス利用が前提) query, session_id (任意), limit: 整数 results: 配列(メッセージオブジェクト) +4 delete_message SET/MANAGE 特定のメッセージIDを指定してログから完全に削除し、インデックスをクリーンアップする。 message_id: 唯一ID, session_id (任意) status: Enum(OK, NOT_FOUND) +5 rebuild_index SET/MANAGE インデックスファイルを再生成し、データ整合性を回復させる。(メンテナンス) なし status: Enum(OK, ERROR) +6 get_system_health GET/READ システムの稼働状況とログファイルの健全性レポートを提供する。 なし report: JSON (ヘルスステータス) + + +データ構造設計:メッセージオブジェクト (Message Object Definition) +🎯 基本方針 + +全てのデータは**「不変(Immutable)」**であることを原則とします。一度記録されたイベントは変更せず、削除が必要な場合は新しいエントリで処理するか、専用の delete_message コマンドで対応します。 +📝 メッセージオブジェクト (Message) の構造定義案 + +メッセージを構成する要素を分解し、データ型を明確に定義します。(C++でのStructやJSONスキーマとして使用できます) + +{ + "message_id": "UUID/ULID文字列", // ★最重要。削除・参照のユニークキー + "session_id": "ユーザー固有のセッション識別子", + "timestamp": 1720843200000, // UTCミリ秒タイムスタンプ (UTC厳守) + "role": "ENUM (user | assistant | system)", // メッセージの発信元 + "content": "メッセージの本文(テキストデータ)", // 記録したい主要な情報 + "metadata": { // 追加情報格納用。拡張性を持たせるための辞書型 + "tool_call_id": "もしこのメッセージがツール呼び出しの結果ならIDを保持", + "is_system_message": true/false, // システム操作やプロンプト定義など + "checksum": "(オプション)データの完全性を保証するためのハッシュ値" + } +} + +⚙️ 補足的なデータ構造の定義 + +ログシステムをより堅牢にするために、メッセージとは別に必要な型も定義しておくと完璧です。 + + LogEntry (永続化層の最小単位): + 実際にディスクに書き込まれる最も小さいデータ単位。(上記 Message のバイナリ/JSON表現) + SearchIndexEntry (インデックス構造体): + 検索効率を最大化するための、非常に軽量なメタデータ。 + message_id: どのメッセージかを示すキー + session_id: 属するセッションID + keywords_hash: メッセージ内容から抽出されたキーワードのハッシュ値(またはリスト) + file_offset: ログファイル内での物理的な開始位置 + data_size: そのメッセージが占めるバイトサイズ + +📐 Tiny-ConMem-MCP: ストレージ制御 統合設計仕様 +1. アーキテクチャ概観 + +┌─────────────────────────────────────────────────────────────┐ +│ 外部プロセス (LLM Host) │ +└──────────────────────────┬──────────────────────────────────┘ + │ stdio (JSON-RPC) +┌──────────────────────────▼──────────────────────────────────┐ +│ L1: Protocol Layer (mcpc) │ +│ MCPメッセージの受信・ツールディスパッチ │ +└──────────────────────────┬──────────────────────────────────┘ + │ +┌──────────────────────────▼──────────────────────────────────┐ +│ L2: Memory Core (ビジネスロジック) │ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Write Queue │ │ In-Memory │ │ Search │ │ +│ │ (保留中キュー) │ │ Index │ │ Engine │ │ +│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ +│ │ │ │ │ +└─────────┼────────────────┼────────────────┼──────────────────┘ + │ │ │ +┌─────────▼────────────────▼────────────────▼──────────────────┐ +│ L3: Persistence Layer (永続化層) │ +│ │ +│ ┌──────────┐ ┌────────────┐ ┌───────────────────┐ │ +│ │ active.log│ │snapshot.bin│ │ compaction worker │ │ +│ │(append- │ │(状態保存) │ │(ログ整理・容量削減) │ │ +│ │ only) │ └────────────┘ └───────────────────┘ │ +│ └──────────┘ │ +└─────────────────────────────────────────────────────────────┘ + +2. データ構造定義 +2.1 Message Object(永続化対象の基本単位) + +全ツールの入力・出力の中心となるデータです。一度書き込まれたら不変(Immutable)で、変更は新エントリの追記で表現します。 + +// Message (ログ1レコードの論理構造) +{ + "message_id": "ULID文字列", // 削除・参照のユニークキー + "session_id": "セッション識別子", // セッション単位のグルーピング + "timestamp": 1720843200000, // UTC ミリ秒 + "role": "user | assistant | system", + "content": "メッセージ本文", + "metadata": { + "tool_call_id": "(任意)ツール呼び出しID", + "tags": ["カレー", "設計"], //(任意)検索用タグ + "memory_tier": "short | mid | long", // 後述のメモリ階層フラグ + "deleted": false // 論理削除フラグ + } +} + +2.2 LogRecord(ディスク上の物理フォーマット) + +active.log に追記される物理レコードです。Message Object をシリアライズしたものに、検証用のメタ情報を付加します。 + +┌──────────┬──────────┬─────────────────────┬──────────┐ +│ record │ payload │ payload (serialized │ checksum │ +│ length │ type │ Message Object) │ (CRC32) │ +│ (4 byte) │ (1 byte) │ (可変長) │ (4 byte) │ +└──────────┴──────────┴─────────────────────┴──────────┘ + +payload_type: + 0x01 = PUT (メッセージの書き込み) + 0x02 = DEL (メッセージの論理削除) + 0x03 = SNAPSHOT_MARKER (snapshot境界マーカー) + +2.3 In-Memory Index(メモリ上のインデックス) + +メモリ上に保持する高速検索用の目次です。これが SearchIndexEntry の実体で、search_messages / read_messages の高速化を担います。 + +struct IndexEntry { + uint64_t file_id; // ログファイル識別子 (1 = active.log 等) + uint64_t offset; // レコードの開始バイト位置 + uint32_t length; // レコード長 + int64_t timestamp; // メッセージのタイムスタンプ + uint8_t type; // PUT / DEL + // セッションID・メッセージID は + // ハッシュテーブルの Key として使用 +}; + +// メモリ上のデータ構造(ハッシュテーブル + リスト) +HashTable<message_id, IndexEntry*> // message_id → 位置情報 +HashTable<session_id, List<message_id>> // session_id → 所属メッセージ一覧 +HashTable<keyword_hash, List<message_id>> // 検索用キーワード逆引き + +3. 書き込みパイプライン(write_message / delete_message) + +┌─────────────┐ +│ MCP Tool 呼出 │ write_message / delete_message +└──────┬──────┘ + │ + ▼ +┌─────────────────────┐ +│ Step 1: メモリ更新 │ In-Memory Index に即座に反映 +│ │ (PUT → 追加, DEL → フラグ更新) +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Step 2: キュー追加 │ Write Queue に LogRecord を積む +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Step 3: バッチ書き込み │ flush_interval_ms (1000ms) または +│ │ max_pending_records (128件) に +│ │ 達したら write() で一括追記 +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Step 4: 定期 fsync │ fsync_interval_ms (5000ms) ごとに +│ │ fsync() でディスクに確実に落とす +└─────────────────────┘ + +設計パラメータ: + +#define FLUSH_INTERVAL_MS 1000 // キュー→ファイルの書き出し間隔 +#define FSYNC_INTERVAL_MS 5000 // fsync() の実行間隔 +#define MAX_PENDING_RECORDS 128 // キューの最大保留件数 + +4. 読み込みパイプライン(read_messages / search_messages) + +┌─────────────┐ +│ MCP Tool 呼出 │ read_messages / search_messages +└──────┬──────┘ + │ + ▼ +┌─────────────────────┐ +│ Step 1: インデックス │ In-Memory Index から該当レコードの +│ 検索 │ (file_id, offset, length) を取得 +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Step 2: pread() │ ログファイルの該当オフセットのみ +│ 部分読み出し │ pread(fd, buf, length, offset) +│ │ ※ログ全体は読み込まない +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Step 3: デシリアライズ│ LogRecord → Message Object に復元 +│ │ deleted フラグを確認し、 +│ │ 削除済みなら除外 +└──────┬──────────────┘ + │ + ▼ +┌─────────────────────┐ +│ Step 4: レスポンス │ JSON配列として MCP 経由で返却 +│ 返却 │ +└─────────────────────┘ + +5. 起動時復旧シーケンス + +プロセス起動 + │ + ▼ +┌──────────────────┐ +│ Step 1: │ +│ snapshot.bin 読込 │ 最後のスナップショットから +│ │ In-Memory Index を復元 +└──────┬───────────┘ + │ + ▼ +┌──────────────────┐ +│ Step 2: │ snapshot 作成時点以降の +│ ログ再生 │ active.log を先頭から順に読み、 +│ │ PUT / DEL レコードを適用 +│ │ (index を更新していく) +└──────┬───────────┘ + │ + ▼ +┌──────────────────┐ +│ Step 3: │ インデックスが完全に再構築された状態で +│ サービス開始 │ stdio 受付を開始 +└──────────────────┘ + +6. Snapshot 処理(rebuild_index の一部としても動作) + +現在有効なデータ (deleted=false) を列挙 + │ + ▼ +snapshot.tmp にシリアライズして書き出し + │ + ▼ +fsync(snapshot.tmp) + │ + ▼ +rename(snapshot.tmp → snapshot.bin) + │ ← アトミック操作。クラッシュしても古い snapshot.bin が残る + ▼ +完了 + +役割: 起動時のログ再生量を減らし、起動時間を短縮する。 +7. Compaction 処理 + +トリガー: ログファイルサイズが閾値を超えた または rebuild_index 呼出 + │ + ▼ +現在有効なデータ (deleted=false) のみを列挙 + │ + ▼ +new_active.log に有効レコードのみを書き出し + │ + ▼ +fsync(new_active.log) + │ + ▼ +rename(new_active.log → active.log) + │ ← アトミック操作 + ▼ +古いログファイルを削除 + │ + ▼ +新しい snapshot.bin も生成 (compaction 後の状態を保存) + │ + ▼ +In-Memory Index の file_id / offset を更新 + +役割: ログ肥大化を防ぎ、ディスク使用量を削減する。 +8. メモリ階層とデータライフサイクル + +Message Object の metadata.memory_tier を用いて、LLM会話の寿命を管理します。 + +┌─────────────┬──────────────────────────┬──────────────────┐ +│ 階層 │ 内容 │ 保存場所 │ +├─────────────┼──────────────────────────┼──────────────────┤ +│ short (短期) │ 直近の会話メッセージ │ メモリ + active.log│ +│ │ │ (即座に参照可能) │ +├─────────────┼──────────────────────────┼──────────────────┤ +│ mid (中期) │ セッションの要約・抽象化 │ active.log + │ +│ │ │ snapshot │ +├─────────────┼──────────────────────────┼──────────────────┤ +│ long (長期) │ 重要なユーザー設定・ │ snapshot 優先で │ +│ │ 継続的記憶 (key-value) │ 永続化 │ +├─────────────┼──────────────────────────┼──────────────────┤ +│ discard │ 一時的な雑談・重複情報・ │ compaction 時に │ +│ (破棄対象) │ 古い全文ログ │ 削除対象 │ +└─────────────┴──────────────────────────┴──────────────────┘ + +ライフサイクル: -No conversation schema, retention policy, search behavior, or storage engine is -implied by the current scaffold. +write_message (memory_tier=short) + │ + │ セッション終了・要約トリガー + ▼ +要約生成 → write_message (memory_tier=mid) + │ 元の short メッセージを delete_message で論理削除 + ▼ +compaction 時に short の deleted レコードが物理削除される + │ + │ 長期的に重要と判断されたもの + ▼ +write_message (memory_tier=long) + │ snapshot 優先で永続化 + ▼ +discard レコードは compaction 時に消える +9. MCP Tool とストレージ層の対応表(最終版) +Tool L2処理 L3処理 関連コンポーネント +write_message Index更新 + Queue追加 flush時に active.log へ追記 Write Queue, In-Memory Index +read_messages session_id で Index 検索 pread() で該当レコード読み出し In-Memory Index, active.log +search_messages キーワードハッシュで Index 検索 pread() で該当レコード読み出し In-Memory Index, active.log +delete_message Index の type を DEL に更新 DEL レコードを active.log へ追記 Write Queue, In-Memory Index +rebuild_index 全有効データを再列挙 snapshot 作成 + compaction 実行 Snapshot, Compaction Worker +get_system_health 各種カウンタ収集 ログファイルサイズ・整合性チェック 全コンポーネント diff --git a/docs/implementation-status.md b/docs/implementation-status.md new file mode 100644 index 0000000..9812ac9 --- /dev/null +++ b/docs/implementation-status.md @@ -0,0 +1,227 @@ +# 設計・実装・テスト状況一覧 + +最終更新日: 2026-06-29 + +この文書は、[architecture.md](architecture.md) の初期案、確定したプロトタイプ実装計画、 +現在のソースコード、CTestを突き合わせた追跡表です。設計案に存在しても確定したv1の +対象外となった項目は、単純な未実装と区別しています。 + +## ステータスの読み方 + +| 表記 | 実装状況 | テスト作成状況 | テスト実施状況 | +|---|---|---|---| +| ✅ | 実装済 | 作成済 | 実施済・成功 | +| 🟡 | 部分実装、または設計との差異あり | 一部のみ作成済 | 一部条件のみ実施済 | +| ⬜ | 未実装 | 未作成 | 未実施 | +| ➖ | v1対象外、または採用しない設計 | 該当なし | 該当なし | + +「テスト作成済」はテストコードが存在すること、「実施済」は記載した構成で実際に成功した +ことを意味します。テスト済みでも、電源断などテスト環境で再現していない保証までは含みません。 + +## 現在の検証結果 + +| 構成 | ビルド | CTest | 実施日 | 結果 | +|---|---|---|---|---| +| Windows x64 / MSVC / Debug | ✅ 実施済 | ✅ 実施済 | 2026-06-29 | 5/5成功 | +| Windows x64 / MSVC / Release | ✅ 実施済 | ✅ 実施済 | 2026-06-29 | 5/5成功 | +| Windows以外 | ⬜ 未実施 | ⬜ 未実施 | ― | v1対象外 | + +実行コマンド: + +```powershell +cmake --build --preset windows-vs2026-debug +ctest --preset windows-vs2026-debug +cmake --build --preset windows-vs2026-release +ctest --preset windows-vs2026-release +``` + +## アーキテクチャとプロセス + +| 設計項目 | 実装 | 実装根拠・現在の挙動 | テスト作成 | テスト実施 | 補足・残作業 | +|---|---|---|---|---|---| +| C17単一プロセスMCPサーバ | ✅ 実装済 | `src/app/main.c` と `src/mcp/server.c` | ✅ 作成済 | ✅ Debug/Release | 単一stdioクライアント前提 | +| MCP、ドメイン、ストレージの分離 | 🟡 部分実装 | ディレクトリと型は分離済み | ⬜ 未作成 | ⬜ 未実施 | MCP層がStore APIを直接呼ぶ。専用Memory Core/Application Serviceは未実装 | +| Domainのmcpc/Win32非依存 | ✅ 実装済 | `message.*`, `search.*` は標準Cのみ | ✅ 作成済 | ✅ Debug/Release | 依存方向の自動検査は未作成 | +| Repository function table | ⬜ 未実装 | MCP層へ `tiny_store_t` を直接渡す | ⬜ 未作成 | ⬜ 未実施 | ストレージ差替え契約は未抽象化 | +| `--data-dir <path>` 必須化 | ✅ 実装済 | 相対パスを絶対化し、不足時は終了コード2 | ✅ 作成済 | ✅ Debug/Release | `--help` の自動テストは未作成 | +| data directory自動作成 | ✅ 実装済 | `CreateDirectoryW` | 🟡 間接テスト | ✅ Debug/Release | 深い親ディレクトリの再帰作成は未実装 | +| data directory排他ロック | ✅ 実装済 | `store.lock` を共有なしで保持 | ✅ 作成済 | ✅ Debug/Release | 複数writerはv1対象外 | +| stdoutをJSON-RPC専用、stderrを診断専用 | ✅ 実装済 | プロトコル応答と起動エラーを分離 | 🟡 間接テスト | ✅ Debug/Release | stderr汚染の専用テストは未作成 | +| Windows x64/MSVC対応 | ✅ 実装済 | Win32 file API、BCrypt、CMake preset | ✅ 作成済 | ✅ Debug/Release | Windows SDK差異のCIマトリクスは未作成 | + +## MCP互換境界と外部契約 + +| 設計項目 | 実装 | 実装根拠・現在の挙動 | テスト作成 | テスト実施 | 補足・残作業 | +|---|---|---|---|---|---| +| `initialize` / `tools/list` / `tools/call` | ✅ 実装済 | MCP 2024-11-05の必要フローをローカルdispatcherで処理 | ✅ 作成済 | ✅ Debug/Release | MCP全メソッド対応ではない | +| `notifications/initialized` | ✅ 実装済 | notificationとして応答せず継続 | 🟡 smokeに入力あり | ✅ Debug/Release | 単独の応答なし検証は未作成 | +| raw JSON arguments | ✅ 実装済 | mcpc同梱mjsonでobjectを直接解析 | ✅ 作成済 | ✅ Debug/Release | malformed JSON/JSON-RPCの網羅試験は未作成 | +| required/optional、object、array schema | ✅ 実装済 | `tools/list` に正しいschemaを返す | ✅ 作成済 | ✅ Debug/Release | schema全文のgolden testではなく主要断片を検証 | +| 64-bit timestamp/counter | ✅ 実装済 | JSON builderとバイナリ形式で64-bitを使用 | 🟡 間接テスト | ✅ Debug/Release | 最大/最小境界値は未作成 | +| callbackへのアプリ状態受渡し | ✅ 実装済 | server/handlerへStoreとSearch Indexを明示的に渡す | 🟡 間接テスト | ✅ Debug/Release | 汎用callback APIではない | +| upstream `server.c`/serializerのoverlay | ➖ 設計変更 | submoduleを変更せず、プロジェクト側dispatcherを実装 | ✅ 統合テスト | ✅ Debug/Release | 固定版mcpcのmjsonのみ利用する方針へ変更 | +| TextContent内JSON成功応答 | ✅ 実装済 | 全ツール結果をescape済みJSON文字列で返す | ✅ 作成済 | ✅ Debug/Release | `structuredContent` は未対応 | +| MCP application error (`isError=true`) | ✅ 実装済 | 検証/I/O/内部エラーで設定 | ✅ 検証エラー作成済 | ✅ Debug/Release | 全エラーコードのMCP経由試験は未作成 | +| 安定エラーコード | 🟡 部分実装 | `VALIDATION_ERROR`, `IO_ERROR`, `CORRUPT_STORE`, `BUSY`, `INTERNAL_ERROR` を定義・使用 | 🟡 一部作成済 | 🟡 一部実施済 | `BUSY`/`CORRUPT_STORE` は起動前に発生し、MCP応答にはならない | +| 不正JSON-RPCへの標準error object | ⬜ 未実装 | 一部の不正requestは応答せず無視する | ⬜ 未作成 | ⬜ 未実施 | parse error、invalid request、method not foundの標準応答が必要 | + +## MCPツール別状況 + +| Tool / 要件 | 実装 | テスト作成 | テスト実施 | 現在の確認範囲・不足 | +|---|---|---|---|---| +| `write_message` 基本書込み | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | MCPから書込み、応答Message、再起動後読込みを確認 | +| サーバ生成ULID | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | Crockford形式と同一プロセス内の単調順を確認 | +| サーバ生成UTCミリ秒 | ✅ 実装済 | 🟡 間接テスト | ✅ Debug/Release | 実時刻との許容差・時計巻戻り専用テストは未作成 | +| role検証 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 有効/無効値をDomainで確認 | +| metadata `tool_call_id` | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | round-tripと上限境界が未確認 | +| metadata `tags` | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 日本語tag検索をDomainで確認。32件/128 byte境界は未確認 | +| metadata `memory_tier` | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | enumとMCP入力を確認。自動ライフサイクルは対象外 | +| `read_messages` 全履歴取得 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 1件/0件を確認 | +| timestamp、message_id安定順序 | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 複数件・同一timestampの順序試験が必要 | +| `search_messages` content検索 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | ASCII大小文字とMCP検索を確認 | +| 日本語・tag検索 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | Domainで確認 | +| `session_id` 絞込み | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 複数sessionを使う統合試験が必要 | +| `limit` 既定20・最大100 | ✅ 実装済 | 🟡 一部作成済 | ✅ Debug/Release | limit=10は使用。既定値、0、100、101、非整数境界は未確認 | +| 検索順位とtie-break | ✅ 実装済 | 🟡 基本scoreのみ | 🟡 一部実施済 | 複数候補で exact match、timestamp、ID順を検証していない | +| `delete_message` 即時不可視化 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | lifecycle testでdelete後のreadから消えることを確認 | +| `delete_message` session照合 | ✅ 実装済 | 🟡 正しいsessionのみ | 🟡 一部実施済 | session不一致時のNOT_FOUND未確認 | +| `delete_message` NOT_FOUND | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 存在しないID、二重削除の統合試験が必要 | +| `rebuild_index` 同期compaction | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | MCP呼出しと削除本文の物理除去を確認 | +| rebuild結果カウンタ | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | `records_removed=1` を確認 | +| `get_system_health` 基本カウンタ | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | log/snapshot size、件数、sequenceを出力 | +| health `OK` | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | smokeで確認 | +| health `DEGRADED` | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 不正snapshot/末尾復旧後に確認 | +| health `ERROR` | ⬜ 未実装 | ⬜ 未作成 | ⬜ 未実施 | 致命的Store破損ではサーバが起動しないためtoolを呼べない | +| 復旧履歴 | 🟡 部分実装 | 🟡 一部作成済 | ✅ Debug/Release | `recovered_tail` と直近 `last_error` のみ。履歴配列は未実装 | + +## Messageと入力制約 + +| 設計項目 | 実装 | テスト作成 | テスト実施 | 補足・残作業 | +|---|---|---|---|---| +| immutable Messageモデル | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 更新APIを持たず、deep copyで所有権を分離 | +| `message_id` 26文字ULID | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 不正alphabet/先頭値を確認 | +| `session_id` 最大256 bytes | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 0、256、257 bytes境界が未確認 | +| `content` 最大1 MiB | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 1 MiB前後、長大JSON lineの統合試験が未作成 | +| `tool_call_id` 最大256 bytes | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 境界試験なし | +| tags最大32件、各128 bytes | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 境界試験なし | +| query最大4096 bytes | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 境界試験なし | +| UTF-8妥当性 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 基本的なvalid/overlong invalidを確認。全不正系列は未網羅 | +| `metadata.deleted` の排除 | ✅ 実装済 | 🟡 schemaで間接確認 | ✅ Debug/Release | 削除状態はStore entry/DELのみ | +| checksumを論理metadataから排除 | ✅ 実装済 | ➖ 該当なし | ➖ 該当なし | CRC32を物理recordで保持 | + +## 検索索引 + +| 設計項目 | 実装 | テスト作成 | テスト実施 | 補足・残作業 | +|---|---|---|---|---| +| Unicode codepoint 2/3-gram逆引き索引 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | contentとtagsを索引化 | +| ASCII case folding | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | `CURRY`/`curry` を確認 | +| 日本語検索 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | `設計` tagを確認 | +| hash衝突時の実codepoint照合 | ✅ 実装済 | 🟡 実装経路のみ | 🟡 間接実施 | 意図的なhash collision fixtureは未作成 | +| 1 codepoint query fallback | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 全候補からscore計算する経路が未確認 | +| 起動時索引構築 | ✅ 実装済 | 🟡 再起動で間接確認 | ✅ Debug/Release | 大量データ時の時間/メモリ試験なし | +| write時索引更新 | ✅ 実装済 | ✅ MCP検索で確認 | ✅ Debug/Release | index allocation失敗のfault injectionなし | +| compaction後索引再構築 | ✅ 実装済 | 🟡 rebuild後health/readのみ | 🟡 一部実施済 | rebuild直後のsearch専用試験が必要 | +| `message_id -> entry` hash map | ⬜ 未実装 | ⬜ 未作成 | ⬜ 未実施 | 現在は配列の線形探索 | +| `session_id -> message list` index | ⬜ 未実装 | ⬜ 未作成 | ⬜ 未実施 | 現在は全entry走査 | +| Unicode normalization/full case folding | ➖ v1対象外 | ➖ 該当なし | ➖ 該当なし | NFKC、非ASCII大文字小文字変換は未対応 | + +## バイナリログと耐久性 + +| 設計項目 | 実装 | テスト作成 | テスト実施 | 補足・残作業 | +|---|---|---|---|---| +| `active.log` append-only | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | compaction時のみ置換 | +| little-endian versioned header | ✅ 実装済 | 🟡 round-tripのみ | ✅ Debug/Release | byte単位golden fixtureは未作成 | +| magic/version/type/length/sequence/CRC32 | ✅ 実装済 | 🟡 CRC破損あり | ✅ Debug/Release | 各header fieldの単独異常試験は未作成 | +| PUT payload round-trip | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | write→close→reopenで確認 | +| DEL payload round-trip | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | delete→再起動→compactionで確認 | +| generation marker | ✅ 実装済 | 🟡 lifecycleで間接確認 | ✅ Debug/Release | marker単独のbinary testは未作成 | +| 64-bit file offset API | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 4 GiB超のsparse file試験は未作成 | +| record長上限とoverflow防止 | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 巨大length/整数overflow fixtureが必要 | +| PUT/DEL後の `FlushFileBuffers` | ✅ 実装済 | 🟡 API成功経路のみ | ✅ Debug/Release | flush失敗fault injection、実電源断は未実施 | +| 成功前のメモリ確保 | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | allocation failure注入は未作成 | +| OS書込み後・flush前クラッシュ | 🟡 コード上復旧可能 | ⬜ 未作成 | ⬜ 未実施 | プロセス強制終了/電源断試験が必要 | +| record offset/lengthだけを保持するIndexEntry | ⬜ 未実装 | ⬜ 未作成 | ⬜ 未実施 | 現在はMessage全体をメモリ保持 | +| offset指定部分読出し | ⬜ 未実装 | ⬜ 未作成 | ⬜ 未実施 | 現在のread/searchはメモリ上Messageを使用 | + +## 起動時復旧とsnapshot + +| 設計項目 | 実装 | テスト作成 | テスト実施 | 補足・残作業 | +|---|---|---|---|---| +| 空Store起動 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | smokeで確認 | +| ログ全再生 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | storage/lifecycleで再起動確認 | +| snapshot生成 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | compaction時にlive Messageを保存 | +| generation/CRC/log offset検証 | ✅ 実装済 | ✅ 不正snapshot作成済 | ✅ Debug/Release | generation不一致そのものの専用fixtureは未作成 | +| 有効snapshotから状態復元 | ✅ 実装済 | 🟡 snapshot生成のみ | 🟡 一部実施済 | 有効snapshotを使った再起動を明示判定するテストは未作成 | +| snapshot offset以降のtail再生 | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | snapshot後にPUT/DELして再起動するテストが必要 | +| 不正snapshotを無視して全ログ再生 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | lifecycleで不正内容へ置換して確認 | +| EOF末尾の不完全record切詰め | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | `TC` を追記して確認 | +| 完全recordのCRC不一致で起動失敗 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 最終complete recordを破損して確認 | +| ログ途中recordの破損 | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | 複数recordの中間を破損する専用試験が必要 | +| 未知version/type拒否 | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | binary fixture未作成 | +| 非単調sequence拒否 | ✅ 実装済 | ⬜ 未作成 | ⬜ 未実施 | binary fixture未作成 | +| snapshot作成中クラッシュ | 🟡 原子的置換で設計 | ⬜ 未作成 | ⬜ 未実施 | `snapshot.tmp` 各段階の強制終了試験が必要 | + +## Compactionと削除保証 + +| 設計項目 | 実装 | テスト作成 | テスト実施 | 補足・残作業 | +|---|---|---|---|---| +| DEL後の即時不可視化 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | readから除外されることを確認 | +| live PUTだけの新ログ生成 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 削除本文のbyte列が消えることを確認 | +| 一時ログのflush | ✅ 実装済 | 🟡 成功経路のみ | ✅ Debug/Release | Flush失敗注入は未作成 | +| `MoveFileExW` によるログ置換 | ✅ 実装済 | 🟡 成功経路のみ | ✅ Debug/Release | 置換失敗/直前クラッシュ試験は未作成 | +| ログ置換後のsnapshot置換 | ✅ 実装済 | 🟡 成功経路のみ | ✅ Debug/Release | 2ファイル間クラッシュ窓の強制終了試験は未作成 | +| compaction失敗時の旧Store維持 | 🟡 部分確認 | ✅ temp作成失敗を作成 | ✅ Debug/Release | replacement段階の失敗は未確認 | +| `rebuild_index` 手動トリガー | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | 同期実行中はstdio処理をブロック | +| ログサイズ閾値による自動compaction | ⬜ 未実装 | ⬜ 未作成 | ⬜ 未実施 | v1実装では手動rebuildのみ | +| background compaction worker | ➖ 採用しない | ➖ 該当なし | ➖ 該当なし | 単一スレッド同期方式へ確定変更 | +| SSD/バックアップからの安全消去 | ➖ v1対象外 | ➖ 該当なし | ➖ 該当なし | APIの物理削除保証には含めない | + +## メモリ階層とライフサイクル + +| 設計項目 | 実装 | テスト作成 | テスト実施 | 補足・残作業 | +|---|---|---|---|---| +| `short/mid/long/discard` 属性保存 | ✅ 実装済 | ✅ 作成済 | ✅ Debug/Release | tierは検索/削除ポリシーに影響しない | +| session終了トリガー | ➖ v1対象外 | ➖ 該当なし | ➖ 該当なし | 未実装 | +| LLM要約生成 | ➖ v1対象外 | ➖ 該当なし | ➖ 該当なし | 外部LLM連携なし | +| short→mid→long自動昇格 | ➖ v1対象外 | ➖ 該当なし | ➖ 該当なし | 未実装 | +| discard自動破棄 | ➖ v1対象外 | ➖ 該当なし | ➖ 該当なし | discardも明示deleteまで保持 | +| retention/TTL | ➖ v1対象外 | ➖ 該当なし | ➖ 該当なし | 未実装 | + +## テスト資産一覧 + +| CTest名 | 種別 | テスト作成 | Debug | Release | 主な対象 | +|---|---|---|---|---|---| +| `tiny-convmem-mcp.smoke` | MCP E2E | ✅ 作成済 | ✅ 成功 | ✅ 成功 | initialize、6 tool schema、write/read/search/rebuild/health、検証エラー | +| `tiny-convmem-mcp.domain` | Unit | ✅ 作成済 | ✅ 成功 | ✅ 成功 | role/tier、UTF-8、ULID、deep copy、ASCII/日本語n-gram | +| `tiny-convmem-mcp.storage` | Unit/Integration | ✅ 作成済 | ✅ 成功 | ✅ 成功 | lock、write/reopen、ULID順、temp失敗、CRC破損拒否 | +| `tiny-convmem-mcp.lifecycle` | MCP/Storage E2E | ✅ 作成済 | ✅ 成功 | ✅ 成功 | delete、不可視化、compaction、本文除去、不正snapshot、末尾復旧 | +| `tiny-convmem-mcp.requires-data-dir` | Process | ✅ 作成済 | ✅ 成功 | ✅ 成功 | data-dir必須、非0終了 | + +## 未実装・テスト不足の優先バックログ + +| 優先度 | 項目 | 現在の状態 | 完了条件 | +|---|---|---|---| +| P1 | 専用Memory Core/Application Service | ⬜ 未実装 | MCPがStoreへ直接依存せず、domain repository contract経由になる | +| P1 | message/session索引 | ⬜ 未実装 | ID削除とsession読込みが線形走査でなくなる | +| P1 | 有効snapshot+tail replay試験 | ⬜ 未作成・未実施 | snapshot後のPUT/DELだけがtailから適用されることを確認 | +| P1 | binary corruption matrix | 🟡 一部のみ | unknown version/type、巨大length、非単調sequence、中間CRC破損を自動試験 | +| P1 | 入力サイズ境界 | ⬜ 未作成・未実施 | 全上限の境界内成功/境界外拒否をMCP経由で確認 | +| P2 | read/searchの安定順序 | 🟡 実装済・試験不足 | 複数session/同時刻/同score/limitを網羅 | +| P2 | compaction crash/failure matrix | 🟡 一部のみ | flush、rename前後、log/snapshot間の各停止点から復旧 | +| P2 | 4 GiB超offset | ⬜ 未作成・未実施 | sparse logで64-bit seek/replayを確認 | +| P2 | JSON-RPC標準エラー | ⬜ 未実装 | malformed/invalid/method-not-foundへ規格準拠errorを返す | +| P3 | ログサイズ閾値compaction | ⬜ 未実装 | 設定閾値超過時に安全に同期compactionする | +| P3 | 性能・メモリ・長時間試験 | ⬜ 未作成・未実施 | 大量message、巨大session、繰返しrebuildの基準値を定義・測定 | + +## 明示的なv1対象外 + +| 項目 | 状態 | 理由 | +|---|---|---| +| 非同期write queue、定期flush/fsync | ➖ 非採用 | `OK` 後の消失を避ける同期耐久性を優先 | +| background worker | ➖ 非採用 | 単一stdio・単一スレッドの初期実装に限定 | +| 自動要約、tier昇格、discard削除 | ➖ 対象外 | LLMライフサイクルは6 tool完成後の別機能 | +| encryption at rest | ➖ 対象外 | 鍵管理を含む別設計が必要 | +| pagination | ➖ 対象外 | `read_messages` は現状フル履歴契約 | +| 複数writer/network server | ➖ 対象外 | data directoryを単一プロセスが排他所有 | +| SSD/バックアップの安全消去 | ➖ 対象外 | filesystem compactionだけでは保証不能 | +| Linux/macOS | ➖ 対象外 | Win32 durability/atomic replacementを採用 | diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef54a08..e7768d7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,14 +1,19 @@ add_executable(tiny-convmem-mcp app/main.c + domain/message.c + domain/search.c + storage/store.c + mcp/json.c mcp/server.c - mcp/tools/health.c ) -target_include_directories(tiny-convmem-mcp PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") -target_link_libraries(tiny-convmem-mcp PRIVATE mcpc::mcpc) +target_include_directories(tiny-convmem-mcp PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}" + "${PROJECT_SOURCE_DIR}/third_party/mcpc/src" +) +target_link_libraries(tiny-convmem-mcp PRIVATE mcpc::mcpc bcrypt) target_compile_features(tiny-convmem-mcp PRIVATE c_std_17) if(MSVC) target_compile_options(tiny-convmem-mcp PRIVATE /utf-8 /W4) endif() - diff --git a/src/app/main.c b/src/app/main.c index ab8688f..ce43b61 100644 --- a/src/app/main.c +++ b/src/app/main.c @@ -1,15 +1,57 @@ +#define WIN32_LEAN_AND_MEAN +#include <windows.h> + #include <stdio.h> +#include <stdlib.h> +#include <wchar.h> #include "mcp/server.h" +#include "storage/store.h" + +static void +usage(FILE *stream) +{ + fwprintf(stream, L"Usage: tiny-convmem-mcp --data-dir <path>\n"); +} int -main(void) +wmain(int argc, wchar_t **argv) { - const int result = tiny_convmem_server_run(stdin, stdout); + wchar_t *absolute = NULL; + DWORD required; + tiny_store_t *store = NULL; + tiny_store_status_t status; + int result; + if (argc == 2 && wcscmp(argv[1], L"--help") == 0) + { + usage(stdout); + return 0; + } + if (argc != 3 || wcscmp(argv[1], L"--data-dir") != 0 || argv[2][0] == L'\0') + { + usage(stderr); + return 2; + } + required = GetFullPathNameW(argv[2], 0, NULL, NULL); + if (required == 0 || (absolute = malloc((size_t)required * sizeof(wchar_t))) == NULL + || GetFullPathNameW(argv[2], required, absolute, NULL) == 0) + { + fprintf(stderr, "unable to resolve --data-dir\n"); + free(absolute); + return 2; + } + status = tiny_store_open(&store, absolute); + free(absolute); + if (status != TINY_STORE_OK) + { + fprintf(stderr, "unable to open conversation store: %s\n", tiny_store_status_name(status)); + return status == TINY_STORE_BUSY ? 3 : 1; + } + result = tiny_convmem_server_run(stdin, stdout, store); + tiny_store_close(store); if (result != 0) { fprintf(stderr, "tiny-convmem-mcp stopped with error code %d\n", result); } return result; } - diff --git a/src/domain/README.md b/src/domain/README.md index 0d359af..a30039e 100644 --- a/src/domain/README.md +++ b/src/domain/README.md @@ -1,8 +1,11 @@ # Domain layer -This directory is reserved for the conversation-memory domain model and its -business rules. Conversation, message, retention, and search contracts will be -defined here only after the next design phase. +`message.h` defines immutable Message and metadata concepts, role/tier enums, +input limits, deep-copy ownership, and UTF-8 validation. This layer has no mcpc +or Win32 dependency. -The domain layer must not depend on mcpc or on a concrete storage engine. +`search.h` owns the ASCII-case-folded Unicode codepoint 2/3-gram inverted index, +collision verification, candidate selection, and deterministic relevance score. +`memory_tier` is descriptive in v1. Retention, summarization, and automatic tier +transitions are intentionally not domain behaviors yet. diff --git a/src/domain/message.c b/src/domain/message.c new file mode 100644 index 0000000..d973be4 --- /dev/null +++ b/src/domain/message.c @@ -0,0 +1,255 @@ +#include "domain/message.h" + +#include <stdlib.h> +#include <string.h> + +static char * +duplicate_string(const char *value) +{ + const size_t length = strlen(value); + char *copy = malloc(length + 1); + if (copy != NULL) + { + memcpy(copy, value, length + 1); + } + return copy; +} + +void +tiny_message_init(tiny_message_t *message) +{ + memset(message, 0, sizeof(*message)); + message->metadata.memory_tier = TINY_TIER_SHORT; +} + +void +tiny_message_free(tiny_message_t *message) +{ + size_t index; + free(message->session_id); + free(message->content); + free(message->metadata.tool_call_id); + for (index = 0; index < message->metadata.tag_count; ++index) + { + free(message->metadata.tags[index]); + } + free(message->metadata.tags); + tiny_message_init(message); +} + +int +tiny_message_copy(tiny_message_t *destination, const tiny_message_t *source) +{ + size_t index; + tiny_message_init(destination); + memcpy(destination->message_id, source->message_id, sizeof(destination->message_id)); + destination->timestamp_ms = source->timestamp_ms; + destination->role = source->role; + destination->metadata.memory_tier = source->metadata.memory_tier; + destination->session_id = duplicate_string(source->session_id); + destination->content = duplicate_string(source->content); + if (destination->session_id == NULL || destination->content == NULL) + { + goto failure; + } + if (source->metadata.tool_call_id != NULL) + { + destination->metadata.tool_call_id = duplicate_string(source->metadata.tool_call_id); + if (destination->metadata.tool_call_id == NULL) + { + goto failure; + } + } + if (source->metadata.tag_count > 0) + { + destination->metadata.tags = calloc(source->metadata.tag_count, sizeof(char *)); + if (destination->metadata.tags == NULL) + { + goto failure; + } + for (index = 0; index < source->metadata.tag_count; ++index) + { + destination->metadata.tags[index] = duplicate_string(source->metadata.tags[index]); + if (destination->metadata.tags[index] == NULL) + { + goto failure; + } + destination->metadata.tag_count++; + } + } + return 0; + +failure: + tiny_message_free(destination); + return -1; +} + +const char * +tiny_role_name(tiny_role_t role) +{ + switch (role) + { + case TINY_ROLE_USER: + return "user"; + case TINY_ROLE_ASSISTANT: + return "assistant"; + case TINY_ROLE_SYSTEM: + return "system"; + default: + return "unknown"; + } +} + +bool +tiny_role_parse(const char *value, tiny_role_t *role) +{ + if (strcmp(value, "user") == 0) + { + *role = TINY_ROLE_USER; + } + else if (strcmp(value, "assistant") == 0) + { + *role = TINY_ROLE_ASSISTANT; + } + else if (strcmp(value, "system") == 0) + { + *role = TINY_ROLE_SYSTEM; + } + else + { + return false; + } + return true; +} + +const char * +tiny_tier_name(tiny_memory_tier_t tier) +{ + switch (tier) + { + case TINY_TIER_SHORT: + return "short"; + case TINY_TIER_MID: + return "mid"; + case TINY_TIER_LONG: + return "long"; + case TINY_TIER_DISCARD: + return "discard"; + default: + return "unknown"; + } +} + +bool +tiny_tier_parse(const char *value, tiny_memory_tier_t *tier) +{ + if (strcmp(value, "short") == 0) + { + *tier = TINY_TIER_SHORT; + } + else if (strcmp(value, "mid") == 0) + { + *tier = TINY_TIER_MID; + } + else if (strcmp(value, "long") == 0) + { + *tier = TINY_TIER_LONG; + } + else if (strcmp(value, "discard") == 0) + { + *tier = TINY_TIER_DISCARD; + } + else + { + return false; + } + return true; +} + +bool +tiny_utf8_valid(const char *value, size_t length) +{ + size_t index = 0; + while (index < length) + { + const unsigned char first = (unsigned char)value[index++]; + unsigned needed; + uint32_t codepoint; + if (first < 0x80) + { + if (first == 0) + { + return false; + } + continue; + } + if ((first & 0xe0) == 0xc0) + { + needed = 1; + codepoint = first & 0x1f; + } + else if ((first & 0xf0) == 0xe0) + { + needed = 2; + codepoint = first & 0x0f; + } + else if ((first & 0xf8) == 0xf0) + { + needed = 3; + codepoint = first & 0x07; + } + else + { + return false; + } + if (index + needed > length) + { + return false; + } + while (needed-- > 0) + { + const unsigned char next = (unsigned char)value[index++]; + if ((next & 0xc0) != 0x80) + { + return false; + } + codepoint = (codepoint << 6) | (next & 0x3f); + } + if (codepoint > 0x10ffff || (codepoint >= 0xd800 && codepoint <= 0xdfff)) + { + return false; + } + if ((first & 0xe0) == 0xc0 && codepoint < 0x80) + { + return false; + } + if ((first & 0xf0) == 0xe0 && codepoint < 0x800) + { + return false; + } + if ((first & 0xf8) == 0xf0 && codepoint < 0x10000) + { + return false; + } + } + return true; +} + +bool +tiny_message_id_valid(const char *value) +{ + static const char alphabet[] = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + size_t index; + if (strlen(value) != TINY_MESSAGE_ID_LENGTH || value[0] > '7') + { + return false; + } + for (index = 0; index < TINY_MESSAGE_ID_LENGTH; ++index) + { + if (strchr(alphabet, value[index]) == NULL) + { + return false; + } + } + return true; +} diff --git a/src/domain/message.h b/src/domain/message.h new file mode 100644 index 0000000..fa36d2a --- /dev/null +++ b/src/domain/message.h @@ -0,0 +1,60 @@ +#ifndef TINY_CONVMEM_DOMAIN_MESSAGE_H +#define TINY_CONVMEM_DOMAIN_MESSAGE_H + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +#define TINY_MESSAGE_ID_LENGTH 26 +#define TINY_SESSION_ID_MAX 256 +#define TINY_CONTENT_MAX (1024U * 1024U) +#define TINY_TOOL_CALL_ID_MAX 256 +#define TINY_TAG_MAX 128 +#define TINY_TAG_COUNT_MAX 32 +#define TINY_QUERY_MAX 4096 + +typedef enum tiny_role +{ + TINY_ROLE_USER = 1, + TINY_ROLE_ASSISTANT = 2, + TINY_ROLE_SYSTEM = 3 +} tiny_role_t; + +typedef enum tiny_memory_tier +{ + TINY_TIER_SHORT = 1, + TINY_TIER_MID = 2, + TINY_TIER_LONG = 3, + TINY_TIER_DISCARD = 4 +} tiny_memory_tier_t; + +typedef struct tiny_metadata +{ + char *tool_call_id; + char **tags; + size_t tag_count; + tiny_memory_tier_t memory_tier; +} tiny_metadata_t; + +typedef struct tiny_message +{ + char message_id[TINY_MESSAGE_ID_LENGTH + 1]; + char *session_id; + int64_t timestamp_ms; + tiny_role_t role; + char *content; + tiny_metadata_t metadata; +} tiny_message_t; + +void tiny_message_init(tiny_message_t *message); +void tiny_message_free(tiny_message_t *message); +int tiny_message_copy(tiny_message_t *destination, const tiny_message_t *source); + +const char *tiny_role_name(tiny_role_t role); +bool tiny_role_parse(const char *value, tiny_role_t *role); +const char *tiny_tier_name(tiny_memory_tier_t tier); +bool tiny_tier_parse(const char *value, tiny_memory_tier_t *tier); +bool tiny_utf8_valid(const char *value, size_t length); +bool tiny_message_id_valid(const char *value); + +#endif diff --git a/src/domain/search.c b/src/domain/search.c new file mode 100644 index 0000000..a24b667 --- /dev/null +++ b/src/domain/search.c @@ -0,0 +1,350 @@ +#include "domain/search.h" + +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +typedef struct gram_bucket +{ + uint64_t hash; + uint32_t points[3]; + unsigned width; + size_t *messages; + size_t count; + size_t capacity; +} gram_bucket_t; + +struct tiny_search_index +{ + gram_bucket_t *buckets; + size_t count; + size_t capacity; + size_t message_count; +}; + +static bool +decode_codepoints(const char *text, uint32_t **output, size_t *count) +{ + const size_t bytes = strlen(text); + size_t offset = 0, used = 0; + uint32_t *points = malloc((bytes + 1) * sizeof(uint32_t)); + if (points == NULL) + { + return false; + } + while (offset < bytes) + { + unsigned char first = (unsigned char)text[offset++]; + uint32_t point; + unsigned continuation; + if (first < 0x80) + { + point = first; + continuation = 0; + } + else if ((first & 0xe0) == 0xc0) + { + point = first & 0x1f; + continuation = 1; + } + else if ((first & 0xf0) == 0xe0) + { + point = first & 0x0f; + continuation = 2; + } + else + { + point = first & 0x07; + continuation = 3; + } + while (continuation-- > 0) + { + point = (point << 6) | ((unsigned char)text[offset++] & 0x3f); + } + if (point >= 'A' && point <= 'Z') + { + point += 'a' - 'A'; + } + points[used++] = point; + } + *output = points; + *count = used; + return true; +} + +static uint64_t +gram_hash(const uint32_t *points, size_t width) +{ + uint64_t hash = 1469598103934665603ULL; + size_t index; + for (index = 0; index < width; ++index) + { + hash ^= points[index]; + hash *= 1099511628211ULL; + } + return hash; +} + +static gram_bucket_t * +find_bucket(const tiny_search_index_t *index, const uint32_t *points, unsigned width) +{ + const uint64_t hash = gram_hash(points, width); + size_t position; + for (position = 0; position < index->count; ++position) + { + if (index->buckets[position].hash == hash && index->buckets[position].width == width + && memcmp(index->buckets[position].points, points, width * sizeof(uint32_t)) == 0) + { + return &index->buckets[position]; + } + } + return NULL; +} + +static bool +add_gram(tiny_search_index_t *index, const uint32_t *points, unsigned width, size_t message_index) +{ + gram_bucket_t *bucket = find_bucket(index, points, width); + if (bucket == NULL) + { + gram_bucket_t *resized; + size_t capacity; + if (index->count == index->capacity) + { + capacity = index->capacity ? index->capacity * 2 : 64; + resized = realloc(index->buckets, capacity * sizeof(*resized)); + if (!resized) + { + return false; + } + index->buckets = resized; + index->capacity = capacity; + } + bucket = &index->buckets[index->count++]; + memset(bucket, 0, sizeof(*bucket)); + bucket->hash = gram_hash(points, width); + bucket->width = width; + memcpy(bucket->points, points, width * sizeof(uint32_t)); + } + if (bucket->count > 0 && bucket->messages[bucket->count - 1] == message_index) + { + return true; + } + if (bucket->count == bucket->capacity) + { + const size_t capacity = bucket->capacity ? bucket->capacity * 2 : 8; + size_t *resized = realloc(bucket->messages, capacity * sizeof(*resized)); + if (!resized) + { + return false; + } + bucket->messages = resized; + bucket->capacity = capacity; + } + bucket->messages[bucket->count++] = message_index; + return true; +} + +static bool +index_text(tiny_search_index_t *index, const char *text, size_t message_index) +{ + uint32_t *points = NULL; + size_t count = 0, offset; + unsigned width; + if (!decode_codepoints(text, &points, &count)) + { + return false; + } + for (width = 2; width <= 3; ++width) + { + for (offset = 0; offset + width <= count; ++offset) + { + if (!add_gram(index, points + offset, width, message_index)) + { + free(points); + return false; + } + } + } + free(points); + return true; +} + +tiny_search_index_t * +tiny_search_index_create(void) +{ + return calloc(1, sizeof(tiny_search_index_t)); +} + +void +tiny_search_index_free(tiny_search_index_t *index) +{ + size_t position; + if (!index) + { + return; + } + for (position = 0; position < index->count; ++position) + { + free(index->buckets[position].messages); + } + free(index->buckets); + free(index); +} + +bool +tiny_search_index_add(tiny_search_index_t *index, + const tiny_message_t *message, + size_t message_index) +{ + size_t tag; + if (!index_text(index, message->content, message_index)) + { + return false; + } + for (tag = 0; tag < message->metadata.tag_count; ++tag) + { + if (!index_text(index, message->metadata.tags[tag], message_index)) + { + return false; + } + } + if (message_index >= index->message_count) + { + index->message_count = message_index + 1; + } + return true; +} + +bool +tiny_search_index_candidates(const tiny_search_index_t *index, + const char *query, + size_t **output, + size_t *count) +{ + uint32_t *points = NULL; + size_t point_count = 0, width, offset, used = 0; + bool *seen; + size_t *items; + *output = NULL; + *count = 0; + if (!decode_codepoints(query, &points, &point_count)) + { + return false; + } + items = malloc((index->message_count ? index->message_count : 1) * sizeof(*items)); + seen = calloc(index->message_count ? index->message_count : 1, sizeof(*seen)); + if (!items || !seen) + { + free(points); + free(items); + free(seen); + return false; + } + if (point_count < 2) + { + for (offset = 0; offset < index->message_count; ++offset) + { + seen[offset] = true; + items[used++] = offset; + } + } + else + { + gram_bucket_t *bucket; + width = point_count >= 3 ? 3 : 2; + for (offset = 0; offset + width <= point_count; ++offset) + { + if ((bucket = find_bucket(index, points + offset, (unsigned)width)) != NULL) + { + size_t item; + for (item = 0; item < bucket->count; ++item) + { + if (!seen[bucket->messages[item]]) + { + seen[bucket->messages[item]] = true; + items[used++] = bucket->messages[item]; + } + } + } + } + } + free(points); + free(seen); + *output = items; + *count = used; + return true; +} + +double +tiny_search_score(const char *query, const tiny_message_t *message) +{ + uint32_t *query_points = NULL, *message_points = NULL; + size_t query_count = 0, message_count = 0, width, query_offset, message_offset, matches = 0, + total, tag; + double score = 0.0; + if (!decode_codepoints(query, &query_points, &query_count) + || !decode_codepoints(message->content, &message_points, &message_count) || query_count == 0) + { + goto done; + } + width = query_count >= 3 ? 3 : (query_count >= 2 ? 2 : 1); + total = query_count - width + 1; + for (query_offset = 0; query_offset < total; ++query_offset) + { + bool found = false; + for (message_offset = 0; message_offset + width <= message_count; ++message_offset) + { + if (memcmp(query_points + query_offset, + message_points + message_offset, + width * sizeof(uint32_t)) + == 0) + { + found = true; + break; + } + } + for (tag = 0; tag < message->metadata.tag_count && !found; ++tag) + { + uint32_t *tag_points = NULL; + size_t tag_count = 0; + if (!decode_codepoints(message->metadata.tags[tag], &tag_points, &tag_count)) + { + continue; + } + for (message_offset = 0; message_offset + width <= tag_count; ++message_offset) + { + if (memcmp(query_points + query_offset, + tag_points + message_offset, + width * sizeof(uint32_t)) + == 0) + { + found = true; + break; + } + } + free(tag_points); + } + if (found) + { + matches++; + } + } + score = (double)matches / (double)total; + if (query_count <= message_count) + { + for (message_offset = 0; message_offset + query_count <= message_count; ++message_offset) + { + if (memcmp(query_points, message_points + message_offset, query_count * sizeof(uint32_t)) + == 0) + { + score += 1.0; + break; + } + } + } +done: + free(query_points); + free(message_points); + return score; +} diff --git a/src/domain/search.h b/src/domain/search.h new file mode 100644 index 0000000..3408998 --- /dev/null +++ b/src/domain/search.h @@ -0,0 +1,22 @@ +#ifndef TINY_CONVMEM_DOMAIN_SEARCH_H +#define TINY_CONVMEM_DOMAIN_SEARCH_H + +#include <stdbool.h> +#include <stddef.h> + +#include "domain/message.h" + +typedef struct tiny_search_index tiny_search_index_t; + +tiny_search_index_t *tiny_search_index_create(void); +void tiny_search_index_free(tiny_search_index_t *index); +bool tiny_search_index_add(tiny_search_index_t *index, + const tiny_message_t *message, + size_t message_index); +bool tiny_search_index_candidates(const tiny_search_index_t *index, + const char *query, + size_t **message_indices, + size_t *count); +double tiny_search_score(const char *query, const tiny_message_t *message); + +#endif diff --git a/src/mcp/README.md b/src/mcp/README.md new file mode 100644 index 0000000..dc7ac46 --- /dev/null +++ b/src/mcp/README.md @@ -0,0 +1,14 @@ +# MCP compatibility boundary + +The pinned mcpc revision cannot describe required versus optional properties, +nested objects/arrays, 64-bit values, or application callback context. The +project therefore keeps mcpc immutable and implements the stdio dispatch and +tool schemas in `server.c`, using mcpc's bundled `mjson` parser for raw JSON. + +This boundary supports the MCP 2024-11-05 initialize, tools/list, and tools/call +flow needed by the server. Tool success and application errors are returned as +JSON serialized into TextContent; application errors also set `isError`. + +If mcpc is upgraded, this boundary should be removed only after its integration +tests demonstrate equivalent optional/object/array schema and callback-state +behavior. diff --git a/src/mcp/json.c b/src/mcp/json.c new file mode 100644 index 0000000..dcef91b --- /dev/null +++ b/src/mcp/json.c @@ -0,0 +1,137 @@ +#include "mcp/json.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +static bool +reserve(tiny_json_buffer_t *buffer, size_t addition) +{ + size_t needed, capacity; + char *resized; + if (buffer->failed || addition > SIZE_MAX - buffer->length - 1) + { + return false; + } + needed = buffer->length + addition + 1; + if (needed <= buffer->capacity) + { + return true; + } + capacity = buffer->capacity == 0 ? 256 : buffer->capacity; + while (capacity < needed) + { + if (capacity > SIZE_MAX / 2) + { + buffer->failed = true; + return false; + } + capacity *= 2; + } + resized = realloc(buffer->data, capacity); + if (resized == NULL) + { + buffer->failed = true; + return false; + } + buffer->data = resized; + buffer->capacity = capacity; + return true; +} + +void +tiny_json_init(tiny_json_buffer_t *buffer) +{ + memset(buffer, 0, sizeof(*buffer)); +} +void +tiny_json_free(tiny_json_buffer_t *buffer) +{ + free(buffer->data); + tiny_json_init(buffer); +} + +bool +tiny_json_raw_n(tiny_json_buffer_t *buffer, const char *value, size_t length) +{ + if (!reserve(buffer, length)) + { + return false; + } + memcpy(buffer->data + buffer->length, value, length); + buffer->length += length; + buffer->data[buffer->length] = '\0'; + return true; +} +bool +tiny_json_raw(tiny_json_buffer_t *buffer, const char *value) +{ + return tiny_json_raw_n(buffer, value, strlen(value)); +} + +bool +tiny_json_string(tiny_json_buffer_t *buffer, const char *value) +{ + const unsigned char *position = (const unsigned char *)value; + if (!tiny_json_raw(buffer, "\"")) + { + return false; + } + while (*position) + { + char escaped[7]; + if (*position == '"' || *position == '\\') + { + escaped[0] = '\\'; + escaped[1] = (char)*position; + escaped[2] = '\0'; + tiny_json_raw(buffer, escaped); + } + else if (*position == '\b') + { + tiny_json_raw(buffer, "\\b"); + } + else if (*position == '\f') + { + tiny_json_raw(buffer, "\\f"); + } + else if (*position == '\n') + { + tiny_json_raw(buffer, "\\n"); + } + else if (*position == '\r') + { + tiny_json_raw(buffer, "\\r"); + } + else if (*position == '\t') + { + tiny_json_raw(buffer, "\\t"); + } + else if (*position < 0x20) + { + snprintf(escaped, sizeof(escaped), "\\u%04x", *position); + tiny_json_raw(buffer, escaped); + } + else + { + tiny_json_raw_n(buffer, (const char *)position, 1); + } + ++position; + } + return tiny_json_raw(buffer, "\""); +} + +bool +tiny_json_u64(tiny_json_buffer_t *buffer, unsigned long long value) +{ + char text[32]; + snprintf(text, sizeof(text), "%llu", value); + return tiny_json_raw(buffer, text); +} +bool +tiny_json_i64(tiny_json_buffer_t *buffer, long long value) +{ + char text[32]; + snprintf(text, sizeof(text), "%lld", value); + return tiny_json_raw(buffer, text); +} diff --git a/src/mcp/json.h b/src/mcp/json.h new file mode 100644 index 0000000..a46e6c7 --- /dev/null +++ b/src/mcp/json.h @@ -0,0 +1,23 @@ +#ifndef TINY_CONVMEM_MCP_JSON_H +#define TINY_CONVMEM_MCP_JSON_H + +#include <stdbool.h> +#include <stddef.h> + +typedef struct tiny_json_buffer +{ + char *data; + size_t length; + size_t capacity; + bool failed; +} tiny_json_buffer_t; + +void tiny_json_init(tiny_json_buffer_t *buffer); +void tiny_json_free(tiny_json_buffer_t *buffer); +bool tiny_json_raw(tiny_json_buffer_t *buffer, const char *value); +bool tiny_json_raw_n(tiny_json_buffer_t *buffer, const char *value, size_t length); +bool tiny_json_string(tiny_json_buffer_t *buffer, const char *value); +bool tiny_json_u64(tiny_json_buffer_t *buffer, unsigned long long value); +bool tiny_json_i64(tiny_json_buffer_t *buffer, long long value); + +#endif diff --git a/src/mcp/server.c b/src/mcp/server.c index 57d197d..80d8307 100644 --- a/src/mcp/server.c +++ b/src/mcp/server.c @@ -1,52 +1,757 @@ #include "mcp/server.h" -#include <mcpc/mcpc.h> -#include "mcp/tools/health.h" +#include "domain/search.h" +#include "mcp/json.h" +#include "mjson.h" -int -tiny_convmem_server_run(FILE *input, FILE *output) +#include <math.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#define MCP_PROTOCOL_VERSION "2024-11-05" + +typedef struct result_item { - mcpc_server_t *server = mcpc_server_new_iostrm(input, output); - if (server == NULL) + const tiny_store_entry_t *entry; + double score; +} result_item_t; + +static char * +read_line(FILE *input) +{ + size_t length = 0, capacity = 4096; + int character; + char *line = malloc(capacity); + if (!line) + { + return NULL; + } + while ((character = fgetc(input)) != EOF) { - return MCPC_EC_BUG; + if (length + 1 >= capacity) + { + char *r; + capacity *= 2; + r = realloc(line, capacity); + if (!r) + { + free(line); + return NULL; + } + line = r; + } + line[length++] = (char)character; + if (character == '\n') + { + break; + } } + if (length == 0 && character == EOF) + { + free(line); + return NULL; + } + line[length] = '\0'; + return line; +} - mcpc_errcode_t result = mcpc_server_set_nament( - server, - (const char8_t *)"tiny-convmem-mcp" - ); - if (result == MCPC_EC_0) +static char * +json_string_value( + const char *json, int length, const char *path, size_t maximum, bool required, bool *valid) +{ + const char *token = NULL; + int token_length = 0; + int type = mjson_find(json, length, path, &token, &token_length); + char *value; + int decoded; + if (type == MJSON_TOK_INVALID) + { + if (required) + { + *valid = false; + } + return NULL; + } + if (type != MJSON_TOK_STRING || token_length < 2 || (size_t)token_length > maximum * 6 + 2) + { + *valid = false; + return NULL; + } + value = malloc(maximum + 1); + if (!value) + { + *valid = false; + return NULL; + } + decoded = mjson_get_string(json, length, path, value, (int)maximum + 1); + if (decoded < 0 || (size_t)decoded > maximum || !tiny_utf8_valid(value, (size_t)decoded)) { - result = mcpc_server_capa_enable_tool(server); + free(value); + *valid = false; + return NULL; } + value[decoded] = '\0'; + return value; +} - mcpc_tool_t *health = NULL; - if (result == MCPC_EC_0) +static bool +object_has_only(const char *json, int length, const char *const *allowed, size_t allowed_count) +{ + int offset = 0, koff, klen, voff, vlen, vtype; + while ((offset = mjson_next(json, length, offset, &koff, &klen, &voff, &vlen, &vtype)) != 0) { - health = tiny_convmem_health_tool_create(); - if (health == NULL) + char key[64]; + int decoded; + size_t index; + if (klen <= 0) + { + return false; + } + decoded = mjson_get_string(json + koff, klen, "$", key, sizeof(key)); + if (decoded <= 0 || decoded >= (int)sizeof(key)) + { + return false; + } + key[decoded] = '\0'; + for (index = 0; index < allowed_count; ++index) { - result = MCPC_EC_BUG; + if (strcmp(key, allowed[index]) == 0) + { + break; + } + } + if (index == allowed_count) + { + return false; } } + return true; +} + +static void +free_input_message(tiny_message_t *message) +{ + tiny_message_free(message); +} - if (result == MCPC_EC_0) +static bool +parse_tags(const char *json, int length, tiny_message_t *message) +{ + const char *array = NULL; + int array_length = 0; + int type = mjson_find(json, length, "$.metadata.tags", &array, &array_length); + int offset = 0, koff, klen, voff, vlen, vtype; + if (type == MJSON_TOK_INVALID) { - result = mcpc_server_add_tool(server, health); + return true; } - if (result == MCPC_EC_0) + if (type != MJSON_TOK_ARRAY) { - result = mcpc_server_start(server); + return false; } + while ((offset = mjson_next(array, array_length, offset, &koff, &klen, &voff, &vlen, &vtype)) + != 0) + { + char *tag; + int decoded; + char **resized; + if (vtype != MJSON_TOK_STRING || message->metadata.tag_count >= TINY_TAG_COUNT_MAX) + { + return false; + } + tag = malloc(TINY_TAG_MAX + 1); + if (!tag) + { + return false; + } + decoded = mjson_get_string(array + voff, vlen, "$", tag, TINY_TAG_MAX + 1); + if (decoded <= 0 || decoded > TINY_TAG_MAX || !tiny_utf8_valid(tag, (size_t)decoded)) + { + free(tag); + return false; + } + tag[decoded] = '\0'; + resized = realloc(message->metadata.tags, (message->metadata.tag_count + 1) * sizeof(char *)); + if (!resized) + { + free(tag); + return false; + } + message->metadata.tags = resized; + message->metadata.tags[message->metadata.tag_count++] = tag; + } + return true; +} + +static bool +parse_write(const char *args, int length, tiny_message_t *message) +{ + static const char *const top_keys[] = { "session_id", "role", "content", "metadata" }; + static const char *const metadata_keys[] = { "tool_call_id", "tags", "memory_tier" }; + bool valid = true; + char *role = NULL, *tier = NULL; + const char *metadata = NULL; + int metadata_length = 0, metadata_type; + tiny_message_init(message); + if (!object_has_only(args, length, top_keys, sizeof(top_keys) / sizeof(top_keys[0]))) + { + valid = false; + } + metadata_type = mjson_find(args, length, "$.metadata", &metadata, &metadata_length); + if (metadata_type != MJSON_TOK_INVALID + && (metadata_type != MJSON_TOK_OBJECT + || !object_has_only(metadata, + metadata_length, + metadata_keys, + sizeof(metadata_keys) / sizeof(metadata_keys[0])))) + { + valid = false; + } + message->session_id + = json_string_value(args, length, "$.session_id", TINY_SESSION_ID_MAX, true, &valid); + role = json_string_value(args, length, "$.role", 16, true, &valid); + message->content = json_string_value(args, length, "$.content", TINY_CONTENT_MAX, true, &valid); + message->metadata.tool_call_id = json_string_value( + args, length, "$.metadata.tool_call_id", TINY_TOOL_CALL_ID_MAX, false, &valid); + tier = json_string_value(args, length, "$.metadata.memory_tier", 16, false, &valid); + if (!valid || !message->session_id || !message->content || !role || message->session_id[0] == '\0' + || message->content[0] == '\0' || !tiny_role_parse(role, &message->role) + || (tier && !tiny_tier_parse(tier, &message->metadata.memory_tier)) + || !parse_tags(args, length, message)) + { + valid = false; + } + free(role); + free(tier); + if (!valid) + { + free_input_message(message); + } + return valid; +} + +static void +append_message(tiny_json_buffer_t *json, const tiny_message_t *message) +{ + size_t index; + tiny_json_raw(json, "{\"message_id\":"); + tiny_json_string(json, message->message_id); + tiny_json_raw(json, ",\"session_id\":"); + tiny_json_string(json, message->session_id); + tiny_json_raw(json, ",\"timestamp\":"); + tiny_json_i64(json, message->timestamp_ms); + tiny_json_raw(json, ",\"role\":"); + tiny_json_string(json, tiny_role_name(message->role)); + tiny_json_raw(json, ",\"content\":"); + tiny_json_string(json, message->content); + tiny_json_raw(json, ",\"metadata\":{"); + if (message->metadata.tool_call_id) + { + tiny_json_raw(json, "\"tool_call_id\":"); + tiny_json_string(json, message->metadata.tool_call_id); + tiny_json_raw(json, ","); + } + tiny_json_raw(json, "\"tags\":["); + for (index = 0; index < message->metadata.tag_count; ++index) + { + if (index) + { + tiny_json_raw(json, ","); + } + tiny_json_string(json, message->metadata.tags[index]); + } + tiny_json_raw(json, "],\"memory_tier\":"); + tiny_json_string(json, tiny_tier_name(message->metadata.memory_tier)); + tiny_json_raw(json, "}}"); +} - { - const mcpc_errcode_t close_result = mcpc_server_close(server); - if (result == MCPC_EC_0) +static void +send_json_result( + FILE *output, const char *id, int id_length, tiny_json_buffer_t *inner, bool is_error) +{ + tiny_json_buffer_t response; + tiny_json_init(&response); + tiny_json_raw(&response, "{\"jsonrpc\":\"2.0\",\"id\":"); + tiny_json_raw_n(&response, id, (size_t)id_length); + tiny_json_raw(&response, ",\"result\":{\"content\":[{\"type\":\"text\",\"text\":"); + tiny_json_string(&response, inner->data ? inner->data : "{}"); + tiny_json_raw(&response, "}]"); + if (is_error) + { + tiny_json_raw(&response, ",\"isError\":true"); + } + tiny_json_raw(&response, "}}\n"); + if (!response.failed) + { + fwrite(response.data, 1, response.length, output); + fflush(output); + } + tiny_json_free(&response); +} + +static void +send_error(FILE *output, const char *id, int id_length, const char *code, const char *message) +{ + tiny_json_buffer_t inner; + tiny_json_init(&inner); + tiny_json_raw(&inner, "{\"status\":\"ERROR\",\"error\":{\"code\":"); + tiny_json_string(&inner, code); + tiny_json_raw(&inner, ",\"message\":"); + tiny_json_string(&inner, message); + tiny_json_raw(&inner, "}}"); + send_json_result(output, id, id_length, &inner, true); + tiny_json_free(&inner); +} + +static int +compare_history(const void *left, const void *right) +{ + const tiny_store_entry_t *a = *(const tiny_store_entry_t *const *)left, + *b = *(const tiny_store_entry_t *const *)right; + if (a->message.timestamp_ms < b->message.timestamp_ms) + { + return -1; + } + if (a->message.timestamp_ms > b->message.timestamp_ms) + { + return 1; + } + return strcmp(a->message.message_id, b->message.message_id); +} + +static int +compare_results(const void *l, const void *r) +{ + const result_item_t *a = l, *b = r; + if (a->score > b->score) + { + return -1; + } + if (a->score < b->score) + { + return 1; + } + if (a->entry->message.timestamp_ms > b->entry->message.timestamp_ms) + { + return -1; + } + if (a->entry->message.timestamp_ms < b->entry->message.timestamp_ms) + { + return 1; + } + return strcmp(a->entry->message.message_id, b->entry->message.message_id); +} + +static tiny_search_index_t * +build_search_index(const tiny_store_t *store) +{ + tiny_search_index_t *index = tiny_search_index_create(); + size_t position; + if (!index) + { + return NULL; + } + for (position = 0; position < tiny_store_entry_count(store); ++position) + { + if (!tiny_search_index_add(index, &tiny_store_entry(store, position)->message, position)) + { + tiny_search_index_free(index); + return NULL; + } + } + return index; +} + +static void +handle_call(FILE *output, + const char *id, + int id_length, + const char *name, + const char *args, + int args_length, + tiny_store_t *store, + tiny_search_index_t **search_index) +{ + tiny_json_buffer_t result; + tiny_store_status_t status; + tiny_json_init(&result); + if (strcmp(name, "write_message") == 0) + { + tiny_message_t input, written; + if (!parse_write(args, args_length, &input)) + { + send_error(output, id, id_length, "VALIDATION_ERROR", "invalid write_message arguments"); + return; + } + tiny_message_init(&written); + status = tiny_store_write(store, &input, &written); + tiny_message_free(&input); + if (status != TINY_STORE_OK) + { + send_error( + output, id, id_length, tiny_store_status_name(status), "message was not written"); + return; + } + if (*search_index + && !tiny_search_index_add(*search_index, &written, tiny_store_entry_count(store) - 1)) + { + tiny_search_index_free(*search_index); + *search_index = NULL; + } + tiny_json_raw(&result, "{\"status\":\"OK\",\"message\":"); + append_message(&result, &written); + tiny_json_raw(&result, "}"); + tiny_message_free(&written); + } + else if (strcmp(name, "read_messages") == 0) + { + static const char *const keys[] = { "session_id" }; + bool valid = object_has_only(args, args_length, keys, 1); + char *session + = json_string_value(args, args_length, "$.session_id", TINY_SESSION_ID_MAX, true, &valid); + size_t i, count = 0; + const tiny_store_entry_t **items; + if (!valid || !session || !session[0]) + { + free(session); + send_error(output, id, id_length, "VALIDATION_ERROR", "session_id is required"); + return; + } + items = malloc(tiny_store_entry_count(store) * sizeof(*items)); + if (!items) + { + free(session); + send_error(output, id, id_length, "INTERNAL_ERROR", "out of memory"); + return; + } + for (i = 0; i < tiny_store_entry_count(store); ++i) + { + const tiny_store_entry_t *e = tiny_store_entry(store, i); + if (!e->deleted && strcmp(e->message.session_id, session) == 0) + { + items[count++] = e; + } + } + qsort(items, count, sizeof(*items), compare_history); + tiny_json_raw(&result, "{\"history\":["); + for (i = 0; i < count; ++i) + { + if (i) + { + tiny_json_raw(&result, ","); + } + append_message(&result, &items[i]->message); + } + tiny_json_raw(&result, "]}"); + free(items); + free(session); + } + else if (strcmp(name, "search_messages") == 0) + { + static const char *const keys[] = { "query", "session_id", "limit" }; + bool valid = object_has_only(args, args_length, keys, 3); + char *query = json_string_value(args, args_length, "$.query", TINY_QUERY_MAX, true, &valid); + char *session = json_string_value( + args, args_length, "$.session_id", TINY_SESSION_ID_MAX, false, &valid); + double number = 20; + size_t limit = 20, i, count = 0, candidate_count = 0, *candidates = NULL; + result_item_t *items; + if (mjson_find(args, args_length, "$.limit", NULL, NULL) != MJSON_TOK_INVALID) + { + if (!mjson_get_number(args, args_length, "$.limit", &number) || number < 1 || number > 100 + || floor(number) != number) + { + valid = false; + } + else + { + limit = (size_t)number; + } + } + if (!valid || !query || !query[0]) + { + free(query); + free(session); + send_error(output, id, id_length, "VALIDATION_ERROR", "query or limit is invalid"); + return; + } + if (!*search_index) + { + *search_index = build_search_index(store); + } + if (!*search_index + || !tiny_search_index_candidates(*search_index, query, &candidates, &candidate_count)) + { + free(query); + free(session); + send_error(output, id, id_length, "INTERNAL_ERROR", "search index allocation failed"); + return; + } + items = malloc((candidate_count ? candidate_count : 1) * sizeof(*items)); + if (!items) + { + free(candidates); + free(query); + free(session); + send_error(output, id, id_length, "INTERNAL_ERROR", "out of memory"); + return; + } + for (i = 0; i < candidate_count; ++i) + { + const tiny_store_entry_t *e = tiny_store_entry(store, candidates[i]); + double score; + if (!e || e->deleted || (session && strcmp(e->message.session_id, session) != 0)) + { + continue; + } + score = tiny_search_score(query, &e->message); + if (score > 0) + { + items[count].entry = e; + items[count].score = score; + count++; + } + } + qsort(items, count, sizeof(*items), compare_results); + if (count > limit) + { + count = limit; + } + tiny_json_raw(&result, "{\"results\":["); + for (i = 0; i < count; ++i) + { + if (i) + { + tiny_json_raw(&result, ","); + } + append_message(&result, &items[i].entry->message); + } + tiny_json_raw(&result, "]}"); + free(candidates); + free(items); + free(query); + free(session); + } + else if (strcmp(name, "delete_message") == 0) + { + static const char *const keys[] = { "message_id", "session_id" }; + bool valid = object_has_only(args, args_length, keys, 2); + char *mid = json_string_value( + args, args_length, "$.message_id", TINY_MESSAGE_ID_LENGTH, true, &valid); + char *session = json_string_value( + args, args_length, "$.session_id", TINY_SESSION_ID_MAX, false, &valid); + if (!valid || !mid || !tiny_message_id_valid(mid)) + { + free(mid); + free(session); + send_error(output, + id, + id_length, + "VALIDATION_ERROR", + "message_id must be a canonical 26-character ULID"); + return; + } + status = tiny_store_delete(store, mid, session); + free(mid); + free(session); + if (status != TINY_STORE_OK && status != TINY_STORE_NOT_FOUND) + { + send_error(output, + id, + id_length, + tiny_store_status_name(status), + "delete record was not written"); + return; + } + tiny_json_raw(&result, + status == TINY_STORE_OK ? "{\"status\":\"OK\"}" : "{\"status\":\"NOT_FOUND\"}"); + } + else if (strcmp(name, "rebuild_index") == 0) + { + tiny_rebuild_report_t report; + tiny_search_index_t *rebuilt; + if (!object_has_only(args, args_length, NULL, 0)) + { + send_error(output, id, id_length, "VALIDATION_ERROR", "rebuild_index takes no arguments"); + return; + } + status = tiny_store_rebuild(store, &report); + if (status != TINY_STORE_OK) + { + send_error(output, id, id_length, tiny_store_status_name(status), "index rebuild failed"); + return; + } + rebuilt = build_search_index(store); + if (!rebuilt) + { + tiny_search_index_free(*search_index); + *search_index = NULL; + send_error(output, + id, + id_length, + "INTERNAL_ERROR", + "log was compacted but the search index could not be allocated"); + return; + } + tiny_search_index_free(*search_index); + *search_index = rebuilt; + tiny_json_raw(&result, "{\"status\":\"OK\",\"messages_indexed\":"); + tiny_json_u64(&result, report.messages_indexed); + tiny_json_raw(&result, ",\"records_removed\":"); + tiny_json_u64(&result, report.records_removed); + tiny_json_raw(&result, ",\"bytes_before\":"); + tiny_json_u64(&result, report.bytes_before); + tiny_json_raw(&result, ",\"bytes_after\":"); + tiny_json_u64(&result, report.bytes_after); + tiny_json_raw(&result, "}"); + } + else if (strcmp(name, "get_system_health") == 0) + { + tiny_store_health_t health; + if (!object_has_only(args, args_length, NULL, 0)) + { + send_error( + output, id, id_length, "VALIDATION_ERROR", "get_system_health takes no arguments"); + return; + } + tiny_store_health(store, &health); + tiny_json_raw(&result, "{\"status\":"); + tiny_json_string(&result, health.last_error[0] ? "DEGRADED" : "OK"); + tiny_json_raw(&result, ",\"format_version\":1,\"log_size\":"); + tiny_json_u64(&result, health.log_size); + tiny_json_raw(&result, ",\"snapshot_size\":"); + tiny_json_u64(&result, health.snapshot_size); + tiny_json_raw(&result, ",\"active_messages\":"); + tiny_json_u64(&result, health.active_messages); + tiny_json_raw(&result, ",\"deleted_messages\":"); + tiny_json_u64(&result, health.deleted_messages); + tiny_json_raw(&result, ",\"last_sequence\":"); + tiny_json_u64(&result, health.last_sequence); + tiny_json_raw(&result, ",\"recovered_tail\":"); + tiny_json_raw(&result, health.recovered_tail ? "true" : "false"); + tiny_json_raw(&result, ",\"snapshot_valid\":"); + tiny_json_raw(&result, health.snapshot_valid ? "true" : "false"); + tiny_json_raw(&result, ",\"last_error\":"); + tiny_json_string(&result, health.last_error); + tiny_json_raw(&result, "}"); + } + else + { + send_error(output, id, id_length, "VALIDATION_ERROR", "unknown tool"); + return; + } + if (result.failed) + { + send_error(output, id, id_length, "INTERNAL_ERROR", "response allocation failed"); + } + else + { + send_json_result(output, id, id_length, &result, false); + } + tiny_json_free(&result); +} + +static const char *tools_json + = "{\"tools\":[" + "{\"name\":\"write_message\",\"description\":\"Durably append a conversation " + "message\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"session_id\":{\"type\":" + "\"string\",\"maxLength\":256},\"role\":{\"type\":\"string\",\"enum\":[\"user\"," + "\"assistant\",\"system\"]},\"content\":{\"type\":\"string\",\"maxLength\":1048576}," + "\"metadata\":{\"type\":\"object\",\"properties\":{\"tool_call_id\":{\"type\":\"string\"," + "\"maxLength\":256},\"tags\":{\"type\":\"array\",\"maxItems\":32,\"items\":{\"type\":" + "\"string\",\"maxLength\":128}},\"memory_tier\":{\"type\":\"string\",\"enum\":[\"short\"," + "\"mid\",\"long\",\"discard\"]}},\"additionalProperties\":false}},\"required\":[\"session_" + "id\",\"role\",\"content\"],\"additionalProperties\":false}}," + "{\"name\":\"read_messages\",\"description\":\"Read a session in chronological " + "order\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"session_id\":{\"type\":" + "\"string\",\"maxLength\":256}},\"required\":[\"session_id\"],\"additionalProperties\":false}" + "}," + "{\"name\":\"search_messages\",\"description\":\"Search message content and " + "tags\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\"," + "\"maxLength\":4096},\"session_id\":{\"type\":\"string\",\"maxLength\":256},\"limit\":{" + "\"type\":\"integer\",\"minimum\":1,\"maximum\":100}},\"required\":[\"query\"]," + "\"additionalProperties\":false}}," + "{\"name\":\"delete_message\",\"description\":\"Durably tombstone a " + "message\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"message_id\":{\"type\":" + "\"string\",\"minLength\":26,\"maxLength\":26},\"session_id\":{\"type\":\"string\"," + "\"maxLength\":256}},\"required\":[\"message_id\"],\"additionalProperties\":false}}," + "{\"name\":\"rebuild_index\",\"description\":\"Compact the log and rebuild the " + "snapshot\",\"inputSchema\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":" + "false}}," + "{\"name\":\"get_system_health\",\"description\":\"Report persistence health and " + "counters\",\"inputSchema\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":" + "false}}]}"; + +int +tiny_convmem_server_run(FILE *input, FILE *output, tiny_store_t *store) +{ + char *line; + tiny_search_index_t *search_index = build_search_index(store); + if (!search_index) + { + return 1; + } + while ((line = read_line(input)) != NULL) + { + int length = (int)strlen(line); + const char *method = NULL, *id = NULL, *params = NULL; + int ml = 0, il = 0, pl = 0; + if (mjson_find(line, length, "$.method", &method, &ml) != MJSON_TOK_STRING) + { + free(line); + continue; + } + mjson_find(line, length, "$.id", &id, &il); + mjson_find(line, length, "$.params", ¶ms, &pl); { - result = close_result; + char method_value[64]; + int decoded + = mjson_get_string(line, length, "$.method", method_value, sizeof(method_value)); + if (decoded < 0) + { + free(line); + continue; + } + method_value[decoded] = '\0'; + if (strcmp(method_value, "initialize") == 0 && id) + { + fprintf(output, + "{\"jsonrpc\":\"2.0\",\"id\":%.*s,\"result\":{\"protocolVersion\":\"%s\"," + "\"capabilities\":{\"tools\":{}},\"serverInfo\":{\"name\":\"tiny-convmem-mcp\"," + "\"version\":\"0.1.0\"}}}\n", + il, + id, + MCP_PROTOCOL_VERSION); + } + else if (strcmp(method_value, "tools/list") == 0 && id) + { + fprintf( + output, "{\"jsonrpc\":\"2.0\",\"id\":%.*s,\"result\":%s}\n", il, id, tools_json); + } + else if (strcmp(method_value, "tools/call") == 0 && id && params) + { + char name[64]; + const char *args = NULL; + int al = 0; + int n = mjson_get_string(params, pl, "$.name", name, sizeof(name)); + if (n > 0) + { + name[n] = '\0'; + if (mjson_find(params, pl, "$.arguments", &args, &al) == MJSON_TOK_OBJECT) + { + handle_call(output, id, il, name, args, al, store, &search_index); + } + else + { + send_error(output, id, il, "VALIDATION_ERROR", "arguments must be an object"); + } + } + } + fflush(output); } - } - - return (int)result; + free(line); + } + tiny_search_index_free(search_index); + return 0; } diff --git a/src/mcp/server.h b/src/mcp/server.h index e3ff2c4..4ae76e4 100644 --- a/src/mcp/server.h +++ b/src/mcp/server.h @@ -2,8 +2,8 @@ #define TINY_CONVMEM_MCP_SERVER_H #include <stdio.h> +#include "storage/store.h" -int tiny_convmem_server_run(FILE *input, FILE *output); +int tiny_convmem_server_run(FILE *input, FILE *output, tiny_store_t *store); #endif - diff --git a/src/mcp/tools/health.c b/src/mcp/tools/health.c deleted file mode 100644 index 0309ea5..0000000 --- a/src/mcp/tools/health.c +++ /dev/null @@ -1,29 +0,0 @@ -#include "mcp/tools/health.h" - -#include <mcpc/ucbr.h> - -static void -health_call(const mcpc_tool_t *tool, mcpc_ucbr_t *result) -{ - (void)tool; - mcpc_ucbr_toolcall_add_text_printf8(result, (const char8_t *)"ok"); -} - -mcpc_tool_t * -tiny_convmem_health_tool_create(void) -{ - mcpc_tool_t *tool = mcpc_tool_new2( - (const char8_t *)"health", - (const char8_t *)"Check whether the tiny conversation memory server is running" - ); - if (tool == NULL) - { - return NULL; - } - - if (mcpc_tool_set_call_cb(tool, health_call) != MCPC_EC_0) - { - return NULL; - } - return tool; -} diff --git a/src/mcp/tools/health.h b/src/mcp/tools/health.h deleted file mode 100644 index f4bfe0a..0000000 --- a/src/mcp/tools/health.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef TINY_CONVMEM_MCP_HEALTH_H -#define TINY_CONVMEM_MCP_HEALTH_H - -#include <mcpc/tool.h> - -mcpc_tool_t *tiny_convmem_health_tool_create(void); - -#endif - diff --git a/src/storage/README.md b/src/storage/README.md index ceb8965..3c03dba 100644 --- a/src/storage/README.md +++ b/src/storage/README.md @@ -1,6 +1,9 @@ # Storage layer -This directory is reserved for persistence adapters. The storage engine and its -schema are intentionally undecided; SQLite or another dependency will be added -to `vcpkg.json` only after the storage contract is designed. +The storage adapter owns `active.log`, `snapshot.bin`, the data-directory lock, +in-memory live/deleted state, synchronous PUT/DEL writes, startup replay, and +compaction. It uses Win32 64-bit file APIs and `FlushFileBuffers`. +The binary format is private but versioned. Invalid complete records are never +skipped; only an incomplete EOF record is truncated. Snapshot generation must +match the active log before snapshot state is trusted. diff --git a/src/storage/store.c b/src/storage/store.c new file mode 100644 index 0000000..9d26e0f --- /dev/null +++ b/src/storage/store.c @@ -0,0 +1,1111 @@ +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <bcrypt.h> + +#include "storage/store.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define RECORD_HEADER_SIZE 24U +#define RECORD_VERSION 1U +#define RECORD_PUT 1U +#define RECORD_DEL 2U +#define RECORD_MARKER 3U +#define RECORD_PAYLOAD_MAX (TINY_CONTENT_MAX + 16384U) +#define SNAPSHOT_HEADER_SIZE 52U + +struct tiny_store +{ + HANDLE lock; + HANDLE log; + wchar_t *directory; + wchar_t *log_path; + wchar_t *snapshot_path; + tiny_store_entry_t *entries; + size_t entry_count; + size_t entry_capacity; + uint64_t last_sequence; + uint64_t last_ulid_ms; + unsigned char last_random[10]; + unsigned char generation[16]; + bool has_generation; + bool recovered_tail; + bool snapshot_valid; + char last_error[160]; +}; + +static void +put_u32(unsigned char *p, uint32_t v) +{ + p[0] = (unsigned char)v; + p[1] = (unsigned char)(v >> 8); + p[2] = (unsigned char)(v >> 16); + p[3] = (unsigned char)(v >> 24); +} +static void +put_u64(unsigned char *p, uint64_t v) +{ + unsigned i; + for (i = 0; i < 8; ++i) + { + p[i] = (unsigned char)(v >> (i * 8)); + } +} +static uint32_t +get_u32(const unsigned char *p) +{ + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} +static uint64_t +get_u64(const unsigned char *p) +{ + uint64_t v = 0; + int i; + for (i = 7; i >= 0; --i) + { + v = (v << 8) | p[i]; + } + return v; +} + +static uint32_t +crc32_bytes(const unsigned char *data, size_t length) +{ + uint32_t crc = 0xffffffffU; + size_t i; + for (i = 0; i < length; ++i) + { + unsigned bit; + crc ^= data[i]; + for (bit = 0; bit < 8; ++bit) + { + crc = (crc >> 1) ^ (0xedb88320U & (uint32_t)-(int32_t)(crc & 1)); + } + } + return ~crc; +} + +static wchar_t * +join_path(const wchar_t *directory, const wchar_t *name) +{ + const size_t left = wcslen(directory), right = wcslen(name); + const bool separator = left > 0 && directory[left - 1] != L'\\' && directory[left - 1] != L'/'; + wchar_t *result = malloc((left + right + (separator ? 2 : 1)) * sizeof(wchar_t)); + if (result == NULL) + { + return NULL; + } + wcscpy_s(result, left + right + (separator ? 2 : 1), directory); + if (separator) + { + wcscat_s(result, left + right + 2, L"\\"); + } + wcscat_s(result, left + right + (separator ? 2 : 1), name); + return result; +} + +static void +set_error(tiny_store_t *store, const char *message) +{ + strncpy_s(store->last_error, sizeof(store->last_error), message, _TRUNCATE); +} + +static bool +write_all(HANDLE file, const void *data, uint32_t length) +{ + const unsigned char *position = data; + while (length > 0) + { + DWORD written = 0; + if (!WriteFile(file, position, length, &written, NULL) || written == 0) + { + return false; + } + position += written; + length -= written; + } + return true; +} + +static bool +read_all(HANDLE file, void *data, uint32_t length, uint32_t *actual) +{ + unsigned char *position = data; + uint32_t total = 0; + while (total < length) + { + DWORD count = 0; + if (!ReadFile(file, position + total, length - total, &count, NULL)) + { + return false; + } + if (count == 0) + { + break; + } + total += count; + } + *actual = total; + return true; +} + +static uint64_t +now_milliseconds(void) +{ + FILETIME ft; + ULARGE_INTEGER ticks; + GetSystemTimeAsFileTime(&ft); + ticks.LowPart = ft.dwLowDateTime; + ticks.HighPart = ft.dwHighDateTime; + return ticks.QuadPart / 10000ULL - 11644473600000ULL; +} + +static void +make_ulid(tiny_store_t *store, uint64_t timestamp, char output[27]) +{ + static const char alphabet[] = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + unsigned char bytes[16]; + unsigned index; + if (timestamp > store->last_ulid_ms) + { + store->last_ulid_ms = timestamp; + if (BCryptGenRandom( + NULL, store->last_random, sizeof(store->last_random), BCRYPT_USE_SYSTEM_PREFERRED_RNG) + != 0) + { + uint64_t fallback = GetTickCount64() ^ timestamp; + for (index = 0; index < sizeof(store->last_random); ++index) + { + store->last_random[index] = (unsigned char)(fallback >> ((index % 8) * 8)); + } + } + } + else + { + timestamp = store->last_ulid_ms; + for (index = 10; index-- > 0;) + { + if (++store->last_random[index] != 0) + { + break; + } + } + } + bytes[0] = (unsigned char)(timestamp >> 40); + bytes[1] = (unsigned char)(timestamp >> 32); + bytes[2] = (unsigned char)(timestamp >> 24); + bytes[3] = (unsigned char)(timestamp >> 16); + bytes[4] = (unsigned char)(timestamp >> 8); + bytes[5] = (unsigned char)timestamp; + memcpy(bytes + 6, store->last_random, 10); + for (index = 0; index < 26; ++index) + { + const unsigned bit = index * 5; + unsigned value = 0, part; + for (part = 0; part < 5; ++part) + { + const int source_bit = (int)bit + (int)part - 2; + value <<= 1; + if (source_bit >= 0) + { + value |= (bytes[source_bit / 8] >> (7 - source_bit % 8)) & 1U; + } + } + output[index] = alphabet[value]; + } + output[26] = '\0'; +} + +static bool +ensure_entries(tiny_store_t *store) +{ + tiny_store_entry_t *resized; + size_t capacity; + if (store->entry_count < store->entry_capacity) + { + return true; + } + capacity = store->entry_capacity == 0 ? 32 : store->entry_capacity * 2; + resized = realloc(store->entries, capacity * sizeof(*resized)); + if (resized == NULL) + { + return false; + } + store->entries = resized; + store->entry_capacity = capacity; + return true; +} + +static tiny_store_entry_t * +find_entry(tiny_store_t *store, const char *message_id) +{ + size_t index; + for (index = 0; index < store->entry_count; ++index) + { + if (strcmp(store->entries[index].message.message_id, message_id) == 0) + { + return &store->entries[index]; + } + } + return NULL; +} + +static bool +serialize_put(const tiny_message_t *message, unsigned char **payload, uint32_t *length) +{ + size_t session_len = strlen(message->session_id), content_len = strlen(message->content); + size_t tool_len = message->metadata.tool_call_id ? strlen(message->metadata.tool_call_id) : 0; + size_t total = 8 + 26 + 2 + 16 + session_len + content_len + tool_len; + size_t index, offset; + unsigned char *data; + for (index = 0; index < message->metadata.tag_count; ++index) + { + total += 4 + strlen(message->metadata.tags[index]); + } + if (total > UINT32_MAX || total > RECORD_PAYLOAD_MAX) + { + return false; + } + data = malloc(total); + if (data == NULL) + { + return false; + } + offset = 0; + put_u64(data + offset, (uint64_t)message->timestamp_ms); + offset += 8; + memcpy(data + offset, message->message_id, 26); + offset += 26; + data[offset++] = (unsigned char)message->role; + data[offset++] = (unsigned char)message->metadata.memory_tier; + put_u32(data + offset, (uint32_t)session_len); + offset += 4; + put_u32(data + offset, (uint32_t)content_len); + offset += 4; + put_u32(data + offset, (uint32_t)tool_len); + offset += 4; + put_u32(data + offset, (uint32_t)message->metadata.tag_count); + offset += 4; + memcpy(data + offset, message->session_id, session_len); + offset += session_len; + memcpy(data + offset, message->content, content_len); + offset += content_len; + if (tool_len > 0) + { + memcpy(data + offset, message->metadata.tool_call_id, tool_len); + offset += tool_len; + } + for (index = 0; index < message->metadata.tag_count; ++index) + { + const size_t tag_len = strlen(message->metadata.tags[index]); + put_u32(data + offset, (uint32_t)tag_len); + offset += 4; + memcpy(data + offset, message->metadata.tags[index], tag_len); + offset += tag_len; + } + *payload = data; + *length = (uint32_t)total; + return true; +} + +static char * +copy_field(const unsigned char *data, size_t length) +{ + char *value = malloc(length + 1); + if (value != NULL) + { + memcpy(value, data, length); + value[length] = '\0'; + } + return value; +} + +static bool +deserialize_put(const unsigned char *data, uint32_t length, tiny_message_t *message) +{ + uint32_t session_len, content_len, tool_len, tag_count; + size_t offset = 0, index; + tiny_message_init(message); + if (length < 8 + 26 + 2 + 16) + { + return false; + } + message->timestamp_ms = (int64_t)get_u64(data + offset); + offset += 8; + memcpy(message->message_id, data + offset, 26); + message->message_id[26] = '\0'; + offset += 26; + message->role = (tiny_role_t)data[offset++]; + message->metadata.memory_tier = (tiny_memory_tier_t)data[offset++]; + session_len = get_u32(data + offset); + offset += 4; + content_len = get_u32(data + offset); + offset += 4; + tool_len = get_u32(data + offset); + offset += 4; + tag_count = get_u32(data + offset); + offset += 4; + if (!tiny_message_id_valid(message->message_id) + || (message->role < TINY_ROLE_USER || message->role > TINY_ROLE_SYSTEM) + || (message->metadata.memory_tier < TINY_TIER_SHORT + || message->metadata.memory_tier > TINY_TIER_DISCARD) + || session_len == 0 || session_len > TINY_SESSION_ID_MAX || content_len == 0 + || content_len > TINY_CONTENT_MAX || tool_len > TINY_TOOL_CALL_ID_MAX + || tag_count > TINY_TAG_COUNT_MAX || offset + session_len + content_len + tool_len > length) + { + return false; + } + message->session_id = copy_field(data + offset, session_len); + offset += session_len; + message->content = copy_field(data + offset, content_len); + offset += content_len; + if (tool_len > 0) + { + message->metadata.tool_call_id = copy_field(data + offset, tool_len); + offset += tool_len; + } + if (message->session_id == NULL || message->content == NULL + || (tool_len && !message->metadata.tool_call_id)) + { + goto fail; + } + if (tag_count > 0) + { + message->metadata.tags = calloc(tag_count, sizeof(char *)); + if (message->metadata.tags == NULL) + { + goto fail; + } + for (index = 0; index < tag_count; ++index) + { + uint32_t tag_len; + if (offset + 4 > length) + { + goto fail; + } + tag_len = get_u32(data + offset); + offset += 4; + if (tag_len == 0 || tag_len > TINY_TAG_MAX || offset + tag_len > length) + { + goto fail; + } + message->metadata.tags[index] = copy_field(data + offset, tag_len); + offset += tag_len; + if (message->metadata.tags[index] == NULL) + { + goto fail; + } + message->metadata.tag_count++; + } + } + if (offset != length || !tiny_utf8_valid(message->session_id, session_len) + || !tiny_utf8_valid(message->content, content_len)) + { + goto fail; + } + return true; +fail: + tiny_message_free(message); + return false; +} + +static tiny_store_status_t +append_record_to(HANDLE file, + uint8_t type, + uint64_t sequence, + const unsigned char *payload, + uint32_t payload_length, + bool flush) +{ + unsigned char header[RECORD_HEADER_SIZE] = { 0 }; + LARGE_INTEGER end; + memcpy(header, "TCMR", 4); + header[4] = RECORD_VERSION; + header[5] = type; + put_u32(header + 8, payload_length); + put_u64(header + 12, sequence); + put_u32(header + 20, crc32_bytes(payload, payload_length)); + end.QuadPart = 0; + if (!SetFilePointerEx(file, end, NULL, FILE_END) || !write_all(file, header, sizeof(header)) + || !write_all(file, payload, payload_length) || (flush && !FlushFileBuffers(file))) + { + return TINY_STORE_IO_ERROR; + } + return TINY_STORE_OK; +} + +static tiny_store_status_t +apply_put(tiny_store_t *store, tiny_message_t *message, uint64_t sequence) +{ + tiny_store_entry_t *entry = find_entry(store, message->message_id); + if (entry != NULL) + { + return TINY_STORE_CORRUPT; + } + if (!ensure_entries(store)) + { + return TINY_STORE_NO_MEMORY; + } + entry = &store->entries[store->entry_count++]; + memset(entry, 0, sizeof(*entry)); + entry->message = *message; + tiny_message_init(message); + entry->sequence = sequence; + return TINY_STORE_OK; +} + +static void +clear_entries(tiny_store_t *store) +{ + size_t index; + for (index = 0; index < store->entry_count; ++index) + { + tiny_message_free(&store->entries[index].message); + } + store->entry_count = 0; + store->last_sequence = 0; +} + +static tiny_store_status_t +replay_log(tiny_store_t *store, uint64_t starting_offset) +{ + LARGE_INTEGER start, file_size, boundary; + start.QuadPart = (LONGLONG)starting_offset; + boundary = start; + if (!SetFilePointerEx(store->log, start, NULL, FILE_BEGIN) + || !GetFileSizeEx(store->log, &file_size)) + { + return TINY_STORE_IO_ERROR; + } + while (boundary.QuadPart < file_size.QuadPart) + { + unsigned char header[RECORD_HEADER_SIZE], *payload = NULL; + uint32_t actual = 0, payload_length, expected_crc; + uint64_t sequence; + tiny_store_status_t status = TINY_STORE_OK; + if (!read_all(store->log, header, sizeof(header), &actual)) + { + return TINY_STORE_IO_ERROR; + } + if (actual < sizeof(header)) + { + goto torn_tail; + } + if (memcmp(header, "TCMR", 4) != 0 || header[4] != RECORD_VERSION + || (header[5] != RECORD_PUT && header[5] != RECORD_DEL && header[5] != RECORD_MARKER)) + { + return TINY_STORE_CORRUPT; + } + payload_length = get_u32(header + 8); + sequence = get_u64(header + 12); + expected_crc = get_u32(header + 20); + if (payload_length > RECORD_PAYLOAD_MAX + || (header[5] != RECORD_MARKER && sequence <= store->last_sequence)) + { + return TINY_STORE_CORRUPT; + } + payload = malloc(payload_length == 0 ? 1 : payload_length); + if (payload == NULL) + { + return TINY_STORE_NO_MEMORY; + } + if (!read_all(store->log, payload, payload_length, &actual)) + { + free(payload); + return TINY_STORE_IO_ERROR; + } + if (actual < payload_length) + { + free(payload); + goto torn_tail; + } + if (crc32_bytes(payload, payload_length) != expected_crc) + { + free(payload); + return TINY_STORE_CORRUPT; + } + if (header[5] == RECORD_MARKER) + { + if (boundary.QuadPart != 0 || sequence != 0 + || payload_length != sizeof(store->generation)) + { + free(payload); + return TINY_STORE_CORRUPT; + } + memcpy(store->generation, payload, sizeof(store->generation)); + store->has_generation = true; + } + else if (header[5] == RECORD_PUT) + { + tiny_message_t message; + if (!deserialize_put(payload, payload_length, &message)) + { + free(payload); + return TINY_STORE_CORRUPT; + } + status = apply_put(store, &message, sequence); + tiny_message_free(&message); + } + else + { + char id[27]; + tiny_store_entry_t *entry; + if (payload_length != 34) + { + free(payload); + return TINY_STORE_CORRUPT; + } + memcpy(id, payload, 26); + id[26] = '\0'; + entry = find_entry(store, id); + if (entry == NULL || entry->deleted || !tiny_message_id_valid(id)) + { + free(payload); + return TINY_STORE_CORRUPT; + } + entry->deleted = true; + } + free(payload); + if (status != TINY_STORE_OK) + { + return status; + } + if (header[5] != RECORD_MARKER) + { + store->last_sequence = sequence; + } + boundary.QuadPart += RECORD_HEADER_SIZE + payload_length; + } + return TINY_STORE_OK; + +torn_tail: + if (!SetFilePointerEx(store->log, boundary, NULL, FILE_BEGIN) || !SetEndOfFile(store->log) + || !FlushFileBuffers(store->log)) + { + return TINY_STORE_IO_ERROR; + } + store->recovered_tail = true; + set_error(store, "an incomplete trailing record was truncated during recovery"); + return TINY_STORE_OK; +} + +static tiny_store_status_t +write_snapshot(tiny_store_t *store) +{ + wchar_t *temporary = join_path(store->directory, L"snapshot.tmp"); + HANDLE file; + unsigned char data[SNAPSHOT_HEADER_SIZE] = { 0 }; + LARGE_INTEGER size; + bool ok; + size_t index; + if (temporary == NULL) + { + return TINY_STORE_NO_MEMORY; + } + file = CreateFileW(temporary, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) + { + free(temporary); + return TINY_STORE_IO_ERROR; + } + GetFileSizeEx(store->log, &size); + memcpy(data, "TCMS", 4); + data[4] = 1; + memcpy(data + 8, store->generation, 16); + put_u64(data + 24, store->last_sequence); + put_u64(data + 32, (uint64_t)size.QuadPart); + put_u64(data + 40, (uint64_t)store->entry_count); + put_u32(data + 48, crc32_bytes(data, 48)); + ok = write_all(file, data, sizeof(data)); + for (index = 0; ok && index < store->entry_count; ++index) + { + unsigned char prefix[16], *payload = NULL; + uint32_t length = 0; + if (!serialize_put(&store->entries[index].message, &payload, &length)) + { + ok = false; + break; + } + put_u64(prefix, store->entries[index].sequence); + put_u32(prefix + 8, length); + put_u32(prefix + 12, crc32_bytes(payload, length)); + ok = write_all(file, prefix, sizeof(prefix)) && write_all(file, payload, length); + free(payload); + } + if (ok) + { + ok = FlushFileBuffers(file) != 0; + } + CloseHandle(file); + if (ok) + { + ok = MoveFileExW( + temporary, store->snapshot_path, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) + != 0; + } + free(temporary); + store->snapshot_valid = ok; + return ok ? TINY_STORE_OK : TINY_STORE_IO_ERROR; +} + +static bool +log_generation_matches(tiny_store_t *store, const unsigned char generation[16]) +{ + unsigned char header[RECORD_HEADER_SIZE], payload[16]; + uint32_t actual = 0; + LARGE_INTEGER zero = { 0 }; + if (!SetFilePointerEx(store->log, zero, NULL, FILE_BEGIN) + || !read_all(store->log, header, sizeof(header), &actual) || actual != sizeof(header) + || memcmp(header, "TCMR", 4) != 0 || header[4] != RECORD_VERSION || header[5] != RECORD_MARKER + || get_u64(header + 12) != 0 || get_u32(header + 8) != 16 + || !read_all(store->log, payload, sizeof(payload), &actual) || actual != sizeof(payload) + || get_u32(header + 20) != crc32_bytes(payload, sizeof(payload))) + { + return false; + } + return memcmp(payload, generation, 16) == 0; +} + +static bool +load_snapshot(tiny_store_t *store, uint64_t *log_offset) +{ + HANDLE file = CreateFileW(store->snapshot_path, + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + unsigned char header[SNAPSHOT_HEADER_SIZE]; + uint32_t actual = 0; + uint64_t count, index, expected_sequence, offset; + LARGE_INTEGER log_size; + if (file == INVALID_HANDLE_VALUE) + { + return false; + } + if (!read_all(file, header, sizeof(header), &actual) || actual != sizeof(header) + || memcmp(header, "TCMS", 4) != 0 || header[4] != 1 + || get_u32(header + 48) != crc32_bytes(header, 48) || !GetFileSizeEx(store->log, &log_size)) + { + goto invalid; + } + expected_sequence = get_u64(header + 24); + offset = get_u64(header + 32); + count = get_u64(header + 40); + if (offset > (uint64_t)log_size.QuadPart || count > SIZE_MAX + || !log_generation_matches(store, header + 8)) + { + goto invalid; + } + memcpy(store->generation, header + 8, 16); + store->has_generation = true; + for (index = 0; index < count; ++index) + { + unsigned char prefix[16], *payload; + uint32_t length, crc; + tiny_message_t message; + tiny_store_status_t status; + if (!read_all(file, prefix, sizeof(prefix), &actual) || actual != sizeof(prefix)) + { + goto invalid_entries; + } + length = get_u32(prefix + 8); + crc = get_u32(prefix + 12); + if (length > RECORD_PAYLOAD_MAX) + { + goto invalid_entries; + } + payload = malloc(length ? length : 1); + if (!payload) + { + goto invalid_entries; + } + if (!read_all(file, payload, length, &actual) || actual != length + || crc32_bytes(payload, length) != crc || !deserialize_put(payload, length, &message)) + { + free(payload); + goto invalid_entries; + } + free(payload); + status = apply_put(store, &message, get_u64(prefix)); + tiny_message_free(&message); + if (status != TINY_STORE_OK) + { + goto invalid_entries; + } + store->last_sequence = get_u64(prefix); + } + if (store->last_sequence != expected_sequence) + { + goto invalid_entries; + } + CloseHandle(file); + *log_offset = offset; + store->snapshot_valid = true; + return true; +invalid_entries: + clear_entries(store); + store->has_generation = false; +invalid: + set_error(store, "snapshot validation failed; the active log was replayed from the beginning"); + CloseHandle(file); + return false; +} + +tiny_store_status_t +tiny_store_open(tiny_store_t **output, const wchar_t *directory) +{ + tiny_store_t *store = calloc(1, sizeof(*store)); + wchar_t *lock_path = NULL; + tiny_store_status_t status; + if (store == NULL) + { + return TINY_STORE_NO_MEMORY; + } + store->lock = INVALID_HANDLE_VALUE; + store->log = INVALID_HANDLE_VALUE; + store->directory = _wcsdup(directory); + store->log_path = join_path(directory, L"active.log"); + store->snapshot_path = join_path(directory, L"snapshot.bin"); + lock_path = join_path(directory, L"store.lock"); + if (!store->directory || !store->log_path || !store->snapshot_path || !lock_path) + { + status = TINY_STORE_NO_MEMORY; + goto fail; + } + if (!CreateDirectoryW(directory, NULL) && GetLastError() != ERROR_ALREADY_EXISTS) + { + status = TINY_STORE_IO_ERROR; + goto fail; + } + store->lock = CreateFileW( + lock_path, GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + free(lock_path); + lock_path = NULL; + if (store->lock == INVALID_HANDLE_VALUE) + { + status = GetLastError() == ERROR_SHARING_VIOLATION ? TINY_STORE_BUSY : TINY_STORE_IO_ERROR; + goto fail; + } + store->log = CreateFileW(store->log_path, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (store->log == INVALID_HANDLE_VALUE) + { + status = TINY_STORE_IO_ERROR; + goto fail; + } + { + uint64_t replay_offset = 0; + (void)load_snapshot(store, &replay_offset); + status = replay_log(store, replay_offset); + } + if (status != TINY_STORE_OK) + { + goto fail; + } + *output = store; + return TINY_STORE_OK; +fail: + free(lock_path); + tiny_store_close(store); + return status; +} + +void +tiny_store_close(tiny_store_t *store) +{ + size_t index; + if (!store) + { + return; + } + if (store->log != INVALID_HANDLE_VALUE) + { + CloseHandle(store->log); + } + if (store->lock != INVALID_HANDLE_VALUE) + { + CloseHandle(store->lock); + } + for (index = 0; index < store->entry_count; ++index) + { + tiny_message_free(&store->entries[index].message); + } + free(store->entries); + free(store->directory); + free(store->log_path); + free(store->snapshot_path); + free(store); +} + +const char * +tiny_store_status_name(tiny_store_status_t status) +{ + switch (status) + { + case TINY_STORE_OK: + return "OK"; + case TINY_STORE_NOT_FOUND: + return "NOT_FOUND"; + case TINY_STORE_BUSY: + return "BUSY"; + case TINY_STORE_IO_ERROR: + return "IO_ERROR"; + case TINY_STORE_CORRUPT: + return "CORRUPT_STORE"; + default: + return "INTERNAL_ERROR"; + } +} + +tiny_store_status_t +tiny_store_write(tiny_store_t *store, const tiny_message_t *input, tiny_message_t *written) +{ + tiny_message_t message, indexed; + unsigned char *payload = NULL; + uint32_t length = 0; + tiny_store_status_t status; + tiny_message_init(&message); + tiny_message_init(&indexed); + if (tiny_message_copy(&message, input) != 0) + { + return TINY_STORE_NO_MEMORY; + } + message.timestamp_ms = (int64_t)now_milliseconds(); + make_ulid(store, (uint64_t)message.timestamp_ms, message.message_id); + if (!serialize_put(&message, &payload, &length) || !ensure_entries(store) + || tiny_message_copy(&indexed, &message) != 0) + { + free(payload); + tiny_message_free(&message); + tiny_message_free(&indexed); + return TINY_STORE_NO_MEMORY; + } + status + = append_record_to(store->log, RECORD_PUT, store->last_sequence + 1, payload, length, true); + free(payload); + if (status != TINY_STORE_OK) + { + tiny_message_free(&message); + tiny_message_free(&indexed); + set_error(store, "failed to append and flush a PUT record"); + return status; + } + store->last_sequence++; + memset(&store->entries[store->entry_count], 0, sizeof(store->entries[store->entry_count])); + store->entries[store->entry_count].message = indexed; + store->entries[store->entry_count].sequence = store->last_sequence; + store->entry_count++; + *written = message; + return TINY_STORE_OK; +} + +tiny_store_status_t +tiny_store_delete(tiny_store_t *store, const char *message_id, const char *session_id) +{ + tiny_store_entry_t *entry = find_entry(store, message_id); + unsigned char payload[34]; + tiny_store_status_t status; + if (!entry || entry->deleted + || (session_id && strcmp(entry->message.session_id, session_id) != 0)) + { + return TINY_STORE_NOT_FOUND; + } + memcpy(payload, message_id, 26); + put_u64(payload + 26, now_milliseconds()); + status = append_record_to( + store->log, RECORD_DEL, store->last_sequence + 1, payload, sizeof(payload), true); + if (status != TINY_STORE_OK) + { + set_error(store, "failed to append and flush a DEL record"); + return status; + } + store->last_sequence++; + entry->deleted = true; + return TINY_STORE_OK; +} + +tiny_store_status_t +tiny_store_rebuild(tiny_store_t *store, tiny_rebuild_report_t *report) +{ + wchar_t *temporary = join_path(store->directory, L"active.tmp"); + HANDLE file; + LARGE_INTEGER before, after; + size_t read_index, write_index = 0; + uint64_t removed = 0, max_sequence = 0; + tiny_store_status_t status = TINY_STORE_OK; + if (!temporary) + { + return TINY_STORE_NO_MEMORY; + } + GetFileSizeEx(store->log, &before); + file = CreateFileW( + temporary, GENERIC_WRITE | GENERIC_READ, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) + { + free(temporary); + return TINY_STORE_IO_ERROR; + } + if (BCryptGenRandom( + NULL, store->generation, sizeof(store->generation), BCRYPT_USE_SYSTEM_PREFERRED_RNG) + != 0) + { + CloseHandle(file); + free(temporary); + return TINY_STORE_IO_ERROR; + } + store->has_generation = true; + status = append_record_to( + file, RECORD_MARKER, 0, store->generation, sizeof(store->generation), false); + for (read_index = 0; read_index < store->entry_count; ++read_index) + { + if (status == TINY_STORE_OK && !store->entries[read_index].deleted) + { + unsigned char *payload = NULL; + uint32_t length = 0; + if (!serialize_put(&store->entries[read_index].message, &payload, &length)) + { + status = TINY_STORE_NO_MEMORY; + break; + } + status = append_record_to( + file, RECORD_PUT, store->entries[read_index].sequence, payload, length, false); + free(payload); + if (status != TINY_STORE_OK) + { + break; + } + if (store->entries[read_index].sequence > max_sequence) + { + max_sequence = store->entries[read_index].sequence; + } + } + else + { + removed++; + } + } + if (status == TINY_STORE_OK && !FlushFileBuffers(file)) + { + status = TINY_STORE_IO_ERROR; + } + CloseHandle(file); + if (status == TINY_STORE_OK) + { + CloseHandle(store->log); + store->log = INVALID_HANDLE_VALUE; + if (!MoveFileExW( + temporary, store->log_path, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) + { + status = TINY_STORE_IO_ERROR; + } + store->log = CreateFileW(store->log_path, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (store->log == INVALID_HANDLE_VALUE) + { + status = TINY_STORE_IO_ERROR; + } + } + if (status == TINY_STORE_OK) + { + for (read_index = 0; read_index < store->entry_count; ++read_index) + { + if (store->entries[read_index].deleted) + { + tiny_message_free(&store->entries[read_index].message); + } + else + { + if (write_index != read_index) + { + store->entries[write_index] = store->entries[read_index]; + } + write_index++; + } + } + store->entry_count = write_index; + store->last_sequence = max_sequence; + GetFileSizeEx(store->log, &after); + report->messages_indexed = write_index; + report->records_removed = removed; + report->bytes_before = (uint64_t)before.QuadPart; + report->bytes_after = (uint64_t)after.QuadPart; + status = write_snapshot(store); + } + if (status != TINY_STORE_OK) + { + set_error(store, + "compaction did not complete; the previous valid log is retained when replacement " + "had not begun"); + } + free(temporary); + return status; +} + +void +tiny_store_health(const tiny_store_t *store, tiny_store_health_t *health) +{ + LARGE_INTEGER size; + size_t index; + memset(health, 0, sizeof(*health)); + if (GetFileSizeEx(store->log, &size)) + { + health->log_size = (uint64_t)size.QuadPart; + } + { + HANDLE f = CreateFileW(store->snapshot_path, + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (f != INVALID_HANDLE_VALUE) + { + if (GetFileSizeEx(f, &size)) + { + health->snapshot_size = (uint64_t)size.QuadPart; + } + CloseHandle(f); + } + } + for (index = 0; index < store->entry_count; ++index) + { + if (store->entries[index].deleted) + { + health->deleted_messages++; + } + else + { + health->active_messages++; + } + } + health->last_sequence = store->last_sequence; + health->recovered_tail = store->recovered_tail; + health->snapshot_valid = store->snapshot_valid; + strncpy_s(health->last_error, sizeof(health->last_error), store->last_error, _TRUNCATE); +} + +size_t +tiny_store_entry_count(const tiny_store_t *store) +{ + return store->entry_count; +} +const tiny_store_entry_t * +tiny_store_entry(const tiny_store_t *store, size_t index) +{ + return index < store->entry_count ? &store->entries[index] : NULL; +} diff --git a/src/storage/store.h b/src/storage/store.h new file mode 100644 index 0000000..cc14a80 --- /dev/null +++ b/src/storage/store.h @@ -0,0 +1,64 @@ +#ifndef TINY_CONVMEM_STORAGE_STORE_H +#define TINY_CONVMEM_STORAGE_STORE_H + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <wchar.h> + +#include "domain/message.h" + +typedef enum tiny_store_status +{ + TINY_STORE_OK = 0, + TINY_STORE_NOT_FOUND, + TINY_STORE_BUSY, + TINY_STORE_IO_ERROR, + TINY_STORE_CORRUPT, + TINY_STORE_NO_MEMORY +} tiny_store_status_t; + +typedef struct tiny_store_entry +{ + tiny_message_t message; + bool deleted; + uint64_t sequence; +} tiny_store_entry_t; + +typedef struct tiny_store tiny_store_t; + +typedef struct tiny_rebuild_report +{ + uint64_t messages_indexed; + uint64_t records_removed; + uint64_t bytes_before; + uint64_t bytes_after; +} tiny_rebuild_report_t; + +typedef struct tiny_store_health +{ + uint64_t log_size; + uint64_t snapshot_size; + uint64_t active_messages; + uint64_t deleted_messages; + uint64_t last_sequence; + bool recovered_tail; + bool snapshot_valid; + char last_error[160]; +} tiny_store_health_t; + +tiny_store_status_t tiny_store_open(tiny_store_t **store, const wchar_t *directory); +void tiny_store_close(tiny_store_t *store); +const char *tiny_store_status_name(tiny_store_status_t status); + +tiny_store_status_t +tiny_store_write(tiny_store_t *store, const tiny_message_t *input, tiny_message_t *written); +tiny_store_status_t +tiny_store_delete(tiny_store_t *store, const char *message_id, const char *session_id); +tiny_store_status_t tiny_store_rebuild(tiny_store_t *store, tiny_rebuild_report_t *report); +void tiny_store_health(const tiny_store_t *store, tiny_store_health_t *health); + +size_t tiny_store_entry_count(const tiny_store_t *store); +const tiny_store_entry_t *tiny_store_entry(const tiny_store_t *store, size_t index); + +#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8f79a91..1407fc7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,8 +3,47 @@ add_test( COMMAND "${CMAKE_COMMAND}" "-DSERVER=$<TARGET_FILE:tiny-convmem-mcp>" "-DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/fixtures/smoke-input.jsonl" + "-DDATA_DIR=${CMAKE_CURRENT_BINARY_DIR}/smoke-data" -P "${CMAKE_CURRENT_SOURCE_DIR}/smoke.cmake" ) set_tests_properties(tiny-convmem-mcp.smoke PROPERTIES TIMEOUT 15) +add_executable(tiny-convmem-domain-test + domain_test.c + "${PROJECT_SOURCE_DIR}/src/domain/message.c" + "${PROJECT_SOURCE_DIR}/src/domain/search.c" +) +target_include_directories(tiny-convmem-domain-test PRIVATE "${PROJECT_SOURCE_DIR}/src") +target_compile_features(tiny-convmem-domain-test PRIVATE c_std_17) +if(MSVC) + target_compile_options(tiny-convmem-domain-test PRIVATE /utf-8 /W4) +endif() +add_test(NAME tiny-convmem-mcp.domain COMMAND tiny-convmem-domain-test) + +add_executable(tiny-convmem-storage-test + storage_test.c + "${PROJECT_SOURCE_DIR}/src/domain/message.c" + "${PROJECT_SOURCE_DIR}/src/storage/store.c" +) +target_include_directories(tiny-convmem-storage-test PRIVATE "${PROJECT_SOURCE_DIR}/src") +target_compile_features(tiny-convmem-storage-test PRIVATE c_std_17) +target_link_libraries(tiny-convmem-storage-test PRIVATE bcrypt) +if(MSVC) + target_compile_options(tiny-convmem-storage-test PRIVATE /utf-8 /W4) +endif() +add_test(NAME tiny-convmem-mcp.storage COMMAND tiny-convmem-storage-test) + +add_test( + NAME tiny-convmem-mcp.lifecycle + COMMAND "${CMAKE_COMMAND}" + "-DSERVER=$<TARGET_FILE:tiny-convmem-mcp>" + "-DWRITE_INPUT=${CMAKE_CURRENT_SOURCE_DIR}/fixtures/write-once.jsonl" + "-DHEALTH_INPUT=${CMAKE_CURRENT_SOURCE_DIR}/fixtures/health.jsonl" + "-DDATA_DIR=${CMAKE_CURRENT_BINARY_DIR}/lifecycle-data" + -P "${CMAKE_CURRENT_SOURCE_DIR}/lifecycle.cmake" +) +set_tests_properties(tiny-convmem-mcp.lifecycle PROPERTIES TIMEOUT 15) + +add_test(NAME tiny-convmem-mcp.requires-data-dir COMMAND $<TARGET_FILE:tiny-convmem-mcp>) +set_tests_properties(tiny-convmem-mcp.requires-data-dir PROPERTIES WILL_FAIL TRUE) diff --git a/tests/domain_test.c b/tests/domain_test.c new file mode 100644 index 0000000..d50e4bd --- /dev/null +++ b/tests/domain_test.c @@ -0,0 +1,71 @@ +#include "domain/message.h" +#include "domain/search.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +static int failures; +#define CHECK(value) \ + do \ + { \ + if (!(value)) \ + { \ + fprintf(stderr, "CHECK failed at line %d: %s\n", __LINE__, #value); \ + failures++; \ + } \ + } \ + while (0) + +int +main(void) +{ + tiny_role_t role; + tiny_memory_tier_t tier; + const char valid_utf8[] = "\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; + const char invalid_utf8[] = "\xc0\xaf"; + tiny_message_t original, copy; + tiny_search_index_t *search; + size_t *candidates = NULL, candidate_count = 0; + + CHECK(tiny_role_parse("user", &role) && role == TINY_ROLE_USER); + CHECK(!tiny_role_parse("tool", &role)); + CHECK(tiny_tier_parse("discard", &tier) && tier == TINY_TIER_DISCARD); + CHECK(!tiny_tier_parse("archive", &tier)); + CHECK(tiny_utf8_valid(valid_utf8, strlen(valid_utf8))); + CHECK(!tiny_utf8_valid(invalid_utf8, sizeof(invalid_utf8) - 1)); + CHECK(tiny_message_id_valid("01ARZ3NDEKTSV4RRFFQ69G5FAV")); + CHECK(!tiny_message_id_valid("81ARZ3NDEKTSV4RRFFQ69G5FAV")); + CHECK(!tiny_message_id_valid("01ARZ3NDEKTSV4RRFFQ69G5FAI")); + + tiny_message_init(&original); + tiny_message_init(©); + original.session_id = _strdup("session"); + original.content = _strdup("content"); + original.role = TINY_ROLE_ASSISTANT; + strcpy_s(original.message_id, sizeof(original.message_id), "01ARZ3NDEKTSV4RRFFQ69G5FAV"); + CHECK(tiny_message_copy(©, &original) == 0); + CHECK(copy.session_id != original.session_id); + CHECK(strcmp(copy.content, "content") == 0); + search = tiny_search_index_create(); + CHECK(search != NULL); + free(original.content); + original.content = _strdup("Curry and coffee"); + original.metadata.tags = calloc(1, sizeof(char *)); + original.metadata.tags[0] = _strdup("設計メモ"); + original.metadata.tag_count = 1; + CHECK(tiny_search_index_add(search, &original, 7)); + CHECK(tiny_search_index_candidates(search, "CURRY", &candidates, &candidate_count)); + CHECK(candidate_count == 1 && candidates[0] == 7); + CHECK(tiny_search_score("curry", &original) > 1.0); + free(candidates); + candidates = NULL; + CHECK(tiny_search_index_candidates(search, "設計", &candidates, &candidate_count)); + CHECK(candidate_count == 1 && candidates[0] == 7); + CHECK(tiny_search_score("設計", &original) > 0.0); + free(candidates); + tiny_search_index_free(search); + tiny_message_free(&original); + tiny_message_free(©); + return failures == 0 ? 0 : 1; +} diff --git a/tests/fixtures/health.jsonl b/tests/fixtures/health.jsonl new file mode 100644 index 0000000..8d1a19f --- /dev/null +++ b/tests/fixtures/health.jsonl @@ -0,0 +1,3 @@ +{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"lifecycle","version":"1"}}} +{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"read_messages","arguments":{"session_id":"lifecycle"}}} +{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"get_system_health","arguments":{}}} diff --git a/tests/fixtures/smoke-input.jsonl b/tests/fixtures/smoke-input.jsonl index 5c046a4..64bf18f 100644 --- a/tests/fixtures/smoke-input.jsonl +++ b/tests/fixtures/smoke-input.jsonl @@ -1,5 +1,9 @@ {"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"tiny-convmem-smoke","version":"0.1.0"}}} {"jsonrpc":"2.0","method":"notifications/initialized","params":{}} {"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}} -{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"health","arguments":{}}} - +{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"write_message","arguments":{"session_id":"smoke","role":"user","content":"hello durable memory","metadata":{"tags":["hello","memory"],"memory_tier":"short"}}}} +{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"read_messages","arguments":{"session_id":"smoke"}}} +{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"search_messages","arguments":{"query":"durable","limit":10}}} +{"jsonrpc":"2.0","id":6,"method":"tools/call","params":{"name":"rebuild_index","arguments":{}}} +{"jsonrpc":"2.0","id":7,"method":"tools/call","params":{"name":"get_system_health","arguments":{}}} +{"jsonrpc":"2.0","id":8,"method":"tools/call","params":{"name":"write_message","arguments":{"session_id":"smoke","role":"user","content":"must not be stored","unexpected":true}}} diff --git a/tests/fixtures/write-once.jsonl b/tests/fixtures/write-once.jsonl new file mode 100644 index 0000000..0378968 --- /dev/null +++ b/tests/fixtures/write-once.jsonl @@ -0,0 +1,2 @@ +{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"lifecycle","version":"1"}}} +{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"write_message","arguments":{"session_id":"lifecycle","role":"user","content":"secret-to-remove","metadata":{"tags":["秘密","remove"],"memory_tier":"long"}}}} diff --git a/tests/lifecycle.cmake b/tests/lifecycle.cmake new file mode 100644 index 0000000..8d2f97d --- /dev/null +++ b/tests/lifecycle.cmake @@ -0,0 +1,51 @@ +foreach(variable SERVER WRITE_INPUT HEALTH_INPUT DATA_DIR) + if(NOT DEFINED ${variable}) + message(FATAL_ERROR "${variable} is required") + endif() +endforeach() + +file(REMOVE_RECURSE "${DATA_DIR}") +execute_process( + COMMAND "${SERVER}" --data-dir "${DATA_DIR}" + INPUT_FILE "${WRITE_INPUT}" + OUTPUT_VARIABLE write_output + ERROR_VARIABLE write_error + RESULT_VARIABLE write_result +) +if(NOT write_result EQUAL 0) + message(FATAL_ERROR "initial write failed: ${write_error}") +endif() +string(REGEX MATCH "message_id[^0-9A-Z]+([0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z][0-9A-Z])" match "${write_output}") +set(message_id "${CMAKE_MATCH_1}") +if(NOT message_id) + message(FATAL_ERROR "could not extract message ID from: ${write_output}") +endif() + +set(delete_input "${DATA_DIR}/delete.jsonl") +file(WRITE "${delete_input}" + "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{\"protocolVersion\":\"2024-11-05\",\"capabilities\":{},\"clientInfo\":{\"name\":\"lifecycle\",\"version\":\"1\"}}}\n" + "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/call\",\"params\":{\"name\":\"delete_message\",\"arguments\":{\"message_id\":\"${message_id}\",\"session_id\":\"lifecycle\"}}}\n" + "{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"tools/call\",\"params\":{\"name\":\"read_messages\",\"arguments\":{\"session_id\":\"lifecycle\"}}}\n" + "{\"jsonrpc\":\"2.0\",\"id\":4,\"method\":\"tools/call\",\"params\":{\"name\":\"rebuild_index\",\"arguments\":{}}}\n" +) +execute_process(COMMAND "${SERVER}" --data-dir "${DATA_DIR}" INPUT_FILE "${delete_input}" + OUTPUT_VARIABLE delete_output ERROR_VARIABLE delete_error RESULT_VARIABLE delete_result) +if(NOT delete_result EQUAL 0 OR NOT delete_output MATCHES "records_removed[^0-9]+1") + message(FATAL_ERROR "delete/compaction failed: ${delete_output} ${delete_error}") +endif() + +file(READ "${DATA_DIR}/active.log" compacted_hex HEX) +string(FIND "${compacted_hex}" "7365637265742d746f2d72656d6f7665" secret_position) +if(NOT secret_position EQUAL -1) + message(FATAL_ERROR "compacted log still contains deleted content") +endif() + +file(APPEND "${DATA_DIR}/active.log" "TC") +file(WRITE "${DATA_DIR}/snapshot.bin" "invalid snapshot") +execute_process(COMMAND "${SERVER}" --data-dir "${DATA_DIR}" INPUT_FILE "${HEALTH_INPUT}" + OUTPUT_VARIABLE recovery_output ERROR_VARIABLE recovery_error RESULT_VARIABLE recovery_result) +string(FIND "${recovery_output}" "secret-to-remove" recovered_secret_position) +if(NOT recovery_result EQUAL 0 OR NOT recovery_output MATCHES "recovered_tail[^a-z]+true" OR + NOT recovery_output MATCHES "snapshot_valid[^a-z]+false" OR NOT recovered_secret_position EQUAL -1) + message(FATAL_ERROR "restart/tail recovery failed: ${recovery_output} ${recovery_error}") +endif() diff --git a/tests/smoke.cmake b/tests/smoke.cmake index b741d4b..2481297 100644 --- a/tests/smoke.cmake +++ b/tests/smoke.cmake @@ -4,9 +4,14 @@ endif() if(NOT DEFINED INPUT OR NOT EXISTS "${INPUT}") message(FATAL_ERROR "INPUT does not name the smoke-test fixture") endif() +if(NOT DEFINED DATA_DIR) + message(FATAL_ERROR "DATA_DIR is required") +endif() + +file(REMOVE_RECURSE "${DATA_DIR}") execute_process( - COMMAND "${SERVER}" + COMMAND "${SERVER}" --data-dir "${DATA_DIR}" INPUT_FILE "${INPUT}" OUTPUT_VARIABLE server_output ERROR_VARIABLE server_error @@ -24,9 +29,17 @@ foreach(expected "\"protocolVersion\":\"2024-11-05\"" "\"serverInfo\"" "\"capabilities\"" - "\"name\":\"health\"" + "\"name\":\"write_message\"" + "\"name\":\"read_messages\"" + "\"name\":\"search_messages\"" + "\"name\":\"delete_message\"" + "\"name\":\"rebuild_index\"" + "\"name\":\"get_system_health\"" + "\"required\":[\"session_id\",\"role\",\"content\"]" "\"type\":\"text\"" - "\"text\":\"ok\"" + "\\\"status\\\":\\\"OK\\\"" + "\\\"code\\\":\\\"VALIDATION_ERROR\\\"" + "hello durable memory" ) string(FIND "${server_output}" "${expected}" match_position) if(match_position EQUAL -1) @@ -36,4 +49,4 @@ foreach(expected endif() endforeach() -message(STATUS "MCP initialize, tools/list, and health tools/call succeeded") +message(STATUS "MCP initialize, six-tool listing, write, read, search, rebuild, and health succeeded") diff --git a/tests/storage_test.c b/tests/storage_test.c new file mode 100644 index 0000000..bac0f0b --- /dev/null +++ b/tests/storage_test.c @@ -0,0 +1,97 @@ +#define WIN32_LEAN_AND_MEAN +#include <windows.h> + +#include "storage/store.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +static int failures; +#define CHECK(value) \ + do \ + { \ + if (!(value)) \ + { \ + fprintf(stderr, "CHECK failed at line %d: %s\n", __LINE__, #value); \ + failures++; \ + } \ + } \ + while (0) + +int +wmain(void) +{ + wchar_t temporary[MAX_PATH], directory[MAX_PATH], log_path[MAX_PATH]; + tiny_store_t *store = NULL, *second = NULL; + tiny_message_t input, first, second_message; + tiny_store_status_t status; + HANDLE log; + HANDLE blocker; + tiny_rebuild_report_t report; + LARGE_INTEGER position; + unsigned char byte; + DWORD count; + + CHECK(GetTempPathW(MAX_PATH, temporary) > 0); + CHECK(GetTempFileNameW(temporary, L"tcm", 0, directory) > 0); + DeleteFileW(directory); + CHECK(CreateDirectoryW(directory, NULL) != 0); + status = tiny_store_open(&store, directory); + CHECK(status == TINY_STORE_OK); + CHECK(tiny_store_open(&second, directory) == TINY_STORE_BUSY); + + tiny_message_init(&input); + tiny_message_init(&first); + tiny_message_init(&second_message); + input.session_id = _strdup("unit"); + input.content = _strdup("first"); + input.role = TINY_ROLE_USER; + CHECK(tiny_store_write(store, &input, &first) == TINY_STORE_OK); + free(input.content); + input.content = _strdup("second"); + CHECK(tiny_store_write(store, &input, &second_message) == TINY_STORE_OK); + CHECK(strcmp(first.message_id, second_message.message_id) < 0); + tiny_message_free(&input); + tiny_message_free(&first); + tiny_message_free(&second_message); + tiny_store_close(store); + store = NULL; + + CHECK(tiny_store_open(&store, directory) == TINY_STORE_OK); + CHECK(tiny_store_entry_count(store) == 2); + swprintf_s(log_path, MAX_PATH, L"%s\\active.tmp", directory); + blocker = CreateFileW( + log_path, GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + CHECK(blocker != INVALID_HANDLE_VALUE); + CHECK(tiny_store_rebuild(store, &report) == TINY_STORE_IO_ERROR); + CHECK(tiny_store_entry_count(store) == 2); + CloseHandle(blocker); + DeleteFileW(log_path); + tiny_store_close(store); + store = NULL; + + swprintf_s(log_path, MAX_PATH, L"%s\\active.log", directory); + log = CreateFileW( + log_path, GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + CHECK(log != INVALID_HANDLE_VALUE); + position.QuadPart = -1; + CHECK(SetFilePointerEx(log, position, NULL, FILE_END) != 0); + CHECK(ReadFile(log, &byte, 1, &count, NULL) != 0 && count == 1); + byte ^= 0x5a; + position.QuadPart = -1; + CHECK(SetFilePointerEx(log, position, NULL, FILE_CURRENT) != 0); + CHECK(WriteFile(log, &byte, 1, &count, NULL) != 0 && count == 1); + CloseHandle(log); + CHECK(tiny_store_open(&store, directory) == TINY_STORE_CORRUPT); + + DeleteFileW(log_path); + swprintf_s(log_path, MAX_PATH, L"%s\\active.tmp", directory); + DeleteFileW(log_path); + swprintf_s(log_path, MAX_PATH, L"%s\\store.lock", directory); + DeleteFileW(log_path); + swprintf_s(log_path, MAX_PATH, L"%s\\snapshot.bin", directory); + DeleteFileW(log_path); + RemoveDirectoryW(directory); + return failures == 0 ? 0 : 1; +} |
