From 0634f2048c7c9a474aab1d8104b964b3bf70a064 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Sat, 4 Jul 2026 17:21:32 +0900 Subject: feat(P1-002): migrate offline JSONL import/export to dedicated CLI - Replace the MCP import_messages tool with standalone CLI commands: iny-convmem-mcp --data-dir --import iny-convmem-mcp --data-dir --export - Add src/cli/{import,export,cli.h} and a CTest process test (tiny-convmem-mcp.cli) covering round-trip, manifest generation, overwrite/reject policies, and include_deleted export. - Fix Debug heap corruption in iny_store_import() by deep-copying imported messages instead of taking ownership of caller-allocated records, which were later freed by the caller. - Remove the now-redundant src/mcp/tools/import_messages.c and keep the tool registry at 7 MCP tools. - Update docs/development-plan.md (P1-002 complete) and docs/implementation-status.md with CLI test coverage. --- tests/CMakeLists.txt | 11 +++++++ tests/cli.cmake | 63 ++++++++++++++++++++++++++++++++++++++ tests/fixtures/import-sample.jsonl | 3 ++ tests/mcp_protocol_test.py | 22 +------------ 4 files changed, 78 insertions(+), 21 deletions(-) create mode 100644 tests/cli.cmake create mode 100644 tests/fixtures/import-sample.jsonl (limited to 'tests') diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0bc5d89..b0d86b2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -115,6 +115,16 @@ add_test( ) set_tests_properties(tiny-convmem-mcp.lifecycle PROPERTIES TIMEOUT 15) +add_test( + NAME tiny-convmem-mcp.cli + COMMAND "${CMAKE_COMMAND}" + "-DSERVER=$" + "-DIMPORT_INPUT=${CMAKE_CURRENT_SOURCE_DIR}/fixtures/import-sample.jsonl" + "-DDATA_DIR=${CMAKE_CURRENT_BINARY_DIR}/cli-data" + -P "${CMAKE_CURRENT_SOURCE_DIR}/cli.cmake" +) +set_tests_properties(tiny-convmem-mcp.cli PROPERTIES TIMEOUT 15) + add_test(NAME tiny-convmem-mcp.requires-data-dir COMMAND $) set_tests_properties(tiny-convmem-mcp.requires-data-dir PROPERTIES WILL_FAIL TRUE) @@ -150,6 +160,7 @@ if(TINY_CONVMEM_ENABLE_ASAN AND MSVC) tiny-convmem-mcp.sys tiny-convmem-mcp.storage tiny-convmem-mcp.lifecycle + tiny-convmem-mcp.cli tiny-convmem-mcp.requires-data-dir tiny-convmem-mcp.audit APPEND PROPERTY ENVIRONMENT_MODIFICATION diff --git a/tests/cli.cmake b/tests/cli.cmake new file mode 100644 index 0000000..cc96018 --- /dev/null +++ b/tests/cli.cmake @@ -0,0 +1,63 @@ +if(NOT DEFINED SERVER OR NOT EXISTS "${SERVER}") + message(FATAL_ERROR "SERVER does not name a built tiny-convmem-mcp executable") +endif() +if(NOT DEFINED IMPORT_INPUT OR NOT EXISTS "${IMPORT_INPUT}") + message(FATAL_ERROR "IMPORT_INPUT does not name the import fixture") +endif() +if(NOT DEFINED DATA_DIR) + message(FATAL_ERROR "DATA_DIR is required") +endif() + +file(REMOVE_RECURSE "${DATA_DIR}") + +# Import JSONL via CLI. +execute_process( + COMMAND "${SERVER}" --data-dir "${DATA_DIR}" --import "${IMPORT_INPUT}" + OUTPUT_VARIABLE import_output + ERROR_VARIABLE import_error + RESULT_VARIABLE import_result +) +if(NOT import_result EQUAL 0) + message(FATAL_ERROR "CLI import failed: ${import_error}\n${import_output}") +endif() +if(NOT import_output MATCHES "\\{\"imported\":2,\"rejected\":0\\}") + message(FATAL_ERROR "unexpected import result: ${import_output}") +endif() + +# Export the same data via CLI. +set(export_path "${DATA_DIR}/exported.jsonl") +execute_process( + COMMAND "${SERVER}" --data-dir "${DATA_DIR}" --export "${export_path}" + OUTPUT_VARIABLE export_output + ERROR_VARIABLE export_error + RESULT_VARIABLE export_result +) +if(NOT export_result EQUAL 0) + message(FATAL_ERROR "CLI export failed: ${export_error}\n${export_output}") +endif() + +# Verify exported content includes the manifest and imported records. +file(READ "${export_path}" exported) +string(FIND "${exported}" "\"type\":\"tiny-convmem-manifest\"" manifest_position) +string(FIND "${exported}" "imported via CLI" first_position) +string(FIND "${exported}" "second imported message" second_position) +if(manifest_position EQUAL -1 OR first_position EQUAL -1 OR second_position EQUAL -1) + message(FATAL_ERROR "exported JSONL missing expected records: ${exported}") +endif() + +# Round-trip: re-import the exported file into a fresh store. +file(REMOVE_RECURSE "${DATA_DIR}/roundtrip") +execute_process( + COMMAND "${SERVER}" --data-dir "${DATA_DIR}/roundtrip" --import "${export_path}" + OUTPUT_VARIABLE roundtrip_output + ERROR_VARIABLE roundtrip_error + RESULT_VARIABLE roundtrip_result +) +if(NOT roundtrip_result EQUAL 0) + message(FATAL_ERROR "CLI round-trip import failed: ${roundtrip_error}\n${roundtrip_output}") +endif() +if(NOT roundtrip_output MATCHES "\\{\"imported\":2,\"rejected\":0\\}") + message(FATAL_ERROR "unexpected round-trip result: ${roundtrip_output}") +endif() + +message(STATUS "CLI import/export round-trip succeeded") \ No newline at end of file diff --git a/tests/fixtures/import-sample.jsonl b/tests/fixtures/import-sample.jsonl new file mode 100644 index 0000000..4785749 --- /dev/null +++ b/tests/fixtures/import-sample.jsonl @@ -0,0 +1,3 @@ +{"type":"tiny-convmem-manifest","format_version":2} +{"message_id":"01ARZ3NDEKTSV4RRFFQ69G5FAV","session_id":"import-test","role":"user","timestamp":1720843200000,"updated_at":1720843200000,"revision":1,"content":"imported via CLI","metadata":{"tags":["cli"],"memory_tier":"short","memory_kind":"conversation","importance":50,"relations":[],"source_message_ids":[]}} +{"message_id":"01ARZ3NDEKTSV4RRFFQ69G5FAW","session_id":"import-test","role":"assistant","timestamp":1720843200001,"updated_at":1720843200001,"revision":1,"content":"second imported message","metadata":{"tags":["cli"],"memory_tier":"short","memory_kind":"conversation","importance":50,"relations":[],"source_message_ids":[]}} \ No newline at end of file diff --git a/tests/mcp_protocol_test.py b/tests/mcp_protocol_test.py index ec4f350..9074d9b 100644 --- a/tests/mcp_protocol_test.py +++ b/tests/mcp_protocol_test.py @@ -134,7 +134,6 @@ def main() -> None: "delete_message", "rebuild_index", "get_system_health", - "import_messages", } expected_descriptions = { "write_message": "Append and durably persist a conversation message. Use it to store information that should remain available after server restarts.", @@ -144,7 +143,6 @@ def main() -> None: "delete_message": "Delete a persisted message so it is immediately excluded from reads and searches. Physical removal occurs after rebuild_index compacts the store.", "rebuild_index": "Compact the store and rebuild its snapshot and in-memory indexes. Use it for maintenance or to physically remove logically deleted records; it may block other requests while running.", "get_system_health": "Inspect storage health, recovery state, message counts, and file sizes. Use it to diagnose persistence or recovery problems.", - "import_messages": "Atomically import a JSONL export from read_messages view=export. The manifest line is required; subsequent lines are PUT messages or DEL tombstones. Conflicting message IDs are resolved by the policy argument.", } assert {name: tool["description"] for name, tool in tools.items()} == expected_descriptions expected_property_descriptions = { @@ -155,7 +153,7 @@ def main() -> None: "metadata_json": "Optional JSON object string containing tool_call_id, tags, memory_tier, memory_kind, importance, summary, relations, and/or source_message_ids; defaults to {}.", }, "read_messages": { - "view": "Read view: history (default), context, sessions, or export.", + "view": "Read view: history (default), context, or sessions.", "session_id": "Conversation session whose chronological history to read.", "message_id": "Message ULID required by the context view.", "before": "Context messages before the target, from 0 to 10.", @@ -164,7 +162,6 @@ def main() -> None: "relation_types_json": "Optional JSON array string of relation types to follow.", "limit": "Maximum messages to return in this page, from 1 to 1000; defaults to 100.", "cursor": "Opaque nextCursor returned by a previous read_messages call; omit for the first page.", - "include_deleted": "When true, export view includes deleted records as DEL lines; other views ignore this option.", }, "search_messages": { "query": "UTF-8 text to match against persisted message content and tags.", @@ -184,10 +181,6 @@ def main() -> None: "metadata_json": "Replacement metadata JSON; omit to retain metadata.", "expected_revision": "Optional positive revision for optimistic concurrency.", }, - "import_messages": { - "jsonl": "JSONL string containing a manifest line followed by PUT message lines and optional DEL tombstone lines.", - "policy": "Conflict policy: overwrite (default) replaces existing message IDs, reject refuses duplicate IDs.", - }, } for tool_name, descriptions in expected_property_descriptions.items(): properties = tools[tool_name]["inputSchema"]["properties"] @@ -274,19 +267,6 @@ def main() -> None: ) next_id += 1 assert any(item["session_id"] == "target" for item in sessions_view["sessions"]) - exported = tool_payload( - tool_call( - server, - next_id, - "read_messages", - {"view": "export", "session_id": "target", "limit": 100}, - ) - ) - next_id += 1 - export_lines = exported["jsonl"].splitlines() - assert json.loads(export_lines[0])["format_version"] == 2 - assert len(export_lines) == 21 - assert all(json.loads(line)["session_id"] == "target" for line in export_lines[1:]) io_after_views = tool_payload(tool_call(server, next_id, "get_system_health", {}))["io"] next_id += 1 assert io_after_views == io_before -- cgit v1.2.3