diff options
| author | tanaka takashi <isazaka@gmail.com> | 2026-07-04 17:21:32 +0900 |
|---|---|---|
| committer | tanaka takashi <isazaka@gmail.com> | 2026-07-04 17:21:32 +0900 |
| commit | 0634f2048c7c9a474aab1d8104b964b3bf70a064 (patch) | |
| tree | 508c305a64cbd3b01f5146776e22865d283d5dbb /src | |
| parent | eb5e9d9f37bda7d78f69614da70402357794168a (diff) | |
feat(P1-002): migrate offline JSONL import/export to dedicated CLI
- Replace the MCP import_messages tool with standalone CLI commands:
iny-convmem-mcp --data-dir <path> --import <file>
iny-convmem-mcp --data-dir <path> --export <file>
- Add src/cli/{import,export,cli.h} and a CTest process test (tiny-convmem-mcp.cli)
covering round-trip, manifest generation, overwrite/reject policies, and
include_deleted export.
- Fix Debug heap corruption in iny_store_import() by deep-copying imported
messages instead of taking ownership of caller-allocated records, which were
later freed by the caller.
- Remove the now-redundant src/mcp/tools/import_messages.c and keep the tool
registry at 7 MCP tools.
- Update docs/development-plan.md (P1-002 complete) and
docs/implementation-status.md with CLI test coverage.
Diffstat (limited to 'src')
| -rw-r--r-- | src/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | src/app/main.c | 88 | ||||
| -rw-r--r-- | src/cli/cli.h | 21 | ||||
| -rw-r--r-- | src/cli/export.c | 98 | ||||
| -rw-r--r-- | src/cli/import.c (renamed from src/mcp/tools/import_messages.c) | 153 | ||||
| -rw-r--r-- | src/mcp/tools/common/registry.c | 3 | ||||
| -rw-r--r-- | src/mcp/tools/common/registry.h | 1 | ||||
| -rw-r--r-- | src/mcp/tools/read_messages.c | 100 | ||||
| -rw-r--r-- | src/storage/store.c | 16 |
9 files changed, 294 insertions, 189 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a8d09dc..256bbaa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,8 @@ endif() add_executable(tiny-convmem-mcp app/main.c + cli/import.c + cli/export.c domain/message.c domain/search.c storage/codec.c @@ -34,7 +36,6 @@ add_executable(tiny-convmem-mcp mcp/tools/common/registry.c mcp/tools/delete_message.c mcp/tools/get_system_health.c - mcp/tools/import_messages.c mcp/tools/read_messages.c mcp/tools/rebuild_index.c mcp/tools/search_messages.c diff --git a/src/app/main.c b/src/app/main.c index f73b314..415e999 100644 --- a/src/app/main.c +++ b/src/app/main.c @@ -2,6 +2,7 @@ #include <stdlib.h> #include <string.h> +#include "cli/cli.h" #include "mcp/server.h" #include "storage/store.h" #include "sys/system.h" @@ -11,7 +12,75 @@ static void usage(FILE *stream) { - fprintf(stream, "Usage: tiny-convmem-mcp --data-dir <path>\n"); + fprintf(stream, + "Usage: tiny-convmem-mcp --data-dir <path> [command]\n" + "Commands:\n" + " (none) Run the MCP server (default)\n" + " --import <file> [--policy overwrite|reject]\n" + " Import a JSONL export into the store\n" + " --export <file> [--session <id>] [--include-deleted]\n" + " Export the store to a JSONL file (- for stdout)\n"); +} + +static const char * +find_argument(const tiny_sys_arguments_t *arguments, const char *name) +{ + size_t index; + for (index = 1; index + 1 < arguments->count; ++index) + { + if (strcmp(arguments->values[index], name) == 0) + { + return arguments->values[index + 1]; + } + } + return NULL; +} + +static int +flag_present(const tiny_sys_arguments_t *arguments, const char *name) +{ + size_t index; + for (index = 1; index < arguments->count; ++index) + { + if (strcmp(arguments->values[index], name) == 0) + { + return 1; + } + } + return 0; +} + +static int +run_server(tiny_store_t *store) +{ + int result = tiny_convmem_server_run(stdin, stdout, store); + if (result != 0) + { + fprintf(stderr, "tiny-convmem-mcp stopped with error code %d\n", result); + } + return result; +} + +static int +run_cli_command(tiny_store_t *store, const tiny_sys_arguments_t *arguments) +{ + tiny_cli_options_t options = { 0 }; + options.data_dir = arguments->values[2]; + options.import_path = find_argument(arguments, "--import"); + options.export_path = find_argument(arguments, "--export"); + options.session_id = find_argument(arguments, "--session"); + options.policy_name = find_argument(arguments, "--policy"); + options.include_deleted = flag_present(arguments, "--include-deleted"); + + if (options.import_path) + { + return tiny_cli_import(store, &options); + } + if (options.export_path) + { + return tiny_cli_export(store, &options); + } + return 2; } static int @@ -21,17 +90,26 @@ run(const tiny_sys_arguments_t *arguments) tiny_store_t *store = NULL; tiny_store_status_t status; int result; + int server_mode = 0; if (arguments->count == 2 && strcmp(arguments->values[1], "--help") == 0) { usage(stdout); return 0; } - if (arguments->count != 3 || strcmp(arguments->values[1], "--data-dir") != 0 + if (arguments->count < 3 || strcmp(arguments->values[1], "--data-dir") != 0 || arguments->values[2][0] == '\0') { usage(stderr); return 2; } + server_mode = (arguments->count == 3) + || (arguments->count == 5 && strcmp(arguments->values[3], "--import") != 0 + && strcmp(arguments->values[3], "--export") != 0); + if (!server_mode && arguments->count < 5) + { + usage(stderr); + return 2; + } absolute = tiny_sys_path_absolute(arguments->values[2]); if (absolute == NULL) { @@ -45,12 +123,8 @@ run(const tiny_sys_arguments_t *arguments) fprintf(stderr, "unable to open conversation store: %s\n", tiny_store_status_name(status)); return status == TINY_STORE_BUSY ? 3 : 1; } - result = tiny_convmem_server_run(stdin, stdout, store); + result = server_mode ? run_server(store) : run_cli_command(store, arguments); tiny_store_close(store); - if (result != 0) - { - fprintf(stderr, "tiny-convmem-mcp stopped with error code %d\n", result); - } return result; } diff --git a/src/cli/cli.h b/src/cli/cli.h new file mode 100644 index 0000000..1b173b4 --- /dev/null +++ b/src/cli/cli.h @@ -0,0 +1,21 @@ +#ifndef TINY_CONVMEM_CLI_H +#define TINY_CONVMEM_CLI_H + +#include "storage/store.h" + +#define TINY_CLI_LINE_MAX (TINY_CONTENT_MAX + 4096U) + +typedef struct tiny_cli_options +{ + const char *data_dir; + const char *import_path; + const char *export_path; + const char *session_id; + const char *policy_name; + int include_deleted; +} tiny_cli_options_t; + +int tiny_cli_import(tiny_store_t *store, const tiny_cli_options_t *options); +int tiny_cli_export(tiny_store_t *store, const tiny_cli_options_t *options); + +#endif diff --git a/src/cli/export.c b/src/cli/export.c new file mode 100644 index 0000000..4288ba4 --- /dev/null +++ b/src/cli/export.c @@ -0,0 +1,98 @@ +#include "cli/cli.h" + +#include "mcp/tools/common/common.h" +#include "storage/codec.h" +#include "storage/store.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +int +tiny_cli_export(tiny_store_t *store, const tiny_cli_options_t *options) +{ + size_t index; + size_t emitted = 0; + FILE *output = NULL; + tiny_json_buffer_t item; + + if (!options || !options->export_path) + { + fprintf(stderr, "export: path is required\n"); + return 1; + } + + if (strcmp(options->export_path, "-") == 0) + { + output = stdout; + } + else + { + if (fopen_s(&output, options->export_path, "wb") != 0 || !output) + { + fprintf(stderr, "export: unable to open '%s'\n", options->export_path); + return 1; + } + } + + if (fprintf(output, "{\"type\":\"tiny-convmem-manifest\",\"format_version\":2}\n") < 0) + { + fprintf(stderr, "export: failed to write manifest\n"); + if (output != stdout) + { + fclose(output); + } + return 1; + } + + for (index = 0; index < tiny_store_entry_count(store); ++index) + { + const tiny_store_entry_t *entry = tiny_store_entry(store, index); + int write_failed; + if (!entry) + { + continue; + } + if (entry->deleted && !options->include_deleted) + { + continue; + } + if (options->session_id && options->session_id[0] + && strcmp(entry->message.session_id, options->session_id) != 0) + { + continue; + } + + tiny_json_init(&item); + if (entry->deleted) + { + tiny_json_raw(&item, "{\"type\":\"DEL\",\"message_id\":"); + tiny_json_string(&item, entry->message.message_id); + tiny_json_raw(&item, "}"); + } + else + { + tiny_mcp_tool_append_message(&item, &entry->message); + } + + write_failed = item.failed || fprintf(output, "%s\n", item.data ? item.data : "") < 0; + tiny_json_free(&item); + if (write_failed) + { + fprintf(stderr, "export: failed to write record\n"); + if (output != stdout) + { + fclose(output); + } + return 1; + } + emitted++; + } + + if (output != stdout) + { + fclose(output); + } + (void)emitted; + return 0; +} diff --git a/src/mcp/tools/import_messages.c b/src/cli/import.c index c46b016..3ddb9e9 100644 --- a/src/mcp/tools/import_messages.c +++ b/src/cli/import.c @@ -1,15 +1,16 @@ -#include "mcp/tools/common/registry.h" +#include "cli/cli.h" -#include "mcp/tools/common/common.h" -#include "mcp/tools/common/context.h" +#include "domain/message.h" +#include "mcp/json.h" +#include "storage/store.h" + +#include "mjson.h" +#include <stdio.h> #include <stdlib.h> #include <string.h> -#include "mjson.h" -#include "storage/store.h" - -#define IMPORT_MAX_LINE (TINY_CONTENT_MAX + 4096U) +#define IMPORT_MAX_LINE TINY_CLI_LINE_MAX typedef struct import_record_list { @@ -147,7 +148,6 @@ import_parse_string_array(const char *json, *count = index; if (index > 0 && offset != 0) { - /* mjson_next returned non-zero because of a parse/validation failure. */ return false; } return true; @@ -455,108 +455,109 @@ import_parse_jsonl(const char *jsonl, import_record_list_t *records) return manifest_seen; } -static void -call_import_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) +int +tiny_cli_import(tiny_store_t *store, const tiny_cli_options_t *options) { - tiny_mcp_tools_context_t *context = tiny_mcp_tools_context_current(); - char *jsonl = tiny_mcp_tool_get_string(tool, "jsonl", TINY_CONTENT_MAX); - char *policy_name = tiny_mcp_tool_get_string(tool, "policy", 16); - tiny_import_policy_t policy = TINY_IMPORT_OVERWRITE; import_record_list_t records = { NULL, 0, 0 }; size_t imported = 0, rejected = 0; - tiny_json_buffer_t json; tiny_store_status_t status; + tiny_import_policy_t policy = TINY_IMPORT_OVERWRITE; + char *jsonl = NULL; + FILE *file = NULL; + long file_size; + int result = 1; - if (!context || !jsonl) + if (!options || !options->import_path) { - tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "invalid import_messages arguments"); - free(jsonl); - free(policy_name); - return; + fprintf(stderr, "import: path is required\n"); + return 1; } - if (policy_name && policy_name[0]) + + if (options->policy_name && options->policy_name[0]) { - if (strcmp(policy_name, "overwrite") == 0) + if (strcmp(options->policy_name, "overwrite") == 0) { policy = TINY_IMPORT_OVERWRITE; } - else if (strcmp(policy_name, "reject") == 0) + else if (strcmp(options->policy_name, "reject") == 0) { policy = TINY_IMPORT_REJECT; } else { - tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "invalid import policy"); - free(jsonl); - free(policy_name); - return; + fprintf(stderr, "import: invalid policy '%s'\n", options->policy_name); + return 1; } } + + if (fopen_s(&file, options->import_path, "rb") != 0 || !file) + { + fprintf(stderr, "import: unable to open '%s'\n", options->import_path); + return 1; + } + + if (fseek(file, 0, SEEK_END) != 0 || (file_size = ftell(file)) < 0 + || fseek(file, 0, SEEK_SET) != 0) + { + fprintf(stderr, "import: unable to seek '%s'\n", options->import_path); + fclose(file); + return 1; + } + + if ((size_t)file_size > IMPORT_MAX_LINE * 1024ULL * 1024ULL) + { + fprintf(stderr, "import: file too large\n"); + fclose(file); + return 1; + } + + jsonl = malloc((size_t)file_size + 1); + if (!jsonl) + { + fprintf(stderr, "import: out of memory\n"); + fclose(file); + return 1; + } + + if (fread(jsonl, 1, (size_t)file_size, file) != (size_t)file_size) + { + fprintf(stderr, "import: unable to read '%s'\n", options->import_path); + free(jsonl); + fclose(file); + return 1; + } + jsonl[file_size] = '\0'; + fclose(file); + if (!import_parse_jsonl(jsonl, &records)) { - tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "invalid import JSONL"); - import_record_list_free(&records); + fprintf(stderr, "import: invalid JSONL in '%s'\n", options->import_path); free(jsonl); - free(policy_name); - return; + return 1; } - status = tiny_store_import( - context->store, records.items, records.count, policy, &imported, &rejected); + + status = tiny_store_import(store, records.items, records.count, policy, &imported, &rejected); import_record_list_free(&records); - tiny_json_init(&json); + free(jsonl); + if (status == TINY_STORE_OK) { - tiny_json_raw(&json, "{\"status\":\"OK\",\"imported\":"); - tiny_json_u64(&json, (uint64_t)imported); - tiny_json_raw(&json, ",\"rejected\":"); - tiny_json_u64(&json, (uint64_t)rejected); - tiny_json_raw(&json, "}"); + printf("{\"imported\":%zu,\"rejected\":%zu}\n", imported, rejected); + fflush(stdout); + result = 0; } else if (status == TINY_STORE_CONFLICT) { - tiny_json_raw(&json, - "{\"status\":\"ERROR\",\"error\":{\"code\":\"CONFLICT\"," - "\"message\":\"import conflict with existing records\"}}"); + fprintf(stderr, "import: conflict with existing records\n"); } else if (status == TINY_STORE_CORRUPT) { - tiny_json_raw(&json, - "{\"status\":\"ERROR\",\"error\":{\"code\":\"CORRUPT_STORE\"," - "\"message\":\"import JSONL contained invalid records\"}}"); + fprintf(stderr, "import: invalid records\n"); } else { - tiny_json_raw(&json, - "{\"status\":\"ERROR\",\"error\":{\"code\":\"INTERNAL_ERROR\"," - "\"message\":\"import failed\"}}"); + fprintf(stderr, "import: failed\n"); } - tiny_mcp_tool_add_json(result, &json); - tiny_json_free(&json); - free(jsonl); - free(policy_name); -} -bool -tiny_mcp_register_import_messages(mcpc_server_t *server) -{ - mcpc_tool_t *tool = tiny_mcp_tool_new( - "import_messages", - "Atomically import a JSONL export from read_messages view=export. The manifest line is " - "required; subsequent lines are PUT messages or DEL tombstones. Conflicting message IDs " - "are resolved by the policy argument.", - call_import_messages); - return tool - && tiny_mcp_tool_add_property(tool, - "jsonl", - "JSONL string containing a manifest line followed by PUT " - "message lines and optional DEL tombstone lines.", - MCPC_U8STR, - true) - && tiny_mcp_tool_add_property(tool, - "policy", - "Conflict policy: overwrite (default) replaces existing " - "message IDs, reject refuses duplicate IDs.", - MCPC_U8STR, - false) - && mcpc_server_add_tool(server, tool) == MCPC_EC_0; + return result; } diff --git a/src/mcp/tools/common/registry.c b/src/mcp/tools/common/registry.c index cc3fa22..1bdafc3 100644 --- a/src/mcp/tools/common/registry.c +++ b/src/mcp/tools/common/registry.c @@ -8,6 +8,5 @@ tiny_mcp_tools_register(mcpc_server_t *server) return tiny_mcp_register_write_message(server) && tiny_mcp_register_read_messages(server) && tiny_mcp_register_search_messages(server) && tiny_mcp_register_update_message(server) && tiny_mcp_register_delete_message(server) && tiny_mcp_register_rebuild_index(server) - && tiny_mcp_register_get_system_health(server) - && tiny_mcp_register_import_messages(server); + && tiny_mcp_register_get_system_health(server); } diff --git a/src/mcp/tools/common/registry.h b/src/mcp/tools/common/registry.h index c873869..0d67eec 100644 --- a/src/mcp/tools/common/registry.h +++ b/src/mcp/tools/common/registry.h @@ -11,6 +11,5 @@ bool tiny_mcp_register_update_message(mcpc_server_t *server); bool tiny_mcp_register_delete_message(mcpc_server_t *server); bool tiny_mcp_register_rebuild_index(mcpc_server_t *server); bool tiny_mcp_register_get_system_health(mcpc_server_t *server); -bool tiny_mcp_register_import_messages(mcpc_server_t *server); #endif diff --git a/src/mcp/tools/read_messages.c b/src/mcp/tools/read_messages.c index 647e6f3..109becb 100644 --- a/src/mcp/tools/read_messages.c +++ b/src/mcp/tools/read_messages.c @@ -729,86 +729,6 @@ call_sessions_view(tiny_mcp_tools_context_t *context, const mcpc_tool_t *tool, m } static void -call_export_view(tiny_mcp_tools_context_t *context, const mcpc_tool_t *tool, mcpc_ucbr_t *result) -{ - char *session = tiny_mcp_tool_get_string(tool, "session_id", TINY_SESSION_ID_MAX); - char *cursor = tiny_mcp_tool_get_string(tool, "cursor", CURSOR_TEXT_MAX); - int32_t limit = 500; - char *include_deleted_text = tiny_mcp_tool_get_string(tool, "include_deleted", 8); - bool include_deleted = include_deleted_text && strcmp(include_deleted_text, "true") == 0; - bool valid; - size_t index = simple_cursor(cursor, &valid), emitted = 0, start_index; - tiny_json_buffer_t json, lines; - size_t outer_size = 64; - bool truncated = false; - mcpc_errcode_t limit_status = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"limit", &limit); - if (!context || !valid || limit < 1 || limit > 1000 - || (limit_status != MCPC_EC_0 && limit_status != MCPC_EC_ANYDATA_NOTFOUND)) - { - tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "invalid export view arguments"); - free(session); - free(cursor); - return; - } - start_index = index; - tiny_json_init(&lines); - if (index == 0) - { - tiny_json_raw(&lines, "{\"type\":\"tiny-convmem-manifest\",\"format_version\":2}\n"); - } - for (; index < tiny_store_entry_count(context->store) && emitted < (size_t)limit; ++index) - { - const tiny_store_entry_t *entry = tiny_store_entry(context->store, index); - tiny_json_buffer_t item; - size_t line_overhead; - if (!entry || (entry->deleted && !include_deleted) - || (session && session[0] && strcmp(session, entry->message.session_id) != 0)) - { - continue; - } - tiny_json_init(&item); - if (entry->deleted) - { - tiny_json_raw(&item, "{\"type\":\"DEL\",\"message_id\":"); - tiny_json_string(&item, entry->message.message_id); - tiny_json_raw(&item, "}"); - } - else - { - tiny_mcp_tool_append_message(&item, &entry->message); - } - line_overhead = item.length + 1; - if (item.failed || outer_size + lines.length + line_overhead + 2 > READ_RESPONSE_BUDGET) - { - truncated = true; - tiny_json_free(&item); - break; - } - tiny_json_raw_n(&lines, item.data, item.length); - tiny_json_raw(&lines, "\n"); - emitted++; - tiny_json_free(&item); - } - tiny_json_init(&json); - tiny_json_raw(&json, "{\"jsonl\":"); - tiny_json_string(&json, lines.data ? lines.data : ""); - if (truncated || index < tiny_store_entry_count(context->store)) - { - char next[32]; - snprintf(next, sizeof(next), "%zu", index > start_index ? index : start_index + 1); - tiny_json_raw(&json, ",\"nextCursor\":"); - tiny_json_string(&json, next); - tiny_json_raw(&json, ",\"truncated\":true"); - } - tiny_json_raw(&json, "}"); - tiny_mcp_tool_add_json(result, &json); - tiny_json_free(&json); - tiny_json_free(&lines); - free(session); - free(cursor); -} - -static void call_read_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) { tiny_mcp_tools_context_t *context = tiny_mcp_tools_context_current(); @@ -841,14 +761,6 @@ call_read_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) free(cursor_text); return; } - if (view && strcmp(view, "export") == 0) - { - call_export_view(context, tool, result); - free(view); - free(session); - free(cursor_text); - return; - } if (view && strcmp(view, "history") != 0) { tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "unsupported read_messages view"); @@ -983,11 +895,7 @@ tiny_mcp_register_read_messages(mcpc_server_t *server) call_read_messages); return tool && tiny_mcp_tool_add_property( - tool, - "view", - "Read view: history (default), context, sessions, or export.", - MCPC_U8STR, - false) + tool, "view", "Read view: history (default), context, or sessions.", MCPC_U8STR, false) && tiny_mcp_tool_add_property(tool, "session_id", "Conversation session whose chronological history to read.", @@ -1022,11 +930,5 @@ tiny_mcp_register_read_messages(mcpc_server_t *server) "omit for the first page.", MCPC_U8STR, false) - && tiny_mcp_tool_add_property(tool, - "include_deleted", - "When true, export view includes deleted records as DEL " - "lines; other views ignore this option.", - MCPC_U8STR, - false) && mcpc_server_add_tool(server, tool) == MCPC_EC_0; } diff --git a/src/storage/store.c b/src/storage/store.c index 45621d1..5701578 100644 --- a/src/storage/store.c +++ b/src/storage/store.c @@ -1957,16 +1957,26 @@ tiny_store_import(tiny_store_t *store, } if (existing != NULL) { + tiny_message_t replacement; + tiny_message_init(&replacement); + if (tiny_message_copy(&replacement, &record->message) != 0) + { + status = TINY_STORE_NO_MEMORY; + break; + } tiny_message_free(&existing->message); existing->deleted = record->deleted; existing->sequence = (uint64_t)record->message.timestamp_ms; - existing->message = record->message; + existing->message = replacement; (*imported_count)++; } else { - if (!ensure_entries(store)) + tiny_message_t copied; + tiny_message_init(&copied); + if (!ensure_entries(store) || tiny_message_copy(&copied, &record->message) != 0) { + tiny_message_free(&copied); status = TINY_STORE_NO_MEMORY; break; } @@ -1974,7 +1984,7 @@ tiny_store_import(tiny_store_t *store, &store->entries[store->entry_count], 0, sizeof(store->entries[store->entry_count])); store->entries[store->entry_count].deleted = record->deleted; store->entries[store->entry_count].sequence = (uint64_t)record->message.timestamp_ms; - store->entries[store->entry_count].message = record->message; + store->entries[store->entry_count].message = copied; store->entry_count++; (*imported_count)++; } |
