diff options
Diffstat (limited to 'src/cli')
| -rw-r--r-- | src/cli/cli.h | 21 | ||||
| -rw-r--r-- | src/cli/export.c | 98 | ||||
| -rw-r--r-- | src/cli/import.c | 563 |
3 files changed, 682 insertions, 0 deletions
diff --git a/src/cli/cli.h b/src/cli/cli.h new file mode 100644 index 0000000..1b173b4 --- /dev/null +++ b/src/cli/cli.h @@ -0,0 +1,21 @@ +#ifndef TINY_CONVMEM_CLI_H +#define TINY_CONVMEM_CLI_H + +#include "storage/store.h" + +#define TINY_CLI_LINE_MAX (TINY_CONTENT_MAX + 4096U) + +typedef struct tiny_cli_options +{ + const char *data_dir; + const char *import_path; + const char *export_path; + const char *session_id; + const char *policy_name; + int include_deleted; +} tiny_cli_options_t; + +int tiny_cli_import(tiny_store_t *store, const tiny_cli_options_t *options); +int tiny_cli_export(tiny_store_t *store, const tiny_cli_options_t *options); + +#endif diff --git a/src/cli/export.c b/src/cli/export.c new file mode 100644 index 0000000..4288ba4 --- /dev/null +++ b/src/cli/export.c @@ -0,0 +1,98 @@ +#include "cli/cli.h" + +#include "mcp/tools/common/common.h" +#include "storage/codec.h" +#include "storage/store.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +int +tiny_cli_export(tiny_store_t *store, const tiny_cli_options_t *options) +{ + size_t index; + size_t emitted = 0; + FILE *output = NULL; + tiny_json_buffer_t item; + + if (!options || !options->export_path) + { + fprintf(stderr, "export: path is required\n"); + return 1; + } + + if (strcmp(options->export_path, "-") == 0) + { + output = stdout; + } + else + { + if (fopen_s(&output, options->export_path, "wb") != 0 || !output) + { + fprintf(stderr, "export: unable to open '%s'\n", options->export_path); + return 1; + } + } + + if (fprintf(output, "{\"type\":\"tiny-convmem-manifest\",\"format_version\":2}\n") < 0) + { + fprintf(stderr, "export: failed to write manifest\n"); + if (output != stdout) + { + fclose(output); + } + return 1; + } + + for (index = 0; index < tiny_store_entry_count(store); ++index) + { + const tiny_store_entry_t *entry = tiny_store_entry(store, index); + int write_failed; + if (!entry) + { + continue; + } + if (entry->deleted && !options->include_deleted) + { + continue; + } + if (options->session_id && options->session_id[0] + && strcmp(entry->message.session_id, options->session_id) != 0) + { + continue; + } + + tiny_json_init(&item); + if (entry->deleted) + { + tiny_json_raw(&item, "{\"type\":\"DEL\",\"message_id\":"); + tiny_json_string(&item, entry->message.message_id); + tiny_json_raw(&item, "}"); + } + else + { + tiny_mcp_tool_append_message(&item, &entry->message); + } + + write_failed = item.failed || fprintf(output, "%s\n", item.data ? item.data : "") < 0; + tiny_json_free(&item); + if (write_failed) + { + fprintf(stderr, "export: failed to write record\n"); + if (output != stdout) + { + fclose(output); + } + return 1; + } + emitted++; + } + + if (output != stdout) + { + fclose(output); + } + (void)emitted; + return 0; +} diff --git a/src/cli/import.c b/src/cli/import.c new file mode 100644 index 0000000..3ddb9e9 --- /dev/null +++ b/src/cli/import.c @@ -0,0 +1,563 @@ +#include "cli/cli.h" + +#include "domain/message.h" +#include "mcp/json.h" +#include "storage/store.h" + +#include "mjson.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define IMPORT_MAX_LINE TINY_CLI_LINE_MAX + +typedef struct import_record_list +{ + tiny_import_record_t *items; + size_t count; + size_t capacity; +} import_record_list_t; + +static void +import_record_list_free(import_record_list_t *list) +{ + size_t index; + if (!list) + { + return; + } + for (index = 0; index < list->count; ++index) + { + tiny_message_free(&list->items[index].message); + } + free(list->items); + list->items = NULL; + list->count = 0; + list->capacity = 0; +} + +static bool +import_record_list_push(import_record_list_t *list) +{ + tiny_import_record_t *resized; + size_t capacity; + if (list->count < list->capacity) + { + return true; + } + capacity = list->capacity == 0 ? 32 : list->capacity * 2; + resized = realloc(list->items, capacity * sizeof(*resized)); + if (!resized) + { + return false; + } + list->items = resized; + list->capacity = capacity; + return true; +} + +static bool +import_get_string(const char *json, int len, const char *path, char *out, size_t out_size) +{ + int decoded = mjson_get_string(json, len, path, out, (int)out_size); + return decoded > 0 && (size_t)decoded < out_size; +} + +static bool +import_get_i64(const char *json, int len, const char *path, int64_t *value) +{ + double number; + if (mjson_get_number(json, len, path, &number) != 1) + { + return false; + } + *value = (int64_t)number; + return true; +} + +static bool +import_get_u64(const char *json, int len, const char *path, uint64_t *value) +{ + double number; + if (mjson_get_number(json, len, path, &number) != 1 || number < 0) + { + return false; + } + *value = (uint64_t)number; + return true; +} + +static bool +import_parse_string_array(const char *json, + int len, + const char *path, + char ***output, + size_t *count, + size_t max_count, + size_t max_length) +{ + int offset = 0, key_offset, key_length, value_offset, value_length, value_type; + char *array_json = NULL; + int array_len = 0; + size_t index = 0; + *output = NULL; + *count = 0; + if (mjson_find(json, len, path, &array_json, &array_len) == 0) + { + return true; + } + if (array_json == NULL || mjson_find(array_json, array_len, "$", NULL, NULL) != MJSON_TOK_ARRAY) + { + return false; + } + *output = calloc(max_count, sizeof(char *)); + if (!*output) + { + return false; + } + while ((offset = mjson_next(array_json, + array_len, + offset, + &key_offset, + &key_length, + &value_offset, + &value_length, + &value_type)) + != 0) + { + char value[512]; + int decoded; + if (value_type != MJSON_TOK_STRING || index >= max_count + || (decoded = mjson_get_string( + array_json + value_offset, value_length, "$", value, sizeof(value))) + <= 0 + || (size_t)decoded > max_length || decoded >= (int)sizeof(value)) + { + break; + } + (*output)[index] = malloc((size_t)decoded + 1); + if (!(*output)[index]) + { + break; + } + memcpy((*output)[index], value, (size_t)decoded); + (*output)[index][decoded] = '\0'; + index++; + } + *count = index; + if (index > 0 && offset != 0) + { + return false; + } + return true; +} + +static bool +import_parse_relations(const char *json, int len, const char *path, tiny_metadata_t *metadata) +{ + int offset = 0, key_offset, key_length, value_offset, value_length, value_type; + char *array_json = NULL; + int array_len = 0; + size_t index = 0; + if (mjson_find(json, len, path, &array_json, &array_len) == 0) + { + return true; + } + if (array_json == NULL || mjson_find(array_json, array_len, "$", NULL, NULL) != MJSON_TOK_ARRAY) + { + return false; + } + metadata->relations = calloc(TINY_RELATION_COUNT_MAX, sizeof(tiny_relation_t)); + if (!metadata->relations) + { + return false; + } + while ((offset = mjson_next(array_json, + array_len, + offset, + &key_offset, + &key_length, + &value_offset, + &value_length, + &value_type)) + != 0) + { + char type_name[32]; + char message_id[32]; + int decoded; + tiny_relation_t *relation; + if (value_type != MJSON_TOK_OBJECT || index >= TINY_RELATION_COUNT_MAX + || mjson_get_string( + array_json + value_offset, value_length, "$.type", type_name, sizeof(type_name)) + <= 0 + || (decoded = mjson_get_string(array_json + value_offset, + value_length, + "$.message_id", + message_id, + sizeof(message_id))) + <= 0) + { + return false; + } + relation = &metadata->relations[index]; + if (!tiny_relation_type_parse(type_name, &relation->type) + || !tiny_message_id_valid(message_id)) + { + return false; + } + memcpy(relation->message_id, message_id, (size_t)decoded); + relation->message_id[decoded] = '\0'; + index++; + } + metadata->relation_count = index; + return true; +} + +static bool +import_parse_metadata(const char *json, int len, tiny_message_t *message) +{ + char tier_name[32] = { 0 }; + char kind_name[32] = { 0 }; + char tool_call_id[TINY_TOOL_CALL_ID_MAX + 1] = { 0 }; + char summary[TINY_SUMMARY_MAX + 1] = { 0 }; + double importance_number = 0; + bool has_tool_call_id, has_summary, has_importance; + has_tool_call_id = mjson_find(json, len, "$.metadata.tool_call_id", NULL, NULL) != 0 + && import_get_string( + json, len, "$.metadata.tool_call_id", tool_call_id, sizeof(tool_call_id)); + has_summary = mjson_find(json, len, "$.metadata.summary", NULL, NULL) != 0 + && import_get_string(json, len, "$.metadata.summary", summary, sizeof(summary)); + has_importance = mjson_get_number(json, len, "$.metadata.importance", &importance_number) == 1; + if (!import_get_string(json, len, "$.metadata.memory_tier", tier_name, sizeof(tier_name)) + || !import_get_string(json, len, "$.metadata.memory_kind", kind_name, sizeof(kind_name)) + || !import_parse_string_array(json, + len, + "$.metadata.tags", + &message->metadata.tags, + &message->metadata.tag_count, + TINY_TAG_COUNT_MAX, + TINY_TAG_MAX) + || !import_parse_relations(json, len, "$.metadata.relations", &message->metadata) + || !import_parse_string_array(json, + len, + "$.metadata.source_message_ids", + &message->metadata.source_message_ids, + &message->metadata.source_count, + TINY_SOURCE_COUNT_MAX, + TINY_MESSAGE_ID_LENGTH)) + { + return false; + } + if (!tiny_tier_parse(tier_name, &message->metadata.memory_tier) + || !tiny_kind_parse(kind_name, &message->metadata.memory_kind)) + { + return false; + } + if (has_tool_call_id) + { + message->metadata.tool_call_id = malloc(strlen(tool_call_id) + 1); + if (!message->metadata.tool_call_id) + { + return false; + } + memcpy(message->metadata.tool_call_id, tool_call_id, strlen(tool_call_id) + 1); + } + if (has_summary) + { + message->metadata.summary = malloc(strlen(summary) + 1); + if (!message->metadata.summary) + { + return false; + } + memcpy(message->metadata.summary, summary, strlen(summary) + 1); + } + if (has_importance) + { + if (importance_number < 0 || importance_number > 100) + { + return false; + } + message->metadata.importance = (uint8_t)importance_number; + } + return true; +} + +static bool +import_manifest_valid(const char *line, int len) +{ + char type_name[64]; + double version = 0; + if (!import_get_string(line, len, "$.type", type_name, sizeof(type_name)) + || strcmp(type_name, "tiny-convmem-manifest") != 0 + || mjson_get_number(line, len, "$.format_version", &version) != 1) + { + return false; + } + return (int)version == 2; +} + +static bool +import_parse_del_record(const char *line, int len, tiny_import_record_t *record) +{ + char type_name[32]; + tiny_message_init(&record->message); + if (!import_get_string(line, len, "$.type", type_name, sizeof(type_name)) + || strcmp(type_name, "DEL") != 0 + || !import_get_string( + line, len, "$.message_id", record->message.message_id, sizeof(record->message.message_id)) + || !tiny_message_id_valid(record->message.message_id)) + { + return false; + } + record->deleted = true; + return true; +} + +static bool +import_parse_message_record(const char *line, int len, tiny_import_record_t *record) +{ + char role_name[32]; + char session_id[TINY_SESSION_ID_MAX + 1]; + char content_buffer[4096]; + tiny_message_init(&record->message); + record->deleted = false; + if (!import_get_string( + line, len, "$.message_id", record->message.message_id, sizeof(record->message.message_id)) + || !import_get_string(line, len, "$.session_id", session_id, sizeof(session_id)) + || !import_get_string(line, len, "$.role", role_name, sizeof(role_name)) + || !import_get_i64(line, len, "$.timestamp", &record->message.timestamp_ms) + || !import_get_i64(line, len, "$.updated_at", &record->message.updated_at_ms) + || !import_get_u64(line, len, "$.revision", &record->message.revision)) + { + return false; + } + if (!tiny_message_id_valid(record->message.message_id) + || !tiny_role_parse(role_name, &record->message.role) || record->message.timestamp_ms < 0 + || record->message.updated_at_ms < 0 || record->message.revision == 0 + || record->message.revision > INT32_MAX) + { + return false; + } + if (mjson_find(line, len, "$.content", NULL, NULL) == 0) + { + return false; + } + if (mjson_get_string(line, len, "$.content", content_buffer, sizeof(content_buffer)) > 0) + { + size_t length = strlen(content_buffer); + if (length == 0 || length > TINY_CONTENT_MAX) + { + return false; + } + record->message.content = malloc(length + 1); + if (!record->message.content) + { + return false; + } + memcpy(record->message.content, content_buffer, length + 1); + } + else + { + const char *content = NULL; + int content_len = 0; + if (mjson_find(line, len, "$.content", &content, &content_len) == 0 || !content + || content_len < 2) + { + return false; + } + if ((size_t)(content_len - 2) == 0 || (size_t)(content_len - 2) > TINY_CONTENT_MAX) + { + return false; + } + record->message.content = malloc((size_t)content_len - 1); + if (!record->message.content) + { + return false; + } + memcpy(record->message.content, content + 1, (size_t)content_len - 2); + record->message.content[(size_t)content_len - 2] = '\0'; + } + if (!tiny_utf8_valid(record->message.content, strlen(record->message.content))) + { + return false; + } + record->message.session_id = malloc(strlen(session_id) + 1); + if (!record->message.session_id) + { + return false; + } + memcpy(record->message.session_id, session_id, strlen(session_id) + 1); + if (!import_parse_metadata(line, len, &record->message) + || !tiny_metadata_valid(&record->message.metadata)) + { + tiny_message_free(&record->message); + tiny_message_init(&record->message); + return false; + } + return true; +} + +static bool +import_parse_jsonl(const char *jsonl, import_record_list_t *records) +{ + const char *line_start = jsonl; + const char *line_end; + bool manifest_seen = false; + while (*line_start) + { + int length; + tiny_import_record_t record; + line_end = strchr(line_start, '\n'); + if (line_end == NULL) + { + line_end = line_start + strlen(line_start); + } + length = (int)(line_end - line_start); + if (length > 0) + { + if (!manifest_seen) + { + if (!import_manifest_valid(line_start, length)) + { + return false; + } + manifest_seen = true; + } + else if (import_parse_del_record(line_start, length, &record)) + { + if (!import_record_list_push(records)) + { + return false; + } + records->items[records->count++] = record; + } + else if (import_parse_message_record(line_start, length, &record)) + { + if (!import_record_list_push(records)) + { + tiny_message_free(&record.message); + return false; + } + records->items[records->count++] = record; + } + else + { + return false; + } + } + if (*line_end == '\0') + { + break; + } + line_start = line_end + 1; + } + return manifest_seen; +} + +int +tiny_cli_import(tiny_store_t *store, const tiny_cli_options_t *options) +{ + import_record_list_t records = { NULL, 0, 0 }; + size_t imported = 0, rejected = 0; + tiny_store_status_t status; + tiny_import_policy_t policy = TINY_IMPORT_OVERWRITE; + char *jsonl = NULL; + FILE *file = NULL; + long file_size; + int result = 1; + + if (!options || !options->import_path) + { + fprintf(stderr, "import: path is required\n"); + return 1; + } + + if (options->policy_name && options->policy_name[0]) + { + if (strcmp(options->policy_name, "overwrite") == 0) + { + policy = TINY_IMPORT_OVERWRITE; + } + else if (strcmp(options->policy_name, "reject") == 0) + { + policy = TINY_IMPORT_REJECT; + } + else + { + fprintf(stderr, "import: invalid policy '%s'\n", options->policy_name); + return 1; + } + } + + if (fopen_s(&file, options->import_path, "rb") != 0 || !file) + { + fprintf(stderr, "import: unable to open '%s'\n", options->import_path); + return 1; + } + + if (fseek(file, 0, SEEK_END) != 0 || (file_size = ftell(file)) < 0 + || fseek(file, 0, SEEK_SET) != 0) + { + fprintf(stderr, "import: unable to seek '%s'\n", options->import_path); + fclose(file); + return 1; + } + + if ((size_t)file_size > IMPORT_MAX_LINE * 1024ULL * 1024ULL) + { + fprintf(stderr, "import: file too large\n"); + fclose(file); + return 1; + } + + jsonl = malloc((size_t)file_size + 1); + if (!jsonl) + { + fprintf(stderr, "import: out of memory\n"); + fclose(file); + return 1; + } + + if (fread(jsonl, 1, (size_t)file_size, file) != (size_t)file_size) + { + fprintf(stderr, "import: unable to read '%s'\n", options->import_path); + free(jsonl); + fclose(file); + return 1; + } + jsonl[file_size] = '\0'; + fclose(file); + + if (!import_parse_jsonl(jsonl, &records)) + { + fprintf(stderr, "import: invalid JSONL in '%s'\n", options->import_path); + free(jsonl); + return 1; + } + + status = tiny_store_import(store, records.items, records.count, policy, &imported, &rejected); + import_record_list_free(&records); + free(jsonl); + + if (status == TINY_STORE_OK) + { + printf("{\"imported\":%zu,\"rejected\":%zu}\n", imported, rejected); + fflush(stdout); + result = 0; + } + else if (status == TINY_STORE_CONFLICT) + { + fprintf(stderr, "import: conflict with existing records\n"); + } + else if (status == TINY_STORE_CORRUPT) + { + fprintf(stderr, "import: invalid records\n"); + } + else + { + fprintf(stderr, "import: failed\n"); + } + + return result; +} |