diff options
| author | tanaka takashi <isazaka@gmail.com> | 2026-06-29 02:36:40 +0900 |
|---|---|---|
| committer | tanaka takashi <isazaka@gmail.com> | 2026-06-29 02:36:40 +0900 |
| commit | 4c6a831aa2a57e2bcc20aa3fc2f1b7172f5be285 (patch) | |
| tree | 074021381abf57cad434430f25027115b681b8a9 /src | |
| parent | 8d38b06f11c713e547d70468340242f23322152c (diff) | |
feat: 会話メモリのプロトタイプを実装
- 6種類のMCPツールと入力検証を追加
- バイナリログ、スナップショット、復旧、圧縮処理を実装
- Unicode n-gram検索とドメインモデルを追加
- 単体テストと統合テストを追加
- アーキテクチャと実装状況を文書化
- Cコードの記述スタイルを統一
Diffstat (limited to 'src')
| -rw-r--r-- | src/CMakeLists.txt | 13 | ||||
| -rw-r--r-- | src/app/main.c | 48 | ||||
| -rw-r--r-- | src/domain/README.md | 11 | ||||
| -rw-r--r-- | src/domain/message.c | 255 | ||||
| -rw-r--r-- | src/domain/message.h | 60 | ||||
| -rw-r--r-- | src/domain/search.c | 350 | ||||
| -rw-r--r-- | src/domain/search.h | 22 | ||||
| -rw-r--r-- | src/mcp/README.md | 14 | ||||
| -rw-r--r-- | src/mcp/json.c | 137 | ||||
| -rw-r--r-- | src/mcp/json.h | 23 | ||||
| -rw-r--r-- | src/mcp/server.c | 763 | ||||
| -rw-r--r-- | src/mcp/server.h | 4 | ||||
| -rw-r--r-- | src/mcp/tools/health.c | 29 | ||||
| -rw-r--r-- | src/mcp/tools/health.h | 9 | ||||
| -rw-r--r-- | src/storage/README.md | 9 | ||||
| -rw-r--r-- | src/storage/store.c | 1111 | ||||
| -rw-r--r-- | src/storage/store.h | 64 |
17 files changed, 2839 insertions, 83 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef54a08..e7768d7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,14 +1,19 @@ add_executable(tiny-convmem-mcp app/main.c + domain/message.c + domain/search.c + storage/store.c + mcp/json.c mcp/server.c - mcp/tools/health.c ) -target_include_directories(tiny-convmem-mcp PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") -target_link_libraries(tiny-convmem-mcp PRIVATE mcpc::mcpc) +target_include_directories(tiny-convmem-mcp PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}" + "${PROJECT_SOURCE_DIR}/third_party/mcpc/src" +) +target_link_libraries(tiny-convmem-mcp PRIVATE mcpc::mcpc bcrypt) target_compile_features(tiny-convmem-mcp PRIVATE c_std_17) if(MSVC) target_compile_options(tiny-convmem-mcp PRIVATE /utf-8 /W4) endif() - diff --git a/src/app/main.c b/src/app/main.c index ab8688f..ce43b61 100644 --- a/src/app/main.c +++ b/src/app/main.c @@ -1,15 +1,57 @@ +#define WIN32_LEAN_AND_MEAN +#include <windows.h> + #include <stdio.h> +#include <stdlib.h> +#include <wchar.h> #include "mcp/server.h" +#include "storage/store.h" + +static void +usage(FILE *stream) +{ + fwprintf(stream, L"Usage: tiny-convmem-mcp --data-dir <path>\n"); +} int -main(void) +wmain(int argc, wchar_t **argv) { - const int result = tiny_convmem_server_run(stdin, stdout); + wchar_t *absolute = NULL; + DWORD required; + tiny_store_t *store = NULL; + tiny_store_status_t status; + int result; + if (argc == 2 && wcscmp(argv[1], L"--help") == 0) + { + usage(stdout); + return 0; + } + if (argc != 3 || wcscmp(argv[1], L"--data-dir") != 0 || argv[2][0] == L'\0') + { + usage(stderr); + return 2; + } + required = GetFullPathNameW(argv[2], 0, NULL, NULL); + if (required == 0 || (absolute = malloc((size_t)required * sizeof(wchar_t))) == NULL + || GetFullPathNameW(argv[2], required, absolute, NULL) == 0) + { + fprintf(stderr, "unable to resolve --data-dir\n"); + free(absolute); + return 2; + } + status = tiny_store_open(&store, absolute); + free(absolute); + if (status != TINY_STORE_OK) + { + fprintf(stderr, "unable to open conversation store: %s\n", tiny_store_status_name(status)); + return status == TINY_STORE_BUSY ? 3 : 1; + } + result = tiny_convmem_server_run(stdin, stdout, store); + tiny_store_close(store); if (result != 0) { fprintf(stderr, "tiny-convmem-mcp stopped with error code %d\n", result); } return result; } - diff --git a/src/domain/README.md b/src/domain/README.md index 0d359af..a30039e 100644 --- a/src/domain/README.md +++ b/src/domain/README.md @@ -1,8 +1,11 @@ # Domain layer -This directory is reserved for the conversation-memory domain model and its -business rules. Conversation, message, retention, and search contracts will be -defined here only after the next design phase. +`message.h` defines immutable Message and metadata concepts, role/tier enums, +input limits, deep-copy ownership, and UTF-8 validation. This layer has no mcpc +or Win32 dependency. -The domain layer must not depend on mcpc or on a concrete storage engine. +`search.h` owns the ASCII-case-folded Unicode codepoint 2/3-gram inverted index, +collision verification, candidate selection, and deterministic relevance score. +`memory_tier` is descriptive in v1. Retention, summarization, and automatic tier +transitions are intentionally not domain behaviors yet. diff --git a/src/domain/message.c b/src/domain/message.c new file mode 100644 index 0000000..d973be4 --- /dev/null +++ b/src/domain/message.c @@ -0,0 +1,255 @@ +#include "domain/message.h" + +#include <stdlib.h> +#include <string.h> + +static char * +duplicate_string(const char *value) +{ + const size_t length = strlen(value); + char *copy = malloc(length + 1); + if (copy != NULL) + { + memcpy(copy, value, length + 1); + } + return copy; +} + +void +tiny_message_init(tiny_message_t *message) +{ + memset(message, 0, sizeof(*message)); + message->metadata.memory_tier = TINY_TIER_SHORT; +} + +void +tiny_message_free(tiny_message_t *message) +{ + size_t index; + free(message->session_id); + free(message->content); + free(message->metadata.tool_call_id); + for (index = 0; index < message->metadata.tag_count; ++index) + { + free(message->metadata.tags[index]); + } + free(message->metadata.tags); + tiny_message_init(message); +} + +int +tiny_message_copy(tiny_message_t *destination, const tiny_message_t *source) +{ + size_t index; + tiny_message_init(destination); + memcpy(destination->message_id, source->message_id, sizeof(destination->message_id)); + destination->timestamp_ms = source->timestamp_ms; + destination->role = source->role; + destination->metadata.memory_tier = source->metadata.memory_tier; + destination->session_id = duplicate_string(source->session_id); + destination->content = duplicate_string(source->content); + if (destination->session_id == NULL || destination->content == NULL) + { + goto failure; + } + if (source->metadata.tool_call_id != NULL) + { + destination->metadata.tool_call_id = duplicate_string(source->metadata.tool_call_id); + if (destination->metadata.tool_call_id == NULL) + { + goto failure; + } + } + if (source->metadata.tag_count > 0) + { + destination->metadata.tags = calloc(source->metadata.tag_count, sizeof(char *)); + if (destination->metadata.tags == NULL) + { + goto failure; + } + for (index = 0; index < source->metadata.tag_count; ++index) + { + destination->metadata.tags[index] = duplicate_string(source->metadata.tags[index]); + if (destination->metadata.tags[index] == NULL) + { + goto failure; + } + destination->metadata.tag_count++; + } + } + return 0; + +failure: + tiny_message_free(destination); + return -1; +} + +const char * +tiny_role_name(tiny_role_t role) +{ + switch (role) + { + case TINY_ROLE_USER: + return "user"; + case TINY_ROLE_ASSISTANT: + return "assistant"; + case TINY_ROLE_SYSTEM: + return "system"; + default: + return "unknown"; + } +} + +bool +tiny_role_parse(const char *value, tiny_role_t *role) +{ + if (strcmp(value, "user") == 0) + { + *role = TINY_ROLE_USER; + } + else if (strcmp(value, "assistant") == 0) + { + *role = TINY_ROLE_ASSISTANT; + } + else if (strcmp(value, "system") == 0) + { + *role = TINY_ROLE_SYSTEM; + } + else + { + return false; + } + return true; +} + +const char * +tiny_tier_name(tiny_memory_tier_t tier) +{ + switch (tier) + { + case TINY_TIER_SHORT: + return "short"; + case TINY_TIER_MID: + return "mid"; + case TINY_TIER_LONG: + return "long"; + case TINY_TIER_DISCARD: + return "discard"; + default: + return "unknown"; + } +} + +bool +tiny_tier_parse(const char *value, tiny_memory_tier_t *tier) +{ + if (strcmp(value, "short") == 0) + { + *tier = TINY_TIER_SHORT; + } + else if (strcmp(value, "mid") == 0) + { + *tier = TINY_TIER_MID; + } + else if (strcmp(value, "long") == 0) + { + *tier = TINY_TIER_LONG; + } + else if (strcmp(value, "discard") == 0) + { + *tier = TINY_TIER_DISCARD; + } + else + { + return false; + } + return true; +} + +bool +tiny_utf8_valid(const char *value, size_t length) +{ + size_t index = 0; + while (index < length) + { + const unsigned char first = (unsigned char)value[index++]; + unsigned needed; + uint32_t codepoint; + if (first < 0x80) + { + if (first == 0) + { + return false; + } + continue; + } + if ((first & 0xe0) == 0xc0) + { + needed = 1; + codepoint = first & 0x1f; + } + else if ((first & 0xf0) == 0xe0) + { + needed = 2; + codepoint = first & 0x0f; + } + else if ((first & 0xf8) == 0xf0) + { + needed = 3; + codepoint = first & 0x07; + } + else + { + return false; + } + if (index + needed > length) + { + return false; + } + while (needed-- > 0) + { + const unsigned char next = (unsigned char)value[index++]; + if ((next & 0xc0) != 0x80) + { + return false; + } + codepoint = (codepoint << 6) | (next & 0x3f); + } + if (codepoint > 0x10ffff || (codepoint >= 0xd800 && codepoint <= 0xdfff)) + { + return false; + } + if ((first & 0xe0) == 0xc0 && codepoint < 0x80) + { + return false; + } + if ((first & 0xf0) == 0xe0 && codepoint < 0x800) + { + return false; + } + if ((first & 0xf8) == 0xf0 && codepoint < 0x10000) + { + return false; + } + } + return true; +} + +bool +tiny_message_id_valid(const char *value) +{ + static const char alphabet[] = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + size_t index; + if (strlen(value) != TINY_MESSAGE_ID_LENGTH || value[0] > '7') + { + return false; + } + for (index = 0; index < TINY_MESSAGE_ID_LENGTH; ++index) + { + if (strchr(alphabet, value[index]) == NULL) + { + return false; + } + } + return true; +} diff --git a/src/domain/message.h b/src/domain/message.h new file mode 100644 index 0000000..fa36d2a --- /dev/null +++ b/src/domain/message.h @@ -0,0 +1,60 @@ +#ifndef TINY_CONVMEM_DOMAIN_MESSAGE_H +#define TINY_CONVMEM_DOMAIN_MESSAGE_H + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +#define TINY_MESSAGE_ID_LENGTH 26 +#define TINY_SESSION_ID_MAX 256 +#define TINY_CONTENT_MAX (1024U * 1024U) +#define TINY_TOOL_CALL_ID_MAX 256 +#define TINY_TAG_MAX 128 +#define TINY_TAG_COUNT_MAX 32 +#define TINY_QUERY_MAX 4096 + +typedef enum tiny_role +{ + TINY_ROLE_USER = 1, + TINY_ROLE_ASSISTANT = 2, + TINY_ROLE_SYSTEM = 3 +} tiny_role_t; + +typedef enum tiny_memory_tier +{ + TINY_TIER_SHORT = 1, + TINY_TIER_MID = 2, + TINY_TIER_LONG = 3, + TINY_TIER_DISCARD = 4 +} tiny_memory_tier_t; + +typedef struct tiny_metadata +{ + char *tool_call_id; + char **tags; + size_t tag_count; + tiny_memory_tier_t memory_tier; +} tiny_metadata_t; + +typedef struct tiny_message +{ + char message_id[TINY_MESSAGE_ID_LENGTH + 1]; + char *session_id; + int64_t timestamp_ms; + tiny_role_t role; + char *content; + tiny_metadata_t metadata; +} tiny_message_t; + +void tiny_message_init(tiny_message_t *message); +void tiny_message_free(tiny_message_t *message); +int tiny_message_copy(tiny_message_t *destination, const tiny_message_t *source); + +const char *tiny_role_name(tiny_role_t role); +bool tiny_role_parse(const char *value, tiny_role_t *role); +const char *tiny_tier_name(tiny_memory_tier_t tier); +bool tiny_tier_parse(const char *value, tiny_memory_tier_t *tier); +bool tiny_utf8_valid(const char *value, size_t length); +bool tiny_message_id_valid(const char *value); + +#endif diff --git a/src/domain/search.c b/src/domain/search.c new file mode 100644 index 0000000..a24b667 --- /dev/null +++ b/src/domain/search.c @@ -0,0 +1,350 @@ +#include "domain/search.h" + +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +typedef struct gram_bucket +{ + uint64_t hash; + uint32_t points[3]; + unsigned width; + size_t *messages; + size_t count; + size_t capacity; +} gram_bucket_t; + +struct tiny_search_index +{ + gram_bucket_t *buckets; + size_t count; + size_t capacity; + size_t message_count; +}; + +static bool +decode_codepoints(const char *text, uint32_t **output, size_t *count) +{ + const size_t bytes = strlen(text); + size_t offset = 0, used = 0; + uint32_t *points = malloc((bytes + 1) * sizeof(uint32_t)); + if (points == NULL) + { + return false; + } + while (offset < bytes) + { + unsigned char first = (unsigned char)text[offset++]; + uint32_t point; + unsigned continuation; + if (first < 0x80) + { + point = first; + continuation = 0; + } + else if ((first & 0xe0) == 0xc0) + { + point = first & 0x1f; + continuation = 1; + } + else if ((first & 0xf0) == 0xe0) + { + point = first & 0x0f; + continuation = 2; + } + else + { + point = first & 0x07; + continuation = 3; + } + while (continuation-- > 0) + { + point = (point << 6) | ((unsigned char)text[offset++] & 0x3f); + } + if (point >= 'A' && point <= 'Z') + { + point += 'a' - 'A'; + } + points[used++] = point; + } + *output = points; + *count = used; + return true; +} + +static uint64_t +gram_hash(const uint32_t *points, size_t width) +{ + uint64_t hash = 1469598103934665603ULL; + size_t index; + for (index = 0; index < width; ++index) + { + hash ^= points[index]; + hash *= 1099511628211ULL; + } + return hash; +} + +static gram_bucket_t * +find_bucket(const tiny_search_index_t *index, const uint32_t *points, unsigned width) +{ + const uint64_t hash = gram_hash(points, width); + size_t position; + for (position = 0; position < index->count; ++position) + { + if (index->buckets[position].hash == hash && index->buckets[position].width == width + && memcmp(index->buckets[position].points, points, width * sizeof(uint32_t)) == 0) + { + return &index->buckets[position]; + } + } + return NULL; +} + +static bool +add_gram(tiny_search_index_t *index, const uint32_t *points, unsigned width, size_t message_index) +{ + gram_bucket_t *bucket = find_bucket(index, points, width); + if (bucket == NULL) + { + gram_bucket_t *resized; + size_t capacity; + if (index->count == index->capacity) + { + capacity = index->capacity ? index->capacity * 2 : 64; + resized = realloc(index->buckets, capacity * sizeof(*resized)); + if (!resized) + { + return false; + } + index->buckets = resized; + index->capacity = capacity; + } + bucket = &index->buckets[index->count++]; + memset(bucket, 0, sizeof(*bucket)); + bucket->hash = gram_hash(points, width); + bucket->width = width; + memcpy(bucket->points, points, width * sizeof(uint32_t)); + } + if (bucket->count > 0 && bucket->messages[bucket->count - 1] == message_index) + { + return true; + } + if (bucket->count == bucket->capacity) + { + const size_t capacity = bucket->capacity ? bucket->capacity * 2 : 8; + size_t *resized = realloc(bucket->messages, capacity * sizeof(*resized)); + if (!resized) + { + return false; + } + bucket->messages = resized; + bucket->capacity = capacity; + } + bucket->messages[bucket->count++] = message_index; + return true; +} + +static bool +index_text(tiny_search_index_t *index, const char *text, size_t message_index) +{ + uint32_t *points = NULL; + size_t count = 0, offset; + unsigned width; + if (!decode_codepoints(text, &points, &count)) + { + return false; + } + for (width = 2; width <= 3; ++width) + { + for (offset = 0; offset + width <= count; ++offset) + { + if (!add_gram(index, points + offset, width, message_index)) + { + free(points); + return false; + } + } + } + free(points); + return true; +} + +tiny_search_index_t * +tiny_search_index_create(void) +{ + return calloc(1, sizeof(tiny_search_index_t)); +} + +void +tiny_search_index_free(tiny_search_index_t *index) +{ + size_t position; + if (!index) + { + return; + } + for (position = 0; position < index->count; ++position) + { + free(index->buckets[position].messages); + } + free(index->buckets); + free(index); +} + +bool +tiny_search_index_add(tiny_search_index_t *index, + const tiny_message_t *message, + size_t message_index) +{ + size_t tag; + if (!index_text(index, message->content, message_index)) + { + return false; + } + for (tag = 0; tag < message->metadata.tag_count; ++tag) + { + if (!index_text(index, message->metadata.tags[tag], message_index)) + { + return false; + } + } + if (message_index >= index->message_count) + { + index->message_count = message_index + 1; + } + return true; +} + +bool +tiny_search_index_candidates(const tiny_search_index_t *index, + const char *query, + size_t **output, + size_t *count) +{ + uint32_t *points = NULL; + size_t point_count = 0, width, offset, used = 0; + bool *seen; + size_t *items; + *output = NULL; + *count = 0; + if (!decode_codepoints(query, &points, &point_count)) + { + return false; + } + items = malloc((index->message_count ? index->message_count : 1) * sizeof(*items)); + seen = calloc(index->message_count ? index->message_count : 1, sizeof(*seen)); + if (!items || !seen) + { + free(points); + free(items); + free(seen); + return false; + } + if (point_count < 2) + { + for (offset = 0; offset < index->message_count; ++offset) + { + seen[offset] = true; + items[used++] = offset; + } + } + else + { + gram_bucket_t *bucket; + width = point_count >= 3 ? 3 : 2; + for (offset = 0; offset + width <= point_count; ++offset) + { + if ((bucket = find_bucket(index, points + offset, (unsigned)width)) != NULL) + { + size_t item; + for (item = 0; item < bucket->count; ++item) + { + if (!seen[bucket->messages[item]]) + { + seen[bucket->messages[item]] = true; + items[used++] = bucket->messages[item]; + } + } + } + } + } + free(points); + free(seen); + *output = items; + *count = used; + return true; +} + +double +tiny_search_score(const char *query, const tiny_message_t *message) +{ + uint32_t *query_points = NULL, *message_points = NULL; + size_t query_count = 0, message_count = 0, width, query_offset, message_offset, matches = 0, + total, tag; + double score = 0.0; + if (!decode_codepoints(query, &query_points, &query_count) + || !decode_codepoints(message->content, &message_points, &message_count) || query_count == 0) + { + goto done; + } + width = query_count >= 3 ? 3 : (query_count >= 2 ? 2 : 1); + total = query_count - width + 1; + for (query_offset = 0; query_offset < total; ++query_offset) + { + bool found = false; + for (message_offset = 0; message_offset + width <= message_count; ++message_offset) + { + if (memcmp(query_points + query_offset, + message_points + message_offset, + width * sizeof(uint32_t)) + == 0) + { + found = true; + break; + } + } + for (tag = 0; tag < message->metadata.tag_count && !found; ++tag) + { + uint32_t *tag_points = NULL; + size_t tag_count = 0; + if (!decode_codepoints(message->metadata.tags[tag], &tag_points, &tag_count)) + { + continue; + } + for (message_offset = 0; message_offset + width <= tag_count; ++message_offset) + { + if (memcmp(query_points + query_offset, + tag_points + message_offset, + width * sizeof(uint32_t)) + == 0) + { + found = true; + break; + } + } + free(tag_points); + } + if (found) + { + matches++; + } + } + score = (double)matches / (double)total; + if (query_count <= message_count) + { + for (message_offset = 0; message_offset + query_count <= message_count; ++message_offset) + { + if (memcmp(query_points, message_points + message_offset, query_count * sizeof(uint32_t)) + == 0) + { + score += 1.0; + break; + } + } + } +done: + free(query_points); + free(message_points); + return score; +} diff --git a/src/domain/search.h b/src/domain/search.h new file mode 100644 index 0000000..3408998 --- /dev/null +++ b/src/domain/search.h @@ -0,0 +1,22 @@ +#ifndef TINY_CONVMEM_DOMAIN_SEARCH_H +#define TINY_CONVMEM_DOMAIN_SEARCH_H + +#include <stdbool.h> +#include <stddef.h> + +#include "domain/message.h" + +typedef struct tiny_search_index tiny_search_index_t; + +tiny_search_index_t *tiny_search_index_create(void); +void tiny_search_index_free(tiny_search_index_t *index); +bool tiny_search_index_add(tiny_search_index_t *index, + const tiny_message_t *message, + size_t message_index); +bool tiny_search_index_candidates(const tiny_search_index_t *index, + const char *query, + size_t **message_indices, + size_t *count); +double tiny_search_score(const char *query, const tiny_message_t *message); + +#endif diff --git a/src/mcp/README.md b/src/mcp/README.md new file mode 100644 index 0000000..dc7ac46 --- /dev/null +++ b/src/mcp/README.md @@ -0,0 +1,14 @@ +# MCP compatibility boundary + +The pinned mcpc revision cannot describe required versus optional properties, +nested objects/arrays, 64-bit values, or application callback context. The +project therefore keeps mcpc immutable and implements the stdio dispatch and +tool schemas in `server.c`, using mcpc's bundled `mjson` parser for raw JSON. + +This boundary supports the MCP 2024-11-05 initialize, tools/list, and tools/call +flow needed by the server. Tool success and application errors are returned as +JSON serialized into TextContent; application errors also set `isError`. + +If mcpc is upgraded, this boundary should be removed only after its integration +tests demonstrate equivalent optional/object/array schema and callback-state +behavior. diff --git a/src/mcp/json.c b/src/mcp/json.c new file mode 100644 index 0000000..dcef91b --- /dev/null +++ b/src/mcp/json.c @@ -0,0 +1,137 @@ +#include "mcp/json.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +static bool +reserve(tiny_json_buffer_t *buffer, size_t addition) +{ + size_t needed, capacity; + char *resized; + if (buffer->failed || addition > SIZE_MAX - buffer->length - 1) + { + return false; + } + needed = buffer->length + addition + 1; + if (needed <= buffer->capacity) + { + return true; + } + capacity = buffer->capacity == 0 ? 256 : buffer->capacity; + while (capacity < needed) + { + if (capacity > SIZE_MAX / 2) + { + buffer->failed = true; + return false; + } + capacity *= 2; + } + resized = realloc(buffer->data, capacity); + if (resized == NULL) + { + buffer->failed = true; + return false; + } + buffer->data = resized; + buffer->capacity = capacity; + return true; +} + +void +tiny_json_init(tiny_json_buffer_t *buffer) +{ + memset(buffer, 0, sizeof(*buffer)); +} +void +tiny_json_free(tiny_json_buffer_t *buffer) +{ + free(buffer->data); + tiny_json_init(buffer); +} + +bool +tiny_json_raw_n(tiny_json_buffer_t *buffer, const char *value, size_t length) +{ + if (!reserve(buffer, length)) + { + return false; + } + memcpy(buffer->data + buffer->length, value, length); + buffer->length += length; + buffer->data[buffer->length] = '\0'; + return true; +} +bool +tiny_json_raw(tiny_json_buffer_t *buffer, const char *value) +{ + return tiny_json_raw_n(buffer, value, strlen(value)); +} + +bool +tiny_json_string(tiny_json_buffer_t *buffer, const char *value) +{ + const unsigned char *position = (const unsigned char *)value; + if (!tiny_json_raw(buffer, "\"")) + { + return false; + } + while (*position) + { + char escaped[7]; + if (*position == '"' || *position == '\\') + { + escaped[0] = '\\'; + escaped[1] = (char)*position; + escaped[2] = '\0'; + tiny_json_raw(buffer, escaped); + } + else if (*position == '\b') + { + tiny_json_raw(buffer, "\\b"); + } + else if (*position == '\f') + { + tiny_json_raw(buffer, "\\f"); + } + else if (*position == '\n') + { + tiny_json_raw(buffer, "\\n"); + } + else if (*position == '\r') + { + tiny_json_raw(buffer, "\\r"); + } + else if (*position == '\t') + { + tiny_json_raw(buffer, "\\t"); + } + else if (*position < 0x20) + { + snprintf(escaped, sizeof(escaped), "\\u%04x", *position); + tiny_json_raw(buffer, escaped); + } + else + { + tiny_json_raw_n(buffer, (const char *)position, 1); + } + ++position; + } + return tiny_json_raw(buffer, "\""); +} + +bool +tiny_json_u64(tiny_json_buffer_t *buffer, unsigned long long value) +{ + char text[32]; + snprintf(text, sizeof(text), "%llu", value); + return tiny_json_raw(buffer, text); +} +bool +tiny_json_i64(tiny_json_buffer_t *buffer, long long value) +{ + char text[32]; + snprintf(text, sizeof(text), "%lld", value); + return tiny_json_raw(buffer, text); +} diff --git a/src/mcp/json.h b/src/mcp/json.h new file mode 100644 index 0000000..a46e6c7 --- /dev/null +++ b/src/mcp/json.h @@ -0,0 +1,23 @@ +#ifndef TINY_CONVMEM_MCP_JSON_H +#define TINY_CONVMEM_MCP_JSON_H + +#include <stdbool.h> +#include <stddef.h> + +typedef struct tiny_json_buffer +{ + char *data; + size_t length; + size_t capacity; + bool failed; +} tiny_json_buffer_t; + +void tiny_json_init(tiny_json_buffer_t *buffer); +void tiny_json_free(tiny_json_buffer_t *buffer); +bool tiny_json_raw(tiny_json_buffer_t *buffer, const char *value); +bool tiny_json_raw_n(tiny_json_buffer_t *buffer, const char *value, size_t length); +bool tiny_json_string(tiny_json_buffer_t *buffer, const char *value); +bool tiny_json_u64(tiny_json_buffer_t *buffer, unsigned long long value); +bool tiny_json_i64(tiny_json_buffer_t *buffer, long long value); + +#endif diff --git a/src/mcp/server.c b/src/mcp/server.c index 57d197d..80d8307 100644 --- a/src/mcp/server.c +++ b/src/mcp/server.c @@ -1,52 +1,757 @@ #include "mcp/server.h" -#include <mcpc/mcpc.h> -#include "mcp/tools/health.h" +#include "domain/search.h" +#include "mcp/json.h" +#include "mjson.h" -int -tiny_convmem_server_run(FILE *input, FILE *output) +#include <math.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#define MCP_PROTOCOL_VERSION "2024-11-05" + +typedef struct result_item { - mcpc_server_t *server = mcpc_server_new_iostrm(input, output); - if (server == NULL) + const tiny_store_entry_t *entry; + double score; +} result_item_t; + +static char * +read_line(FILE *input) +{ + size_t length = 0, capacity = 4096; + int character; + char *line = malloc(capacity); + if (!line) + { + return NULL; + } + while ((character = fgetc(input)) != EOF) { - return MCPC_EC_BUG; + if (length + 1 >= capacity) + { + char *r; + capacity *= 2; + r = realloc(line, capacity); + if (!r) + { + free(line); + return NULL; + } + line = r; + } + line[length++] = (char)character; + if (character == '\n') + { + break; + } } + if (length == 0 && character == EOF) + { + free(line); + return NULL; + } + line[length] = '\0'; + return line; +} - mcpc_errcode_t result = mcpc_server_set_nament( - server, - (const char8_t *)"tiny-convmem-mcp" - ); - if (result == MCPC_EC_0) +static char * +json_string_value( + const char *json, int length, const char *path, size_t maximum, bool required, bool *valid) +{ + const char *token = NULL; + int token_length = 0; + int type = mjson_find(json, length, path, &token, &token_length); + char *value; + int decoded; + if (type == MJSON_TOK_INVALID) + { + if (required) + { + *valid = false; + } + return NULL; + } + if (type != MJSON_TOK_STRING || token_length < 2 || (size_t)token_length > maximum * 6 + 2) + { + *valid = false; + return NULL; + } + value = malloc(maximum + 1); + if (!value) + { + *valid = false; + return NULL; + } + decoded = mjson_get_string(json, length, path, value, (int)maximum + 1); + if (decoded < 0 || (size_t)decoded > maximum || !tiny_utf8_valid(value, (size_t)decoded)) { - result = mcpc_server_capa_enable_tool(server); + free(value); + *valid = false; + return NULL; } + value[decoded] = '\0'; + return value; +} - mcpc_tool_t *health = NULL; - if (result == MCPC_EC_0) +static bool +object_has_only(const char *json, int length, const char *const *allowed, size_t allowed_count) +{ + int offset = 0, koff, klen, voff, vlen, vtype; + while ((offset = mjson_next(json, length, offset, &koff, &klen, &voff, &vlen, &vtype)) != 0) { - health = tiny_convmem_health_tool_create(); - if (health == NULL) + char key[64]; + int decoded; + size_t index; + if (klen <= 0) + { + return false; + } + decoded = mjson_get_string(json + koff, klen, "$", key, sizeof(key)); + if (decoded <= 0 || decoded >= (int)sizeof(key)) + { + return false; + } + key[decoded] = '\0'; + for (index = 0; index < allowed_count; ++index) { - result = MCPC_EC_BUG; + if (strcmp(key, allowed[index]) == 0) + { + break; + } + } + if (index == allowed_count) + { + return false; } } + return true; +} + +static void +free_input_message(tiny_message_t *message) +{ + tiny_message_free(message); +} - if (result == MCPC_EC_0) +static bool +parse_tags(const char *json, int length, tiny_message_t *message) +{ + const char *array = NULL; + int array_length = 0; + int type = mjson_find(json, length, "$.metadata.tags", &array, &array_length); + int offset = 0, koff, klen, voff, vlen, vtype; + if (type == MJSON_TOK_INVALID) { - result = mcpc_server_add_tool(server, health); + return true; } - if (result == MCPC_EC_0) + if (type != MJSON_TOK_ARRAY) { - result = mcpc_server_start(server); + return false; } + while ((offset = mjson_next(array, array_length, offset, &koff, &klen, &voff, &vlen, &vtype)) + != 0) + { + char *tag; + int decoded; + char **resized; + if (vtype != MJSON_TOK_STRING || message->metadata.tag_count >= TINY_TAG_COUNT_MAX) + { + return false; + } + tag = malloc(TINY_TAG_MAX + 1); + if (!tag) + { + return false; + } + decoded = mjson_get_string(array + voff, vlen, "$", tag, TINY_TAG_MAX + 1); + if (decoded <= 0 || decoded > TINY_TAG_MAX || !tiny_utf8_valid(tag, (size_t)decoded)) + { + free(tag); + return false; + } + tag[decoded] = '\0'; + resized = realloc(message->metadata.tags, (message->metadata.tag_count + 1) * sizeof(char *)); + if (!resized) + { + free(tag); + return false; + } + message->metadata.tags = resized; + message->metadata.tags[message->metadata.tag_count++] = tag; + } + return true; +} + +static bool +parse_write(const char *args, int length, tiny_message_t *message) +{ + static const char *const top_keys[] = { "session_id", "role", "content", "metadata" }; + static const char *const metadata_keys[] = { "tool_call_id", "tags", "memory_tier" }; + bool valid = true; + char *role = NULL, *tier = NULL; + const char *metadata = NULL; + int metadata_length = 0, metadata_type; + tiny_message_init(message); + if (!object_has_only(args, length, top_keys, sizeof(top_keys) / sizeof(top_keys[0]))) + { + valid = false; + } + metadata_type = mjson_find(args, length, "$.metadata", &metadata, &metadata_length); + if (metadata_type != MJSON_TOK_INVALID + && (metadata_type != MJSON_TOK_OBJECT + || !object_has_only(metadata, + metadata_length, + metadata_keys, + sizeof(metadata_keys) / sizeof(metadata_keys[0])))) + { + valid = false; + } + message->session_id + = json_string_value(args, length, "$.session_id", TINY_SESSION_ID_MAX, true, &valid); + role = json_string_value(args, length, "$.role", 16, true, &valid); + message->content = json_string_value(args, length, "$.content", TINY_CONTENT_MAX, true, &valid); + message->metadata.tool_call_id = json_string_value( + args, length, "$.metadata.tool_call_id", TINY_TOOL_CALL_ID_MAX, false, &valid); + tier = json_string_value(args, length, "$.metadata.memory_tier", 16, false, &valid); + if (!valid || !message->session_id || !message->content || !role || message->session_id[0] == '\0' + || message->content[0] == '\0' || !tiny_role_parse(role, &message->role) + || (tier && !tiny_tier_parse(tier, &message->metadata.memory_tier)) + || !parse_tags(args, length, message)) + { + valid = false; + } + free(role); + free(tier); + if (!valid) + { + free_input_message(message); + } + return valid; +} + +static void +append_message(tiny_json_buffer_t *json, const tiny_message_t *message) +{ + size_t index; + tiny_json_raw(json, "{\"message_id\":"); + tiny_json_string(json, message->message_id); + tiny_json_raw(json, ",\"session_id\":"); + tiny_json_string(json, message->session_id); + tiny_json_raw(json, ",\"timestamp\":"); + tiny_json_i64(json, message->timestamp_ms); + tiny_json_raw(json, ",\"role\":"); + tiny_json_string(json, tiny_role_name(message->role)); + tiny_json_raw(json, ",\"content\":"); + tiny_json_string(json, message->content); + tiny_json_raw(json, ",\"metadata\":{"); + if (message->metadata.tool_call_id) + { + tiny_json_raw(json, "\"tool_call_id\":"); + tiny_json_string(json, message->metadata.tool_call_id); + tiny_json_raw(json, ","); + } + tiny_json_raw(json, "\"tags\":["); + for (index = 0; index < message->metadata.tag_count; ++index) + { + if (index) + { + tiny_json_raw(json, ","); + } + tiny_json_string(json, message->metadata.tags[index]); + } + tiny_json_raw(json, "],\"memory_tier\":"); + tiny_json_string(json, tiny_tier_name(message->metadata.memory_tier)); + tiny_json_raw(json, "}}"); +} - { - const mcpc_errcode_t close_result = mcpc_server_close(server); - if (result == MCPC_EC_0) +static void +send_json_result( + FILE *output, const char *id, int id_length, tiny_json_buffer_t *inner, bool is_error) +{ + tiny_json_buffer_t response; + tiny_json_init(&response); + tiny_json_raw(&response, "{\"jsonrpc\":\"2.0\",\"id\":"); + tiny_json_raw_n(&response, id, (size_t)id_length); + tiny_json_raw(&response, ",\"result\":{\"content\":[{\"type\":\"text\",\"text\":"); + tiny_json_string(&response, inner->data ? inner->data : "{}"); + tiny_json_raw(&response, "}]"); + if (is_error) + { + tiny_json_raw(&response, ",\"isError\":true"); + } + tiny_json_raw(&response, "}}\n"); + if (!response.failed) + { + fwrite(response.data, 1, response.length, output); + fflush(output); + } + tiny_json_free(&response); +} + +static void +send_error(FILE *output, const char *id, int id_length, const char *code, const char *message) +{ + tiny_json_buffer_t inner; + tiny_json_init(&inner); + tiny_json_raw(&inner, "{\"status\":\"ERROR\",\"error\":{\"code\":"); + tiny_json_string(&inner, code); + tiny_json_raw(&inner, ",\"message\":"); + tiny_json_string(&inner, message); + tiny_json_raw(&inner, "}}"); + send_json_result(output, id, id_length, &inner, true); + tiny_json_free(&inner); +} + +static int +compare_history(const void *left, const void *right) +{ + const tiny_store_entry_t *a = *(const tiny_store_entry_t *const *)left, + *b = *(const tiny_store_entry_t *const *)right; + if (a->message.timestamp_ms < b->message.timestamp_ms) + { + return -1; + } + if (a->message.timestamp_ms > b->message.timestamp_ms) + { + return 1; + } + return strcmp(a->message.message_id, b->message.message_id); +} + +static int +compare_results(const void *l, const void *r) +{ + const result_item_t *a = l, *b = r; + if (a->score > b->score) + { + return -1; + } + if (a->score < b->score) + { + return 1; + } + if (a->entry->message.timestamp_ms > b->entry->message.timestamp_ms) + { + return -1; + } + if (a->entry->message.timestamp_ms < b->entry->message.timestamp_ms) + { + return 1; + } + return strcmp(a->entry->message.message_id, b->entry->message.message_id); +} + +static tiny_search_index_t * +build_search_index(const tiny_store_t *store) +{ + tiny_search_index_t *index = tiny_search_index_create(); + size_t position; + if (!index) + { + return NULL; + } + for (position = 0; position < tiny_store_entry_count(store); ++position) + { + if (!tiny_search_index_add(index, &tiny_store_entry(store, position)->message, position)) + { + tiny_search_index_free(index); + return NULL; + } + } + return index; +} + +static void +handle_call(FILE *output, + const char *id, + int id_length, + const char *name, + const char *args, + int args_length, + tiny_store_t *store, + tiny_search_index_t **search_index) +{ + tiny_json_buffer_t result; + tiny_store_status_t status; + tiny_json_init(&result); + if (strcmp(name, "write_message") == 0) + { + tiny_message_t input, written; + if (!parse_write(args, args_length, &input)) + { + send_error(output, id, id_length, "VALIDATION_ERROR", "invalid write_message arguments"); + return; + } + tiny_message_init(&written); + status = tiny_store_write(store, &input, &written); + tiny_message_free(&input); + if (status != TINY_STORE_OK) + { + send_error( + output, id, id_length, tiny_store_status_name(status), "message was not written"); + return; + } + if (*search_index + && !tiny_search_index_add(*search_index, &written, tiny_store_entry_count(store) - 1)) + { + tiny_search_index_free(*search_index); + *search_index = NULL; + } + tiny_json_raw(&result, "{\"status\":\"OK\",\"message\":"); + append_message(&result, &written); + tiny_json_raw(&result, "}"); + tiny_message_free(&written); + } + else if (strcmp(name, "read_messages") == 0) + { + static const char *const keys[] = { "session_id" }; + bool valid = object_has_only(args, args_length, keys, 1); + char *session + = json_string_value(args, args_length, "$.session_id", TINY_SESSION_ID_MAX, true, &valid); + size_t i, count = 0; + const tiny_store_entry_t **items; + if (!valid || !session || !session[0]) + { + free(session); + send_error(output, id, id_length, "VALIDATION_ERROR", "session_id is required"); + return; + } + items = malloc(tiny_store_entry_count(store) * sizeof(*items)); + if (!items) + { + free(session); + send_error(output, id, id_length, "INTERNAL_ERROR", "out of memory"); + return; + } + for (i = 0; i < tiny_store_entry_count(store); ++i) + { + const tiny_store_entry_t *e = tiny_store_entry(store, i); + if (!e->deleted && strcmp(e->message.session_id, session) == 0) + { + items[count++] = e; + } + } + qsort(items, count, sizeof(*items), compare_history); + tiny_json_raw(&result, "{\"history\":["); + for (i = 0; i < count; ++i) + { + if (i) + { + tiny_json_raw(&result, ","); + } + append_message(&result, &items[i]->message); + } + tiny_json_raw(&result, "]}"); + free(items); + free(session); + } + else if (strcmp(name, "search_messages") == 0) + { + static const char *const keys[] = { "query", "session_id", "limit" }; + bool valid = object_has_only(args, args_length, keys, 3); + char *query = json_string_value(args, args_length, "$.query", TINY_QUERY_MAX, true, &valid); + char *session = json_string_value( + args, args_length, "$.session_id", TINY_SESSION_ID_MAX, false, &valid); + double number = 20; + size_t limit = 20, i, count = 0, candidate_count = 0, *candidates = NULL; + result_item_t *items; + if (mjson_find(args, args_length, "$.limit", NULL, NULL) != MJSON_TOK_INVALID) + { + if (!mjson_get_number(args, args_length, "$.limit", &number) || number < 1 || number > 100 + || floor(number) != number) + { + valid = false; + } + else + { + limit = (size_t)number; + } + } + if (!valid || !query || !query[0]) + { + free(query); + free(session); + send_error(output, id, id_length, "VALIDATION_ERROR", "query or limit is invalid"); + return; + } + if (!*search_index) + { + *search_index = build_search_index(store); + } + if (!*search_index + || !tiny_search_index_candidates(*search_index, query, &candidates, &candidate_count)) + { + free(query); + free(session); + send_error(output, id, id_length, "INTERNAL_ERROR", "search index allocation failed"); + return; + } + items = malloc((candidate_count ? candidate_count : 1) * sizeof(*items)); + if (!items) + { + free(candidates); + free(query); + free(session); + send_error(output, id, id_length, "INTERNAL_ERROR", "out of memory"); + return; + } + for (i = 0; i < candidate_count; ++i) + { + const tiny_store_entry_t *e = tiny_store_entry(store, candidates[i]); + double score; + if (!e || e->deleted || (session && strcmp(e->message.session_id, session) != 0)) + { + continue; + } + score = tiny_search_score(query, &e->message); + if (score > 0) + { + items[count].entry = e; + items[count].score = score; + count++; + } + } + qsort(items, count, sizeof(*items), compare_results); + if (count > limit) + { + count = limit; + } + tiny_json_raw(&result, "{\"results\":["); + for (i = 0; i < count; ++i) + { + if (i) + { + tiny_json_raw(&result, ","); + } + append_message(&result, &items[i].entry->message); + } + tiny_json_raw(&result, "]}"); + free(candidates); + free(items); + free(query); + free(session); + } + else if (strcmp(name, "delete_message") == 0) + { + static const char *const keys[] = { "message_id", "session_id" }; + bool valid = object_has_only(args, args_length, keys, 2); + char *mid = json_string_value( + args, args_length, "$.message_id", TINY_MESSAGE_ID_LENGTH, true, &valid); + char *session = json_string_value( + args, args_length, "$.session_id", TINY_SESSION_ID_MAX, false, &valid); + if (!valid || !mid || !tiny_message_id_valid(mid)) + { + free(mid); + free(session); + send_error(output, + id, + id_length, + "VALIDATION_ERROR", + "message_id must be a canonical 26-character ULID"); + return; + } + status = tiny_store_delete(store, mid, session); + free(mid); + free(session); + if (status != TINY_STORE_OK && status != TINY_STORE_NOT_FOUND) + { + send_error(output, + id, + id_length, + tiny_store_status_name(status), + "delete record was not written"); + return; + } + tiny_json_raw(&result, + status == TINY_STORE_OK ? "{\"status\":\"OK\"}" : "{\"status\":\"NOT_FOUND\"}"); + } + else if (strcmp(name, "rebuild_index") == 0) + { + tiny_rebuild_report_t report; + tiny_search_index_t *rebuilt; + if (!object_has_only(args, args_length, NULL, 0)) + { + send_error(output, id, id_length, "VALIDATION_ERROR", "rebuild_index takes no arguments"); + return; + } + status = tiny_store_rebuild(store, &report); + if (status != TINY_STORE_OK) + { + send_error(output, id, id_length, tiny_store_status_name(status), "index rebuild failed"); + return; + } + rebuilt = build_search_index(store); + if (!rebuilt) + { + tiny_search_index_free(*search_index); + *search_index = NULL; + send_error(output, + id, + id_length, + "INTERNAL_ERROR", + "log was compacted but the search index could not be allocated"); + return; + } + tiny_search_index_free(*search_index); + *search_index = rebuilt; + tiny_json_raw(&result, "{\"status\":\"OK\",\"messages_indexed\":"); + tiny_json_u64(&result, report.messages_indexed); + tiny_json_raw(&result, ",\"records_removed\":"); + tiny_json_u64(&result, report.records_removed); + tiny_json_raw(&result, ",\"bytes_before\":"); + tiny_json_u64(&result, report.bytes_before); + tiny_json_raw(&result, ",\"bytes_after\":"); + tiny_json_u64(&result, report.bytes_after); + tiny_json_raw(&result, "}"); + } + else if (strcmp(name, "get_system_health") == 0) + { + tiny_store_health_t health; + if (!object_has_only(args, args_length, NULL, 0)) + { + send_error( + output, id, id_length, "VALIDATION_ERROR", "get_system_health takes no arguments"); + return; + } + tiny_store_health(store, &health); + tiny_json_raw(&result, "{\"status\":"); + tiny_json_string(&result, health.last_error[0] ? "DEGRADED" : "OK"); + tiny_json_raw(&result, ",\"format_version\":1,\"log_size\":"); + tiny_json_u64(&result, health.log_size); + tiny_json_raw(&result, ",\"snapshot_size\":"); + tiny_json_u64(&result, health.snapshot_size); + tiny_json_raw(&result, ",\"active_messages\":"); + tiny_json_u64(&result, health.active_messages); + tiny_json_raw(&result, ",\"deleted_messages\":"); + tiny_json_u64(&result, health.deleted_messages); + tiny_json_raw(&result, ",\"last_sequence\":"); + tiny_json_u64(&result, health.last_sequence); + tiny_json_raw(&result, ",\"recovered_tail\":"); + tiny_json_raw(&result, health.recovered_tail ? "true" : "false"); + tiny_json_raw(&result, ",\"snapshot_valid\":"); + tiny_json_raw(&result, health.snapshot_valid ? "true" : "false"); + tiny_json_raw(&result, ",\"last_error\":"); + tiny_json_string(&result, health.last_error); + tiny_json_raw(&result, "}"); + } + else + { + send_error(output, id, id_length, "VALIDATION_ERROR", "unknown tool"); + return; + } + if (result.failed) + { + send_error(output, id, id_length, "INTERNAL_ERROR", "response allocation failed"); + } + else + { + send_json_result(output, id, id_length, &result, false); + } + tiny_json_free(&result); +} + +static const char *tools_json + = "{\"tools\":[" + "{\"name\":\"write_message\",\"description\":\"Durably append a conversation " + "message\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"session_id\":{\"type\":" + "\"string\",\"maxLength\":256},\"role\":{\"type\":\"string\",\"enum\":[\"user\"," + "\"assistant\",\"system\"]},\"content\":{\"type\":\"string\",\"maxLength\":1048576}," + "\"metadata\":{\"type\":\"object\",\"properties\":{\"tool_call_id\":{\"type\":\"string\"," + "\"maxLength\":256},\"tags\":{\"type\":\"array\",\"maxItems\":32,\"items\":{\"type\":" + "\"string\",\"maxLength\":128}},\"memory_tier\":{\"type\":\"string\",\"enum\":[\"short\"," + "\"mid\",\"long\",\"discard\"]}},\"additionalProperties\":false}},\"required\":[\"session_" + "id\",\"role\",\"content\"],\"additionalProperties\":false}}," + "{\"name\":\"read_messages\",\"description\":\"Read a session in chronological " + "order\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"session_id\":{\"type\":" + "\"string\",\"maxLength\":256}},\"required\":[\"session_id\"],\"additionalProperties\":false}" + "}," + "{\"name\":\"search_messages\",\"description\":\"Search message content and " + "tags\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\"," + "\"maxLength\":4096},\"session_id\":{\"type\":\"string\",\"maxLength\":256},\"limit\":{" + "\"type\":\"integer\",\"minimum\":1,\"maximum\":100}},\"required\":[\"query\"]," + "\"additionalProperties\":false}}," + "{\"name\":\"delete_message\",\"description\":\"Durably tombstone a " + "message\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"message_id\":{\"type\":" + "\"string\",\"minLength\":26,\"maxLength\":26},\"session_id\":{\"type\":\"string\"," + "\"maxLength\":256}},\"required\":[\"message_id\"],\"additionalProperties\":false}}," + "{\"name\":\"rebuild_index\",\"description\":\"Compact the log and rebuild the " + "snapshot\",\"inputSchema\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":" + "false}}," + "{\"name\":\"get_system_health\",\"description\":\"Report persistence health and " + "counters\",\"inputSchema\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":" + "false}}]}"; + +int +tiny_convmem_server_run(FILE *input, FILE *output, tiny_store_t *store) +{ + char *line; + tiny_search_index_t *search_index = build_search_index(store); + if (!search_index) + { + return 1; + } + while ((line = read_line(input)) != NULL) + { + int length = (int)strlen(line); + const char *method = NULL, *id = NULL, *params = NULL; + int ml = 0, il = 0, pl = 0; + if (mjson_find(line, length, "$.method", &method, &ml) != MJSON_TOK_STRING) + { + free(line); + continue; + } + mjson_find(line, length, "$.id", &id, &il); + mjson_find(line, length, "$.params", ¶ms, &pl); { - result = close_result; + char method_value[64]; + int decoded + = mjson_get_string(line, length, "$.method", method_value, sizeof(method_value)); + if (decoded < 0) + { + free(line); + continue; + } + method_value[decoded] = '\0'; + if (strcmp(method_value, "initialize") == 0 && id) + { + fprintf(output, + "{\"jsonrpc\":\"2.0\",\"id\":%.*s,\"result\":{\"protocolVersion\":\"%s\"," + "\"capabilities\":{\"tools\":{}},\"serverInfo\":{\"name\":\"tiny-convmem-mcp\"," + "\"version\":\"0.1.0\"}}}\n", + il, + id, + MCP_PROTOCOL_VERSION); + } + else if (strcmp(method_value, "tools/list") == 0 && id) + { + fprintf( + output, "{\"jsonrpc\":\"2.0\",\"id\":%.*s,\"result\":%s}\n", il, id, tools_json); + } + else if (strcmp(method_value, "tools/call") == 0 && id && params) + { + char name[64]; + const char *args = NULL; + int al = 0; + int n = mjson_get_string(params, pl, "$.name", name, sizeof(name)); + if (n > 0) + { + name[n] = '\0'; + if (mjson_find(params, pl, "$.arguments", &args, &al) == MJSON_TOK_OBJECT) + { + handle_call(output, id, il, name, args, al, store, &search_index); + } + else + { + send_error(output, id, il, "VALIDATION_ERROR", "arguments must be an object"); + } + } + } + fflush(output); } - } - - return (int)result; + free(line); + } + tiny_search_index_free(search_index); + return 0; } diff --git a/src/mcp/server.h b/src/mcp/server.h index e3ff2c4..4ae76e4 100644 --- a/src/mcp/server.h +++ b/src/mcp/server.h @@ -2,8 +2,8 @@ #define TINY_CONVMEM_MCP_SERVER_H #include <stdio.h> +#include "storage/store.h" -int tiny_convmem_server_run(FILE *input, FILE *output); +int tiny_convmem_server_run(FILE *input, FILE *output, tiny_store_t *store); #endif - diff --git a/src/mcp/tools/health.c b/src/mcp/tools/health.c deleted file mode 100644 index 0309ea5..0000000 --- a/src/mcp/tools/health.c +++ /dev/null @@ -1,29 +0,0 @@ -#include "mcp/tools/health.h" - -#include <mcpc/ucbr.h> - -static void -health_call(const mcpc_tool_t *tool, mcpc_ucbr_t *result) -{ - (void)tool; - mcpc_ucbr_toolcall_add_text_printf8(result, (const char8_t *)"ok"); -} - -mcpc_tool_t * -tiny_convmem_health_tool_create(void) -{ - mcpc_tool_t *tool = mcpc_tool_new2( - (const char8_t *)"health", - (const char8_t *)"Check whether the tiny conversation memory server is running" - ); - if (tool == NULL) - { - return NULL; - } - - if (mcpc_tool_set_call_cb(tool, health_call) != MCPC_EC_0) - { - return NULL; - } - return tool; -} diff --git a/src/mcp/tools/health.h b/src/mcp/tools/health.h deleted file mode 100644 index f4bfe0a..0000000 --- a/src/mcp/tools/health.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef TINY_CONVMEM_MCP_HEALTH_H -#define TINY_CONVMEM_MCP_HEALTH_H - -#include <mcpc/tool.h> - -mcpc_tool_t *tiny_convmem_health_tool_create(void); - -#endif - diff --git a/src/storage/README.md b/src/storage/README.md index ceb8965..3c03dba 100644 --- a/src/storage/README.md +++ b/src/storage/README.md @@ -1,6 +1,9 @@ # Storage layer -This directory is reserved for persistence adapters. The storage engine and its -schema are intentionally undecided; SQLite or another dependency will be added -to `vcpkg.json` only after the storage contract is designed. +The storage adapter owns `active.log`, `snapshot.bin`, the data-directory lock, +in-memory live/deleted state, synchronous PUT/DEL writes, startup replay, and +compaction. It uses Win32 64-bit file APIs and `FlushFileBuffers`. +The binary format is private but versioned. Invalid complete records are never +skipped; only an incomplete EOF record is truncated. Snapshot generation must +match the active log before snapshot state is trusted. diff --git a/src/storage/store.c b/src/storage/store.c new file mode 100644 index 0000000..9d26e0f --- /dev/null +++ b/src/storage/store.c @@ -0,0 +1,1111 @@ +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <bcrypt.h> + +#include "storage/store.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define RECORD_HEADER_SIZE 24U +#define RECORD_VERSION 1U +#define RECORD_PUT 1U +#define RECORD_DEL 2U +#define RECORD_MARKER 3U +#define RECORD_PAYLOAD_MAX (TINY_CONTENT_MAX + 16384U) +#define SNAPSHOT_HEADER_SIZE 52U + +struct tiny_store +{ + HANDLE lock; + HANDLE log; + wchar_t *directory; + wchar_t *log_path; + wchar_t *snapshot_path; + tiny_store_entry_t *entries; + size_t entry_count; + size_t entry_capacity; + uint64_t last_sequence; + uint64_t last_ulid_ms; + unsigned char last_random[10]; + unsigned char generation[16]; + bool has_generation; + bool recovered_tail; + bool snapshot_valid; + char last_error[160]; +}; + +static void +put_u32(unsigned char *p, uint32_t v) +{ + p[0] = (unsigned char)v; + p[1] = (unsigned char)(v >> 8); + p[2] = (unsigned char)(v >> 16); + p[3] = (unsigned char)(v >> 24); +} +static void +put_u64(unsigned char *p, uint64_t v) +{ + unsigned i; + for (i = 0; i < 8; ++i) + { + p[i] = (unsigned char)(v >> (i * 8)); + } +} +static uint32_t +get_u32(const unsigned char *p) +{ + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} +static uint64_t +get_u64(const unsigned char *p) +{ + uint64_t v = 0; + int i; + for (i = 7; i >= 0; --i) + { + v = (v << 8) | p[i]; + } + return v; +} + +static uint32_t +crc32_bytes(const unsigned char *data, size_t length) +{ + uint32_t crc = 0xffffffffU; + size_t i; + for (i = 0; i < length; ++i) + { + unsigned bit; + crc ^= data[i]; + for (bit = 0; bit < 8; ++bit) + { + crc = (crc >> 1) ^ (0xedb88320U & (uint32_t)-(int32_t)(crc & 1)); + } + } + return ~crc; +} + +static wchar_t * +join_path(const wchar_t *directory, const wchar_t *name) +{ + const size_t left = wcslen(directory), right = wcslen(name); + const bool separator = left > 0 && directory[left - 1] != L'\\' && directory[left - 1] != L'/'; + wchar_t *result = malloc((left + right + (separator ? 2 : 1)) * sizeof(wchar_t)); + if (result == NULL) + { + return NULL; + } + wcscpy_s(result, left + right + (separator ? 2 : 1), directory); + if (separator) + { + wcscat_s(result, left + right + 2, L"\\"); + } + wcscat_s(result, left + right + (separator ? 2 : 1), name); + return result; +} + +static void +set_error(tiny_store_t *store, const char *message) +{ + strncpy_s(store->last_error, sizeof(store->last_error), message, _TRUNCATE); +} + +static bool +write_all(HANDLE file, const void *data, uint32_t length) +{ + const unsigned char *position = data; + while (length > 0) + { + DWORD written = 0; + if (!WriteFile(file, position, length, &written, NULL) || written == 0) + { + return false; + } + position += written; + length -= written; + } + return true; +} + +static bool +read_all(HANDLE file, void *data, uint32_t length, uint32_t *actual) +{ + unsigned char *position = data; + uint32_t total = 0; + while (total < length) + { + DWORD count = 0; + if (!ReadFile(file, position + total, length - total, &count, NULL)) + { + return false; + } + if (count == 0) + { + break; + } + total += count; + } + *actual = total; + return true; +} + +static uint64_t +now_milliseconds(void) +{ + FILETIME ft; + ULARGE_INTEGER ticks; + GetSystemTimeAsFileTime(&ft); + ticks.LowPart = ft.dwLowDateTime; + ticks.HighPart = ft.dwHighDateTime; + return ticks.QuadPart / 10000ULL - 11644473600000ULL; +} + +static void +make_ulid(tiny_store_t *store, uint64_t timestamp, char output[27]) +{ + static const char alphabet[] = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + unsigned char bytes[16]; + unsigned index; + if (timestamp > store->last_ulid_ms) + { + store->last_ulid_ms = timestamp; + if (BCryptGenRandom( + NULL, store->last_random, sizeof(store->last_random), BCRYPT_USE_SYSTEM_PREFERRED_RNG) + != 0) + { + uint64_t fallback = GetTickCount64() ^ timestamp; + for (index = 0; index < sizeof(store->last_random); ++index) + { + store->last_random[index] = (unsigned char)(fallback >> ((index % 8) * 8)); + } + } + } + else + { + timestamp = store->last_ulid_ms; + for (index = 10; index-- > 0;) + { + if (++store->last_random[index] != 0) + { + break; + } + } + } + bytes[0] = (unsigned char)(timestamp >> 40); + bytes[1] = (unsigned char)(timestamp >> 32); + bytes[2] = (unsigned char)(timestamp >> 24); + bytes[3] = (unsigned char)(timestamp >> 16); + bytes[4] = (unsigned char)(timestamp >> 8); + bytes[5] = (unsigned char)timestamp; + memcpy(bytes + 6, store->last_random, 10); + for (index = 0; index < 26; ++index) + { + const unsigned bit = index * 5; + unsigned value = 0, part; + for (part = 0; part < 5; ++part) + { + const int source_bit = (int)bit + (int)part - 2; + value <<= 1; + if (source_bit >= 0) + { + value |= (bytes[source_bit / 8] >> (7 - source_bit % 8)) & 1U; + } + } + output[index] = alphabet[value]; + } + output[26] = '\0'; +} + +static bool +ensure_entries(tiny_store_t *store) +{ + tiny_store_entry_t *resized; + size_t capacity; + if (store->entry_count < store->entry_capacity) + { + return true; + } + capacity = store->entry_capacity == 0 ? 32 : store->entry_capacity * 2; + resized = realloc(store->entries, capacity * sizeof(*resized)); + if (resized == NULL) + { + return false; + } + store->entries = resized; + store->entry_capacity = capacity; + return true; +} + +static tiny_store_entry_t * +find_entry(tiny_store_t *store, const char *message_id) +{ + size_t index; + for (index = 0; index < store->entry_count; ++index) + { + if (strcmp(store->entries[index].message.message_id, message_id) == 0) + { + return &store->entries[index]; + } + } + return NULL; +} + +static bool +serialize_put(const tiny_message_t *message, unsigned char **payload, uint32_t *length) +{ + size_t session_len = strlen(message->session_id), content_len = strlen(message->content); + size_t tool_len = message->metadata.tool_call_id ? strlen(message->metadata.tool_call_id) : 0; + size_t total = 8 + 26 + 2 + 16 + session_len + content_len + tool_len; + size_t index, offset; + unsigned char *data; + for (index = 0; index < message->metadata.tag_count; ++index) + { + total += 4 + strlen(message->metadata.tags[index]); + } + if (total > UINT32_MAX || total > RECORD_PAYLOAD_MAX) + { + return false; + } + data = malloc(total); + if (data == NULL) + { + return false; + } + offset = 0; + put_u64(data + offset, (uint64_t)message->timestamp_ms); + offset += 8; + memcpy(data + offset, message->message_id, 26); + offset += 26; + data[offset++] = (unsigned char)message->role; + data[offset++] = (unsigned char)message->metadata.memory_tier; + put_u32(data + offset, (uint32_t)session_len); + offset += 4; + put_u32(data + offset, (uint32_t)content_len); + offset += 4; + put_u32(data + offset, (uint32_t)tool_len); + offset += 4; + put_u32(data + offset, (uint32_t)message->metadata.tag_count); + offset += 4; + memcpy(data + offset, message->session_id, session_len); + offset += session_len; + memcpy(data + offset, message->content, content_len); + offset += content_len; + if (tool_len > 0) + { + memcpy(data + offset, message->metadata.tool_call_id, tool_len); + offset += tool_len; + } + for (index = 0; index < message->metadata.tag_count; ++index) + { + const size_t tag_len = strlen(message->metadata.tags[index]); + put_u32(data + offset, (uint32_t)tag_len); + offset += 4; + memcpy(data + offset, message->metadata.tags[index], tag_len); + offset += tag_len; + } + *payload = data; + *length = (uint32_t)total; + return true; +} + +static char * +copy_field(const unsigned char *data, size_t length) +{ + char *value = malloc(length + 1); + if (value != NULL) + { + memcpy(value, data, length); + value[length] = '\0'; + } + return value; +} + +static bool +deserialize_put(const unsigned char *data, uint32_t length, tiny_message_t *message) +{ + uint32_t session_len, content_len, tool_len, tag_count; + size_t offset = 0, index; + tiny_message_init(message); + if (length < 8 + 26 + 2 + 16) + { + return false; + } + message->timestamp_ms = (int64_t)get_u64(data + offset); + offset += 8; + memcpy(message->message_id, data + offset, 26); + message->message_id[26] = '\0'; + offset += 26; + message->role = (tiny_role_t)data[offset++]; + message->metadata.memory_tier = (tiny_memory_tier_t)data[offset++]; + session_len = get_u32(data + offset); + offset += 4; + content_len = get_u32(data + offset); + offset += 4; + tool_len = get_u32(data + offset); + offset += 4; + tag_count = get_u32(data + offset); + offset += 4; + if (!tiny_message_id_valid(message->message_id) + || (message->role < TINY_ROLE_USER || message->role > TINY_ROLE_SYSTEM) + || (message->metadata.memory_tier < TINY_TIER_SHORT + || message->metadata.memory_tier > TINY_TIER_DISCARD) + || session_len == 0 || session_len > TINY_SESSION_ID_MAX || content_len == 0 + || content_len > TINY_CONTENT_MAX || tool_len > TINY_TOOL_CALL_ID_MAX + || tag_count > TINY_TAG_COUNT_MAX || offset + session_len + content_len + tool_len > length) + { + return false; + } + message->session_id = copy_field(data + offset, session_len); + offset += session_len; + message->content = copy_field(data + offset, content_len); + offset += content_len; + if (tool_len > 0) + { + message->metadata.tool_call_id = copy_field(data + offset, tool_len); + offset += tool_len; + } + if (message->session_id == NULL || message->content == NULL + || (tool_len && !message->metadata.tool_call_id)) + { + goto fail; + } + if (tag_count > 0) + { + message->metadata.tags = calloc(tag_count, sizeof(char *)); + if (message->metadata.tags == NULL) + { + goto fail; + } + for (index = 0; index < tag_count; ++index) + { + uint32_t tag_len; + if (offset + 4 > length) + { + goto fail; + } + tag_len = get_u32(data + offset); + offset += 4; + if (tag_len == 0 || tag_len > TINY_TAG_MAX || offset + tag_len > length) + { + goto fail; + } + message->metadata.tags[index] = copy_field(data + offset, tag_len); + offset += tag_len; + if (message->metadata.tags[index] == NULL) + { + goto fail; + } + message->metadata.tag_count++; + } + } + if (offset != length || !tiny_utf8_valid(message->session_id, session_len) + || !tiny_utf8_valid(message->content, content_len)) + { + goto fail; + } + return true; +fail: + tiny_message_free(message); + return false; +} + +static tiny_store_status_t +append_record_to(HANDLE file, + uint8_t type, + uint64_t sequence, + const unsigned char *payload, + uint32_t payload_length, + bool flush) +{ + unsigned char header[RECORD_HEADER_SIZE] = { 0 }; + LARGE_INTEGER end; + memcpy(header, "TCMR", 4); + header[4] = RECORD_VERSION; + header[5] = type; + put_u32(header + 8, payload_length); + put_u64(header + 12, sequence); + put_u32(header + 20, crc32_bytes(payload, payload_length)); + end.QuadPart = 0; + if (!SetFilePointerEx(file, end, NULL, FILE_END) || !write_all(file, header, sizeof(header)) + || !write_all(file, payload, payload_length) || (flush && !FlushFileBuffers(file))) + { + return TINY_STORE_IO_ERROR; + } + return TINY_STORE_OK; +} + +static tiny_store_status_t +apply_put(tiny_store_t *store, tiny_message_t *message, uint64_t sequence) +{ + tiny_store_entry_t *entry = find_entry(store, message->message_id); + if (entry != NULL) + { + return TINY_STORE_CORRUPT; + } + if (!ensure_entries(store)) + { + return TINY_STORE_NO_MEMORY; + } + entry = &store->entries[store->entry_count++]; + memset(entry, 0, sizeof(*entry)); + entry->message = *message; + tiny_message_init(message); + entry->sequence = sequence; + return TINY_STORE_OK; +} + +static void +clear_entries(tiny_store_t *store) +{ + size_t index; + for (index = 0; index < store->entry_count; ++index) + { + tiny_message_free(&store->entries[index].message); + } + store->entry_count = 0; + store->last_sequence = 0; +} + +static tiny_store_status_t +replay_log(tiny_store_t *store, uint64_t starting_offset) +{ + LARGE_INTEGER start, file_size, boundary; + start.QuadPart = (LONGLONG)starting_offset; + boundary = start; + if (!SetFilePointerEx(store->log, start, NULL, FILE_BEGIN) + || !GetFileSizeEx(store->log, &file_size)) + { + return TINY_STORE_IO_ERROR; + } + while (boundary.QuadPart < file_size.QuadPart) + { + unsigned char header[RECORD_HEADER_SIZE], *payload = NULL; + uint32_t actual = 0, payload_length, expected_crc; + uint64_t sequence; + tiny_store_status_t status = TINY_STORE_OK; + if (!read_all(store->log, header, sizeof(header), &actual)) + { + return TINY_STORE_IO_ERROR; + } + if (actual < sizeof(header)) + { + goto torn_tail; + } + if (memcmp(header, "TCMR", 4) != 0 || header[4] != RECORD_VERSION + || (header[5] != RECORD_PUT && header[5] != RECORD_DEL && header[5] != RECORD_MARKER)) + { + return TINY_STORE_CORRUPT; + } + payload_length = get_u32(header + 8); + sequence = get_u64(header + 12); + expected_crc = get_u32(header + 20); + if (payload_length > RECORD_PAYLOAD_MAX + || (header[5] != RECORD_MARKER && sequence <= store->last_sequence)) + { + return TINY_STORE_CORRUPT; + } + payload = malloc(payload_length == 0 ? 1 : payload_length); + if (payload == NULL) + { + return TINY_STORE_NO_MEMORY; + } + if (!read_all(store->log, payload, payload_length, &actual)) + { + free(payload); + return TINY_STORE_IO_ERROR; + } + if (actual < payload_length) + { + free(payload); + goto torn_tail; + } + if (crc32_bytes(payload, payload_length) != expected_crc) + { + free(payload); + return TINY_STORE_CORRUPT; + } + if (header[5] == RECORD_MARKER) + { + if (boundary.QuadPart != 0 || sequence != 0 + || payload_length != sizeof(store->generation)) + { + free(payload); + return TINY_STORE_CORRUPT; + } + memcpy(store->generation, payload, sizeof(store->generation)); + store->has_generation = true; + } + else if (header[5] == RECORD_PUT) + { + tiny_message_t message; + if (!deserialize_put(payload, payload_length, &message)) + { + free(payload); + return TINY_STORE_CORRUPT; + } + status = apply_put(store, &message, sequence); + tiny_message_free(&message); + } + else + { + char id[27]; + tiny_store_entry_t *entry; + if (payload_length != 34) + { + free(payload); + return TINY_STORE_CORRUPT; + } + memcpy(id, payload, 26); + id[26] = '\0'; + entry = find_entry(store, id); + if (entry == NULL || entry->deleted || !tiny_message_id_valid(id)) + { + free(payload); + return TINY_STORE_CORRUPT; + } + entry->deleted = true; + } + free(payload); + if (status != TINY_STORE_OK) + { + return status; + } + if (header[5] != RECORD_MARKER) + { + store->last_sequence = sequence; + } + boundary.QuadPart += RECORD_HEADER_SIZE + payload_length; + } + return TINY_STORE_OK; + +torn_tail: + if (!SetFilePointerEx(store->log, boundary, NULL, FILE_BEGIN) || !SetEndOfFile(store->log) + || !FlushFileBuffers(store->log)) + { + return TINY_STORE_IO_ERROR; + } + store->recovered_tail = true; + set_error(store, "an incomplete trailing record was truncated during recovery"); + return TINY_STORE_OK; +} + +static tiny_store_status_t +write_snapshot(tiny_store_t *store) +{ + wchar_t *temporary = join_path(store->directory, L"snapshot.tmp"); + HANDLE file; + unsigned char data[SNAPSHOT_HEADER_SIZE] = { 0 }; + LARGE_INTEGER size; + bool ok; + size_t index; + if (temporary == NULL) + { + return TINY_STORE_NO_MEMORY; + } + file = CreateFileW(temporary, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) + { + free(temporary); + return TINY_STORE_IO_ERROR; + } + GetFileSizeEx(store->log, &size); + memcpy(data, "TCMS", 4); + data[4] = 1; + memcpy(data + 8, store->generation, 16); + put_u64(data + 24, store->last_sequence); + put_u64(data + 32, (uint64_t)size.QuadPart); + put_u64(data + 40, (uint64_t)store->entry_count); + put_u32(data + 48, crc32_bytes(data, 48)); + ok = write_all(file, data, sizeof(data)); + for (index = 0; ok && index < store->entry_count; ++index) + { + unsigned char prefix[16], *payload = NULL; + uint32_t length = 0; + if (!serialize_put(&store->entries[index].message, &payload, &length)) + { + ok = false; + break; + } + put_u64(prefix, store->entries[index].sequence); + put_u32(prefix + 8, length); + put_u32(prefix + 12, crc32_bytes(payload, length)); + ok = write_all(file, prefix, sizeof(prefix)) && write_all(file, payload, length); + free(payload); + } + if (ok) + { + ok = FlushFileBuffers(file) != 0; + } + CloseHandle(file); + if (ok) + { + ok = MoveFileExW( + temporary, store->snapshot_path, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) + != 0; + } + free(temporary); + store->snapshot_valid = ok; + return ok ? TINY_STORE_OK : TINY_STORE_IO_ERROR; +} + +static bool +log_generation_matches(tiny_store_t *store, const unsigned char generation[16]) +{ + unsigned char header[RECORD_HEADER_SIZE], payload[16]; + uint32_t actual = 0; + LARGE_INTEGER zero = { 0 }; + if (!SetFilePointerEx(store->log, zero, NULL, FILE_BEGIN) + || !read_all(store->log, header, sizeof(header), &actual) || actual != sizeof(header) + || memcmp(header, "TCMR", 4) != 0 || header[4] != RECORD_VERSION || header[5] != RECORD_MARKER + || get_u64(header + 12) != 0 || get_u32(header + 8) != 16 + || !read_all(store->log, payload, sizeof(payload), &actual) || actual != sizeof(payload) + || get_u32(header + 20) != crc32_bytes(payload, sizeof(payload))) + { + return false; + } + return memcmp(payload, generation, 16) == 0; +} + +static bool +load_snapshot(tiny_store_t *store, uint64_t *log_offset) +{ + HANDLE file = CreateFileW(store->snapshot_path, + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + unsigned char header[SNAPSHOT_HEADER_SIZE]; + uint32_t actual = 0; + uint64_t count, index, expected_sequence, offset; + LARGE_INTEGER log_size; + if (file == INVALID_HANDLE_VALUE) + { + return false; + } + if (!read_all(file, header, sizeof(header), &actual) || actual != sizeof(header) + || memcmp(header, "TCMS", 4) != 0 || header[4] != 1 + || get_u32(header + 48) != crc32_bytes(header, 48) || !GetFileSizeEx(store->log, &log_size)) + { + goto invalid; + } + expected_sequence = get_u64(header + 24); + offset = get_u64(header + 32); + count = get_u64(header + 40); + if (offset > (uint64_t)log_size.QuadPart || count > SIZE_MAX + || !log_generation_matches(store, header + 8)) + { + goto invalid; + } + memcpy(store->generation, header + 8, 16); + store->has_generation = true; + for (index = 0; index < count; ++index) + { + unsigned char prefix[16], *payload; + uint32_t length, crc; + tiny_message_t message; + tiny_store_status_t status; + if (!read_all(file, prefix, sizeof(prefix), &actual) || actual != sizeof(prefix)) + { + goto invalid_entries; + } + length = get_u32(prefix + 8); + crc = get_u32(prefix + 12); + if (length > RECORD_PAYLOAD_MAX) + { + goto invalid_entries; + } + payload = malloc(length ? length : 1); + if (!payload) + { + goto invalid_entries; + } + if (!read_all(file, payload, length, &actual) || actual != length + || crc32_bytes(payload, length) != crc || !deserialize_put(payload, length, &message)) + { + free(payload); + goto invalid_entries; + } + free(payload); + status = apply_put(store, &message, get_u64(prefix)); + tiny_message_free(&message); + if (status != TINY_STORE_OK) + { + goto invalid_entries; + } + store->last_sequence = get_u64(prefix); + } + if (store->last_sequence != expected_sequence) + { + goto invalid_entries; + } + CloseHandle(file); + *log_offset = offset; + store->snapshot_valid = true; + return true; +invalid_entries: + clear_entries(store); + store->has_generation = false; +invalid: + set_error(store, "snapshot validation failed; the active log was replayed from the beginning"); + CloseHandle(file); + return false; +} + +tiny_store_status_t +tiny_store_open(tiny_store_t **output, const wchar_t *directory) +{ + tiny_store_t *store = calloc(1, sizeof(*store)); + wchar_t *lock_path = NULL; + tiny_store_status_t status; + if (store == NULL) + { + return TINY_STORE_NO_MEMORY; + } + store->lock = INVALID_HANDLE_VALUE; + store->log = INVALID_HANDLE_VALUE; + store->directory = _wcsdup(directory); + store->log_path = join_path(directory, L"active.log"); + store->snapshot_path = join_path(directory, L"snapshot.bin"); + lock_path = join_path(directory, L"store.lock"); + if (!store->directory || !store->log_path || !store->snapshot_path || !lock_path) + { + status = TINY_STORE_NO_MEMORY; + goto fail; + } + if (!CreateDirectoryW(directory, NULL) && GetLastError() != ERROR_ALREADY_EXISTS) + { + status = TINY_STORE_IO_ERROR; + goto fail; + } + store->lock = CreateFileW( + lock_path, GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + free(lock_path); + lock_path = NULL; + if (store->lock == INVALID_HANDLE_VALUE) + { + status = GetLastError() == ERROR_SHARING_VIOLATION ? TINY_STORE_BUSY : TINY_STORE_IO_ERROR; + goto fail; + } + store->log = CreateFileW(store->log_path, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (store->log == INVALID_HANDLE_VALUE) + { + status = TINY_STORE_IO_ERROR; + goto fail; + } + { + uint64_t replay_offset = 0; + (void)load_snapshot(store, &replay_offset); + status = replay_log(store, replay_offset); + } + if (status != TINY_STORE_OK) + { + goto fail; + } + *output = store; + return TINY_STORE_OK; +fail: + free(lock_path); + tiny_store_close(store); + return status; +} + +void +tiny_store_close(tiny_store_t *store) +{ + size_t index; + if (!store) + { + return; + } + if (store->log != INVALID_HANDLE_VALUE) + { + CloseHandle(store->log); + } + if (store->lock != INVALID_HANDLE_VALUE) + { + CloseHandle(store->lock); + } + for (index = 0; index < store->entry_count; ++index) + { + tiny_message_free(&store->entries[index].message); + } + free(store->entries); + free(store->directory); + free(store->log_path); + free(store->snapshot_path); + free(store); +} + +const char * +tiny_store_status_name(tiny_store_status_t status) +{ + switch (status) + { + case TINY_STORE_OK: + return "OK"; + case TINY_STORE_NOT_FOUND: + return "NOT_FOUND"; + case TINY_STORE_BUSY: + return "BUSY"; + case TINY_STORE_IO_ERROR: + return "IO_ERROR"; + case TINY_STORE_CORRUPT: + return "CORRUPT_STORE"; + default: + return "INTERNAL_ERROR"; + } +} + +tiny_store_status_t +tiny_store_write(tiny_store_t *store, const tiny_message_t *input, tiny_message_t *written) +{ + tiny_message_t message, indexed; + unsigned char *payload = NULL; + uint32_t length = 0; + tiny_store_status_t status; + tiny_message_init(&message); + tiny_message_init(&indexed); + if (tiny_message_copy(&message, input) != 0) + { + return TINY_STORE_NO_MEMORY; + } + message.timestamp_ms = (int64_t)now_milliseconds(); + make_ulid(store, (uint64_t)message.timestamp_ms, message.message_id); + if (!serialize_put(&message, &payload, &length) || !ensure_entries(store) + || tiny_message_copy(&indexed, &message) != 0) + { + free(payload); + tiny_message_free(&message); + tiny_message_free(&indexed); + return TINY_STORE_NO_MEMORY; + } + status + = append_record_to(store->log, RECORD_PUT, store->last_sequence + 1, payload, length, true); + free(payload); + if (status != TINY_STORE_OK) + { + tiny_message_free(&message); + tiny_message_free(&indexed); + set_error(store, "failed to append and flush a PUT record"); + return status; + } + store->last_sequence++; + memset(&store->entries[store->entry_count], 0, sizeof(store->entries[store->entry_count])); + store->entries[store->entry_count].message = indexed; + store->entries[store->entry_count].sequence = store->last_sequence; + store->entry_count++; + *written = message; + return TINY_STORE_OK; +} + +tiny_store_status_t +tiny_store_delete(tiny_store_t *store, const char *message_id, const char *session_id) +{ + tiny_store_entry_t *entry = find_entry(store, message_id); + unsigned char payload[34]; + tiny_store_status_t status; + if (!entry || entry->deleted + || (session_id && strcmp(entry->message.session_id, session_id) != 0)) + { + return TINY_STORE_NOT_FOUND; + } + memcpy(payload, message_id, 26); + put_u64(payload + 26, now_milliseconds()); + status = append_record_to( + store->log, RECORD_DEL, store->last_sequence + 1, payload, sizeof(payload), true); + if (status != TINY_STORE_OK) + { + set_error(store, "failed to append and flush a DEL record"); + return status; + } + store->last_sequence++; + entry->deleted = true; + return TINY_STORE_OK; +} + +tiny_store_status_t +tiny_store_rebuild(tiny_store_t *store, tiny_rebuild_report_t *report) +{ + wchar_t *temporary = join_path(store->directory, L"active.tmp"); + HANDLE file; + LARGE_INTEGER before, after; + size_t read_index, write_index = 0; + uint64_t removed = 0, max_sequence = 0; + tiny_store_status_t status = TINY_STORE_OK; + if (!temporary) + { + return TINY_STORE_NO_MEMORY; + } + GetFileSizeEx(store->log, &before); + file = CreateFileW( + temporary, GENERIC_WRITE | GENERIC_READ, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) + { + free(temporary); + return TINY_STORE_IO_ERROR; + } + if (BCryptGenRandom( + NULL, store->generation, sizeof(store->generation), BCRYPT_USE_SYSTEM_PREFERRED_RNG) + != 0) + { + CloseHandle(file); + free(temporary); + return TINY_STORE_IO_ERROR; + } + store->has_generation = true; + status = append_record_to( + file, RECORD_MARKER, 0, store->generation, sizeof(store->generation), false); + for (read_index = 0; read_index < store->entry_count; ++read_index) + { + if (status == TINY_STORE_OK && !store->entries[read_index].deleted) + { + unsigned char *payload = NULL; + uint32_t length = 0; + if (!serialize_put(&store->entries[read_index].message, &payload, &length)) + { + status = TINY_STORE_NO_MEMORY; + break; + } + status = append_record_to( + file, RECORD_PUT, store->entries[read_index].sequence, payload, length, false); + free(payload); + if (status != TINY_STORE_OK) + { + break; + } + if (store->entries[read_index].sequence > max_sequence) + { + max_sequence = store->entries[read_index].sequence; + } + } + else + { + removed++; + } + } + if (status == TINY_STORE_OK && !FlushFileBuffers(file)) + { + status = TINY_STORE_IO_ERROR; + } + CloseHandle(file); + if (status == TINY_STORE_OK) + { + CloseHandle(store->log); + store->log = INVALID_HANDLE_VALUE; + if (!MoveFileExW( + temporary, store->log_path, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) + { + status = TINY_STORE_IO_ERROR; + } + store->log = CreateFileW(store->log_path, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (store->log == INVALID_HANDLE_VALUE) + { + status = TINY_STORE_IO_ERROR; + } + } + if (status == TINY_STORE_OK) + { + for (read_index = 0; read_index < store->entry_count; ++read_index) + { + if (store->entries[read_index].deleted) + { + tiny_message_free(&store->entries[read_index].message); + } + else + { + if (write_index != read_index) + { + store->entries[write_index] = store->entries[read_index]; + } + write_index++; + } + } + store->entry_count = write_index; + store->last_sequence = max_sequence; + GetFileSizeEx(store->log, &after); + report->messages_indexed = write_index; + report->records_removed = removed; + report->bytes_before = (uint64_t)before.QuadPart; + report->bytes_after = (uint64_t)after.QuadPart; + status = write_snapshot(store); + } + if (status != TINY_STORE_OK) + { + set_error(store, + "compaction did not complete; the previous valid log is retained when replacement " + "had not begun"); + } + free(temporary); + return status; +} + +void +tiny_store_health(const tiny_store_t *store, tiny_store_health_t *health) +{ + LARGE_INTEGER size; + size_t index; + memset(health, 0, sizeof(*health)); + if (GetFileSizeEx(store->log, &size)) + { + health->log_size = (uint64_t)size.QuadPart; + } + { + HANDLE f = CreateFileW(store->snapshot_path, + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + NULL); + if (f != INVALID_HANDLE_VALUE) + { + if (GetFileSizeEx(f, &size)) + { + health->snapshot_size = (uint64_t)size.QuadPart; + } + CloseHandle(f); + } + } + for (index = 0; index < store->entry_count; ++index) + { + if (store->entries[index].deleted) + { + health->deleted_messages++; + } + else + { + health->active_messages++; + } + } + health->last_sequence = store->last_sequence; + health->recovered_tail = store->recovered_tail; + health->snapshot_valid = store->snapshot_valid; + strncpy_s(health->last_error, sizeof(health->last_error), store->last_error, _TRUNCATE); +} + +size_t +tiny_store_entry_count(const tiny_store_t *store) +{ + return store->entry_count; +} +const tiny_store_entry_t * +tiny_store_entry(const tiny_store_t *store, size_t index) +{ + return index < store->entry_count ? &store->entries[index] : NULL; +} diff --git a/src/storage/store.h b/src/storage/store.h new file mode 100644 index 0000000..cc14a80 --- /dev/null +++ b/src/storage/store.h @@ -0,0 +1,64 @@ +#ifndef TINY_CONVMEM_STORAGE_STORE_H +#define TINY_CONVMEM_STORAGE_STORE_H + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <wchar.h> + +#include "domain/message.h" + +typedef enum tiny_store_status +{ + TINY_STORE_OK = 0, + TINY_STORE_NOT_FOUND, + TINY_STORE_BUSY, + TINY_STORE_IO_ERROR, + TINY_STORE_CORRUPT, + TINY_STORE_NO_MEMORY +} tiny_store_status_t; + +typedef struct tiny_store_entry +{ + tiny_message_t message; + bool deleted; + uint64_t sequence; +} tiny_store_entry_t; + +typedef struct tiny_store tiny_store_t; + +typedef struct tiny_rebuild_report +{ + uint64_t messages_indexed; + uint64_t records_removed; + uint64_t bytes_before; + uint64_t bytes_after; +} tiny_rebuild_report_t; + +typedef struct tiny_store_health +{ + uint64_t log_size; + uint64_t snapshot_size; + uint64_t active_messages; + uint64_t deleted_messages; + uint64_t last_sequence; + bool recovered_tail; + bool snapshot_valid; + char last_error[160]; +} tiny_store_health_t; + +tiny_store_status_t tiny_store_open(tiny_store_t **store, const wchar_t *directory); +void tiny_store_close(tiny_store_t *store); +const char *tiny_store_status_name(tiny_store_status_t status); + +tiny_store_status_t +tiny_store_write(tiny_store_t *store, const tiny_message_t *input, tiny_message_t *written); +tiny_store_status_t +tiny_store_delete(tiny_store_t *store, const char *message_id, const char *session_id); +tiny_store_status_t tiny_store_rebuild(tiny_store_t *store, tiny_rebuild_report_t *report); +void tiny_store_health(const tiny_store_t *store, tiny_store_health_t *health); + +size_t tiny_store_entry_count(const tiny_store_t *store); +const tiny_store_entry_t *tiny_store_entry(const tiny_store_t *store, size_t index); + +#endif |
