diff options
Diffstat (limited to 'src')
27 files changed, 2537 insertions, 68 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1998150..31a5338 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,12 +30,14 @@ add_executable(tiny-convmem-mcp mcp/tools/common/common.c mcp/tools/common/context.c mcp/tools/common/history.c + mcp/tools/common/memory_index.c mcp/tools/common/registry.c mcp/tools/delete_message.c mcp/tools/get_system_health.c mcp/tools/read_messages.c mcp/tools/rebuild_index.c mcp/tools/search_messages.c + mcp/tools/update_message.c mcp/tools/write_message.c ) diff --git a/src/domain/message.c b/src/domain/message.c index b1db3e1..ec019c4 100644 --- a/src/domain/message.c +++ b/src/domain/message.c @@ -33,6 +33,9 @@ tiny_message_init(tiny_message_t *message) } memset(message, 0, sizeof(*message)); message->metadata.memory_tier = TINY_TIER_SHORT; + message->metadata.memory_kind = TINY_KIND_CONVERSATION; + message->metadata.importance = 50; + message->revision = 1; } void @@ -46,11 +49,18 @@ tiny_message_free(tiny_message_t *message) free(message->session_id); free(message->content); free(message->metadata.tool_call_id); + free(message->metadata.summary); for (index = 0; index < message->metadata.tag_count; ++index) { free(message->metadata.tags[index]); } free(message->metadata.tags); + free(message->metadata.relations); + for (index = 0; index < message->metadata.source_count; ++index) + { + free(message->metadata.source_message_ids[index]); + } + free(message->metadata.source_message_ids); tiny_message_init(message); } @@ -59,8 +69,7 @@ tiny_message_copy(tiny_message_t *destination, const tiny_message_t *source) { size_t index; if (destination == NULL || source == NULL || destination == source || source->session_id == NULL - || source->content == NULL - || (source->metadata.tag_count != 0 && source->metadata.tags == NULL)) + || source->content == NULL || !tiny_metadata_valid(&source->metadata)) { return -1; } @@ -69,12 +78,56 @@ tiny_message_copy(tiny_message_t *destination, const tiny_message_t *source) destination->timestamp_ms = source->timestamp_ms; destination->role = source->role; destination->metadata.memory_tier = source->metadata.memory_tier; + destination->metadata.memory_kind = source->metadata.memory_kind; + destination->metadata.importance = source->metadata.importance; + destination->revision = source->revision; + destination->updated_at_ms = source->updated_at_ms; destination->session_id = duplicate_string(source->session_id); destination->content = duplicate_string(source->content); if (destination->session_id == NULL || destination->content == NULL) { return copy_failed(destination); } + if (source->metadata.summary != NULL) + { + destination->metadata.summary = duplicate_string(source->metadata.summary); + if (destination->metadata.summary == NULL) + { + return copy_failed(destination); + } + } + if (source->metadata.relation_count > 0) + { + destination->metadata.relations + = malloc(source->metadata.relation_count * sizeof(*destination->metadata.relations)); + if (destination->metadata.relations == NULL) + { + return copy_failed(destination); + } + memcpy(destination->metadata.relations, + source->metadata.relations, + source->metadata.relation_count * sizeof(*destination->metadata.relations)); + destination->metadata.relation_count = source->metadata.relation_count; + } + if (source->metadata.source_count > 0) + { + destination->metadata.source_message_ids + = calloc(source->metadata.source_count, sizeof(char *)); + if (destination->metadata.source_message_ids == NULL) + { + return copy_failed(destination); + } + for (index = 0; index < source->metadata.source_count; ++index) + { + destination->metadata.source_message_ids[index] + = duplicate_string(source->metadata.source_message_ids[index]); + if (destination->metadata.source_message_ids[index] == NULL) + { + return copy_failed(destination); + } + destination->metadata.source_count++; + } + } if (source->metadata.tool_call_id != NULL) { destination->metadata.tool_call_id = duplicate_string(source->metadata.tool_call_id); @@ -103,6 +156,70 @@ tiny_message_copy(tiny_message_t *destination, const tiny_message_t *source) return 0; } +bool +tiny_metadata_valid(const tiny_metadata_t *metadata) +{ + size_t index, other; + if (!metadata || metadata->memory_tier < TINY_TIER_SHORT + || metadata->memory_tier > TINY_TIER_DISCARD || metadata->memory_kind < TINY_KIND_CONVERSATION + || metadata->memory_kind > TINY_KIND_SUMMARY || metadata->importance > 100 + || metadata->tag_count > TINY_TAG_COUNT_MAX || (metadata->tag_count && !metadata->tags) + || metadata->relation_count > TINY_RELATION_COUNT_MAX + || (metadata->relation_count && !metadata->relations) + || metadata->source_count > TINY_SOURCE_COUNT_MAX + || (metadata->source_count && !metadata->source_message_ids)) + { + return false; + } + if (metadata->summary + && (strlen(metadata->summary) > TINY_SUMMARY_MAX + || !tiny_utf8_valid(metadata->summary, strlen(metadata->summary)))) + { + return false; + } + for (index = 0; index < metadata->tag_count; ++index) + { + if (!metadata->tags[index] || !metadata->tags[index][0] + || strlen(metadata->tags[index]) > TINY_TAG_MAX) + { + return false; + } + } + for (index = 0; index < metadata->relation_count; ++index) + { + const tiny_relation_t *relation = &metadata->relations[index]; + if (relation->type < TINY_RELATION_DEPENDS_ON || relation->type > TINY_RELATION_RELATED_TO + || !tiny_message_id_valid(relation->message_id)) + { + return false; + } + for (other = 0; other < index; ++other) + { + if (relation->type == metadata->relations[other].type + && strcmp(relation->message_id, metadata->relations[other].message_id) == 0) + { + return false; + } + } + } + for (index = 0; index < metadata->source_count; ++index) + { + if (!metadata->source_message_ids[index] + || !tiny_message_id_valid(metadata->source_message_ids[index])) + { + return false; + } + for (other = 0; other < index; ++other) + { + if (strcmp(metadata->source_message_ids[index], metadata->source_message_ids[other]) == 0) + { + return false; + } + } + } + return true; +} + const char * tiny_role_name(tiny_role_t role) { @@ -193,6 +310,61 @@ tiny_tier_parse(const char *value, tiny_memory_tier_t *tier) return true; } +const char * +tiny_kind_name(tiny_memory_kind_t kind) +{ + static const char *names[] + = { "unknown", "conversation", "episode", "semantic", "procedure", "preference", "summary" }; + return kind >= TINY_KIND_CONVERSATION && kind <= TINY_KIND_SUMMARY ? names[kind] : names[0]; +} + +bool +tiny_kind_parse(const char *value, tiny_memory_kind_t *kind) +{ + tiny_memory_kind_t candidate; + if (value == NULL || kind == NULL) + { + return false; + } + for (candidate = TINY_KIND_CONVERSATION; candidate <= TINY_KIND_SUMMARY; ++candidate) + { + if (strcmp(value, tiny_kind_name(candidate)) == 0) + { + *kind = candidate; + return true; + } + } + return false; +} + +const char * +tiny_relation_type_name(tiny_relation_type_t type) +{ + static const char *names[] + = { "unknown", "depends_on", "derived_from", "supersedes", "related_to" }; + return type >= TINY_RELATION_DEPENDS_ON && type <= TINY_RELATION_RELATED_TO ? names[type] + : names[0]; +} + +bool +tiny_relation_type_parse(const char *value, tiny_relation_type_t *type) +{ + tiny_relation_type_t candidate; + if (value == NULL || type == NULL) + { + return false; + } + for (candidate = TINY_RELATION_DEPENDS_ON; candidate <= TINY_RELATION_RELATED_TO; ++candidate) + { + if (strcmp(value, tiny_relation_type_name(candidate)) == 0) + { + *type = candidate; + return true; + } + } + return false; +} + bool tiny_utf8_valid(const char *value, size_t length) { diff --git a/src/domain/message.h b/src/domain/message.h index fa36d2a..cece77c 100644 --- a/src/domain/message.h +++ b/src/domain/message.h @@ -12,6 +12,9 @@ #define TINY_TAG_MAX 128 #define TINY_TAG_COUNT_MAX 32 #define TINY_QUERY_MAX 4096 +#define TINY_SUMMARY_MAX 4096 +#define TINY_RELATION_COUNT_MAX 32 +#define TINY_SOURCE_COUNT_MAX 64 typedef enum tiny_role { @@ -28,14 +31,56 @@ typedef enum tiny_memory_tier TINY_TIER_DISCARD = 4 } tiny_memory_tier_t; +typedef enum tiny_memory_kind +{ + TINY_KIND_CONVERSATION = 1, + TINY_KIND_EPISODE, + TINY_KIND_SEMANTIC, + TINY_KIND_PROCEDURE, + TINY_KIND_PREFERENCE, + TINY_KIND_SUMMARY +} tiny_memory_kind_t; + +typedef enum tiny_relation_type +{ + TINY_RELATION_DEPENDS_ON = 1, + TINY_RELATION_DERIVED_FROM, + TINY_RELATION_SUPERSEDES, + TINY_RELATION_RELATED_TO +} tiny_relation_type_t; + +typedef struct tiny_relation +{ + tiny_relation_type_t type; + char message_id[TINY_MESSAGE_ID_LENGTH + 1]; +} tiny_relation_t; + typedef struct tiny_metadata { char *tool_call_id; char **tags; size_t tag_count; tiny_memory_tier_t memory_tier; + tiny_memory_kind_t memory_kind; + uint8_t importance; + char *summary; + tiny_relation_t *relations; + size_t relation_count; + char **source_message_ids; + size_t source_count; } tiny_metadata_t; +typedef uint32_t tiny_metadata_mask_t; +#define TINY_METADATA_TOOL_CALL_ID UINT32_C(0x01) +#define TINY_METADATA_TAGS UINT32_C(0x02) +#define TINY_METADATA_TIER UINT32_C(0x04) +#define TINY_METADATA_KIND UINT32_C(0x08) +#define TINY_METADATA_IMPORTANCE UINT32_C(0x10) +#define TINY_METADATA_SUMMARY UINT32_C(0x20) +#define TINY_METADATA_RELATIONS UINT32_C(0x40) +#define TINY_METADATA_SOURCES UINT32_C(0x80) +#define TINY_METADATA_ALL UINT32_C(0xff) + typedef struct tiny_message { char message_id[TINY_MESSAGE_ID_LENGTH + 1]; @@ -44,6 +89,8 @@ typedef struct tiny_message tiny_role_t role; char *content; tiny_metadata_t metadata; + uint64_t revision; + int64_t updated_at_ms; } tiny_message_t; void tiny_message_init(tiny_message_t *message); @@ -54,7 +101,12 @@ const char *tiny_role_name(tiny_role_t role); bool tiny_role_parse(const char *value, tiny_role_t *role); const char *tiny_tier_name(tiny_memory_tier_t tier); bool tiny_tier_parse(const char *value, tiny_memory_tier_t *tier); +const char *tiny_kind_name(tiny_memory_kind_t kind); +bool tiny_kind_parse(const char *value, tiny_memory_kind_t *kind); +const char *tiny_relation_type_name(tiny_relation_type_t type); +bool tiny_relation_type_parse(const char *value, tiny_relation_type_t *type); bool tiny_utf8_valid(const char *value, size_t length); bool tiny_message_id_valid(const char *value); +bool tiny_metadata_valid(const tiny_metadata_t *metadata); #endif diff --git a/src/domain/search.c b/src/domain/search.c index 0f9179d..10109a7 100644 --- a/src/domain/search.c +++ b/src/domain/search.c @@ -458,6 +458,12 @@ tiny_search_index_add(tiny_search_index_t *index, tiny_search_index_free(delta); return false; } + if (message->metadata.summary + && !index_text_direct(delta, message->metadata.summary, message_index)) + { + tiny_search_index_free(delta); + return false; + } for (tag = 0; tag < message->metadata.tag_count; ++tag) { if (!index_text_direct(delta, message->metadata.tags[tag], message_index)) @@ -609,6 +615,26 @@ tiny_search_score(const char *query, const tiny_message_t *message) } free(tag_points); } + if (!found && message->metadata.summary) + { + uint32_t *summary_points = NULL; + size_t summary_count = 0; + if (decode_codepoints(message->metadata.summary, &summary_points, &summary_count)) + { + for (message_offset = 0; message_offset + width <= summary_count; ++message_offset) + { + if (memcmp(query_points + query_offset, + summary_points + message_offset, + width * sizeof(uint32_t)) + == 0) + { + found = true; + break; + } + } + } + free(summary_points); + } if (found) { matches++; diff --git a/src/mcp/json.c b/src/mcp/json.c index 62f31c9..59e2395 100644 --- a/src/mcp/json.c +++ b/src/mcp/json.c @@ -1,6 +1,7 @@ #include "mcp/json.h" #include <stdio.h> +#include <math.h> #include <stdlib.h> #include <string.h> @@ -172,3 +173,15 @@ tiny_json_i64(tiny_json_buffer_t *buffer, long long value) snprintf(text, sizeof(text), "%lld", value); return tiny_json_raw(buffer, text); } + +bool +tiny_json_number(tiny_json_buffer_t *buffer, double value) +{ + char text[32]; + if (!isfinite(value)) + { + return false; + } + snprintf(text, sizeof(text), "%.17g", value); + return tiny_json_raw(buffer, text); +} diff --git a/src/mcp/json.h b/src/mcp/json.h index a46e6c7..fe38a81 100644 --- a/src/mcp/json.h +++ b/src/mcp/json.h @@ -19,5 +19,6 @@ bool tiny_json_raw_n(tiny_json_buffer_t *buffer, const char *value, size_t lengt bool tiny_json_string(tiny_json_buffer_t *buffer, const char *value); bool tiny_json_u64(tiny_json_buffer_t *buffer, unsigned long long value); bool tiny_json_i64(tiny_json_buffer_t *buffer, long long value); +bool tiny_json_number(tiny_json_buffer_t *buffer, double value); #endif diff --git a/src/mcp/tools/common/common.c b/src/mcp/tools/common/common.c index d6669f8..62f507f 100644 --- a/src/mcp/tools/common/common.c +++ b/src/mcp/tools/common/common.c @@ -68,6 +68,10 @@ tiny_mcp_tool_append_message(tiny_json_buffer_t *json, const tiny_message_t *mes tiny_json_string(json, tiny_role_name(message->role)); tiny_json_raw(json, ",\"content\":"); tiny_json_string(json, message->content); + tiny_json_raw(json, ",\"revision\":"); + tiny_json_u64(json, message->revision); + tiny_json_raw(json, ",\"updated_at\":"); + tiny_json_i64(json, message->updated_at_ms); tiny_json_raw(json, ",\"metadata\":{"); if (message->metadata.tool_call_id) { @@ -86,6 +90,38 @@ tiny_mcp_tool_append_message(tiny_json_buffer_t *json, const tiny_message_t *mes } tiny_json_raw(json, "],\"memory_tier\":"); tiny_json_string(json, tiny_tier_name(message->metadata.memory_tier)); + tiny_json_raw(json, ",\"memory_kind\":"); + tiny_json_string(json, tiny_kind_name(message->metadata.memory_kind)); + tiny_json_raw(json, ",\"importance\":"); + tiny_json_u64(json, message->metadata.importance); + if (message->metadata.summary) + { + tiny_json_raw(json, ",\"summary\":"); + tiny_json_string(json, message->metadata.summary); + } + tiny_json_raw(json, ",\"relations\":["); + for (index = 0; index < message->metadata.relation_count; ++index) + { + if (index) + { + tiny_json_raw(json, ","); + } + tiny_json_raw(json, "{\"type\":"); + tiny_json_string(json, tiny_relation_type_name(message->metadata.relations[index].type)); + tiny_json_raw(json, ",\"message_id\":"); + tiny_json_string(json, message->metadata.relations[index].message_id); + tiny_json_raw(json, "}"); + } + tiny_json_raw(json, "],\"source_message_ids\":["); + for (index = 0; index < message->metadata.source_count; ++index) + { + if (index) + { + tiny_json_raw(json, ","); + } + tiny_json_string(json, message->metadata.source_message_ids[index]); + } + tiny_json_raw(json, "]"); tiny_json_raw(json, "}}"); } diff --git a/src/mcp/tools/common/common.h b/src/mcp/tools/common/common.h index 07caea1..741b2ce 100644 --- a/src/mcp/tools/common/common.h +++ b/src/mcp/tools/common/common.h @@ -3,12 +3,15 @@ #include "domain/message.h" #include "mcp/json.h" +#include "domain/message.h" #include <mcpc/mcpc.h> void tiny_mcp_tool_add_error(mcpc_ucbr_t *result, const char *code, const char *message); void tiny_mcp_tool_add_json(mcpc_ucbr_t *result, tiny_json_buffer_t *json); char *tiny_mcp_tool_get_string(const mcpc_tool_t *tool, const char *name, size_t maximum); void tiny_mcp_tool_append_message(tiny_json_buffer_t *json, const tiny_message_t *message); +bool +tiny_mcp_parse_metadata(const char *json, tiny_message_t *message, tiny_metadata_mask_t *present); mcpc_tool_t *tiny_mcp_tool_new(const char *name, const char *description, mcpc_tcallcb_t callback); bool tiny_mcp_tool_add_property(mcpc_tool_t *tool, diff --git a/src/mcp/tools/common/context.c b/src/mcp/tools/common/context.c index d79b5de..e833f70 100644 --- a/src/mcp/tools/common/context.c +++ b/src/mcp/tools/common/context.c @@ -43,10 +43,12 @@ tiny_mcp_tools_context_new(tiny_store_t *store) context->store = store; context->search_index = tiny_mcp_tools_build_search_index(store); context->history_index = tiny_history_index_build(store); - if (!context->search_index || !context->history_index) + context->memory_index = tiny_memory_index_build(store); + if (!context->search_index || !context->history_index || !context->memory_index) { tiny_search_index_free(context->search_index); tiny_history_index_free(context->history_index); + tiny_memory_index_free(context->memory_index); free(context); return NULL; } @@ -62,10 +64,40 @@ tiny_mcp_tools_context_free(tiny_mcp_tools_context_t *context) } tiny_search_index_free(context->search_index); tiny_history_index_free(context->history_index); + tiny_memory_index_free(context->memory_index); free(context); } bool +tiny_mcp_tools_ensure_memory_index(tiny_mcp_tools_context_t *context) +{ + if (!context) + { + return false; + } + if (!context->memory_index) + { + context->memory_index = tiny_memory_index_build(context->store); + } + return context->memory_index != NULL; +} + +void +tiny_mcp_tools_invalidate_indexes(tiny_mcp_tools_context_t *context) +{ + if (!context) + { + return; + } + tiny_search_index_free(context->search_index); + tiny_history_index_free(context->history_index); + tiny_memory_index_free(context->memory_index); + context->search_index = NULL; + context->history_index = NULL; + context->memory_index = NULL; +} + +bool tiny_mcp_tools_context_activate(tiny_mcp_tools_context_t *context) { if (!context || active_context) diff --git a/src/mcp/tools/common/context.h b/src/mcp/tools/common/context.h index 2a6650a..bd167f2 100644 --- a/src/mcp/tools/common/context.h +++ b/src/mcp/tools/common/context.h @@ -3,6 +3,7 @@ #include "domain/search.h" #include "mcp/tools/common/history.h" +#include "mcp/tools/common/memory_index.h" #include "storage/store.h" typedef struct tiny_mcp_tools_context @@ -10,6 +11,7 @@ typedef struct tiny_mcp_tools_context tiny_store_t *store; tiny_search_index_t *search_index; tiny_history_index_t *history_index; + tiny_memory_index_t *memory_index; } tiny_mcp_tools_context_t; tiny_mcp_tools_context_t *tiny_mcp_tools_context_new(tiny_store_t *store); @@ -19,5 +21,7 @@ void tiny_mcp_tools_context_deactivate(tiny_mcp_tools_context_t *context); tiny_mcp_tools_context_t *tiny_mcp_tools_context_current(void); tiny_search_index_t *tiny_mcp_tools_build_search_index(const tiny_store_t *store); bool tiny_mcp_tools_ensure_history_index(tiny_mcp_tools_context_t *context); +bool tiny_mcp_tools_ensure_memory_index(tiny_mcp_tools_context_t *context); +void tiny_mcp_tools_invalidate_indexes(tiny_mcp_tools_context_t *context); #endif diff --git a/src/mcp/tools/common/history.c b/src/mcp/tools/common/history.c index 6254d8c..9f08e84 100644 --- a/src/mcp/tools/common/history.c +++ b/src/mcp/tools/common/history.c @@ -284,3 +284,26 @@ tiny_history_index_session(const tiny_history_index_t *index, } return index && session_id; } + +size_t +tiny_history_index_session_count(const tiny_history_index_t *index) +{ + return index ? index->count : 0; +} + +bool +tiny_history_index_session_at(const tiny_history_index_t *index, + size_t position, + const char **session_id, + const size_t **entries, + size_t *count) +{ + if (!index || position >= index->count || !session_id || !entries || !count) + { + return false; + } + *session_id = index->buckets[position].session_id; + *entries = index->buckets[position].entries; + *count = index->buckets[position].count; + return true; +} diff --git a/src/mcp/tools/common/history.h b/src/mcp/tools/common/history.h index ac08d99..d6f5708 100644 --- a/src/mcp/tools/common/history.h +++ b/src/mcp/tools/common/history.h @@ -13,5 +13,11 @@ bool tiny_history_index_session(const tiny_history_index_t *index, const char *session_id, const size_t **entries, size_t *count); +size_t tiny_history_index_session_count(const tiny_history_index_t *index); +bool tiny_history_index_session_at(const tiny_history_index_t *index, + size_t position, + const char **session_id, + const size_t **entries, + size_t *count); #endif diff --git a/src/mcp/tools/common/memory_index.c b/src/mcp/tools/common/memory_index.c new file mode 100644 index 0000000..36e430b --- /dev/null +++ b/src/mcp/tools/common/memory_index.c @@ -0,0 +1,219 @@ +#include "mcp/tools/common/memory_index.h" + +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "base/allocator.h" + +#define EMPTY_SLOT SIZE_MAX + +typedef struct memory_bucket +{ + uint64_t hash; + char message_id[TINY_MESSAGE_ID_LENGTH + 1]; + size_t entry_index; + tiny_relation_reference_t *incoming; + size_t incoming_count; + size_t incoming_capacity; +} memory_bucket_t; + +struct tiny_memory_index +{ + memory_bucket_t *buckets; + size_t count; + size_t *slots; + size_t slot_capacity; +}; + +static uint64_t +id_hash(const char *value) +{ + uint64_t hash = UINT64_C(1469598103934665603); + while (*value) + { + hash ^= (unsigned char)*value++; + hash *= UINT64_C(1099511628211); + } + return hash; +} + +static size_t +slot_start(uint64_t hash, size_t capacity) +{ + return (size_t)(hash ^ (hash >> 32)) & (capacity - 1); +} + +static memory_bucket_t * +find_bucket(const tiny_memory_index_t *index, const char *message_id) +{ + uint64_t hash; + size_t slot, bucket; + if (!index || !message_id || !index->slot_capacity) + { + return NULL; + } + hash = id_hash(message_id); + slot = slot_start(hash, index->slot_capacity); + while ((bucket = index->slots[slot]) != EMPTY_SLOT) + { + if (bucket >= index->count) + { + return NULL; + } + if (index->buckets[bucket].hash == hash + && strcmp(index->buckets[bucket].message_id, message_id) == 0) + { + return &index->buckets[bucket]; + } + slot = (slot + 1) & (index->slot_capacity - 1); + } + return NULL; +} + +static bool +add_incoming(memory_bucket_t *bucket, size_t source, tiny_relation_type_t type) +{ + tiny_relation_reference_t *resized; + size_t capacity; + if (bucket->incoming_count == bucket->incoming_capacity) + { + capacity = bucket->incoming_capacity ? bucket->incoming_capacity * 2 : 4; + if (capacity < bucket->incoming_capacity + || !(resized = realloc(bucket->incoming, capacity * sizeof(*resized)))) + { + return false; + } + bucket->incoming = resized; + bucket->incoming_capacity = capacity; + } + bucket->incoming[bucket->incoming_count].source_index = source; + bucket->incoming[bucket->incoming_count].type = type; + bucket->incoming_count++; + return true; +} + +tiny_memory_index_t * +tiny_memory_index_build(const tiny_store_t *store) +{ + tiny_memory_index_t *index; + size_t count = tiny_store_entry_count(store), capacity = 64, position; + if (!store || count > SIZE_MAX / sizeof(memory_bucket_t)) + { + return NULL; + } + while (count > (capacity / 10) * 7) + { + if (capacity > SIZE_MAX / 2) + { + return NULL; + } + capacity *= 2; + } + index = calloc(1, sizeof(*index)); + if (!index || (count && !(index->buckets = calloc(count, sizeof(*index->buckets)))) + || !(index->slots = malloc(capacity * sizeof(*index->slots)))) + { + tiny_memory_index_free(index); + return NULL; + } + index->count = count; + index->slot_capacity = capacity; + for (position = 0; position < capacity; ++position) + { + index->slots[position] = EMPTY_SLOT; + } + for (position = 0; position < count; ++position) + { + const tiny_store_entry_t *entry = tiny_store_entry(store, position); + memory_bucket_t *bucket = &index->buckets[position]; + size_t slot; + memcpy(bucket->message_id, entry->message.message_id, sizeof(bucket->message_id)); + bucket->hash = id_hash(bucket->message_id); + bucket->entry_index = position; + slot = slot_start(bucket->hash, capacity); + while (index->slots[slot] != EMPTY_SLOT) + { + slot = (slot + 1) & (capacity - 1); + } + index->slots[slot] = position; + } + for (position = 0; position < count; ++position) + { + const tiny_store_entry_t *entry = tiny_store_entry(store, position); + size_t relation; + if (entry->deleted) + { + continue; + } + for (relation = 0; relation < entry->message.metadata.relation_count; ++relation) + { + const tiny_relation_t *value = &entry->message.metadata.relations[relation]; + memory_bucket_t *target = find_bucket(index, value->message_id); + if (!target || !add_incoming(target, position, value->type)) + { + tiny_memory_index_free(index); + return NULL; + } + } + } + return index; +} + +void +tiny_memory_index_free(tiny_memory_index_t *index) +{ + size_t position; + if (!index) + { + return; + } + for (position = 0; position < index->count; ++position) + { + free(index->buckets[position].incoming); + } + free(index->buckets); + free(index->slots); + free(index); +} + +bool +tiny_memory_index_find(const tiny_memory_index_t *index, + const char *message_id, + size_t *entry_index) +{ + memory_bucket_t *bucket; + if (!entry_index) + { + return false; + } + *entry_index = 0; + bucket = find_bucket(index, message_id); + if (bucket) + { + *entry_index = bucket->entry_index; + } + return bucket != NULL; +} + +bool +tiny_memory_index_incoming(const tiny_memory_index_t *index, + const char *message_id, + const tiny_relation_reference_t **references, + size_t *count) +{ + memory_bucket_t *bucket; + if (!references || !count) + { + return false; + } + *references = NULL; + *count = 0; + bucket = find_bucket(index, message_id); + if (bucket) + { + *references = bucket->incoming; + *count = bucket->incoming_count; + } + return bucket != NULL; +} diff --git a/src/mcp/tools/common/memory_index.h b/src/mcp/tools/common/memory_index.h new file mode 100644 index 0000000..fbe43a7 --- /dev/null +++ b/src/mcp/tools/common/memory_index.h @@ -0,0 +1,24 @@ +#ifndef TINY_CONVMEM_MCP_TOOLS_MEMORY_INDEX_H +#define TINY_CONVMEM_MCP_TOOLS_MEMORY_INDEX_H + +#include "storage/store.h" + +typedef struct tiny_relation_reference +{ + size_t source_index; + tiny_relation_type_t type; +} tiny_relation_reference_t; + +typedef struct tiny_memory_index tiny_memory_index_t; + +tiny_memory_index_t *tiny_memory_index_build(const tiny_store_t *store); +void tiny_memory_index_free(tiny_memory_index_t *index); +bool tiny_memory_index_find(const tiny_memory_index_t *index, + const char *message_id, + size_t *entry_index); +bool tiny_memory_index_incoming(const tiny_memory_index_t *index, + const char *message_id, + const tiny_relation_reference_t **references, + size_t *count); + +#endif diff --git a/src/mcp/tools/common/registry.c b/src/mcp/tools/common/registry.c index 9aa1dff..1bdafc3 100644 --- a/src/mcp/tools/common/registry.c +++ b/src/mcp/tools/common/registry.c @@ -6,6 +6,7 @@ bool tiny_mcp_tools_register(mcpc_server_t *server) { return tiny_mcp_register_write_message(server) && tiny_mcp_register_read_messages(server) - && tiny_mcp_register_search_messages(server) && tiny_mcp_register_delete_message(server) - && tiny_mcp_register_rebuild_index(server) && tiny_mcp_register_get_system_health(server); + && tiny_mcp_register_search_messages(server) && tiny_mcp_register_update_message(server) + && tiny_mcp_register_delete_message(server) && tiny_mcp_register_rebuild_index(server) + && tiny_mcp_register_get_system_health(server); } diff --git a/src/mcp/tools/common/registry.h b/src/mcp/tools/common/registry.h index bbc68d0..451d300 100644 --- a/src/mcp/tools/common/registry.h +++ b/src/mcp/tools/common/registry.h @@ -4,6 +4,7 @@ #include <mcpc/mcpc.h> bool tiny_mcp_register_write_message(mcpc_server_t *server); +bool tiny_mcp_register_update_message(mcpc_server_t *server); bool tiny_mcp_register_read_messages(mcpc_server_t *server); bool tiny_mcp_register_search_messages(mcpc_server_t *server); bool tiny_mcp_register_delete_message(mcpc_server_t *server); diff --git a/src/mcp/tools/delete_message.c b/src/mcp/tools/delete_message.c index f90bbe3..9f70866 100644 --- a/src/mcp/tools/delete_message.c +++ b/src/mcp/tools/delete_message.c @@ -30,6 +30,10 @@ call_delete_message(const mcpc_tool_t *tool, mcpc_ucbr_t *result) } else { + if (status == TINY_STORE_OK) + { + tiny_mcp_tools_invalidate_indexes(context); + } mcpc_toolcall_result_add_text_printf8( result, (const char8_t *)(status == TINY_STORE_OK ? "{\"status\":\"OK\"}" diff --git a/src/mcp/tools/get_system_health.c b/src/mcp/tools/get_system_health.c index 5fd3fc8..f84511f 100644 --- a/src/mcp/tools/get_system_health.c +++ b/src/mcp/tools/get_system_health.c @@ -19,7 +19,9 @@ call_get_system_health(const mcpc_tool_t *tool, mcpc_ucbr_t *result) tiny_json_init(&json); tiny_json_raw(&json, "{\"status\":"); tiny_json_string(&json, health.last_error[0] ? "DEGRADED" : "OK"); - tiny_json_raw(&json, ",\"format_version\":1,\"log_size\":"); + tiny_json_raw(&json, ",\"format_version\":"); + tiny_json_u64(&json, health.format_version); + tiny_json_raw(&json, ",\"log_size\":"); tiny_json_u64(&json, health.log_size); tiny_json_raw(&json, ",\"snapshot_size\":"); tiny_json_u64(&json, health.snapshot_size); @@ -29,6 +31,21 @@ call_get_system_health(const mcpc_tool_t *tool, mcpc_ucbr_t *result) tiny_json_u64(&json, health.deleted_messages); tiny_json_raw(&json, ",\"last_sequence\":"); tiny_json_u64(&json, health.last_sequence); + tiny_json_raw(&json, ",\"update_records\":"); + tiny_json_u64(&json, health.update_records); + tiny_json_raw(&json, ",\"dead_bytes\":"); + tiny_json_u64(&json, health.dead_bytes); + tiny_json_raw(&json, ",\"io\":{\"write_bytes\":"); + tiny_json_u64(&json, health.write_bytes); + tiny_json_raw(&json, ",\"write_calls\":"); + tiny_json_u64(&json, health.write_calls); + tiny_json_raw(&json, ",\"flush_calls\":"); + tiny_json_u64(&json, health.flush_calls); + tiny_json_raw(&json, ",\"truncate_calls\":"); + tiny_json_u64(&json, health.truncate_calls); + tiny_json_raw(&json, ",\"replace_calls\":"); + tiny_json_u64(&json, health.replace_calls); + tiny_json_raw(&json, "}"); tiny_json_raw(&json, ",\"recovered_tail\":"); tiny_json_raw(&json, health.recovered_tail ? "true" : "false"); tiny_json_raw(&json, ",\"snapshot_valid\":"); diff --git a/src/mcp/tools/read_messages.c b/src/mcp/tools/read_messages.c index 44a026f..aa76ce8 100644 --- a/src/mcp/tools/read_messages.c +++ b/src/mcp/tools/read_messages.c @@ -4,11 +4,13 @@ #include "mcp/tools/common/context.h" #include <stdint.h> +#include <stdio.h> #include <stdlib.h> #include <string.h> #include "base/allocator.h" #include "storage/codec.h" +#include "mjson.h" #define READ_DEFAULT_LIMIT 100 #define READ_MAX_LIMIT 1000 @@ -185,10 +187,493 @@ escaped_size(const char *value, size_t length) return total; } +typedef struct context_edge +{ + size_t entry_index; + tiny_relation_type_t type; + size_t depth; + bool incoming; + int64_t timestamp; + char message_id[TINY_MESSAGE_ID_LENGTH + 1]; +} context_edge_t; + +static bool +relation_type_mask(const char *json, uint32_t *mask) +{ + int offset = 0, key_offset, key_length, value_offset, value_length, value_type; + if (!json) + { + *mask = UINT32_MAX; + return true; + } + if (mjson_find(json, (int)strlen(json), "$", NULL, NULL) != MJSON_TOK_ARRAY) + { + return false; + } + *mask = 0; + while ((offset = mjson_next(json, + (int)strlen(json), + offset, + &key_offset, + &key_length, + &value_offset, + &value_length, + &value_type)) + != 0) + { + char value[24]; + int decoded; + tiny_relation_type_t type; + if (value_type != MJSON_TOK_STRING + || (decoded = mjson_get_string( + json + value_offset, value_length, "$", value, sizeof(value))) + <= 0 + || decoded >= (int)sizeof(value)) + { + return false; + } + value[decoded] = '\0'; + if (!tiny_relation_type_parse(value, &type)) + { + return false; + } + *mask |= UINT32_C(1) << (unsigned)type; + } + return true; +} + +static int +compare_context_edges(const void *left, const void *right) +{ + const context_edge_t *a = left, *b = right; + if (a->depth != b->depth) + return a->depth < b->depth ? -1 : 1; + if (a->type != b->type) + return a->type < b->type ? -1 : 1; + if (a->timestamp != b->timestamp) + return a->timestamp < b->timestamp ? -1 : 1; + return strcmp(a->message_id, b->message_id); +} + +static void +call_context_view(tiny_mcp_tools_context_t *context, const mcpc_tool_t *tool, mcpc_ucbr_t *result) +{ + char *message_id = tiny_mcp_tool_get_string(tool, "message_id", TINY_MESSAGE_ID_LENGTH); + char *types_json = tiny_mcp_tool_get_string(tool, "relation_types_json", 256); + int32_t before = 1, after = 1, relation_depth = 1; + mcpc_errcode_t before_status + = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"before", &before); + mcpc_errcode_t after_status = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"after", &after); + mcpc_errcode_t depth_status + = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"relation_depth", &relation_depth); + const tiny_store_entry_t *target = NULL; + const size_t *entries = NULL; + const tiny_relation_reference_t *incoming = NULL; + size_t session_entry_count = 0, position, target_position = SIZE_MAX, emitted, message_count = 1; + size_t incoming_count = 0; + size_t target_index, queue[50], queue_depth[50], queue_head = 0, queue_count = 0; + context_edge_t edges[49]; + size_t edge_count = 0; + bool *visited = NULL, truncated = false; + uint32_t type_mask; + tiny_json_buffer_t json; + if (!message_id || before < 0 || before > 10 || after < 0 || after > 10 + || relation_depth < 0 || relation_depth > 2 + || (before_status != MCPC_EC_0 && before_status != MCPC_EC_ANYDATA_NOTFOUND) + || (after_status != MCPC_EC_0 && after_status != MCPC_EC_ANYDATA_NOTFOUND) + || (depth_status != MCPC_EC_0 && depth_status != MCPC_EC_ANYDATA_NOTFOUND) + || !relation_type_mask(types_json, &type_mask) || !context + || !tiny_mcp_tools_ensure_memory_index(context) + || !tiny_memory_index_find(context->memory_index, message_id, &target_index)) + { + tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "invalid context view arguments"); + free(message_id); + free(types_json); + return; + } + target = tiny_store_entry(context->store, target_index); + if (!target || target->deleted || !tiny_mcp_tools_ensure_history_index(context) + || !tiny_history_index_session( + context->history_index, target->message.session_id, &entries, &session_entry_count)) + { + tiny_mcp_tool_add_error(result, "NOT_FOUND", "message context was not found"); + free(message_id); + free(types_json); + return; + } + for (position = 0; position < session_entry_count; ++position) + { + if (strcmp(tiny_store_entry(context->store, entries[position])->message.message_id, + message_id) + == 0) + { + target_position = position; + break; + } + } + if (target_position == SIZE_MAX + || !(visited = calloc(tiny_store_entry_count(context->store), sizeof(bool)))) + { + tiny_mcp_tool_add_error(result, "INTERNAL_ERROR", "context index allocation failed"); + free(message_id); + free(types_json); + return; + } + visited[target_index] = true; + queue[queue_count] = target_index; + queue_depth[queue_count++] = 0; + while (queue_head < queue_count && edge_count < 49) + { + size_t current_index = queue[queue_head]; + size_t depth = queue_depth[queue_head++]; + const tiny_store_entry_t *current = tiny_store_entry(context->store, current_index); + size_t relation; + if (depth >= (size_t)relation_depth) + { + continue; + } + for (relation = 0; relation < current->message.metadata.relation_count; ++relation) + { + const tiny_relation_t *value = ¤t->message.metadata.relations[relation]; + size_t related_index; + const tiny_store_entry_t *related; + if (!(type_mask & (UINT32_C(1) << (unsigned)value->type)) + || !tiny_memory_index_find(context->memory_index, value->message_id, &related_index) + || visited[related_index]) + { + continue; + } + related = tiny_store_entry(context->store, related_index); + visited[related_index] = true; + edges[edge_count].entry_index = related_index; + edges[edge_count].type = value->type; + edges[edge_count].depth = depth + 1; + edges[edge_count].incoming = false; + edges[edge_count].timestamp = related->message.timestamp_ms; + memcpy(edges[edge_count].message_id, + related->message.message_id, + sizeof(edges[edge_count].message_id)); + edge_count++; + if (!related->deleted && queue_count < 50) + { + queue[queue_count] = related_index; + queue_depth[queue_count++] = depth + 1; + } + if (edge_count == 49) + { + truncated = true; + break; + } + } + if (edge_count == 49) + { + break; + } + if (!tiny_memory_index_incoming( + context->memory_index, + current->message.message_id, + &incoming, + &incoming_count)) + { + continue; + } + for (relation = 0; relation < incoming_count && edge_count < 49; ++relation) + { + const tiny_store_entry_t *related + = tiny_store_entry(context->store, incoming[relation].source_index); + if (!(type_mask & (UINT32_C(1) << (unsigned)incoming[relation].type)) + || visited[incoming[relation].source_index]) + { + continue; + } + visited[incoming[relation].source_index] = true; + edges[edge_count].entry_index = incoming[relation].source_index; + edges[edge_count].type = incoming[relation].type; + edges[edge_count].depth = depth + 1; + edges[edge_count].incoming = true; + edges[edge_count].timestamp = related->message.timestamp_ms; + memcpy(edges[edge_count].message_id, + related->message.message_id, + sizeof(edges[edge_count].message_id)); + edge_count++; + if (!related->deleted && queue_count < 50) + { + queue[queue_count] = incoming[relation].source_index; + queue_depth[queue_count++] = depth + 1; + } + } + } + qsort(edges, edge_count, sizeof(*edges), compare_context_edges); + tiny_json_init(&json); + tiny_json_raw(&json, "{\"message\":"); + tiny_mcp_tool_append_message(&json, &target->message); + tiny_json_raw(&json, ",\"chronological_context\":{\"before\":["); + emitted = 0; + position = target_position > (size_t)before ? target_position - (size_t)before : 0; + for (; position < target_position; ++position) + { + const tiny_store_entry_t *entry = tiny_store_entry(context->store, entries[position]); + if (entry && !entry->deleted) + { + tiny_json_buffer_t item; + tiny_json_init(&item); + tiny_mcp_tool_append_message(&item, &entry->message); + if (message_count >= 50 || item.failed + || json.length + item.length + 256 > READ_RESPONSE_BUDGET) + { + truncated = true; + tiny_json_free(&item); + break; + } + if (emitted++) + { + tiny_json_raw(&json, ","); + } + tiny_json_raw_n(&json, item.data, item.length); + tiny_json_free(&item); + message_count++; + } + } + tiny_json_raw(&json, "],\"after\":["); + emitted = 0; + for (position = target_position + 1; + position < session_entry_count && emitted < (size_t)after; + ++position) + { + const tiny_store_entry_t *entry = tiny_store_entry(context->store, entries[position]); + if (entry && !entry->deleted) + { + tiny_json_buffer_t item; + tiny_json_init(&item); + tiny_mcp_tool_append_message(&item, &entry->message); + if (message_count >= 50 || item.failed + || json.length + item.length + 256 > READ_RESPONSE_BUDGET) + { + truncated = true; + tiny_json_free(&item); + break; + } + if (emitted++) + { + tiny_json_raw(&json, ","); + } + tiny_json_raw_n(&json, item.data, item.length); + tiny_json_free(&item); + message_count++; + } + } + tiny_json_raw(&json, "]},\"relations\":["); + emitted = 0; + for (position = 0; position < edge_count; ++position) + { + const context_edge_t *edge = &edges[position]; + const tiny_store_entry_t *related = tiny_store_entry(context->store, edge->entry_index); + tiny_json_buffer_t item; + tiny_json_init(&item); + tiny_json_raw(&item, "{\"type\":"); + tiny_json_string(&item, tiny_relation_type_name(edge->type)); + tiny_json_raw(&item, edge->incoming ? ",\"direction\":\"incoming\"" : ",\"direction\":\"outgoing\""); + tiny_json_raw(&item, ",\"depth\":"); + tiny_json_u64(&item, edge->depth); + if (related && !related->deleted) + { + tiny_json_raw(&item, ",\"message\":"); + tiny_mcp_tool_append_message(&item, &related->message); + } + else + { + tiny_json_raw(&item, ",\"message_id\":"); + tiny_json_string(&item, edge->message_id); + tiny_json_raw(&item, ",\"status\":\"deleted\""); + } + tiny_json_raw(&item, "}"); + if (message_count >= 50 || item.failed + || json.length + item.length + 256 > READ_RESPONSE_BUDGET) + { + truncated = true; + tiny_json_free(&item); + break; + } + if (emitted++) + { + tiny_json_raw(&json, ","); + } + tiny_json_raw_n(&json, item.data, item.length); + tiny_json_free(&item); + if (related && !related->deleted) + { + message_count++; + } + } + tiny_json_raw(&json, "],\"session\":{\"session_id\":"); + tiny_json_string(&json, target->message.session_id); + tiny_json_raw(&json, ",\"message_count\":"); + tiny_json_u64(&json, session_entry_count); + tiny_json_raw(&json, "},\"truncated\":"); + tiny_json_raw(&json, truncated ? "true" : "false"); + tiny_json_raw(&json, "}"); + tiny_mcp_tool_add_json(result, &json); + tiny_json_free(&json); + free(visited); + free(message_id); + free(types_json); +} + +static size_t +simple_cursor(const char *cursor, bool *valid) +{ + char *end = NULL; + unsigned long long value; + if (!cursor) + { + *valid = true; + return 0; + } + value = strtoull(cursor, &end, 10); + *valid = cursor[0] && end && !*end && value <= SIZE_MAX; + return *valid ? (size_t)value : 0; +} + +static void +call_sessions_view(tiny_mcp_tools_context_t *context, const mcpc_tool_t *tool, mcpc_ucbr_t *result) +{ + char *cursor = tiny_mcp_tool_get_string(tool, "cursor", CURSOR_TEXT_MAX); + int32_t limit = READ_DEFAULT_LIMIT; + bool valid; + size_t start = simple_cursor(cursor, &valid), index, unique = 0, emitted = 0; + tiny_json_buffer_t json; + mcpc_errcode_t limit_status = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"limit", &limit); + if (!context || !valid || limit < 1 || limit > READ_MAX_LIMIT + || (limit_status != MCPC_EC_0 && limit_status != MCPC_EC_ANYDATA_NOTFOUND)) + { + tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "invalid sessions view arguments"); + free(cursor); + return; + } + tiny_json_init(&json); + tiny_json_raw(&json, "{\"sessions\":["); + for (index = 0; index < tiny_store_entry_count(context->store) && emitted < (size_t)limit; + ++index) + { + const tiny_store_entry_t *entry = tiny_store_entry(context->store, index); + size_t prior, scan, count = 0; + int64_t started = INT64_MAX, ended = 0; + if (!entry || entry->deleted) + { + continue; + } + for (prior = 0; prior < index; ++prior) + { + const tiny_store_entry_t *candidate = tiny_store_entry(context->store, prior); + if (candidate && !candidate->deleted + && strcmp(candidate->message.session_id, entry->message.session_id) == 0) + { + break; + } + } + if (prior != index || unique++ < start) + { + continue; + } + for (scan = index; scan < tiny_store_entry_count(context->store); ++scan) + { + const tiny_store_entry_t *candidate = tiny_store_entry(context->store, scan); + if (candidate && !candidate->deleted + && strcmp(candidate->message.session_id, entry->message.session_id) == 0) + { + count++; + started = candidate->message.timestamp_ms < started ? candidate->message.timestamp_ms + : started; + ended = candidate->message.timestamp_ms > ended ? candidate->message.timestamp_ms + : ended; + } + } + if (emitted++) + { + tiny_json_raw(&json, ","); + } + tiny_json_raw(&json, "{\"session_id\":"); + tiny_json_string(&json, entry->message.session_id); + tiny_json_raw(&json, ",\"message_count\":"); + tiny_json_u64(&json, count); + tiny_json_raw(&json, ",\"started_at\":"); + tiny_json_i64(&json, started); + tiny_json_raw(&json, ",\"last_message_at\":"); + tiny_json_i64(&json, ended); + tiny_json_raw(&json, "}"); + } + tiny_json_raw(&json, "]"); + if (emitted == (size_t)limit) + { + char next[32]; + snprintf(next, sizeof(next), "%zu", start + emitted); + tiny_json_raw(&json, ",\"nextCursor\":"); + tiny_json_string(&json, next); + } + tiny_json_raw(&json, "}"); + tiny_mcp_tool_add_json(result, &json); + tiny_json_free(&json); + free(cursor); +} + +static void +call_export_view(tiny_mcp_tools_context_t *context, const mcpc_tool_t *tool, mcpc_ucbr_t *result) +{ + char *session = tiny_mcp_tool_get_string(tool, "session_id", TINY_SESSION_ID_MAX); + char *cursor = tiny_mcp_tool_get_string(tool, "cursor", CURSOR_TEXT_MAX); + int32_t limit = 500; + bool valid; + size_t index = simple_cursor(cursor, &valid), emitted = 0; + tiny_json_buffer_t json, lines; + mcpc_errcode_t limit_status = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"limit", &limit); + if (!context || !valid || limit < 1 || limit > 1000 + || (limit_status != MCPC_EC_0 && limit_status != MCPC_EC_ANYDATA_NOTFOUND)) + { + tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "invalid export view arguments"); + free(session); + free(cursor); + return; + } + tiny_json_init(&lines); + if (index == 0) + { + tiny_json_raw(&lines, "{\"type\":\"tiny-convmem-manifest\",\"format_version\":2}\n"); + } + for (; index < tiny_store_entry_count(context->store) && emitted < (size_t)limit; ++index) + { + const tiny_store_entry_t *entry = tiny_store_entry(context->store, index); + if (!entry || entry->deleted + || (session && session[0] && strcmp(session, entry->message.session_id) != 0)) + { + continue; + } + tiny_mcp_tool_append_message(&lines, &entry->message); + tiny_json_raw(&lines, "\n"); + emitted++; + } + tiny_json_init(&json); + tiny_json_raw(&json, "{\"jsonl\":"); + tiny_json_string(&json, lines.data ? lines.data : ""); + if (index < tiny_store_entry_count(context->store)) + { + char next[32]; + snprintf(next, sizeof(next), "%zu", index); + tiny_json_raw(&json, ",\"nextCursor\":"); + tiny_json_string(&json, next); + } + tiny_json_raw(&json, "}"); + tiny_mcp_tool_add_json(result, &json); + tiny_json_free(&json); + tiny_json_free(&lines); + free(session); + free(cursor); +} + static void call_read_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) { tiny_mcp_tools_context_t *context = tiny_mcp_tools_context_current(); + char *view = tiny_mcp_tool_get_string(tool, "view", 16); char *session = tiny_mcp_tool_get_string(tool, "session_id", TINY_SESSION_ID_MAX); char *cursor_text = tiny_mcp_tool_get_string(tool, "cursor", CURSOR_TEXT_MAX); int32_t requested_limit = READ_DEFAULT_LIMIT; @@ -201,6 +686,40 @@ call_read_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) const tiny_message_t *last = NULL; tiny_json_buffer_t json; + if (view && strcmp(view, "context") == 0) + { + call_context_view(context, tool, result); + free(view); + free(session); + free(cursor_text); + return; + } + if (view && strcmp(view, "sessions") == 0) + { + call_sessions_view(context, tool, result); + free(view); + free(session); + free(cursor_text); + return; + } + if (view && strcmp(view, "export") == 0) + { + call_export_view(context, tool, result); + free(view); + free(session); + free(cursor_text); + return; + } + if (view && strcmp(view, "history") != 0) + { + tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "unsupported read_messages view"); + free(view); + free(session); + free(cursor_text); + return; + } + free(view); + if (!context || !session || !session[0] || (limit_status != MCPC_EC_0 && limit_status != MCPC_EC_ANYDATA_NOTFOUND) || requested_limit < 1 || requested_limit > READ_MAX_LIMIT @@ -324,11 +843,33 @@ tiny_mcp_register_read_messages(mcpc_server_t *server) "Returns a paginated history for reconstructing that conversation.", call_read_messages); return tool + && tiny_mcp_tool_add_property( + tool, + "view", + "Read view: history (default), context, sessions, or export.", + MCPC_U8STR, + false) && tiny_mcp_tool_add_property(tool, "session_id", "Conversation session whose chronological history to read.", MCPC_U8STR, - true) + false) + && tiny_mcp_tool_add_property( + tool, "message_id", "Message ULID required by the context view.", MCPC_U8STR, false) + && tiny_mcp_tool_add_property( + tool, "before", "Context messages before the target, from 0 to 10.", MCPC_I32, false) + && tiny_mcp_tool_add_property( + tool, "after", "Context messages after the target, from 0 to 10.", MCPC_I32, false) + && tiny_mcp_tool_add_property(tool, + "relation_depth", + "Relation traversal depth from 0 to 2; defaults to 1.", + MCPC_I32, + false) + && tiny_mcp_tool_add_property(tool, + "relation_types_json", + "Optional JSON array string of relation types to follow.", + MCPC_U8STR, + false) && tiny_mcp_tool_add_property( tool, "limit", diff --git a/src/mcp/tools/rebuild_index.c b/src/mcp/tools/rebuild_index.c index e6dbf69..d9a5e87 100644 --- a/src/mcp/tools/rebuild_index.c +++ b/src/mcp/tools/rebuild_index.c @@ -10,8 +10,6 @@ call_rebuild_index(const mcpc_tool_t *tool, mcpc_ucbr_t *result) { tiny_mcp_tools_context_t *context = tiny_mcp_tools_context_current(); tiny_rebuild_report_t report; - tiny_search_index_t *rebuilt; - tiny_history_index_t *history; tiny_store_status_t status; tiny_json_buffer_t json; (void)tool; @@ -26,22 +24,7 @@ call_rebuild_index(const mcpc_tool_t *tool, mcpc_ucbr_t *result) tiny_mcp_tool_add_error(result, tiny_store_status_name(status), "index rebuild failed"); return; } - tiny_history_index_free(context->history_index); - context->history_index = NULL; - rebuilt = tiny_mcp_tools_build_search_index(context->store); - history = tiny_history_index_build(context->store); - if (!rebuilt || !history) - { - tiny_search_index_free(rebuilt); - tiny_history_index_free(history); - tiny_search_index_free(context->search_index); - context->search_index = NULL; - tiny_mcp_tool_add_error(result, "INTERNAL_ERROR", "search index allocation failed"); - return; - } - tiny_search_index_free(context->search_index); - context->search_index = rebuilt; - context->history_index = history; + tiny_mcp_tools_invalidate_indexes(context); tiny_json_init(&json); tiny_json_raw(&json, "{\"status\":\"OK\",\"messages_indexed\":"); tiny_json_u64(&json, report.messages_indexed); diff --git a/src/mcp/tools/search_messages.c b/src/mcp/tools/search_messages.c index 44db1ef..4b7f870 100644 --- a/src/mcp/tools/search_messages.c +++ b/src/mcp/tools/search_messages.c @@ -13,8 +13,31 @@ typedef struct result_item { const tiny_store_entry_t *entry; double score; + double content_score; + double summary_score; + double tag_score; } result_item_t; +static double +score_component(const char *query, const tiny_message_t *message, int component) +{ + tiny_message_t candidate = *message; + candidate.metadata.tags = NULL; + candidate.metadata.tag_count = 0; + candidate.metadata.summary = NULL; + if (component == 1) + { + candidate.content = message->metadata.summary ? message->metadata.summary : ""; + } + else if (component == 2) + { + candidate.content = ""; + candidate.metadata.tags = message->metadata.tags; + candidate.metadata.tag_count = message->metadata.tag_count; + } + return tiny_search_score(query, &candidate); +} + static int compare_results(const void *left, const void *right) { @@ -28,6 +51,22 @@ compare_results(const void *left, const void *right) { return 1; } + if (a->entry->message.metadata.importance > b->entry->message.metadata.importance) + { + return -1; + } + if (a->entry->message.metadata.importance < b->entry->message.metadata.importance) + { + return 1; + } + if (a->entry->message.updated_at_ms > b->entry->message.updated_at_ms) + { + return -1; + } + if (a->entry->message.updated_at_ms < b->entry->message.updated_at_ms) + { + return 1; + } if (a->entry->message.timestamp_ms > b->entry->message.timestamp_ms) { return -1; @@ -45,15 +84,27 @@ call_search_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) tiny_mcp_tools_context_t *context = tiny_mcp_tools_context_current(); char *query = tiny_mcp_tool_get_string(tool, "query", TINY_QUERY_MAX); char *session = tiny_mcp_tool_get_string(tool, "session_id", TINY_SESSION_ID_MAX); + char *kind_text = tiny_mcp_tool_get_string(tool, "memory_kind", 24); + char *tier_text = tiny_mcp_tool_get_string(tool, "memory_tier", 16); + char *tag = tiny_mcp_tool_get_string(tool, "tag", TINY_TAG_MAX); int32_t requested_limit = 20; + int32_t minimum_importance = 0; mcpc_errcode_t limit_status = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"limit", &requested_limit); + mcpc_errcode_t importance_status + = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"min_importance", &minimum_importance); + tiny_memory_kind_t kind = TINY_KIND_CONVERSATION; + tiny_memory_tier_t tier = TINY_TIER_SHORT; size_t limit, index, count = 0, candidate_count = 0, *candidates = NULL; result_item_t *items = NULL; tiny_json_buffer_t json; bool valid = context && query && query[0] && (limit_status == MCPC_EC_0 || limit_status == MCPC_EC_ANYDATA_NOTFOUND) && requested_limit >= 1 && requested_limit <= 100; + valid = valid && minimum_importance >= 0 && minimum_importance <= 100 + && (importance_status == MCPC_EC_0 || importance_status == MCPC_EC_ANYDATA_NOTFOUND) + && (!kind_text || tiny_kind_parse(kind_text, &kind)) + && (!tier_text || tiny_tier_parse(tier_text, &tier)); if (!valid) { tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "query or limit is invalid"); @@ -90,11 +141,37 @@ call_search_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) { continue; } + if ((kind_text && entry->message.metadata.memory_kind != kind) + || (tier_text && entry->message.metadata.memory_tier != tier) + || entry->message.metadata.importance < minimum_importance) + { + continue; + } + if (tag) + { + size_t tag_index; + for (tag_index = 0; tag_index < entry->message.metadata.tag_count; + ++tag_index) + { + if (strcmp(tag, entry->message.metadata.tags[tag_index]) == 0) + { + break; + } + } + if (tag_index == entry->message.metadata.tag_count) + { + continue; + } + } score = tiny_search_score(query, &entry->message); if (score > 0) { items[count].entry = entry; - items[count++].score = score; + items[count].score = score; + items[count].content_score = score_component(query, &entry->message, 0); + items[count].summary_score = score_component(query, &entry->message, 1); + items[count].tag_score = score_component(query, &entry->message, 2); + count++; } } qsort(items, count, sizeof(*items), compare_results); @@ -110,7 +187,47 @@ call_search_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) { tiny_json_raw(&json, ","); } - tiny_mcp_tool_append_message(&json, &items[index].entry->message); + { + tiny_json_buffer_t item; + tiny_json_init(&item); + tiny_mcp_tool_append_message(&item, &items[index].entry->message); + if (item.length > 0) + { + item.length--; + item.data[item.length] = '\0'; + } + tiny_json_raw(&item, ",\"score\":"); + tiny_json_number(&item, items[index].score); + tiny_json_raw(&item, ",\"score_breakdown\":{\"content\":"); + tiny_json_number(&item, items[index].content_score); + tiny_json_raw(&item, ",\"summary\":"); + tiny_json_number(&item, items[index].summary_score); + tiny_json_raw(&item, ",\"tags\":"); + tiny_json_number(&item, items[index].tag_score); + tiny_json_raw(&item, ",\"importance\":0},\"matched_fields\":["); + { + bool matched = false; + if (items[index].content_score > 0) + { + tiny_json_raw(&item, "\"content\""); + matched = true; + } + if (items[index].summary_score > 0) + { + tiny_json_raw(&item, matched ? ",\"summary\"" : "\"summary\""); + matched = true; + } + if (items[index].tag_score > 0) + { + tiny_json_raw(&item, matched ? ",\"tags\"" : "\"tags\""); + } + } + tiny_json_raw(&item, "],\"relation_count\":"); + tiny_json_u64(&item, items[index].entry->message.metadata.relation_count); + tiny_json_raw(&item, "}"); + tiny_json_raw_n(&json, item.data, item.length); + tiny_json_free(&item); + } } tiny_json_raw(&json, "]}"); tiny_mcp_tool_add_json(result, &json); @@ -122,6 +239,9 @@ call_search_messages(const mcpc_tool_t *tool, mcpc_ucbr_t *result) free(candidates); free(query); free(session); + free(kind_text); + free(tier_text); + free(tag); } bool @@ -146,6 +266,16 @@ tiny_mcp_register_search_messages(mcpc_server_t *server) MCPC_U8STR, false) && tiny_mcp_tool_add_property( + tool, "memory_kind", "Optional exact memory kind filter.", MCPC_U8STR, false) + && tiny_mcp_tool_add_property( + tool, "memory_tier", "Optional exact memory tier filter.", MCPC_U8STR, false) + && tiny_mcp_tool_add_property(tool, "tag", "Optional exact tag filter.", MCPC_U8STR, false) + && tiny_mcp_tool_add_property(tool, + "min_importance", + "Minimum importance from 0 to 100; defaults to 0.", + MCPC_I32, + false) + && tiny_mcp_tool_add_property( tool, "limit", "Maximum ranked matches to return, from 1 to 100; defaults to 20.", diff --git a/src/mcp/tools/update_message.c b/src/mcp/tools/update_message.c new file mode 100644 index 0000000..aca1bed --- /dev/null +++ b/src/mcp/tools/update_message.c @@ -0,0 +1,97 @@ +#include "mcp/tools/common/registry.h" + +#include "mcp/tools/common/common.h" +#include "mcp/tools/common/context.h" + +#include <stdint.h> +#include <stdlib.h> + +static void +call_update_message(const mcpc_tool_t *tool, mcpc_ucbr_t *result) +{ + tiny_mcp_tools_context_t *context = tiny_mcp_tools_context_current(); + char *message_id = tiny_mcp_tool_get_string(tool, "message_id", TINY_MESSAGE_ID_LENGTH); + char *content = tiny_mcp_tool_get_string(tool, "content", TINY_CONTENT_MAX); + char *metadata_json = tiny_mcp_tool_get_string(tool, "metadata_json", TINY_CONTENT_MAX); + int32_t expected = 0; + mcpc_errcode_t expected_status + = mcpc_tool_get_tpropval_i32(tool, (const char8_t *)"expected_revision", &expected); + tiny_message_t parsed, updated; + tiny_message_patch_t patch; + tiny_store_status_t status; + tiny_json_buffer_t json; + bool changed = false; + tiny_metadata_mask_t metadata_mask = 0; + tiny_message_init(&parsed); + tiny_message_init(&updated); + patch.content = content; + patch.metadata = metadata_json ? &parsed.metadata : NULL; + patch.metadata_mask = 0; + patch.expected_revision = (uint64_t)expected; + patch.has_expected_revision = expected_status == MCPC_EC_0; + if (!context || !message_id || (!content && !metadata_json) + || (metadata_json && !tiny_mcp_parse_metadata(metadata_json, &parsed, &metadata_mask)) + || (expected_status != MCPC_EC_0 && expected_status != MCPC_EC_ANYDATA_NOTFOUND) + || (patch.has_expected_revision && expected < 1)) + { + tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "invalid update_message arguments"); + } + else + { + patch.metadata_mask = metadata_mask; + status = tiny_store_update(context->store, message_id, &patch, &updated, &changed); + if (status != TINY_STORE_OK) + { + tiny_mcp_tool_add_error( + result, tiny_store_status_name(status), "message was not updated"); + } + else + { + if (changed) + { + tiny_mcp_tools_invalidate_indexes(context); + } + tiny_json_init(&json); + tiny_json_raw(&json, "{\"status\":\"OK\",\"changed\":"); + tiny_json_raw(&json, changed ? "true" : "false"); + tiny_json_raw(&json, ",\"message\":"); + tiny_mcp_tool_append_message(&json, &updated); + tiny_json_raw(&json, "}"); + tiny_mcp_tool_add_json(result, &json); + tiny_json_free(&json); + } + } + free(message_id); + free(content); + free(metadata_json); + tiny_message_free(&parsed); + tiny_message_free(&updated); +} + +bool +tiny_mcp_register_update_message(mcpc_server_t *server) +{ + mcpc_tool_t *tool = tiny_mcp_tool_new("update_message", + "Update message content or metadata while preserving " + "identity, session, role, and creation time.", + call_update_message); + return tool + && tiny_mcp_tool_add_property( + tool, "message_id", "ULID of the live message to update.", MCPC_U8STR, true) + && tiny_mcp_tool_add_property(tool, + "content", + "Replacement UTF-8 content; omit to retain content.", + MCPC_U8STR, + false) + && tiny_mcp_tool_add_property(tool, + "metadata_json", + "Replacement metadata JSON; omit to retain metadata.", + MCPC_U8STR, + false) + && tiny_mcp_tool_add_property(tool, + "expected_revision", + "Optional positive revision for optimistic concurrency.", + MCPC_I32, + false) + && mcpc_server_add_tool(server, tool) == MCPC_EC_0; +} diff --git a/src/mcp/tools/write_message.c b/src/mcp/tools/write_message.c index 7df0058..3d7cb29 100644 --- a/src/mcp/tools/write_message.c +++ b/src/mcp/tools/write_message.c @@ -34,7 +34,9 @@ metadata_has_only_known_fields(const char *json, int length) } key[decoded] = '\0'; if (strcmp(key, "tool_call_id") != 0 && strcmp(key, "tags") != 0 - && strcmp(key, "memory_tier") != 0) + && strcmp(key, "memory_tier") != 0 && strcmp(key, "memory_kind") != 0 + && strcmp(key, "importance") != 0 && strcmp(key, "summary") != 0 + && strcmp(key, "relations") != 0 && strcmp(key, "source_message_ids") != 0) { return false; } @@ -42,14 +44,20 @@ metadata_has_only_known_fields(const char *json, int length) return true; } -static bool -parse_metadata(const char *json, tiny_message_t *message) +bool +tiny_mcp_parse_metadata(const char *json, tiny_message_t *message, tiny_metadata_mask_t *present) { - const char *tool_call = NULL, *tier = NULL, *tags = NULL; - int tool_call_len = 0, tier_len = 0, tags_len = 0; + const char *tool_call = NULL, *tier = NULL, *tags = NULL, *kind = NULL, *summary = NULL; + const char *relations = NULL, *sources = NULL; + int tool_call_len = 0, tier_len = 0, tags_len = 0, kind_len = 0, summary_len = 0; + int relations_len = 0, sources_len = 0; int offset = 0, key_offset, key_length, value_offset, value_length, value_type; int length = (int)strlen(json); int tool_call_type, tier_type, tags_type; + int kind_type, summary_type, importance_type; + int relations_type, sources_type; + double importance = 50; + tiny_metadata_mask_t fields = 0; if (strcmp(json, "{}") == 0) { return true; @@ -62,12 +70,176 @@ parse_metadata(const char *json, tiny_message_t *message) tool_call_type = mjson_find(json, length, "$.tool_call_id", &tool_call, &tool_call_len); tier_type = mjson_find(json, length, "$.memory_tier", &tier, &tier_len); tags_type = mjson_find(json, length, "$.tags", &tags, &tags_len); + kind_type = mjson_find(json, length, "$.memory_kind", &kind, &kind_len); + summary_type = mjson_find(json, length, "$.summary", &summary, &summary_len); + importance_type = mjson_find(json, length, "$.importance", NULL, NULL); + relations_type = mjson_find(json, length, "$.relations", &relations, &relations_len); + sources_type = mjson_find(json, length, "$.source_message_ids", &sources, &sources_len); if ((tool_call_type != MJSON_TOK_INVALID && tool_call_type != MJSON_TOK_STRING) || (tier_type != MJSON_TOK_INVALID && tier_type != MJSON_TOK_STRING) - || (tags_type != MJSON_TOK_INVALID && tags_type != MJSON_TOK_ARRAY)) + || (tags_type != MJSON_TOK_INVALID && tags_type != MJSON_TOK_ARRAY) + || (kind_type != MJSON_TOK_INVALID && kind_type != MJSON_TOK_STRING) + || (summary_type != MJSON_TOK_INVALID && summary_type != MJSON_TOK_STRING) + || (importance_type != MJSON_TOK_INVALID && importance_type != MJSON_TOK_NUMBER) + || (relations_type != MJSON_TOK_INVALID && relations_type != MJSON_TOK_ARRAY) + || (sources_type != MJSON_TOK_INVALID && sources_type != MJSON_TOK_ARRAY)) { return false; } + fields |= tool_call_type != MJSON_TOK_INVALID ? TINY_METADATA_TOOL_CALL_ID : 0; + fields |= tags_type != MJSON_TOK_INVALID ? TINY_METADATA_TAGS : 0; + fields |= tier_type != MJSON_TOK_INVALID ? TINY_METADATA_TIER : 0; + fields |= kind_type != MJSON_TOK_INVALID ? TINY_METADATA_KIND : 0; + fields |= importance_type != MJSON_TOK_INVALID ? TINY_METADATA_IMPORTANCE : 0; + fields |= summary_type != MJSON_TOK_INVALID ? TINY_METADATA_SUMMARY : 0; + fields |= relations_type != MJSON_TOK_INVALID ? TINY_METADATA_RELATIONS : 0; + fields |= sources_type != MJSON_TOK_INVALID ? TINY_METADATA_SOURCES : 0; + if (relations_type == MJSON_TOK_ARRAY) + { + offset = 0; + while ((offset = mjson_next(relations, + relations_len, + offset, + &key_offset, + &key_length, + &value_offset, + &value_length, + &value_type)) + != 0) + { + tiny_relation_t relation; + char type[24]; + int decoded_type, decoded_id; + tiny_relation_t *resized; + if (value_type != MJSON_TOK_OBJECT + || message->metadata.relation_count >= TINY_RELATION_COUNT_MAX) + { + return false; + } + decoded_type = mjson_get_string( + relations + value_offset, value_length, "$.type", type, sizeof(type)); + decoded_id = mjson_get_string(relations + value_offset, + value_length, + "$.message_id", + relation.message_id, + sizeof(relation.message_id)); + if (decoded_type <= 0 || decoded_type >= (int)sizeof(type) + || decoded_id != TINY_MESSAGE_ID_LENGTH) + { + return false; + } + type[decoded_type] = '\0'; + relation.message_id[decoded_id] = '\0'; + if (!tiny_relation_type_parse(type, &relation.type) + || !tiny_message_id_valid(relation.message_id)) + { + return false; + } + resized = realloc(message->metadata.relations, + (message->metadata.relation_count + 1) * sizeof(*resized)); + if (!resized) + { + return false; + } + message->metadata.relations = resized; + message->metadata.relations[message->metadata.relation_count++] = relation; + } + } + if (sources_type == MJSON_TOK_ARRAY) + { + offset = 0; + while ((offset = mjson_next(sources, + sources_len, + offset, + &key_offset, + &key_length, + &value_offset, + &value_length, + &value_type)) + != 0) + { + char *id; + char **resized; + int decoded; + if (value_type != MJSON_TOK_STRING + || message->metadata.source_count >= TINY_SOURCE_COUNT_MAX) + { + return false; + } + id = malloc(TINY_MESSAGE_ID_LENGTH + 1); + if (!id) + { + return false; + } + decoded = mjson_get_string( + sources + value_offset, value_length, "$", id, TINY_MESSAGE_ID_LENGTH + 1); + if (decoded != TINY_MESSAGE_ID_LENGTH) + { + free(id); + return false; + } + id[decoded] = '\0'; + if (!tiny_message_id_valid(id)) + { + free(id); + return false; + } + resized = realloc(message->metadata.source_message_ids, + (message->metadata.source_count + 1) * sizeof(*resized)); + if (!resized) + { + free(id); + return false; + } + message->metadata.source_message_ids = resized; + message->metadata.source_message_ids[message->metadata.source_count++] = id; + } + } + if (importance_type == MJSON_TOK_NUMBER + && mjson_get_number(json, length, "$.importance", &importance) > 0) + { + if (importance < 0 || importance > 100 || importance != (double)(uint8_t)importance) + { + return false; + } + message->metadata.importance = (uint8_t)importance; + } + if (kind_type == MJSON_TOK_STRING) + { + char value[24]; + int decoded = mjson_get_string(json, length, "$.memory_kind", value, sizeof(value)); + if (decoded <= 0 || decoded >= (int)sizeof(value)) + { + return false; + } + value[decoded] = '\0'; + if (!tiny_kind_parse(value, &message->metadata.memory_kind)) + { + return false; + } + } + if (summary_type == MJSON_TOK_STRING) + { + int decoded; + message->metadata.summary = malloc(TINY_SUMMARY_MAX + 1); + if (!message->metadata.summary) + { + return false; + } + decoded = mjson_get_string( + json, length, "$.summary", message->metadata.summary, TINY_SUMMARY_MAX + 1); + if (decoded < 0 || decoded > TINY_SUMMARY_MAX + || (decoded && !tiny_utf8_valid(message->metadata.summary, (size_t)decoded))) + { + return false; + } + message->metadata.summary[decoded] = '\0'; + if (decoded == 0) + { + free(message->metadata.summary); + message->metadata.summary = NULL; + } + } if (tool_call_type == MJSON_TOK_STRING) { int decoded; @@ -144,7 +316,11 @@ parse_metadata(const char *json, tiny_message_t *message) message->metadata.tags[message->metadata.tag_count++] = tag; } } - return true; + if (present) + { + *present = fields; + } + return tiny_metadata_valid(&message->metadata); } static void @@ -163,7 +339,7 @@ call_write_message(const mcpc_tool_t *tool, mcpc_ucbr_t *result) metadata = tiny_mcp_tool_get_string(tool, "metadata_json", TINY_CONTENT_MAX); bool valid = context && input.session_id && input.session_id[0] && role && input.content && input.content[0] && tiny_role_parse(role, &input.role) - && parse_metadata(metadata ? metadata : "{}", &input); + && tiny_mcp_parse_metadata(metadata ? metadata : "{}", &input, NULL); if (!valid) { tiny_mcp_tool_add_error(result, "VALIDATION_ERROR", "invalid write_message arguments"); @@ -178,21 +354,7 @@ call_write_message(const mcpc_tool_t *tool, mcpc_ucbr_t *result) } else { - if (context->search_index - && !tiny_search_index_add( - context->search_index, &written, tiny_store_entry_count(context->store) - 1)) - { - tiny_search_index_free(context->search_index); - context->search_index = NULL; - } - if (context->history_index - && !tiny_history_index_add(context->history_index, - context->store, - tiny_store_entry_count(context->store) - 1)) - { - tiny_history_index_free(context->history_index); - context->history_index = NULL; - } + tiny_mcp_tools_invalidate_indexes(context); tiny_json_init(&json); tiny_json_raw(&json, "{\"status\":\"OK\",\"message\":"); tiny_mcp_tool_append_message(&json, &written); @@ -225,7 +387,9 @@ tiny_mcp_register_write_message(mcpc_server_t *server) && tiny_mcp_tool_add_property(tool, "metadata_json", "Optional JSON object string containing tool_call_id, tags, " - "and/or memory_tier; defaults to {}.", + "memory_tier, memory_kind, importance, summary, relations, " + "and/or source_message_ids; " + "defaults to {}.", MCPC_U8STR, false) && mcpc_server_add_tool(server, tool) == MCPC_EC_0; diff --git a/src/storage/store.c b/src/storage/store.c index 296f4d0..d798d86 100644 --- a/src/storage/store.c +++ b/src/storage/store.c @@ -9,10 +9,14 @@ #include "base/allocator.h" #define RECORD_HEADER_SIZE 24U -#define RECORD_VERSION 1U +#define RECORD_VERSION 2U +#define RECORD_VERSION_LEGACY 1U #define RECORD_PUT 1U #define RECORD_DEL 2U #define RECORD_MARKER 3U +#define RECORD_UPDATE 4U +#define UPDATE_CONTENT 1U +#define UPDATE_METADATA 2U #define RECORD_PAYLOAD_MAX (TINY_CONTENT_MAX + 16384U) #define SNAPSHOT_HEADER_SIZE 52U @@ -27,6 +31,8 @@ struct tiny_store size_t entry_count; size_t entry_capacity; uint64_t last_sequence; + uint64_t update_records; + uint64_t dead_bytes; uint64_t last_ulid_ms; unsigned char last_random[10]; unsigned char generation[16]; @@ -180,6 +186,7 @@ serialize_put(const tiny_message_t *message, unsigned char **payload, uint32_t * { size_t session_len = strlen(message->session_id), content_len = strlen(message->content); size_t tool_len = message->metadata.tool_call_id ? strlen(message->metadata.tool_call_id) : 0; + size_t summary_len = message->metadata.summary ? strlen(message->metadata.summary) : 0; size_t total = 8 + 26 + 2 + 16 + session_len + content_len + tool_len; size_t index, offset; unsigned char *data; @@ -187,6 +194,8 @@ serialize_put(const tiny_message_t *message, unsigned char **payload, uint32_t * { total += 4 + strlen(message->metadata.tags[index]); } + total += 32 + summary_len + message->metadata.relation_count * 27 + + message->metadata.source_count * 26; if (total > UINT32_MAX || total > RECORD_PAYLOAD_MAX) { return false; @@ -228,6 +237,36 @@ serialize_put(const tiny_message_t *message, unsigned char **payload, uint32_t * memcpy(data + offset, message->metadata.tags[index], tag_len); offset += tag_len; } + memcpy(data + offset, "M2", 2); + offset += 2; + tiny_codec_put_u64(data + offset, message->revision); + offset += 8; + tiny_codec_put_u64(data + offset, (uint64_t)message->updated_at_ms); + offset += 8; + data[offset++] = (unsigned char)message->metadata.memory_kind; + data[offset++] = message->metadata.importance; + tiny_codec_put_u32(data + offset, (uint32_t)summary_len); + offset += 4; + tiny_codec_put_u32(data + offset, (uint32_t)message->metadata.relation_count); + offset += 4; + tiny_codec_put_u32(data + offset, (uint32_t)message->metadata.source_count); + offset += 4; + if (summary_len > 0) + { + memcpy(data + offset, message->metadata.summary, summary_len); + offset += summary_len; + } + for (index = 0; index < message->metadata.relation_count; ++index) + { + data[offset++] = (unsigned char)message->metadata.relations[index].type; + memcpy(data + offset, message->metadata.relations[index].message_id, 26); + offset += 26; + } + for (index = 0; index < message->metadata.source_count; ++index) + { + memcpy(data + offset, message->metadata.source_message_ids[index], 26); + offset += 26; + } *payload = data; *length = (uint32_t)total; return true; @@ -330,12 +369,95 @@ deserialize_put(const unsigned char *data, uint32_t length, tiny_message_t *mess message->metadata.tag_count++; } } - if (offset != length || !tiny_utf8_valid(message->session_id, session_len) + if (!tiny_utf8_valid(message->session_id, session_len) || !tiny_utf8_valid(message->content, content_len)) { return deserialize_failed(message); } - return true; + if (offset == length) + { + message->updated_at_ms = message->timestamp_ms; + return true; + } + { + uint32_t summary_len, relation_count, source_count; + if (length - offset < 32 || memcmp(data + offset, "M2", 2) != 0) + { + return deserialize_failed(message); + } + offset += 2; + message->revision = tiny_codec_get_u64(data + offset); + offset += 8; + message->updated_at_ms = (int64_t)tiny_codec_get_u64(data + offset); + offset += 8; + message->metadata.memory_kind = (tiny_memory_kind_t)data[offset++]; + message->metadata.importance = data[offset++]; + summary_len = tiny_codec_get_u32(data + offset); + offset += 4; + relation_count = tiny_codec_get_u32(data + offset); + offset += 4; + source_count = tiny_codec_get_u32(data + offset); + offset += 4; + if (message->revision == 0 || message->metadata.memory_kind < TINY_KIND_CONVERSATION + || message->metadata.memory_kind > TINY_KIND_SUMMARY || summary_len > TINY_SUMMARY_MAX + || relation_count > TINY_RELATION_COUNT_MAX || source_count > TINY_SOURCE_COUNT_MAX + || offset + summary_len + (size_t)relation_count * 27 + (size_t)source_count * 26 != length) + { + return deserialize_failed(message); + } + if (summary_len > 0) + { + message->metadata.summary = copy_field(data + offset, summary_len); + if (!message->metadata.summary || !tiny_utf8_valid(message->metadata.summary, summary_len)) + { + return deserialize_failed(message); + } + offset += summary_len; + } + if (relation_count > 0) + { + message->metadata.relations = calloc(relation_count, sizeof(tiny_relation_t)); + if (!message->metadata.relations) + { + return deserialize_failed(message); + } + for (index = 0; index < relation_count; ++index) + { + tiny_relation_t *relation = &message->metadata.relations[index]; + relation->type = (tiny_relation_type_t)data[offset++]; + memcpy(relation->message_id, data + offset, 26); + relation->message_id[26] = '\0'; + offset += 26; + if (relation->type < TINY_RELATION_DEPENDS_ON + || relation->type > TINY_RELATION_RELATED_TO + || !tiny_message_id_valid(relation->message_id)) + { + return deserialize_failed(message); + } + message->metadata.relation_count++; + } + } + if (source_count > 0) + { + message->metadata.source_message_ids = calloc(source_count, sizeof(char *)); + if (!message->metadata.source_message_ids) + { + return deserialize_failed(message); + } + for (index = 0; index < source_count; ++index) + { + message->metadata.source_message_ids[index] = copy_field(data + offset, 26); + offset += 26; + if (!message->metadata.source_message_ids[index] + || !tiny_message_id_valid(message->metadata.source_message_ids[index])) + { + return deserialize_failed(message); + } + message->metadata.source_count++; + } + } + } + return offset == length; } static tiny_store_status_t @@ -381,6 +503,545 @@ apply_put(tiny_store_t *store, tiny_message_t *message, uint64_t sequence) return TINY_STORE_OK; } +static bool +string_equal(const char *left, const char *right) +{ + return left == right || (left && right && strcmp(left, right) == 0); +} + +static bool +metadata_patch_changes(const tiny_metadata_t *current, + const tiny_metadata_t *patch, + tiny_metadata_mask_t mask) +{ + tiny_metadata_t merged = *current; + if (mask & TINY_METADATA_TOOL_CALL_ID) + { + merged.tool_call_id = patch->tool_call_id; + } + if (mask & TINY_METADATA_TAGS) + { + merged.tags = patch->tags; + merged.tag_count = patch->tag_count; + } + if (mask & TINY_METADATA_TIER) + { + merged.memory_tier = patch->memory_tier; + } + if (mask & TINY_METADATA_KIND) + { + merged.memory_kind = patch->memory_kind; + } + if (mask & TINY_METADATA_IMPORTANCE) + { + merged.importance = patch->importance; + } + if (mask & TINY_METADATA_SUMMARY) + { + merged.summary = patch->summary; + } + if (mask & TINY_METADATA_RELATIONS) + { + merged.relations = patch->relations; + merged.relation_count = patch->relation_count; + } + if (mask & TINY_METADATA_SOURCES) + { + merged.source_message_ids = patch->source_message_ids; + merged.source_count = patch->source_count; + } + if (!string_equal(current->tool_call_id, merged.tool_call_id) + || current->tag_count != merged.tag_count || current->memory_tier != merged.memory_tier + || current->memory_kind != merged.memory_kind || current->importance != merged.importance + || !string_equal(current->summary, merged.summary) + || current->relation_count != merged.relation_count + || current->source_count != merged.source_count) + { + return true; + } + if (current->tag_count) + { + size_t index; + for (index = 0; index < current->tag_count; ++index) + { + if (strcmp(current->tags[index], merged.tags[index]) != 0) + { + return true; + } + } + } + if (current->relation_count + && memcmp(current->relations, + merged.relations, + current->relation_count * sizeof(*current->relations)) + != 0) + { + return true; + } + if (current->source_count) + { + size_t index; + for (index = 0; index < current->source_count; ++index) + { + if (strcmp(current->source_message_ids[index], merged.source_message_ids[index]) != 0) + { + return true; + } + } + } + return false; +} + +static bool +metadata_targets_valid(tiny_store_t *store, const char *self_id, const tiny_metadata_t *metadata) +{ + size_t index; + if (!tiny_metadata_valid(metadata)) + { + return false; + } + for (index = 0; index < metadata->relation_count; ++index) + { + tiny_store_entry_t *target; + if (self_id && strcmp(self_id, metadata->relations[index].message_id) == 0) + { + return false; + } + target = find_entry(store, metadata->relations[index].message_id); + if (!target || target->deleted) + { + return false; + } + } + for (index = 0; index < metadata->source_count; ++index) + { + tiny_store_entry_t *target; + if (self_id && strcmp(self_id, metadata->source_message_ids[index]) == 0) + { + return false; + } + target = find_entry(store, metadata->source_message_ids[index]); + if (!target || target->deleted) + { + return false; + } + } + return true; +} + +static tiny_store_status_t +prepare_update(tiny_store_t *store, + tiny_store_entry_t *entry, + const tiny_message_patch_t *patch, + uint64_t revision, + int64_t updated_at, + tiny_message_t *prepared) +{ + tiny_message_t merged = entry->message; + tiny_metadata_t metadata = entry->message.metadata; + if (patch->content) + { + merged.content = (char *)patch->content; + } + if (patch->metadata_mask & TINY_METADATA_TOOL_CALL_ID) + { + metadata.tool_call_id = patch->metadata->tool_call_id; + } + if (patch->metadata_mask & TINY_METADATA_TAGS) + { + metadata.tags = patch->metadata->tags; + metadata.tag_count = patch->metadata->tag_count; + } + if (patch->metadata_mask & TINY_METADATA_TIER) + { + metadata.memory_tier = patch->metadata->memory_tier; + } + if (patch->metadata_mask & TINY_METADATA_KIND) + { + metadata.memory_kind = patch->metadata->memory_kind; + } + if (patch->metadata_mask & TINY_METADATA_IMPORTANCE) + { + metadata.importance = patch->metadata->importance; + } + if (patch->metadata_mask & TINY_METADATA_SUMMARY) + { + metadata.summary = patch->metadata->summary; + } + if (patch->metadata_mask & TINY_METADATA_RELATIONS) + { + metadata.relations = patch->metadata->relations; + metadata.relation_count = patch->metadata->relation_count; + } + if (patch->metadata_mask & TINY_METADATA_SOURCES) + { + metadata.source_message_ids = patch->metadata->source_message_ids; + metadata.source_count = patch->metadata->source_count; + } + merged.metadata = metadata; + if (!merged.content || !merged.content[0] || strlen(merged.content) > TINY_CONTENT_MAX + || !tiny_utf8_valid(merged.content, strlen(merged.content)) + || !metadata_targets_valid(store, merged.message_id, &metadata)) + { + return TINY_STORE_CONFLICT; + } + if (tiny_message_copy(prepared, &merged) != 0) + { + return TINY_STORE_NO_MEMORY; + } + prepared->revision = revision; + prepared->updated_at_ms = updated_at; + return TINY_STORE_OK; +} + +static bool +delta_put_string(unsigned char *data, size_t capacity, size_t *offset, const char *value) +{ + size_t length = value ? strlen(value) : 0; + if (length > UINT32_MAX || *offset > capacity || capacity - *offset < 4 + length) + { + return false; + } + tiny_codec_put_u32(data + *offset, (uint32_t)length); + *offset += 4; + if (length) + { + memcpy(data + *offset, value, length); + *offset += length; + } + return true; +} + +static bool +serialize_update(const tiny_store_entry_t *entry, + const tiny_message_patch_t *patch, + uint64_t revision, + int64_t updated_at, + unsigned char **payload, + uint32_t *payload_length) +{ + size_t capacity = 64 + (patch->content ? strlen(patch->content) : 0), offset = 0, index; + unsigned char *data; + tiny_metadata_mask_t mask = patch->metadata_mask; + if (mask & TINY_METADATA_TOOL_CALL_ID) + { + capacity += 4 + (patch->metadata->tool_call_id ? strlen(patch->metadata->tool_call_id) : 0); + } + if (mask & TINY_METADATA_TAGS) + { + capacity += 4; + for (index = 0; index < patch->metadata->tag_count; ++index) + { + capacity += 4 + strlen(patch->metadata->tags[index]); + } + } + if (mask & TINY_METADATA_SUMMARY) + { + capacity += 4 + (patch->metadata->summary ? strlen(patch->metadata->summary) : 0); + } + if (mask & TINY_METADATA_RELATIONS) + { + capacity += 4 + patch->metadata->relation_count * 27; + } + if (mask & TINY_METADATA_SOURCES) + { + capacity += 4 + patch->metadata->source_count * 26; + } + capacity += (mask & TINY_METADATA_TIER) ? 1 : 0; + capacity += (mask & TINY_METADATA_KIND) ? 1 : 0; + capacity += (mask & TINY_METADATA_IMPORTANCE) ? 1 : 0; + if (capacity > RECORD_PAYLOAD_MAX || capacity > UINT32_MAX || !(data = malloc(capacity))) + { + return false; + } + memcpy(data, entry->message.message_id, 26); + offset = 26; + tiny_codec_put_u64(data + offset, revision); + offset += 8; + tiny_codec_put_u64(data + offset, (uint64_t)updated_at); + offset += 8; + data[offset++] = patch->content ? UPDATE_CONTENT : 0; + data[offset - 1] |= mask ? UPDATE_METADATA : 0; + tiny_codec_put_u32(data + offset, mask); + offset += 4; + if (patch->content && !delta_put_string(data, capacity, &offset, patch->content)) + { + free(data); + return false; + } + if ((mask & TINY_METADATA_TOOL_CALL_ID) + && !delta_put_string(data, capacity, &offset, patch->metadata->tool_call_id)) + { + free(data); + return false; + } + if (mask & TINY_METADATA_TAGS) + { + tiny_codec_put_u32(data + offset, (uint32_t)patch->metadata->tag_count); + offset += 4; + for (index = 0; index < patch->metadata->tag_count; ++index) + { + if (!delta_put_string(data, capacity, &offset, patch->metadata->tags[index])) + { + free(data); + return false; + } + } + } + if (mask & TINY_METADATA_TIER) + { + data[offset++] = (unsigned char)patch->metadata->memory_tier; + } + if (mask & TINY_METADATA_KIND) + { + data[offset++] = (unsigned char)patch->metadata->memory_kind; + } + if (mask & TINY_METADATA_IMPORTANCE) + { + data[offset++] = patch->metadata->importance; + } + if ((mask & TINY_METADATA_SUMMARY) + && !delta_put_string(data, capacity, &offset, patch->metadata->summary)) + { + free(data); + return false; + } + if (mask & TINY_METADATA_RELATIONS) + { + tiny_codec_put_u32(data + offset, (uint32_t)patch->metadata->relation_count); + offset += 4; + for (index = 0; index < patch->metadata->relation_count; ++index) + { + data[offset++] = (unsigned char)patch->metadata->relations[index].type; + memcpy(data + offset, patch->metadata->relations[index].message_id, 26); + offset += 26; + } + } + if (mask & TINY_METADATA_SOURCES) + { + tiny_codec_put_u32(data + offset, (uint32_t)patch->metadata->source_count); + offset += 4; + for (index = 0; index < patch->metadata->source_count; ++index) + { + memcpy(data + offset, patch->metadata->source_message_ids[index], 26); + offset += 26; + } + } + *payload = data; + *payload_length = (uint32_t)offset; + return true; +} + +static char * +delta_get_string(const unsigned char *data, size_t length, size_t *offset, size_t maximum) +{ + uint32_t field_length; + char *value; + if (*offset > length || length - *offset < 4) + { + return NULL; + } + field_length = tiny_codec_get_u32(data + *offset); + *offset += 4; + if (field_length > maximum || *offset > length || length - *offset < field_length) + { + return NULL; + } + value = copy_field(data + *offset, field_length); + *offset += field_length; + return value; +} + +static tiny_store_status_t +prepare_update_payload(tiny_store_t *store, + const unsigned char *payload, + uint32_t length, + tiny_store_entry_t **target, + tiny_message_t *prepared) +{ + tiny_message_t holder; + tiny_message_patch_t patch; + size_t offset = 0, index; + uint64_t revision; + int64_t updated_at; + uint8_t flags; + uint32_t mask; + char id[27]; + tiny_store_status_t status = TINY_STORE_CORRUPT; + tiny_message_init(&holder); + memset(&patch, 0, sizeof(patch)); +#define UPDATE_PAYLOAD_BAD() \ + do \ + { \ + tiny_message_free(&holder); \ + return TINY_STORE_CORRUPT; \ + } \ + while (0) + if (length < 47) + { + UPDATE_PAYLOAD_BAD(); + } + memcpy(id, payload, 26); + id[26] = '\0'; + offset = 26; + revision = tiny_codec_get_u64(payload + offset); + offset += 8; + updated_at = (int64_t)tiny_codec_get_u64(payload + offset); + offset += 8; + flags = payload[offset++]; + mask = tiny_codec_get_u32(payload + offset); + offset += 4; + *target = find_entry(store, id); + if (!*target || (*target)->deleted || revision != (*target)->message.revision + 1 + || revision > INT32_MAX || !updated_at || (mask & ~TINY_METADATA_ALL) + || flags != ((flags & UPDATE_CONTENT ? UPDATE_CONTENT : 0) | (mask ? UPDATE_METADATA : 0))) + { + UPDATE_PAYLOAD_BAD(); + } + if (flags & UPDATE_CONTENT) + { + holder.content = delta_get_string(payload, length, &offset, TINY_CONTENT_MAX); + if (!holder.content) + { + UPDATE_PAYLOAD_BAD(); + } + patch.content = holder.content; + } + patch.metadata = &holder.metadata; + patch.metadata_mask = mask; + if (mask & TINY_METADATA_TOOL_CALL_ID) + { + holder.metadata.tool_call_id + = delta_get_string(payload, length, &offset, TINY_TOOL_CALL_ID_MAX); + if (!holder.metadata.tool_call_id) + { + UPDATE_PAYLOAD_BAD(); + } + if (!holder.metadata.tool_call_id[0]) + { + free(holder.metadata.tool_call_id); + holder.metadata.tool_call_id = NULL; + } + } + if (mask & TINY_METADATA_TAGS) + { + uint32_t count; + if (offset > length || length - offset < 4) + { + UPDATE_PAYLOAD_BAD(); + } + count = tiny_codec_get_u32(payload + offset); + offset += 4; + if (count > TINY_TAG_COUNT_MAX + || (count && !(holder.metadata.tags = calloc(count, sizeof(char *))))) + { + UPDATE_PAYLOAD_BAD(); + } + for (index = 0; index < count; ++index) + { + holder.metadata.tags[index] = delta_get_string(payload, length, &offset, TINY_TAG_MAX); + if (!holder.metadata.tags[index] || !holder.metadata.tags[index][0]) + { + UPDATE_PAYLOAD_BAD(); + } + holder.metadata.tag_count++; + } + } + if (mask & TINY_METADATA_TIER) + { + if (offset >= length) + { + UPDATE_PAYLOAD_BAD(); + } + holder.metadata.memory_tier = (tiny_memory_tier_t)payload[offset++]; + } + if (mask & TINY_METADATA_KIND) + { + if (offset >= length) + { + UPDATE_PAYLOAD_BAD(); + } + holder.metadata.memory_kind = (tiny_memory_kind_t)payload[offset++]; + } + if (mask & TINY_METADATA_IMPORTANCE) + { + if (offset >= length) + { + UPDATE_PAYLOAD_BAD(); + } + holder.metadata.importance = payload[offset++]; + } + if (mask & TINY_METADATA_SUMMARY) + { + holder.metadata.summary = delta_get_string(payload, length, &offset, TINY_SUMMARY_MAX); + if (!holder.metadata.summary) + { + UPDATE_PAYLOAD_BAD(); + } + if (!holder.metadata.summary[0]) + { + free(holder.metadata.summary); + holder.metadata.summary = NULL; + } + } + if (mask & TINY_METADATA_RELATIONS) + { + uint32_t count; + if (offset > length || length - offset < 4) + { + UPDATE_PAYLOAD_BAD(); + } + count = tiny_codec_get_u32(payload + offset); + offset += 4; + if (count > TINY_RELATION_COUNT_MAX || offset > length || length - offset < (size_t)count * 27 + || (count && !(holder.metadata.relations = calloc(count, sizeof(tiny_relation_t))))) + { + UPDATE_PAYLOAD_BAD(); + } + for (index = 0; index < count; ++index) + { + holder.metadata.relations[index].type = (tiny_relation_type_t)payload[offset++]; + memcpy(holder.metadata.relations[index].message_id, payload + offset, 26); + holder.metadata.relations[index].message_id[26] = '\0'; + offset += 26; + holder.metadata.relation_count++; + } + } + if (mask & TINY_METADATA_SOURCES) + { + uint32_t count; + if (offset > length || length - offset < 4) + { + UPDATE_PAYLOAD_BAD(); + } + count = tiny_codec_get_u32(payload + offset); + offset += 4; + if (count > TINY_SOURCE_COUNT_MAX || offset > length || length - offset < (size_t)count * 26 + || (count && !(holder.metadata.source_message_ids = calloc(count, sizeof(char *))))) + { + UPDATE_PAYLOAD_BAD(); + } + for (index = 0; index < count; ++index) + { + holder.metadata.source_message_ids[index] = copy_field(payload + offset, 26); + if (!holder.metadata.source_message_ids[index]) + { + UPDATE_PAYLOAD_BAD(); + } + offset += 26; + holder.metadata.source_count++; + } + } + if (offset != length) + { + UPDATE_PAYLOAD_BAD(); + } + status = prepare_update(store, *target, &patch, revision, updated_at, prepared); + tiny_message_free(&holder); +#undef UPDATE_PAYLOAD_BAD + return status; +} + static void clear_entries(tiny_store_t *store) { @@ -431,8 +1092,10 @@ replay_log(tiny_store_t *store, uint64_t starting_offset) { return recover_torn_tail(store, boundary); } - if (memcmp(header, "TCMR", 4) != 0 || header[4] != RECORD_VERSION - || (header[5] != RECORD_PUT && header[5] != RECORD_DEL && header[5] != RECORD_MARKER)) + if (memcmp(header, "TCMR", 4) != 0 + || (header[4] != RECORD_VERSION && header[4] != RECORD_VERSION_LEGACY) + || (header[5] != RECORD_PUT && header[5] != RECORD_DEL && header[5] != RECORD_MARKER + && header[5] != RECORD_UPDATE)) { return TINY_STORE_CORRUPT; } @@ -507,9 +1170,28 @@ replay_log(tiny_store_t *store, uint64_t starting_offset) return TINY_STORE_CORRUPT; } entry->deleted = true; + store->dead_bytes += RECORD_HEADER_SIZE + payload_length; } break; + case RECORD_UPDATE: + { + tiny_store_entry_t *target = NULL; + tiny_message_t prepared; + tiny_message_init(&prepared); + status = prepare_update_payload(store, payload, payload_length, &target, &prepared); + if (status == TINY_STORE_OK) + { + tiny_message_free(&target->message); + target->message = prepared; + tiny_message_init(&prepared); + } + tiny_message_free(&prepared); + } + store->update_records++; + store->dead_bytes += RECORD_HEADER_SIZE + payload_length; + break; + default: free(payload); return TINY_STORE_CORRUPT; @@ -596,8 +1278,10 @@ log_generation_matches(tiny_store_t *store, const unsigned char generation[16]) uint32_t actual = 0; if (!tiny_sys_file_seek(&store->log, 0, TINY_SYS_SEEK_BEGIN) || !read_all(&store->log, header, sizeof(header), &actual) || actual != sizeof(header) - || memcmp(header, "TCMR", 4) != 0 || header[4] != RECORD_VERSION || header[5] != RECORD_MARKER - || tiny_codec_get_u64(header + 12) != 0 || tiny_codec_get_u32(header + 8) != 16 + || memcmp(header, "TCMR", 4) != 0 + || (header[4] != RECORD_VERSION && header[4] != RECORD_VERSION_LEGACY) + || header[5] != RECORD_MARKER || tiny_codec_get_u64(header + 12) != 0 + || tiny_codec_get_u32(header + 8) != 16 || !read_all(&store->log, payload, sizeof(payload), &actual) || actual != sizeof(payload) || tiny_codec_get_u32(header + 20) != tiny_codec_crc32(payload, sizeof(payload))) { @@ -799,6 +1483,8 @@ tiny_store_status_name(tiny_store_status_t status) return "IO_ERROR"; case TINY_STORE_CORRUPT: return "CORRUPT_STORE"; + case TINY_STORE_CONFLICT: + return "CONFLICT"; default: return "INTERNAL_ERROR"; } @@ -820,6 +1506,12 @@ tiny_store_write(tiny_store_t *store, const tiny_message_t *input, tiny_message_ { return TINY_STORE_IO_ERROR; } + if (!input->content[0] || strlen(input->content) > TINY_CONTENT_MAX + || !tiny_utf8_valid(input->content, strlen(input->content)) + || !metadata_targets_valid(store, NULL, &input->metadata)) + { + return TINY_STORE_CONFLICT; + } tiny_message_init(&message); tiny_message_init(&indexed); if (tiny_message_copy(&message, input) != 0) @@ -827,6 +1519,8 @@ tiny_store_write(tiny_store_t *store, const tiny_message_t *input, tiny_message_ return TINY_STORE_NO_MEMORY; } message.timestamp_ms = (int64_t)now_milliseconds(); + message.updated_at_ms = message.timestamp_ms; + message.revision = 1; make_ulid(store, (uint64_t)message.timestamp_ms, message.message_id); if (!serialize_put(&message, &payload, &length) || !ensure_entries(store) || tiny_message_copy(&indexed, &message) != 0) @@ -856,6 +1550,127 @@ tiny_store_write(tiny_store_t *store, const tiny_message_t *input, tiny_message_ } tiny_store_status_t +tiny_store_update(tiny_store_t *store, + const char *message_id, + const tiny_message_patch_t *patch, + tiny_message_t *updated, + bool *changed) +{ + tiny_store_entry_t *entry; + tiny_message_t prepared, response; + tiny_message_patch_t effective; + unsigned char *payload = NULL; + uint32_t payload_length = 0; + uint64_t log_size_before = 0; + uint64_t revision; + int64_t updated_at; + tiny_store_status_t status; + if (!store || !message_id || !patch || !updated || !changed || !tiny_message_id_valid(message_id)) + { + return TINY_STORE_NOT_FOUND; + } + tiny_message_init(updated); + tiny_message_init(&prepared); + tiny_message_init(&response); + *changed = false; + entry = find_entry(store, message_id); + if (!entry || entry->deleted) + { + return TINY_STORE_NOT_FOUND; + } + if (patch->has_expected_revision && patch->expected_revision != entry->message.revision) + { + return TINY_STORE_CONFLICT; + } + if (entry->message.revision >= INT32_MAX) + { + return TINY_STORE_CONFLICT; + } + effective = *patch; + if ((effective.metadata_mask & ~TINY_METADATA_ALL) != 0 + || (effective.metadata_mask + && (!effective.metadata || !tiny_metadata_valid(effective.metadata)))) + { + return TINY_STORE_CONFLICT; + } + if (effective.content && strcmp(effective.content, entry->message.content) == 0) + { + effective.content = NULL; + } + if (!effective.metadata || effective.metadata_mask == 0 + || !metadata_patch_changes( + &entry->message.metadata, effective.metadata, effective.metadata_mask)) + { + effective.metadata_mask = 0; + } + if (!effective.content && effective.metadata_mask == 0) + { + return tiny_message_copy(updated, &entry->message) == 0 ? TINY_STORE_OK + : TINY_STORE_NO_MEMORY; + } + revision = entry->message.revision + 1; + updated_at = (int64_t)now_milliseconds(); + status = prepare_update(store, entry, &effective, revision, updated_at, &prepared); + if (status != TINY_STORE_OK) + { + tiny_message_free(&prepared); + return status; + } + if (tiny_message_copy(&response, &prepared) != 0) + { + status = TINY_STORE_NO_MEMORY; + tiny_message_free(&prepared); + return status; + } + if (!serialize_update(entry, &effective, revision, updated_at, &payload, &payload_length)) + { + status = TINY_STORE_NO_MEMORY; + tiny_message_free(&prepared); + tiny_message_free(&response); + return status; + } + if (!tiny_sys_file_size(&store->log, &log_size_before)) + { + free(payload); + tiny_message_free(&prepared); + tiny_message_free(&response); + return TINY_STORE_IO_ERROR; + } + status = append_record_to( + &store->log, RECORD_UPDATE, store->last_sequence + 1, payload, payload_length, true); + if (status != TINY_STORE_OK) + { + bool rolled_back = log_size_before <= INT64_MAX + && tiny_sys_file_seek( + &store->log, (int64_t)log_size_before, TINY_SYS_SEEK_BEGIN) + && tiny_sys_file_truncate(&store->log) + && tiny_sys_file_flush(&store->log); + set_error(store, "failed to append and flush an UPDATE record"); + if (!rolled_back) + { + set_error(store, "UPDATE failed and its incomplete record could not be rolled back"); + } + free(payload); + tiny_message_free(&prepared); + tiny_message_free(&response); + return status; + } + tiny_message_free(&entry->message); + entry->message = prepared; + tiny_message_init(&prepared); + *updated = response; + tiny_message_init(&response); + store->last_sequence++; + store->update_records++; + store->dead_bytes += RECORD_HEADER_SIZE + payload_length; + *changed = true; + free(payload); + tiny_message_free(&prepared); + tiny_message_free(&response); + return TINY_STORE_OK; +} + +tiny_store_status_t tiny_store_delete(tiny_store_t *store, const char *message_id, const char *session_id) { tiny_store_entry_t *entry; @@ -882,6 +1697,7 @@ tiny_store_delete(tiny_store_t *store, const char *message_id, const char *sessi } store->last_sequence++; entry->deleted = true; + store->dead_bytes += RECORD_HEADER_SIZE + sizeof(payload); return TINY_STORE_OK; } @@ -905,6 +1721,15 @@ tiny_store_rebuild(tiny_store_t *store, tiny_rebuild_report_t *report) { return TINY_STORE_IO_ERROR; } + if (store->snapshot_valid && store->update_records == 0 && store->dead_bytes == 0) + { + uint64_t size = 0; + (void)tiny_sys_file_size(&store->log, &size); + report->messages_indexed = store->entry_count; + report->bytes_before = size; + report->bytes_after = size; + return TINY_STORE_OK; + } memcpy(previous_generation, store->generation, sizeof(previous_generation)); previous_has_generation = store->has_generation; temporary = tiny_sys_path_join(store->directory, "active.tmp"); @@ -1002,6 +1827,11 @@ tiny_store_rebuild(tiny_store_t *store, tiny_rebuild_report_t *report) report->bytes_before = before; report->bytes_after = after; status = write_snapshot(store); + if (status == TINY_STORE_OK) + { + store->update_records = 0; + store->dead_bytes = 0; + } } if (status != TINY_STORE_OK) { @@ -1032,6 +1862,7 @@ tiny_store_health(const tiny_store_t *store, tiny_store_health_t *health) { return; } + health->format_version = RECORD_VERSION; if (tiny_sys_file_size(&store->log, &size)) { health->log_size = size; @@ -1059,6 +1890,17 @@ tiny_store_health(const tiny_store_t *store, tiny_store_health_t *health) } } health->last_sequence = store->last_sequence; + health->update_records = store->update_records; + health->dead_bytes = store->dead_bytes; + { + tiny_sys_io_counters_t counters; + tiny_sys_io_counters_get(&counters); + health->write_bytes = counters.write_bytes; + health->write_calls = counters.write_calls; + health->flush_calls = counters.flush_calls; + health->truncate_calls = counters.truncate_calls; + health->replace_calls = counters.replace_calls; + } health->recovered_tail = store->recovered_tail; health->snapshot_valid = store->snapshot_valid; snprintf(health->last_error, sizeof(health->last_error), "%s", store->last_error); diff --git a/src/storage/store.h b/src/storage/store.h index 619f86f..3ddd24d 100644 --- a/src/storage/store.h +++ b/src/storage/store.h @@ -14,7 +14,8 @@ typedef enum tiny_store_status TINY_STORE_BUSY, TINY_STORE_IO_ERROR, TINY_STORE_CORRUPT, - TINY_STORE_NO_MEMORY + TINY_STORE_NO_MEMORY, + TINY_STORE_CONFLICT } tiny_store_status_t; typedef struct tiny_store_entry @@ -36,16 +37,33 @@ typedef struct tiny_rebuild_report typedef struct tiny_store_health { + uint32_t format_version; uint64_t log_size; uint64_t snapshot_size; uint64_t active_messages; uint64_t deleted_messages; uint64_t last_sequence; + uint64_t update_records; + uint64_t dead_bytes; + uint64_t write_bytes; + uint64_t write_calls; + uint64_t flush_calls; + uint64_t truncate_calls; + uint64_t replace_calls; bool recovered_tail; bool snapshot_valid; char last_error[160]; } tiny_store_health_t; +typedef struct tiny_message_patch +{ + const char *content; + const tiny_metadata_t *metadata; + tiny_metadata_mask_t metadata_mask; + uint64_t expected_revision; + bool has_expected_revision; +} tiny_message_patch_t; + tiny_store_status_t tiny_store_open(tiny_store_t **store, const char *directory); void tiny_store_close(tiny_store_t *store); const char *tiny_store_status_name(tiny_store_status_t status); @@ -54,6 +72,11 @@ tiny_store_status_t tiny_store_write(tiny_store_t *store, const tiny_message_t *input, tiny_message_t *written); tiny_store_status_t tiny_store_delete(tiny_store_t *store, const char *message_id, const char *session_id); +tiny_store_status_t tiny_store_update(tiny_store_t *store, + const char *message_id, + const tiny_message_patch_t *patch, + tiny_message_t *updated, + bool *changed); tiny_store_status_t tiny_store_rebuild(tiny_store_t *store, tiny_rebuild_report_t *report); void tiny_store_health(const tiny_store_t *store, tiny_store_health_t *health); diff --git a/src/sys/system.c b/src/sys/system.c index 63d8297..0d99b28 100644 --- a/src/sys/system.c +++ b/src/sys/system.c @@ -16,6 +16,22 @@ typedef struct tiny_sys_test_state } tiny_sys_test_state_t; static tiny_sys_test_state_t tiny_sys_test_state; +static tiny_sys_io_counters_t tiny_sys_io_counters; + +void +tiny_sys_io_counters_reset(void) +{ + memset(&tiny_sys_io_counters, 0, sizeof(tiny_sys_io_counters)); +} + +void +tiny_sys_io_counters_get(tiny_sys_io_counters_t *counters) +{ + if (counters != NULL) + { + *counters = tiny_sys_io_counters; + } +} void tiny_sys_test_reset(void) @@ -349,6 +365,11 @@ tiny_sys_file_write(tiny_sys_file_t *file, const void *data, uint32_t length, ui length = tiny_sys_test_transfer_length(length); result = WriteFile(native_file(file), data, length, &count, NULL) != 0; *actual = count; + if (result) + { + tiny_sys_io_counters.write_calls++; + tiny_sys_io_counters.write_bytes += count; + } return result; } @@ -400,15 +421,19 @@ tiny_sys_file_size(const tiny_sys_file_t *file, uint64_t *size) bool tiny_sys_file_truncate(tiny_sys_file_t *file) { - return tiny_sys_file_valid(file) && !tiny_sys_test_should_fail(TINY_SYS_TEST_TRUNCATE) - && SetEndOfFile(native_file(file)) != 0; + bool result = tiny_sys_file_valid(file) && !tiny_sys_test_should_fail(TINY_SYS_TEST_TRUNCATE) + && SetEndOfFile(native_file(file)) != 0; + tiny_sys_io_counters.truncate_calls += result ? 1U : 0U; + return result; } bool tiny_sys_file_flush(tiny_sys_file_t *file) { - return tiny_sys_file_valid(file) && !tiny_sys_test_should_fail(TINY_SYS_TEST_FLUSH) - && FlushFileBuffers(native_file(file)) != 0; + bool result = tiny_sys_file_valid(file) && !tiny_sys_test_should_fail(TINY_SYS_TEST_FLUSH) + && FlushFileBuffers(native_file(file)) != 0; + tiny_sys_io_counters.flush_calls += result ? 1U : 0U; + return result; } bool @@ -427,6 +452,7 @@ tiny_sys_path_replace(const char *source, const char *destination) } free(wide_source); free(wide_destination); + tiny_sys_io_counters.replace_calls += result ? 1U : 0U; return result; } @@ -703,6 +729,11 @@ tiny_sys_file_read(tiny_sys_file_t *file, void *data, uint32_t length, uint32_t } while (count < 0 && errno == EINTR); *actual = count > 0 ? (uint32_t)count : 0; + if (count >= 0) + { + tiny_sys_io_counters.write_calls++; + tiny_sys_io_counters.write_bytes += *actual; + } return count >= 0; } @@ -769,21 +800,31 @@ tiny_sys_file_truncate(tiny_sys_file_t *file) return false; } position = lseek(native_file(file), 0, SEEK_CUR); - return position >= 0 && ftruncate(native_file(file), position) == 0; + if (position >= 0 && ftruncate(native_file(file), position) == 0) + { + tiny_sys_io_counters.truncate_calls++; + return true; + } + return false; } bool tiny_sys_file_flush(tiny_sys_file_t *file) { - return tiny_sys_file_valid(file) && !tiny_sys_test_should_fail(TINY_SYS_TEST_FLUSH) - && fsync(native_file(file)) == 0; + bool result = tiny_sys_file_valid(file) && !tiny_sys_test_should_fail(TINY_SYS_TEST_FLUSH) + && fsync(native_file(file)) == 0; + tiny_sys_io_counters.flush_calls += result ? 1U : 0U; + return result; } bool tiny_sys_path_replace(const char *source, const char *destination) { - return source != NULL && destination != NULL && !tiny_sys_test_should_fail(TINY_SYS_TEST_REPLACE) - && rename(source, destination) == 0; + bool result = source != NULL && destination != NULL + && !tiny_sys_test_should_fail(TINY_SYS_TEST_REPLACE) + && rename(source, destination) == 0; + tiny_sys_io_counters.replace_calls += result ? 1U : 0U; + return result; } bool diff --git a/src/sys/system.h b/src/sys/system.h index b526ec5..0947205 100644 --- a/src/sys/system.h +++ b/src/sys/system.h @@ -40,6 +40,18 @@ typedef struct tiny_sys_arguments char **values; } tiny_sys_arguments_t; +typedef struct tiny_sys_io_counters +{ + uint64_t write_bytes; + uint64_t write_calls; + uint64_t flush_calls; + uint64_t truncate_calls; + uint64_t replace_calls; +} tiny_sys_io_counters_t; + +void tiny_sys_io_counters_reset(void); +void tiny_sys_io_counters_get(tiny_sys_io_counters_t *counters); + /* Internal test seam. A call index of 1 fails the next matching operation. */ typedef enum tiny_sys_test_operation { |