From 61fa05ec0c4510f1cd437b5d672cd6fd7b996353 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Sat, 4 Jul 2026 18:58:06 +0900 Subject: Bound import memory with staged streaming --- tests/import_export_test.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'tests/import_export_test.py') diff --git a/tests/import_export_test.py b/tests/import_export_test.py index 0ba5c5b..9e7fa02 100644 --- a/tests/import_export_test.py +++ b/tests/import_export_test.py @@ -11,6 +11,16 @@ from pathlib import Path MESSAGE_ID = "01ARZ3NDEKTSV4RRFFQ69G5FAV" +def make_ulid(index: int) -> str: + alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + value = index + output = ["0"] * 26 + for position in range(25, -1, -1): + output[position] = alphabet[value & 31] + value >>= 5 + return "".join(output) + + def record(content: str = "valid", **changes: object) -> dict[str, object]: value: dict[str, object] = { "message_id": MESSAGE_ID, @@ -103,6 +113,48 @@ def main() -> None: ) run(server, root / "long-data", "--import", str(long_line), success=False) + truncated = root / "truncated.jsonl" + write_jsonl(truncated, record()) + truncated.write_bytes(truncated.read_bytes()[:-8]) + run(server, root / "truncated-data", "--import", str(truncated), success=False) + + crlf = root / "crlf.jsonl" + crlf.write_bytes(source.read_bytes().replace(b"\n", b"\r\n", 1).replace(b"\r\n", b"\r\n\r\n", 1)) + run(server, root / "crlf-data", "--import", str(crlf), success=True) + + dangling = root / "dangling.jsonl" + dangling_record = record() + dangling_record["metadata"] = dict(dangling_record["metadata"]) # type: ignore[arg-type] + dangling_record["metadata"]["relations"] = [ # type: ignore[index] + {"type": "derived_from", "message_id": "01ARZ3NDEKTSV4RRFFQ69G5FAW"} + ] + write_jsonl(dangling, dangling_record) + run(server, root / "dangling-data", "--import", str(dangling), success=False) + + duplicate = root / "duplicate.jsonl" + duplicate.write_text( + json.dumps({"type": "tiny-convmem-manifest", "format_version": 2}) + + "\n" + + json.dumps(record("first"), separators=(",", ":")) + + "\n" + + json.dumps(record("last"), separators=(",", ":")) + + "\n", + encoding="utf-8", + ) + duplicate_data = root / "duplicate-data" + run(server, duplicate_data, "--import", str(duplicate), success=True) + duplicate_export = root / "duplicate-export.jsonl" + run(server, duplicate_data, "--export", str(duplicate_export), success=True) + assert json.loads(duplicate_export.read_text(encoding="utf-8").splitlines()[1])["content"] == "last" + + over_count = root / "over-count.jsonl" + with over_count.open("w", encoding="utf-8", newline="\n") as file: + file.write(json.dumps({"type": "tiny-convmem-manifest", "format_version": 2}) + "\n") + for index in range(100_001): + file.write(json.dumps({"type": "DEL", "message_id": make_ulid(index)}) + "\n") + run(server, root / "over-count-data", "--import", str(over_count), success=False) + assert not (root / "over-count-data" / "active.import.tmp").exists() + atomic_target = root / "atomic.jsonl" atomic_target.write_text("old backup", encoding="utf-8") run(server, first, "--export", str(atomic_target), success=True) -- cgit v1.2.3