diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | docs/tools-schema.md | 8 | ||||
| -rw-r--r-- | src/ollama_web/tools/fetch.py | 20 | ||||
| -rw-r--r-- | src/ollama_web/tools/helper/pagination.py | 5 | ||||
| -rw-r--r-- | tests/test_tools.py | 117 |
5 files changed, 144 insertions, 9 deletions
@@ -176,4 +176,5 @@ data/ *.lock *.err .agents/ -.mcp_test_fs/
\ No newline at end of file +.mcp_test_fs/ +plans/
\ No newline at end of file diff --git a/docs/tools-schema.md b/docs/tools-schema.md index ea438d3..b9cf4b9 100644 --- a/docs/tools-schema.md +++ b/docs/tools-schema.md @@ -190,7 +190,7 @@ URL: {url} - ファイルが完了したら次のファイルへ 4. 残り候補 or 残りファイル内容があれば `has_more: true` + `next_token` を付与 -> トークンペイロードは `search_and_fetch` 用に拡張され、`matched`（候補URLリスト）を埋め込むことで再検索なしで続きを取得（API呼び出し増加を防止）。 +> トークンペイロードは `search_and_fetch` 用に拡張され、`matched`（候補リスト）を埋め込むことで再検索なしで続きを取得（API呼び出し増加を防止）。`matched` には [`_slim_matched()`](src/ollama_web/tools/fetch.py:42) で `body`（検索スニペット）を除外した `title` / `href` のみを格納し、トークンサイズの肥大化と末尾欠損による破損を防止する。 ### 設計上の注意点 @@ -223,7 +223,7 @@ URL: {url} ### helper/pagination.py - **実装**: [`pagination.py`](src/ollama_web/tools/helper/pagination.py:1) -- [`make_token()`](src/ollama_web/tools/helper/pagination.py:1): 継続トークン生成（`base64url(JSON)`） +- [`make_token()`](src/ollama_web/tools/helper/pagination.py:1): 継続トークン生成（`base64url(JSON)`、`ensure_ascii=True` で純ASCII化し末尾欠損時の `UnicodeDecodeError` を防止） - [`parse_token()`](src/ollama_web/tools/helper/pagination.py:1): トークン検証・デコード。`tool` 一致・`v` バージョン・必須フィールドを検証し、不正時は `InvalidTokenError` - [`format_paged_result()`](src/ollama_web/tools/helper/pagination.py:1): 戻り値ラッピング。`---page-info---` マーカーで本文とフッターを分離 - [`parse_paged_result()`](src/ollama_web/tools/helper/pagination.py:1): 戻り値分割（テスト・チェーニング用） @@ -303,7 +303,7 @@ flowchart TD } ``` -`search_and_fetch` は追加フィールドを埋め込む: +`search_and_fetch` は追加フィールドを埋め込む。`matched` には検索スニペット（`body`）を含めず `title` / `href` のみを格納する（[`_slim_matched()`](src/ollama_web/tools/fetch.py:42) で生成）。これによりトークンサイズを数百バイトに抑え、LLM が `continue_from` にコピーする際の末尾欠損による破損を防止する: ```json { @@ -311,7 +311,7 @@ flowchart TD "tool": "search_and_fetch", "url": "https://example.com/file.pdf", "offset": 8000, - "matched": [{"title": "...", "href": "...", "body": "..."}], + "matched": [{"title": "...", "href": "..."}], "file_idx": 1, "file_offset": 8000 } diff --git a/src/ollama_web/tools/fetch.py b/src/ollama_web/tools/fetch.py index a9a92f8..65ec511 100644 --- a/src/ollama_web/tools/fetch.py +++ b/src/ollama_web/tools/fetch.py @@ -39,6 +39,20 @@ _FILETYPE_SUFFIXES: dict[str, str] = { _TOOL_NAME = "search_and_fetch" +def _slim_matched(matched: list[dict[str, str]]) -> list[dict[str, str]]: + """Return a token-embeddable view of ``matched`` without the ``body`` field. + + ``search_and_fetch`` embeds the candidate list in its continuation token + so subsequent pages can resume without re-running the web search. The + rendering layer only uses ``title`` and ``href``; the search-engine + snippet (``body``) is unused and can be large (multi-KB for PDF + abstracts). Including it inflates the token to the point where the + model or tool-result trimming may truncate it, corrupting the base64 + payload. Stripping ``body`` keeps the token to a few hundred bytes. + """ + return [{"title": m.get("title", ""), "href": m.get("href", "")} for m in matched] + + def set_session_context(session_id: str | None) -> None: """Set the active session id so fetched files can be persisted later.""" # No-op marker kept for future hooks; storage is keyed by session_id directly. @@ -295,7 +309,7 @@ def _render_file_page( _TOOL_NAME, url, 0, - matched=matched, + matched=_slim_matched(matched), file_idx=file_idx + 1, file_offset=0, ) @@ -328,7 +342,7 @@ def _render_file_page( _TOOL_NAME, url, file_offset + per_file_chars, - matched=matched, + matched=_slim_matched(matched), file_idx=file_idx, file_offset=file_offset + per_file_chars, ) @@ -340,7 +354,7 @@ def _render_file_page( _TOOL_NAME, url, 0, - matched=matched, + matched=_slim_matched(matched), file_idx=file_idx + 1, file_offset=0, ) diff --git a/src/ollama_web/tools/helper/pagination.py b/src/ollama_web/tools/helper/pagination.py index 7f18ac8..748338b 100644 --- a/src/ollama_web/tools/helper/pagination.py +++ b/src/ollama_web/tools/helper/pagination.py @@ -64,7 +64,10 @@ def make_token(tool: str, url: str, offset: int, **extra: Any) -> str: "offset": int(offset), } payload.update(extra) - raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + # ensure_ascii=True keeps the token pure ASCII so truncation of the + # token by the model or tool-result trimming cannot split a multibyte + # UTF-8 sequence (which would raise UnicodeDecodeError on decode). + raw = json.dumps(payload, ensure_ascii=True, separators=(",", ":")).encode("ascii") return _b64url_encode(raw) diff --git a/tests/test_tools.py b/tests/test_tools.py index 67c66ef..a94ea52 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -354,6 +354,123 @@ def test_search_and_fetch_pagination_across_files(monkeypatch): assert seen_titles == ["File1", "File2"] +def test_make_token_ensure_ascii_true(): + """Tokens are pure ASCII even with non-ASCII content (ensure_ascii=True).""" + token = make_token("scrape_url", "https://example.com/日本語", 0, title="論文–要約") + # base64url alphabet is pure ASCII; any non-ASCII byte would break this. + assert all(ord(c) < 128 for c in token) + payload = parse_token(token, "scrape_url") + assert payload["url"] == "https://example.com/日本語" + assert payload["title"] == "論文–要約" + + +def test_search_and_fetch_token_excludes_body(monkeypatch): + """The continuation token must not embed the search snippet (body).""" + from ollama_web.tools import fetch as fetch_mod + + matched = [ + { + "title": "File1", + "href": "https://example.com/f1.txt", + "body": "X" * 5000, # large snippet that would bloat the token + } + ] + monkeypatch.setattr(fetch_mod, "web_search", lambda q, max_results=None: matched) + monkeypatch.setattr( + fetch_mod, + "fetch_public_bytes", + lambda url, timeout=None: (b"A" * 200, "", "text/plain"), + ) + result = fetch_mod.search_and_fetch("query", "txt", max_chars=50) + _, _, next_token = parse_paged_result(result) + assert next_token is not None + payload = parse_token(next_token, "search_and_fetch") + embedded = payload.get("matched", []) + assert embedded, "matched list should be embedded for resume" + for entry in embedded: + assert "body" not in entry, "body must be stripped from the token" + assert "title" in entry and "href" in entry + + +def test_search_and_fetch_token_size_reduced(monkeypatch): + """A large body snippet must not inflate the token.""" + from ollama_web.tools import fetch as fetch_mod + + matched = [ + { + "title": "File1", + "href": "https://example.com/f1.txt", + "body": "B" * 10_000, + } + ] + monkeypatch.setattr(fetch_mod, "web_search", lambda q, max_results=None: matched) + monkeypatch.setattr( + fetch_mod, + "fetch_public_bytes", + lambda url, timeout=None: (b"A" * 200, "", "text/plain"), + ) + result = fetch_mod.search_and_fetch("query", "txt", max_chars=50) + _, _, next_token = parse_paged_result(result) + assert next_token is not None + # Token should be far smaller than the 10KB body it would otherwise carry. + assert len(next_token) < 1000 + + +def test_search_and_fetch_continue_with_slim_token(monkeypatch): + """Resuming with a slim (body-less) token walks files correctly.""" + from ollama_web.tools import fetch as fetch_mod + + matched = [ + {"title": "File1", "href": "https://example.com/f1.txt", "body": "ignored1"}, + {"title": "File2", "href": "https://example.com/f2.txt", "body": "ignored2"}, + ] + monkeypatch.setattr(fetch_mod, "web_search", lambda q, max_results=None: matched) + contents = { + "https://example.com/f1.txt": b"FILE1 " + b"A" * 200, + "https://example.com/f2.txt": b"FILE2 " + b"B" * 200, + } + monkeypatch.setattr( + fetch_mod, + "fetch_public_bytes", + lambda url, timeout=None: (contents[url], "", "text/plain"), + ) + + seen = [] + token = None + for _ in range(20): + result = fetch_mod.search_and_fetch("query", "txt", max_chars=50, continue_from=token) + body_text, has_more, token = parse_paged_result(result) + if "File1" in body_text and "File1" not in seen: + seen.append("File1") + if "File2" in body_text and "File2" not in seen: + seen.append("File2") + if not has_more: + break + assert seen == ["File1", "File2"] + + +def test_search_and_fetch_pagination_with_nonascii_title(monkeypatch): + """Non-ASCII titles/URLs survive a full pagination round-trip.""" + from ollama_web.tools import fetch as fetch_mod + + matched = [ + {"title": "日本語–タイトル", "href": "https://example.com/論文.txt", "body": ""}, + ] + monkeypatch.setattr(fetch_mod, "web_search", lambda q, max_results=None: matched) + monkeypatch.setattr( + fetch_mod, + "fetch_public_bytes", + lambda url, timeout=None: (b"A" * 200, "", "text/plain"), + ) + result = fetch_mod.search_and_fetch("query", "txt", max_chars=50) + body_text, has_more, next_token = parse_paged_result(result) + assert "日本語–タイトル" in body_text + assert has_more is True + # Resuming must not raise even though the token carries non-ASCII fields. + result2 = fetch_mod.search_and_fetch("query", "txt", max_chars=50, continue_from=next_token) + assert isinstance(result2, str) + + # --------------------------------------------------------------------------- # pagination helper tests # --------------------------------------------------------------------------- |