diff options
| author | tanaka takashi <isazaka@gmail.com> | 2026-08-26 19:23:40 +0900 |
|---|---|---|
| committer | tanaka takashi <isazaka@gmail.com> | 2026-08-26 19:23:40 +0900 |
| commit | b495dde6810882efc8481ebebd22ff26a6afb902 (patch) | |
| tree | c255e809c8b4bc23341cd04c8f2d90020e733bda | |
| parent | 33f6b4dbb69e9366e312a8b44dbed9d22e7cba5b (diff) | |
fix(tools): PDFの会話間継続をトークン単独で再開可能にする
長いPDFを複数会話に分けて取得する際、モデルがurl_or_pathを省略したり、過去の添付IDや古い継続トークンを組み合わせたりして、先頭または以前のページへ戻る問題を改善する。PDFカーソル自体は正しく進んでいたため、汎用LLM層やセッション形式を変更せず、pdf_to_textの入出力契約だけを明確化する。
- continue_fromを自己完結した再開情報として扱い、PDFソース、ページ番号、ページ内文字位置をトークンから復元
- 継続時はurl_or_pathを省略可能にし、競合する値が渡されてもトークン内のソースを優先
- Ollama生成スキーマでもurl_or_pathを任意にして、continue_from単独のツール呼び出しを許可
- 未完了結果へ最新トークン、旧トークンの置換、単独再開、上限時の保持を表すPDF専用メタデータを追加
- 継続メタデータを固定キーと真偽値で表現し、会話言語に依存しない形式へ統一
- 無効トークン時は先頭再試行を禁止し、直前の最新トークンを再利用する機械可読情報を返却
- メタデータをpage-info側へ配置して本文連結を汚さず、結果全体を6000文字以内に再調整
- llm.py、チャット処理、セッションJSON、scrape_url、search_and_fetch、MCPツールは変更しない
検証:
- pytest: 132 passed
- ruff check: passed
- mypy src: passed
- 1463891.1463912.pdf: 12ページ、52577文字をcontinue_from単独の10回で完全再構成
- 指定PDFの最大ツール結果長はメタデータ込み5992文字
| -rw-r--r-- | docs/tools-schema.md | 10 | ||||
| -rw-r--r-- | src/ollama_web/tools/pdf.py | 80 | ||||
| -rw-r--r-- | tests/test_tools.py | 62 |
3 files changed, 125 insertions, 27 deletions
diff --git a/docs/tools-schema.md b/docs/tools-schema.md index 874640a..5b920a4 100644 --- a/docs/tools-schema.md +++ b/docs/tools-schema.md @@ -120,13 +120,13 @@ next_token: {token} | パラメータ | 型 | 必須 | デフォルト | 説明 | |------------|----|------|------------|------| -| `url_or_path` | `str` | ✅ | — | HTTP(S) PDF URL。チャット内では現在のセッションの `attachment:<file_id>` も利用可能。任意のローカルパスは拒否 | +| `url_or_path` | `str \| None` | 初回のみ | `None` | 初回取得用のHTTP(S) PDF URL。チャット内では現在のセッションの `attachment:<file_id>` も利用可能。`continue_from` 指定時はトークンから復元されるため省略可能。任意のローカルパスは拒否 | | `max_chars` | `int \| None` | ❌ | [`settings.fetch_max_chars`](src/ollama_web/config.py:52) (=8000) | 1ページあたりの返却文字数上限 | -| `continue_from` | `str \| None` | ❌ | `None` | 前回戻り値の `next_token`。指定時は続きページを取得 | +| `continue_from` | `str \| None` | ❌ | `None` | 同じPDFの直前の戻り値にある `next_token`。PDFソースと次の開始位置を復元して続きページを取得 | ### 戻り値 -`str` — 1ページ分のテキスト、またはエラーメッセージ。続きがある場合は `scrape_url` と同様のページ情報フッターが付与される。 +`str` — 1ページ分のテキスト、またはエラーメッセージ。続きがある場合はページ情報フッターに加え、最新トークンが以前の同一PDF用トークンを置き換えること、上限到達時は最終回答へそのまま残すこと、次回は `continue_from` 単独で再開できることを示すPDF専用メタデータが付与される。このメタデータは会話言語に依存しない固定キーと真偽値で表現し、本文の連結結果には含まれない。 ### 制約 @@ -139,10 +139,10 @@ next_token: {token} ### 処理フロー -1. `continue_from` があればトークンから `(url, page_index, page_char_offset)` を復元 +1. `continue_from` があれば、`url_or_path` の有無や値にかかわらずトークンから `(url, page_index, page_char_offset)` を復元 2. PDF を全件取得・解析 3. カーソル位置から、結果全体の上限内で実際に返せる文字数だけ抽出 -4. 未返却文字または残りページがあれば、その直後を指す `next_token` を付与 +4. 未返却文字または残りページがあれば、その直後を指す `next_token` とPDF専用の再開指示を付与 --- diff --git a/src/ollama_web/tools/pdf.py b/src/ollama_web/tools/pdf.py index 4faf8ac..657edf9 100644 --- a/src/ollama_web/tools/pdf.py +++ b/src/ollama_web/tools/pdf.py @@ -12,6 +12,7 @@ from .helper.pagination import ( PageSizeError, format_page_to_limit, make_token, + parse_paged_result, parse_token, ) from .helper.pdf import extract_pdf_page @@ -23,6 +24,16 @@ logger = logging.getLogger("ollama_web.tools.pdf") AttachmentLoader = Callable[[str], bytes | None] +_RESUME_RULE = ( + "\nresume_tool: pdf_to_text" + "\nresume_page_index: {page_index}" + "\nresume_page_char_offset: {page_char_offset}" + "\nnext_token_is_latest: true" + "\nsupersedes_previous_tokens: true" + "\nresume_with_continue_from_only: true" + "\npreserve_next_token_on_tool_limit: true" +) + def _cursor_value(payload: dict[str, Any], key: str, fallback: int = 0) -> int: raw = payload.get(key, fallback) @@ -38,26 +49,32 @@ def _cursor_value(payload: dict[str, Any], key: str, fallback: int = 0) -> int: def _pdf_to_text( - url_or_path: str, + url_or_path: str | None, max_chars: int | None, continue_from: str | None, attachment_loader: AttachmentLoader | None, ) -> str: - if not url_or_path or not url_or_path.strip(): - return "Invalid PDF source." - - src = url_or_path.strip() + src = (url_or_path or "").strip() page_index = 0 page_char_offset = 0 if continue_from: try: payload = parse_token(continue_from, _TOOL_NAME) - src = str(payload.get("url") or src) + # A continuation token is self-contained. Its source is + # authoritative so a later conversation cannot accidentally pair + # the cursor with a stale attachment ID or URL. + src = str(payload["url"]) # Version 1 used offset as the PDF page index. page_index = _cursor_value(payload, "page_index", int(payload["offset"])) page_char_offset = _cursor_value(payload, "page_char_offset") except ValueError as exc: - return f"Invalid continuation token: {exc}" + return ( + f"Invalid continuation token: {exc}\n" + "restart_from_beginning: false\n" + "required_action: reuse_latest_next_token" + ) + elif not src: + return "Invalid PDF source." if src.startswith(_ATTACHMENT_PREFIX): file_id = src.removeprefix(_ATTACHMENT_PREFIX) @@ -98,12 +115,33 @@ def _pdf_to_text( ) try: - result = format_page_to_limit( - page_factory, - token_factory, - requested_chars=requested, - result_limit=settings.max_tool_result_chars, - ) + result_limit = settings.max_tool_result_chars + render_limit = result_limit + result = "" + for _ in range(4): + result = format_page_to_limit( + page_factory, + token_factory, + requested_chars=requested, + result_limit=render_limit, + ) + _, has_more, next_token = parse_paged_result(result) + if not has_more or next_token is None: + break + next_payload = parse_token(next_token, _TOOL_NAME) + resume_metadata = _RESUME_RULE.format( + page_index=_cursor_value(next_payload, "page_index", next_payload["offset"]), + page_char_offset=_cursor_value(next_payload, "page_char_offset"), + ) + overflow = len(result) + len(resume_metadata) - result_limit + if overflow <= 0: + result += resume_metadata + break + render_limit -= overflow + 8 + if render_limit <= 0: + raise PageSizeError("PDF continuation metadata exceeds the result limit.") + else: + raise PageSizeError("PDF continuation metadata could not fit the result limit.") except PageSizeError as exc: return f"PDF pagination failed: {exc}" @@ -119,17 +157,21 @@ def _pdf_to_text( def pdf_to_text( - url_or_path: str, max_chars: int | None = None, continue_from: str | None = None + url_or_path: str | None = None, + max_chars: int | None = None, + continue_from: str | None = None, ) -> str: """Extract a losslessly paged PDF from an HTTP(S) URL. Args: - url_or_path: HTTP(S) PDF URL. In chat, a session attachment URI may also - be supplied by the attachment context. Arbitrary filesystem paths are - never accepted. + url_or_path: HTTP(S) PDF URL for the first call. In chat, a session + attachment URI may also be supplied by the attachment context. + Omit this argument when continue_from is supplied. Arbitrary + filesystem paths are never accepted. max_chars: Requested maximum source characters for this page. The tool may return fewer characters so metadata and its token remain intact. - continue_from: Opaque continuation token from the preceding result. + continue_from: Opaque, self-contained continuation token from the most + recent result for this PDF. It restores both the source and cursor. """ return _pdf_to_text(url_or_path, max_chars, continue_from, None) @@ -138,7 +180,7 @@ def make_session_pdf_tool(attachment_loader: AttachmentLoader) -> Callable[..., """Create a session-scoped ``pdf_to_text`` supporting attachment IDs.""" def session_pdf_to_text( - url_or_path: str, + url_or_path: str | None = None, max_chars: int | None = None, continue_from: str | None = None, ) -> str: diff --git a/tests/test_tools.py b/tests/test_tools.py index 95f6514..4960d1d 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -312,6 +312,8 @@ def test_pdf_to_text_continue_from_invalid_token(): result = pdf_to_text("https://example.com/doc.pdf", continue_from="bad-token") assert "Invalid continuation token" in result + assert "restart_from_beginning: false" in result + assert "required_action: reuse_latest_next_token" in result def test_pdf_to_text_rejects_negative_page_character_cursor(): @@ -351,16 +353,67 @@ def test_pdf_to_text_pagination_roundtrip(monkeypatch): token = None for _ in range(50): result = pdf_mod.pdf_to_text( - "https://example.com/doc.pdf", max_chars=8000, continue_from=token + "https://example.com/doc.pdf" if token is None else "", + max_chars=8000, + continue_from=token, ) assert len(result) <= 6000 body_text, has_more, token = parse_paged_result(result) + assert "next_token_is_latest:" not in body_text + if has_more: + assert "resume_tool: pdf_to_text" in result + assert "next_token_is_latest: true" in result + assert "supersedes_previous_tokens: true" in result + assert "resume_with_continue_from_only: true" in result + assert "preserve_next_token_on_tool_limit: true" in result + else: + assert "resume_tool:" not in result collected += body_text if not has_more: break assert collected == expected +def test_pdf_to_text_continuation_token_source_is_authoritative(monkeypatch): + """A continuation restores its URL and does not require url_or_path.""" + from ollama_web.tools import pdf as pdf_mod + + data = _make_multipage_pdf(["SOURCE-" + "S" * 9000]) + fetched: list[str] = [] + + def fake_fetch(url: str, timeout=None): + fetched.append(url) + return data, url, "application/pdf" + + monkeypatch.setattr(pdf_mod, "fetch_public_bytes", fake_fetch) + first = pdf_mod.pdf_to_text("https://example.com/source.pdf") + _, has_more, token = parse_paged_result(first) + assert has_more is True + assert token is not None + + resumed = pdf_mod.pdf_to_text(continue_from=token) + body, _, _ = parse_paged_result(resumed) + assert body + assert fetched == [ + "https://example.com/source.pdf", + "https://example.com/source.pdf", + ] + + # Even a conflicting argument cannot redirect a valid continuation. + pdf_mod.pdf_to_text("https://attacker.example/wrong.pdf", continue_from=token) + assert fetched[-1] == "https://example.com/source.pdf" + + +def test_pdf_to_text_ollama_schema_allows_token_only_continuation(): + from ollama._utils import convert_function_to_tool + + from ollama_web.tools.pdf import pdf_to_text + + schema = convert_function_to_tool(pdf_to_text).function.parameters + assert "url_or_path" not in (schema.required or []) + assert "continue_from" not in (schema.required or []) + + def test_session_pdf_tool_reads_original_bytes_and_is_session_scoped(tmp_path): from pypdf import PdfReader @@ -386,7 +439,10 @@ def test_session_pdf_tool_reads_original_bytes_and_is_session_scoped(tmp_path): token = None collected = "" for _ in range(10): - result = tool(f"attachment:{chat_file.id}", continue_from=token) + result = tool( + f"attachment:{chat_file.id}" if token is None else "", + continue_from=token, + ) body, has_more, token = parse_paged_result(result) collected += body if not has_more: @@ -491,7 +547,7 @@ def test_attachment_context_keeps_pdf_continuation_and_registry_can_resume(tmp_p token = first_result.split("next_token:", 1)[1].splitlines()[0].strip() resumed = registry.execute( "pdf_to_text", - {"url_or_path": f"attachment:{chat_file.id}", "continue_from": token}, + {"continue_from": token}, ) assert "unavailable" not in resumed.lower() |
