summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authortanaka takashi <isazaka@gmail.com>2026-08-26 19:23:40 +0900
committertanaka takashi <isazaka@gmail.com>2026-08-26 19:23:40 +0900
commitb495dde6810882efc8481ebebd22ff26a6afb902 (patch)
treec255e809c8b4bc23341cd04c8f2d90020e733bda
parent33f6b4dbb69e9366e312a8b44dbed9d22e7cba5b (diff)
fix(tools): PDFの会話間継続をトークン単独で再開可能にする
長いPDFを複数会話に分けて取得する際、モデルがurl_or_pathを省略したり、過去の添付IDや古い継続トークンを組み合わせたりして、先頭または以前のページへ戻る問題を改善する。PDFカーソル自体は正しく進んでいたため、汎用LLM層やセッション形式を変更せず、pdf_to_textの入出力契約だけを明確化する。 - continue_fromを自己完結した再開情報として扱い、PDFソース、ページ番号、ページ内文字位置をトークンから復元 - 継続時はurl_or_pathを省略可能にし、競合する値が渡されてもトークン内のソースを優先 - Ollama生成スキーマでもurl_or_pathを任意にして、continue_from単独のツール呼び出しを許可 - 未完了結果へ最新トークン、旧トークンの置換、単独再開、上限時の保持を表すPDF専用メタデータを追加 - 継続メタデータを固定キーと真偽値で表現し、会話言語に依存しない形式へ統一 - 無効トークン時は先頭再試行を禁止し、直前の最新トークンを再利用する機械可読情報を返却 - メタデータをpage-info側へ配置して本文連結を汚さず、結果全体を6000文字以内に再調整 - llm.py、チャット処理、セッションJSON、scrape_url、search_and_fetch、MCPツールは変更しない 検証: - pytest: 132 passed - ruff check: passed - mypy src: passed - 1463891.1463912.pdf: 12ページ、52577文字をcontinue_from単独の10回で完全再構成 - 指定PDFの最大ツール結果長はメタデータ込み5992文字
-rw-r--r--docs/tools-schema.md10
-rw-r--r--src/ollama_web/tools/pdf.py80
-rw-r--r--tests/test_tools.py62
3 files changed, 125 insertions, 27 deletions
diff --git a/docs/tools-schema.md b/docs/tools-schema.md
index 874640a..5b920a4 100644
--- a/docs/tools-schema.md
+++ b/docs/tools-schema.md
@@ -120,13 +120,13 @@ next_token: {token}
| パラメータ | 型 | 必須 | デフォルト | 説明 |
|------------|----|------|------------|------|
-| `url_or_path` | `str` | ✅ | — | HTTP(S) PDF URL。チャット内では現在のセッションの `attachment:<file_id>` も利用可能。任意のローカルパスは拒否 |
+| `url_or_path` | `str \| None` | 初回のみ | `None` | 初回取得用のHTTP(S) PDF URL。チャット内では現在のセッションの `attachment:<file_id>` も利用可能。`continue_from` 指定時はトークンから復元されるため省略可能。任意のローカルパスは拒否 |
| `max_chars` | `int \| None` | ❌ | [`settings.fetch_max_chars`](src/ollama_web/config.py:52) (=8000) | 1ページあたりの返却文字数上限 |
-| `continue_from` | `str \| None` | ❌ | `None` | 前回戻り値の `next_token`。指定時は続きページを取得 |
+| `continue_from` | `str \| None` | ❌ | `None` | 同じPDFの直前の戻り値にある `next_token`。PDFソースと次の開始位置を復元して続きページを取得 |
### 戻り値
-`str` — 1ページ分のテキスト、またはエラーメッセージ。続きがある場合は `scrape_url` と同様のページ情報フッターが付与される。
+`str` — 1ページ分のテキスト、またはエラーメッセージ。続きがある場合はページ情報フッターに加え、最新トークンが以前の同一PDF用トークンを置き換えること、上限到達時は最終回答へそのまま残すこと、次回は `continue_from` 単独で再開できることを示すPDF専用メタデータが付与される。このメタデータは会話言語に依存しない固定キーと真偽値で表現し、本文の連結結果には含まれない。
### 制約
@@ -139,10 +139,10 @@ next_token: {token}
### 処理フロー
-1. `continue_from` があればトークンから `(url, page_index, page_char_offset)` を復元
+1. `continue_from` があれば、`url_or_path` の有無や値にかかわらずトークンから `(url, page_index, page_char_offset)` を復元
2. PDF を全件取得・解析
3. カーソル位置から、結果全体の上限内で実際に返せる文字数だけ抽出
-4. 未返却文字または残りページがあれば、その直後を指す `next_token` を付与
+4. 未返却文字または残りページがあれば、その直後を指す `next_token` とPDF専用の再開指示を付与
---
diff --git a/src/ollama_web/tools/pdf.py b/src/ollama_web/tools/pdf.py
index 4faf8ac..657edf9 100644
--- a/src/ollama_web/tools/pdf.py
+++ b/src/ollama_web/tools/pdf.py
@@ -12,6 +12,7 @@ from .helper.pagination import (
PageSizeError,
format_page_to_limit,
make_token,
+ parse_paged_result,
parse_token,
)
from .helper.pdf import extract_pdf_page
@@ -23,6 +24,16 @@ logger = logging.getLogger("ollama_web.tools.pdf")
AttachmentLoader = Callable[[str], bytes | None]
+_RESUME_RULE = (
+ "\nresume_tool: pdf_to_text"
+ "\nresume_page_index: {page_index}"
+ "\nresume_page_char_offset: {page_char_offset}"
+ "\nnext_token_is_latest: true"
+ "\nsupersedes_previous_tokens: true"
+ "\nresume_with_continue_from_only: true"
+ "\npreserve_next_token_on_tool_limit: true"
+)
+
def _cursor_value(payload: dict[str, Any], key: str, fallback: int = 0) -> int:
raw = payload.get(key, fallback)
@@ -38,26 +49,32 @@ def _cursor_value(payload: dict[str, Any], key: str, fallback: int = 0) -> int:
def _pdf_to_text(
- url_or_path: str,
+ url_or_path: str | None,
max_chars: int | None,
continue_from: str | None,
attachment_loader: AttachmentLoader | None,
) -> str:
- if not url_or_path or not url_or_path.strip():
- return "Invalid PDF source."
-
- src = url_or_path.strip()
+ src = (url_or_path or "").strip()
page_index = 0
page_char_offset = 0
if continue_from:
try:
payload = parse_token(continue_from, _TOOL_NAME)
- src = str(payload.get("url") or src)
+ # A continuation token is self-contained. Its source is
+ # authoritative so a later conversation cannot accidentally pair
+ # the cursor with a stale attachment ID or URL.
+ src = str(payload["url"])
# Version 1 used offset as the PDF page index.
page_index = _cursor_value(payload, "page_index", int(payload["offset"]))
page_char_offset = _cursor_value(payload, "page_char_offset")
except ValueError as exc:
- return f"Invalid continuation token: {exc}"
+ return (
+ f"Invalid continuation token: {exc}\n"
+ "restart_from_beginning: false\n"
+ "required_action: reuse_latest_next_token"
+ )
+ elif not src:
+ return "Invalid PDF source."
if src.startswith(_ATTACHMENT_PREFIX):
file_id = src.removeprefix(_ATTACHMENT_PREFIX)
@@ -98,12 +115,33 @@ def _pdf_to_text(
)
try:
- result = format_page_to_limit(
- page_factory,
- token_factory,
- requested_chars=requested,
- result_limit=settings.max_tool_result_chars,
- )
+ result_limit = settings.max_tool_result_chars
+ render_limit = result_limit
+ result = ""
+ for _ in range(4):
+ result = format_page_to_limit(
+ page_factory,
+ token_factory,
+ requested_chars=requested,
+ result_limit=render_limit,
+ )
+ _, has_more, next_token = parse_paged_result(result)
+ if not has_more or next_token is None:
+ break
+ next_payload = parse_token(next_token, _TOOL_NAME)
+ resume_metadata = _RESUME_RULE.format(
+ page_index=_cursor_value(next_payload, "page_index", next_payload["offset"]),
+ page_char_offset=_cursor_value(next_payload, "page_char_offset"),
+ )
+ overflow = len(result) + len(resume_metadata) - result_limit
+ if overflow <= 0:
+ result += resume_metadata
+ break
+ render_limit -= overflow + 8
+ if render_limit <= 0:
+ raise PageSizeError("PDF continuation metadata exceeds the result limit.")
+ else:
+ raise PageSizeError("PDF continuation metadata could not fit the result limit.")
except PageSizeError as exc:
return f"PDF pagination failed: {exc}"
@@ -119,17 +157,21 @@ def _pdf_to_text(
def pdf_to_text(
- url_or_path: str, max_chars: int | None = None, continue_from: str | None = None
+ url_or_path: str | None = None,
+ max_chars: int | None = None,
+ continue_from: str | None = None,
) -> str:
"""Extract a losslessly paged PDF from an HTTP(S) URL.
Args:
- url_or_path: HTTP(S) PDF URL. In chat, a session attachment URI may also
- be supplied by the attachment context. Arbitrary filesystem paths are
- never accepted.
+ url_or_path: HTTP(S) PDF URL for the first call. In chat, a session
+ attachment URI may also be supplied by the attachment context.
+ Omit this argument when continue_from is supplied. Arbitrary
+ filesystem paths are never accepted.
max_chars: Requested maximum source characters for this page. The tool
may return fewer characters so metadata and its token remain intact.
- continue_from: Opaque continuation token from the preceding result.
+ continue_from: Opaque, self-contained continuation token from the most
+ recent result for this PDF. It restores both the source and cursor.
"""
return _pdf_to_text(url_or_path, max_chars, continue_from, None)
@@ -138,7 +180,7 @@ def make_session_pdf_tool(attachment_loader: AttachmentLoader) -> Callable[...,
"""Create a session-scoped ``pdf_to_text`` supporting attachment IDs."""
def session_pdf_to_text(
- url_or_path: str,
+ url_or_path: str | None = None,
max_chars: int | None = None,
continue_from: str | None = None,
) -> str:
diff --git a/tests/test_tools.py b/tests/test_tools.py
index 95f6514..4960d1d 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -312,6 +312,8 @@ def test_pdf_to_text_continue_from_invalid_token():
result = pdf_to_text("https://example.com/doc.pdf", continue_from="bad-token")
assert "Invalid continuation token" in result
+ assert "restart_from_beginning: false" in result
+ assert "required_action: reuse_latest_next_token" in result
def test_pdf_to_text_rejects_negative_page_character_cursor():
@@ -351,16 +353,67 @@ def test_pdf_to_text_pagination_roundtrip(monkeypatch):
token = None
for _ in range(50):
result = pdf_mod.pdf_to_text(
- "https://example.com/doc.pdf", max_chars=8000, continue_from=token
+ "https://example.com/doc.pdf" if token is None else "",
+ max_chars=8000,
+ continue_from=token,
)
assert len(result) <= 6000
body_text, has_more, token = parse_paged_result(result)
+ assert "next_token_is_latest:" not in body_text
+ if has_more:
+ assert "resume_tool: pdf_to_text" in result
+ assert "next_token_is_latest: true" in result
+ assert "supersedes_previous_tokens: true" in result
+ assert "resume_with_continue_from_only: true" in result
+ assert "preserve_next_token_on_tool_limit: true" in result
+ else:
+ assert "resume_tool:" not in result
collected += body_text
if not has_more:
break
assert collected == expected
+def test_pdf_to_text_continuation_token_source_is_authoritative(monkeypatch):
+ """A continuation restores its URL and does not require url_or_path."""
+ from ollama_web.tools import pdf as pdf_mod
+
+ data = _make_multipage_pdf(["SOURCE-" + "S" * 9000])
+ fetched: list[str] = []
+
+ def fake_fetch(url: str, timeout=None):
+ fetched.append(url)
+ return data, url, "application/pdf"
+
+ monkeypatch.setattr(pdf_mod, "fetch_public_bytes", fake_fetch)
+ first = pdf_mod.pdf_to_text("https://example.com/source.pdf")
+ _, has_more, token = parse_paged_result(first)
+ assert has_more is True
+ assert token is not None
+
+ resumed = pdf_mod.pdf_to_text(continue_from=token)
+ body, _, _ = parse_paged_result(resumed)
+ assert body
+ assert fetched == [
+ "https://example.com/source.pdf",
+ "https://example.com/source.pdf",
+ ]
+
+ # Even a conflicting argument cannot redirect a valid continuation.
+ pdf_mod.pdf_to_text("https://attacker.example/wrong.pdf", continue_from=token)
+ assert fetched[-1] == "https://example.com/source.pdf"
+
+
+def test_pdf_to_text_ollama_schema_allows_token_only_continuation():
+ from ollama._utils import convert_function_to_tool
+
+ from ollama_web.tools.pdf import pdf_to_text
+
+ schema = convert_function_to_tool(pdf_to_text).function.parameters
+ assert "url_or_path" not in (schema.required or [])
+ assert "continue_from" not in (schema.required or [])
+
+
def test_session_pdf_tool_reads_original_bytes_and_is_session_scoped(tmp_path):
from pypdf import PdfReader
@@ -386,7 +439,10 @@ def test_session_pdf_tool_reads_original_bytes_and_is_session_scoped(tmp_path):
token = None
collected = ""
for _ in range(10):
- result = tool(f"attachment:{chat_file.id}", continue_from=token)
+ result = tool(
+ f"attachment:{chat_file.id}" if token is None else "",
+ continue_from=token,
+ )
body, has_more, token = parse_paged_result(result)
collected += body
if not has_more:
@@ -491,7 +547,7 @@ def test_attachment_context_keeps_pdf_continuation_and_registry_can_resume(tmp_p
token = first_result.split("next_token:", 1)[1].splitlines()[0].strip()
resumed = registry.execute(
"pdf_to_text",
- {"url_or_path": f"attachment:{chat_file.id}", "continue_from": token},
+ {"continue_from": token},
)
assert "unavailable" not in resumed.lower()