summaryrefslogtreecommitdiff
path: root/src/ollama_web/tools/pdf.py
diff options
context:
space:
mode:
authortanaka takashi <isazaka@gmail.com>2026-08-23 17:28:10 +0900
committertanaka takashi <isazaka@gmail.com>2026-08-23 17:28:39 +0900
commit33f6b4dbb69e9366e312a8b44dbed9d22e7cba5b (patch)
tree29a7480317af94fefe9f69a53095af2f233207ce /src/ollama_web/tools/pdf.py
parent68969bfbeea68ded7ff863d229760562072f910b (diff)
fix(tools): PDFページングを損失なしカーソル方式へ再設計
PDF本文をページ単位で抽出した後に文字数上限で切断しつつ、継続位置だけ次ページへ進めていたため、ページ末尾が復元不能になる問題を修正する。ツール側の本文上限とLLM側の結果上限の不整合、25ページの処理上限、添付経路での継続情報欠落も共通契約で解消する。 - 共通PageResultと、メタデータ込みの結果上限内で本文量を決めるフォーマッターを追加 - v2トークンにPDFのページ番号とページ内文字位置を保持し、v1トークンの読み取り互換性を維持 - pdf_to_text、scrape_url、search_and_fetchを返却済み本文だけカーソルが進む方式へ統一 - max_pdf_pagesを文書全体ではなく1呼び出しの処理上限として扱い、長い単一ページと26ページ目以降の継続取得に対応 - attachment:<file_id>をセッション限定で解決し、既存添付も保存済みプレビューではなく元PDFから再解析 - Web取得ファイルをセッションとURL単位で重複排除し、HTMLおよびテキスト経路の全文ページングも維持 - 正常なページング結果の後段切断を禁止し、結果上限・コンテキスト予算・ラウンド上限では未取得状態を明示 - 日英プロンプト、UIメッセージ、ツールスキーマ、変更履歴を実装へ同期 - 実PDF、30ページ超、長い単一ページ、旧トークン、セッション分離、コンテキスト境界の回帰テストを追加 検証: - pytest: 130 passed - ruff check src tests: passed - mypy src: passed - c2144.pdf: 19ページ、抽出本文46015文字(ページ区切り込み46051文字)を8回で重複・欠落なく完全再構成
Diffstat (limited to 'src/ollama_web/tools/pdf.py')
-rw-r--r--src/ollama_web/tools/pdf.py153
1 files changed, 117 insertions, 36 deletions
diff --git a/src/ollama_web/tools/pdf.py b/src/ollama_web/tools/pdf.py
index 9fdb497..4faf8ac 100644
--- a/src/ollama_web/tools/pdf.py
+++ b/src/ollama_web/tools/pdf.py
@@ -2,67 +2,148 @@
from __future__ import annotations
+import logging
+from collections.abc import Callable
+from typing import Any
+
from ..config import settings
-from .helper.pagination import format_paged_result, make_token, parse_token
-from .helper.pdf import extract_pdf_text
+from .helper.pagination import (
+ PageResult,
+ PageSizeError,
+ format_page_to_limit,
+ make_token,
+ parse_token,
+)
+from .helper.pdf import extract_pdf_page
from .helper.safe_http import UnsafeURL, fetch_public_bytes
_TOOL_NAME = "pdf_to_text"
+_ATTACHMENT_PREFIX = "attachment:"
+logger = logging.getLogger("ollama_web.tools.pdf")
+AttachmentLoader = Callable[[str], bytes | None]
-def pdf_to_text(
- url_or_path: str, max_chars: int | None = None, continue_from: str | None = None
-) -> str:
- """Extract text from a local PDF path or remote URL.
- The text is returned one page at a time. When more pages remain, a
- ``next_token`` is appended that can be passed back via ``continue_from``
- to fetch the next batch of pages.
+def _cursor_value(payload: dict[str, Any], key: str, fallback: int = 0) -> int:
+ raw = payload.get(key, fallback)
+ if isinstance(raw, bool) or not isinstance(raw, (int, str)):
+ raise ValueError(f"Token {key!r} is not an integer.")
+ try:
+ value = int(raw)
+ except (TypeError, ValueError) as exc:
+ raise ValueError(f"Token {key!r} is not an integer.") from exc
+ if value < 0:
+ raise ValueError(f"Token {key!r} must not be negative.")
+ return value
- Args:
- url_or_path: File path or URL pointing to a PDF.
- max_chars: Maximum characters to return per page.
- continue_from: Opaque continuation token returned in a previous call's
- ``next_token``. When provided, extraction resumes from the encoded
- page offset.
-
- Returns:
- Extracted plain text for this page, followed by a ``---page-info---``
- footer when more pages are available, or an error message.
- """
+
+def _pdf_to_text(
+ url_or_path: str,
+ max_chars: int | None,
+ continue_from: str | None,
+ attachment_loader: AttachmentLoader | None,
+) -> str:
if not url_or_path or not url_or_path.strip():
return "Invalid PDF source."
src = url_or_path.strip()
-
- # Resolve the resumption page offset from the continuation token, if any.
- page_offset = 0
+ page_index = 0
+ page_char_offset = 0
if continue_from:
try:
payload = parse_token(continue_from, _TOOL_NAME)
+ src = str(payload.get("url") or src)
+ # Version 1 used offset as the PDF page index.
+ page_index = _cursor_value(payload, "page_index", int(payload["offset"]))
+ page_char_offset = _cursor_value(payload, "page_char_offset")
except ValueError as exc:
return f"Invalid continuation token: {exc}"
- src = str(payload.get("url") or src)
- page_offset = int(payload.get("offset", 0))
- if page_offset < 0:
- page_offset = 0
- data: bytes | None = None
- if src.startswith("http://") or src.startswith("https://"):
+ if src.startswith(_ATTACHMENT_PREFIX):
+ file_id = src.removeprefix(_ATTACHMENT_PREFIX)
+ data = attachment_loader(file_id) if attachment_loader is not None else None
+ if data is None:
+ return "Attachment PDF is unavailable in this session."
+ source_kind = "attachment"
+ elif src.startswith("http://") or src.startswith("https://"):
try:
- data, _, _ = fetch_public_bytes(src, timeout=settings.scrape_timeout)
+ data, final_url, _ = fetch_public_bytes(src, timeout=settings.scrape_timeout)
+ src = final_url or src
except UnsafeURL as exc:
return f"Blocked URL: {exc}"
except Exception as exc: # noqa: BLE001
return f"Failed to fetch PDF: {exc}"
+ source_kind = "remote"
else:
return "Only http(s) PDF URLs are supported."
- if data is None:
- return "No PDF data loaded."
+ requested = max_chars if max_chars is not None and max_chars > 0 else settings.fetch_max_chars
+
+ def page_factory(body_budget: int) -> PageResult:
+ return extract_pdf_page(
+ data,
+ max_chars=body_budget,
+ page_index=page_index,
+ page_char_offset=page_char_offset,
+ max_size_mb=(settings.max_upload_mb if source_kind == "attachment" else None),
+ )
+
+ def token_factory(cursor: dict[str, int]) -> str:
+ return make_token(
+ _TOOL_NAME,
+ src,
+ cursor["page_index"],
+ page_index=cursor["page_index"],
+ page_char_offset=cursor["page_char_offset"],
+ )
+
+ try:
+ result = format_page_to_limit(
+ page_factory,
+ token_factory,
+ requested_chars=requested,
+ result_limit=settings.max_tool_result_chars,
+ )
+ except PageSizeError as exc:
+ return f"PDF pagination failed: {exc}"
- text, has_more, next_page_offset = extract_pdf_text(
- data, max_chars=max_chars, page_offset=page_offset
+ logger.info(
+ "pdf page source=%s page_index=%d page_char_offset=%d result_chars=%d has_more=%s",
+ source_kind,
+ page_index,
+ page_char_offset,
+ len(result),
+ "---page-info---" in result,
)
- next_token = make_token(_TOOL_NAME, src, next_page_offset) if has_more else None
- return format_paged_result(text, has_more=has_more, next_token=next_token)
+ return result
+
+
+def pdf_to_text(
+ url_or_path: str, max_chars: int | None = None, continue_from: str | None = None
+) -> str:
+ """Extract a losslessly paged PDF from an HTTP(S) URL.
+
+ Args:
+ url_or_path: HTTP(S) PDF URL. In chat, a session attachment URI may also
+ be supplied by the attachment context. Arbitrary filesystem paths are
+ never accepted.
+ max_chars: Requested maximum source characters for this page. The tool
+ may return fewer characters so metadata and its token remain intact.
+ continue_from: Opaque continuation token from the preceding result.
+ """
+ return _pdf_to_text(url_or_path, max_chars, continue_from, None)
+
+
+def make_session_pdf_tool(attachment_loader: AttachmentLoader) -> Callable[..., str]:
+ """Create a session-scoped ``pdf_to_text`` supporting attachment IDs."""
+
+ def session_pdf_to_text(
+ url_or_path: str,
+ max_chars: int | None = None,
+ continue_from: str | None = None,
+ ) -> str:
+ return _pdf_to_text(url_or_path, max_chars, continue_from, attachment_loader)
+
+ session_pdf_to_text.__name__ = _TOOL_NAME
+ session_pdf_to_text.__doc__ = pdf_to_text.__doc__
+ return session_pdf_to_text