diff options
| author | tanaka takashi <isazaka@gmail.com> | 2026-08-23 17:28:10 +0900 |
|---|---|---|
| committer | tanaka takashi <isazaka@gmail.com> | 2026-08-23 17:28:39 +0900 |
| commit | 33f6b4dbb69e9366e312a8b44dbed9d22e7cba5b (patch) | |
| tree | 29a7480317af94fefe9f69a53095af2f233207ce /src/ollama_web/tools/pdf.py | |
| parent | 68969bfbeea68ded7ff863d229760562072f910b (diff) | |
fix(tools): PDFページングを損失なしカーソル方式へ再設計
PDF本文をページ単位で抽出した後に文字数上限で切断しつつ、継続位置だけ次ページへ進めていたため、ページ末尾が復元不能になる問題を修正する。ツール側の本文上限とLLM側の結果上限の不整合、25ページの処理上限、添付経路での継続情報欠落も共通契約で解消する。
- 共通PageResultと、メタデータ込みの結果上限内で本文量を決めるフォーマッターを追加
- v2トークンにPDFのページ番号とページ内文字位置を保持し、v1トークンの読み取り互換性を維持
- pdf_to_text、scrape_url、search_and_fetchを返却済み本文だけカーソルが進む方式へ統一
- max_pdf_pagesを文書全体ではなく1呼び出しの処理上限として扱い、長い単一ページと26ページ目以降の継続取得に対応
- attachment:<file_id>をセッション限定で解決し、既存添付も保存済みプレビューではなく元PDFから再解析
- Web取得ファイルをセッションとURL単位で重複排除し、HTMLおよびテキスト経路の全文ページングも維持
- 正常なページング結果の後段切断を禁止し、結果上限・コンテキスト予算・ラウンド上限では未取得状態を明示
- 日英プロンプト、UIメッセージ、ツールスキーマ、変更履歴を実装へ同期
- 実PDF、30ページ超、長い単一ページ、旧トークン、セッション分離、コンテキスト境界の回帰テストを追加
検証:
- pytest: 130 passed
- ruff check src tests: passed
- mypy src: passed
- c2144.pdf: 19ページ、抽出本文46015文字(ページ区切り込み46051文字)を8回で重複・欠落なく完全再構成
Diffstat (limited to 'src/ollama_web/tools/pdf.py')
| -rw-r--r-- | src/ollama_web/tools/pdf.py | 153 |
1 files changed, 117 insertions, 36 deletions
diff --git a/src/ollama_web/tools/pdf.py b/src/ollama_web/tools/pdf.py index 9fdb497..4faf8ac 100644 --- a/src/ollama_web/tools/pdf.py +++ b/src/ollama_web/tools/pdf.py @@ -2,67 +2,148 @@ from __future__ import annotations +import logging +from collections.abc import Callable +from typing import Any + from ..config import settings -from .helper.pagination import format_paged_result, make_token, parse_token -from .helper.pdf import extract_pdf_text +from .helper.pagination import ( + PageResult, + PageSizeError, + format_page_to_limit, + make_token, + parse_token, +) +from .helper.pdf import extract_pdf_page from .helper.safe_http import UnsafeURL, fetch_public_bytes _TOOL_NAME = "pdf_to_text" +_ATTACHMENT_PREFIX = "attachment:" +logger = logging.getLogger("ollama_web.tools.pdf") +AttachmentLoader = Callable[[str], bytes | None] -def pdf_to_text( - url_or_path: str, max_chars: int | None = None, continue_from: str | None = None -) -> str: - """Extract text from a local PDF path or remote URL. - The text is returned one page at a time. When more pages remain, a - ``next_token`` is appended that can be passed back via ``continue_from`` - to fetch the next batch of pages. +def _cursor_value(payload: dict[str, Any], key: str, fallback: int = 0) -> int: + raw = payload.get(key, fallback) + if isinstance(raw, bool) or not isinstance(raw, (int, str)): + raise ValueError(f"Token {key!r} is not an integer.") + try: + value = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"Token {key!r} is not an integer.") from exc + if value < 0: + raise ValueError(f"Token {key!r} must not be negative.") + return value - Args: - url_or_path: File path or URL pointing to a PDF. - max_chars: Maximum characters to return per page. - continue_from: Opaque continuation token returned in a previous call's - ``next_token``. When provided, extraction resumes from the encoded - page offset. - - Returns: - Extracted plain text for this page, followed by a ``---page-info---`` - footer when more pages are available, or an error message. - """ + +def _pdf_to_text( + url_or_path: str, + max_chars: int | None, + continue_from: str | None, + attachment_loader: AttachmentLoader | None, +) -> str: if not url_or_path or not url_or_path.strip(): return "Invalid PDF source." src = url_or_path.strip() - - # Resolve the resumption page offset from the continuation token, if any. - page_offset = 0 + page_index = 0 + page_char_offset = 0 if continue_from: try: payload = parse_token(continue_from, _TOOL_NAME) + src = str(payload.get("url") or src) + # Version 1 used offset as the PDF page index. + page_index = _cursor_value(payload, "page_index", int(payload["offset"])) + page_char_offset = _cursor_value(payload, "page_char_offset") except ValueError as exc: return f"Invalid continuation token: {exc}" - src = str(payload.get("url") or src) - page_offset = int(payload.get("offset", 0)) - if page_offset < 0: - page_offset = 0 - data: bytes | None = None - if src.startswith("http://") or src.startswith("https://"): + if src.startswith(_ATTACHMENT_PREFIX): + file_id = src.removeprefix(_ATTACHMENT_PREFIX) + data = attachment_loader(file_id) if attachment_loader is not None else None + if data is None: + return "Attachment PDF is unavailable in this session." + source_kind = "attachment" + elif src.startswith("http://") or src.startswith("https://"): try: - data, _, _ = fetch_public_bytes(src, timeout=settings.scrape_timeout) + data, final_url, _ = fetch_public_bytes(src, timeout=settings.scrape_timeout) + src = final_url or src except UnsafeURL as exc: return f"Blocked URL: {exc}" except Exception as exc: # noqa: BLE001 return f"Failed to fetch PDF: {exc}" + source_kind = "remote" else: return "Only http(s) PDF URLs are supported." - if data is None: - return "No PDF data loaded." + requested = max_chars if max_chars is not None and max_chars > 0 else settings.fetch_max_chars + + def page_factory(body_budget: int) -> PageResult: + return extract_pdf_page( + data, + max_chars=body_budget, + page_index=page_index, + page_char_offset=page_char_offset, + max_size_mb=(settings.max_upload_mb if source_kind == "attachment" else None), + ) + + def token_factory(cursor: dict[str, int]) -> str: + return make_token( + _TOOL_NAME, + src, + cursor["page_index"], + page_index=cursor["page_index"], + page_char_offset=cursor["page_char_offset"], + ) + + try: + result = format_page_to_limit( + page_factory, + token_factory, + requested_chars=requested, + result_limit=settings.max_tool_result_chars, + ) + except PageSizeError as exc: + return f"PDF pagination failed: {exc}" - text, has_more, next_page_offset = extract_pdf_text( - data, max_chars=max_chars, page_offset=page_offset + logger.info( + "pdf page source=%s page_index=%d page_char_offset=%d result_chars=%d has_more=%s", + source_kind, + page_index, + page_char_offset, + len(result), + "---page-info---" in result, ) - next_token = make_token(_TOOL_NAME, src, next_page_offset) if has_more else None - return format_paged_result(text, has_more=has_more, next_token=next_token) + return result + + +def pdf_to_text( + url_or_path: str, max_chars: int | None = None, continue_from: str | None = None +) -> str: + """Extract a losslessly paged PDF from an HTTP(S) URL. + + Args: + url_or_path: HTTP(S) PDF URL. In chat, a session attachment URI may also + be supplied by the attachment context. Arbitrary filesystem paths are + never accepted. + max_chars: Requested maximum source characters for this page. The tool + may return fewer characters so metadata and its token remain intact. + continue_from: Opaque continuation token from the preceding result. + """ + return _pdf_to_text(url_or_path, max_chars, continue_from, None) + + +def make_session_pdf_tool(attachment_loader: AttachmentLoader) -> Callable[..., str]: + """Create a session-scoped ``pdf_to_text`` supporting attachment IDs.""" + + def session_pdf_to_text( + url_or_path: str, + max_chars: int | None = None, + continue_from: str | None = None, + ) -> str: + return _pdf_to_text(url_or_path, max_chars, continue_from, attachment_loader) + + session_pdf_to_text.__name__ = _TOOL_NAME + session_pdf_to_text.__doc__ = pdf_to_text.__doc__ + return session_pdf_to_text |
