From f2d24cb7d437ad099291262d7c1b021809cd66ae Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Sun, 13 Sep 2026 13:35:03 +0900 Subject: feat(multimedia): MCP画像・音声artifactの安全な表示経路を追加 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MCP Tool結果をテキストと型付きartifactへ分離し、画像・音声のbase64をMIME、サイズ、件数のallowlistで検証する。annotations.audienceを反映し、ユーザー向けartifactだけをセッションへ保存してSSEにはメタデータのみを公開する。 画像はvision capabilityが明示されたモデルだけへ渡し、非対応時は既存の多言語プロンプト機構による固定通知へ置き換える。MCP生成音声とユーザー添付音声はOllamaへ送らず、誤デコードしたテキストやbase64がモデル入力へ混入しないポリシーに統一する。 保存メディア用の認証済み同一オリジン配信ルートを追加し、セッションスコープ、MIME allowlist、nosniff、private/no-store、Rangeリクエストを適用する。UIはMCP由来HTMLを使わずDOM APIでimgとaudio controlsを生成し、CSPへmedia-src selfを追加する。 web_searchは文字列メタデータだけを返す既存仕様を維持する。scrape_urlとsearch_and_fetchは画像・音声・動画レスポンスを拒否し、search_and_fetchの対象拡張子を明示的allowlistへ変更して標準Webツールをテキスト専用の契約に揃える。 責務境界と導入しない過剰対策をdocs/multimedia.mdへ記録し、README、ツール仕様、CHANGELOGを更新する。MCP artifact変換・永続化、vision分岐、音声非送信、認証、Range配信、MIME偽装拒否をテストで追加確認した。 Validation: 148 tests passed; ruff check passed; mypy src passed; node --check src/ollama_web/static/app.js passed. --- src/ollama_web/tools/fetch.py | 23 ++++++++--------- src/ollama_web/tools/registry.py | 56 +++++++++++++++++++++++++++++++++++----- src/ollama_web/tools/scrape.py | 4 +++ 3 files changed, 65 insertions(+), 18 deletions(-) (limited to 'src/ollama_web/tools') diff --git a/src/ollama_web/tools/fetch.py b/src/ollama_web/tools/fetch.py index 6ab17bb..7e2d3b5 100644 --- a/src/ollama_web/tools/fetch.py +++ b/src/ollama_web/tools/fetch.py @@ -80,7 +80,7 @@ def store_fetched_file( "mime": mime, "text": text, "url": url, - } + }, ) @@ -91,7 +91,7 @@ def pop_fetched_files(session_id: str) -> list[_FetchedFile]: def _normalize_file_type(file_type: str) -> str: ft = file_type.strip().lower().lstrip(".") - return _FILETYPE_SUFFIXES.get(ft, ft) + return _FILETYPE_SUFFIXES.get(ft, "") def _filetype_query(query: str, file_type: str) -> str: @@ -117,9 +117,9 @@ def _guess_mime(name: str, file_type: str) -> str: return mime or f"application/{file_type}" -def _fetch_bytes(url: str) -> bytes: - data, _, _ = fetch_public_bytes(url, timeout=settings.scrape_timeout) - return data +def _fetch_bytes(url: str) -> tuple[bytes, str]: + data, _, content_type = fetch_public_bytes(url, timeout=settings.scrape_timeout) + return data, content_type def _full_text_for_file(url: str, data: bytes, file_type: str) -> str: @@ -212,9 +212,7 @@ def search_and_fetch( except (TypeError, ValueError) as exc: return f"Invalid continuation token: cursor is not an integer: {exc}" legacy_pdf_offset = ( - file_offset - if payload.get("v") == 1 and "pdf_page_index" not in payload - else None + file_offset if payload.get("v") == 1 and "pdf_page_index" not in payload else None ) if min(file_idx, file_offset, pdf_page_index, pdf_page_char_offset) < 0: return "Invalid continuation token: cursor values must not be negative." @@ -293,7 +291,10 @@ def _render_file_page( p = get_prompts().get("fetch", {}) try: - data = _fetch_bytes(url) + data, content_type = _fetch_bytes(url) + media_type = content_type.split(";", 1)[0].strip().lower() + if media_type.startswith(("image/", "audio/", "video/")): + raise ValueError(f"unsupported media content type: {media_type}") except Exception as exc: # noqa: BLE001 # On fetch failure, advance to the next candidate rather than aborting # the whole pagination chain. @@ -439,9 +440,7 @@ def _render_file_page( return f"File pagination failed: {exc}" -def _make_fetch_token( - matched: list[dict[str, str]], cursor: dict[str, int], file_type: str -) -> str: +def _make_fetch_token(matched: list[dict[str, str]], cursor: dict[str, int], file_type: str) -> str: file_idx = cursor["file_idx"] url = matched[file_idx].get("href", "") if file_idx < len(matched) else "completed" return make_token( diff --git a/src/ollama_web/tools/registry.py b/src/ollama_web/tools/registry.py index e3b7824..51bdf29 100644 --- a/src/ollama_web/tools/registry.py +++ b/src/ollama_web/tools/registry.py @@ -13,6 +13,37 @@ from .scrape import scrape_url from .search import web_search +@dataclass +class ToolArtifact: + """Binary output produced by a tool, kept outside its text result.""" + + kind: str + mime: str + name: str + data: bytes + file_id: str | None = None + for_user: bool = True + for_assistant: bool = True + + def public_dict(self) -> dict[str, Any]: + """Return browser-safe metadata without the binary payload.""" + return { + "id": self.file_id, + "kind": self.kind, + "mime": self.mime, + "name": self.name, + "size": len(self.data), + } + + +@dataclass +class ToolExecutionResult: + """A tool's LLM-facing text and separately handled binary artifacts.""" + + text: str + artifacts: list[ToolArtifact] = field(default_factory=list) + + @dataclass class ToolRegistry: """A simple registry mapping tool names to callable implementations.""" @@ -75,6 +106,20 @@ class ToolRegistry: def get(self, name: str) -> Callable[..., object] | None: return self._tools.get(name) + def execute_detailed( + self, name: str, arguments: dict[str, object] | str + ) -> ToolExecutionResult: + """Execute a tool while preserving any binary artifacts separately.""" + result = self._execute_raw(name, arguments) + if isinstance(result, ToolExecutionResult): + return result + if isinstance(result, str): + return ToolExecutionResult(result) + try: + return ToolExecutionResult(json.dumps(result, ensure_ascii=False, default=str)) + except (TypeError, ValueError): + return ToolExecutionResult(str(result)) + def execute(self, name: str, arguments: dict[str, object] | str) -> str: """Execute a named tool and return a JSON-encoded string result. @@ -87,6 +132,10 @@ class ToolRegistry: A JSON string representation of the tool's return value, or an error object when the tool is unknown or raises. """ + return self.execute_detailed(name, arguments).text + + def _execute_raw(self, name: str, arguments: dict[str, object] | str) -> object: + """Execute a named tool without flattening a structured result.""" func = self.get(name) if func is None: return json.dumps({"error": f"Unknown tool: {name}"}) @@ -106,12 +155,7 @@ class ToolRegistry: except Exception as exc: # noqa: BLE001 return json.dumps({"error": f"{type(exc).__name__}: {exc}"}) - if isinstance(result, str): - return result - try: - return json.dumps(result, ensure_ascii=False, default=str) - except (TypeError, ValueError): - return str(result) + return result def default_registry() -> ToolRegistry: diff --git a/src/ollama_web/tools/scrape.py b/src/ollama_web/tools/scrape.py index 2f85c38..aa79793 100644 --- a/src/ollama_web/tools/scrape.py +++ b/src/ollama_web/tools/scrape.py @@ -80,6 +80,10 @@ def scrape_url(url: str, max_chars: int | None = None, continue_from: str | None except Exception as exc: # noqa: BLE001 return f"Failed to fetch {url}: {exc}" + media_type = content_type.split(";", 1)[0].strip().lower() + if media_type.startswith(("image/", "audio/", "video/")): + return f"Unsupported content type for text scraping: {media_type}." + cleaned = extract_readable_text(data, content_type) if not cleaned: -- cgit v1.2.3