From e4d421c782a181f421e2987f75b4610845e3ef55 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Wed, 24 Jun 2026 22:47:13 +0900 Subject: Harden web UI security --- src/ollama_web/tools/scrape.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) (limited to 'src/ollama_web/tools/scrape.py') diff --git a/src/ollama_web/tools/scrape.py b/src/ollama_web/tools/scrape.py index d20956d..d5fa2f4 100644 --- a/src/ollama_web/tools/scrape.py +++ b/src/ollama_web/tools/scrape.py @@ -2,10 +2,10 @@ from __future__ import annotations -import httpx from bs4 import BeautifulSoup from ..config import settings +from .helper.safe_http import UnsafeURL, fetch_public_bytes def scrape_url(url: str, max_chars: int | None = None) -> str: @@ -27,17 +27,15 @@ def scrape_url(url: str, max_chars: int | None = None) -> str: return "Invalid URL." limit = max_chars if max_chars is not None and max_chars > 0 else settings.scrape_max_chars - headers = {"User-Agent": "ollama-web/0.1 (+https://github.com/local)"} - try: - with httpx.Client( - timeout=settings.scrape_timeout, - follow_redirects=True, - headers=headers, - ) as client: - resp = client.get(url) - resp.raise_for_status() - html = resp.text + data, final_url, content_type = fetch_public_bytes(url, timeout=settings.scrape_timeout) + encoding = "utf-8" + if "charset=" in content_type: + encoding = content_type.rsplit("charset=", 1)[-1].split(";", 1)[0].strip() + html = data.decode(encoding or "utf-8", errors="replace") + url = final_url + except UnsafeURL as exc: + return f"Blocked URL: {exc}" except Exception as exc: # noqa: BLE001 return f"Failed to fetch {url}: {exc}" @@ -56,4 +54,4 @@ def scrape_url(url: str, max_chars: int | None = None) -> str: if len(cleaned) > limit: cleaned = cleaned[:limit].rsplit(" ", 1)[0] + "\n…[truncated]" - return cleaned or "No readable text content found." \ No newline at end of file + return cleaned or "No readable text content found." -- cgit v1.2.3