From 4e9c66ee25ed77858985765382f41d0cf22ea588 Mon Sep 17 00:00:00 2001 From: tanaka takashi Date: Tue, 20 Oct 2026 02:03:54 +0900 Subject: Improve web search region selection and normalize provider results Update ddgs to 9.16.0 and keep backend="auto" while preserving the LLM-facing query/max_results inputs and title/href/body result schema. Implementation: - Add OLLAMA_WEB_SEARCH_REGION with auto/us-en/jp-jp values, normalized configuration input, and a clear startup error for unsupported values. - In auto mode, detect kana after NFKC normalization and exclusion of URLs and site:/filetype: tokens. Select jp-jp for kana and us-en otherwise, without modifying the original query or sharing per-call region state. - Preserve href/url and body/snippet fallbacks and normalize None values. - Reject malformed HTTP(S) URLs without DNS lookups or extra requests. - Unwrap known Yahoo redirect URLs before excluding Bing aclick ads. - Represent missing and URL-only titles as empty strings rather than inventing titles from snippets. Preserve remaining provider order and allow fewer results after filtering, without additional searches. - Preserve provider exception propagation and identify ddgs in error logs. Coverage and documentation: - Add 72 regression cases for configuration, region routing, concurrency, public schemas, title/URL/ad boundaries, limits, and search_and_fetch continuation. Update existing provider mocks and log assertions. - Document the region environment setting and result semantics in both READMEs and tools-schema.md; update the changelog. - Keep only a short backend availability caveat in tools-schema.md and remove the temporary backend comparison, design, and validation docs. Validation: - pytest: 222 passed; ruff check . and mypy src/ passed. - uv lock --check and git diff --check passed. - Initial live acceptance: 56 searches plus 3 registry smoke checks; default-region auto succeeded 8/18 times and real ad filtering removed two ads. Stable relevance/title quality targets remain unmet because provider acquisition is unreliable. - Follow-up backend study: 234 comparative searches plus 3 Yahoo response diagnostics. Yahoo led individual backends at 9/18; restricted multi- backend combinations did not outperform the contemporaneous auto control. Authentication, access restrictions, temporary upstream failures, and Yahoo HTML/parser mismatches were observed. Do not claim search reliability is fixed or add backend configuration, custom retries, ranking changes, or challenge bypasses. Running app processes were not restarted; restart is needed to load the changes. --- src/ollama_web/config.py | 8 ++++ src/ollama_web/tools/search.py | 99 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 100 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/ollama_web/config.py b/src/ollama_web/config.py index fa07384..484038a 100644 --- a/src/ollama_web/config.py +++ b/src/ollama_web/config.py @@ -12,6 +12,13 @@ def _csv_env(name: str) -> list[str]: return [part.strip() for part in raw.split(",") if part.strip()] +def _search_region_env() -> str: + region = os.environ.get("OLLAMA_WEB_SEARCH_REGION", "auto").strip().lower() + if region not in {"auto", "us-en", "jp-jp"}: + raise ValueError("OLLAMA_WEB_SEARCH_REGION must be one of: auto, us-en, jp-jp") + return region + + @dataclass class Settings: """Runtime settings resolved from environment variables.""" @@ -48,6 +55,7 @@ class Settings: scrape_timeout: float = 15.0 scrape_max_chars: int = 4000 search_max_results: int = 5 + search_region: str = field(default_factory=_search_region_env) fetch_max_results: int = 3 fetch_max_chars: int = 8000 max_fetch_mb: int = 10 diff --git a/src/ollama_web/tools/search.py b/src/ollama_web/tools/search.py index 773f7f3..c3127df 100644 --- a/src/ollama_web/tools/search.py +++ b/src/ollama_web/tools/search.py @@ -1,8 +1,12 @@ -"""Web search tool backed by DuckDuckGo (ddgs).""" +"""Web metasearch with internal region selection and compact summaries.""" from __future__ import annotations import logging +import re +import unicodedata +from ipaddress import ip_address +from urllib.parse import unquote, urlsplit from ddgs import DDGS @@ -10,9 +14,83 @@ from ..config import settings logger = logging.getLogger(__name__) +_QUERY_METADATA = re.compile(r"(?:https?://|www\.)\S+|(? str: + if settings.search_region != "auto": + return settings.search_region + text = _QUERY_METADATA.sub("", unicodedata.normalize("NFKC", query)) + return "jp-jp" if _KANA.search(text) else "us-en" + + +def _is_http_url(value: str) -> bool: + """Validate syntax only; search summaries do not resolve or fetch URLs.""" + if not value or any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in value): + return False + try: + parsed = urlsplit(value) + host = parsed.hostname + if parsed.scheme not in {"http", "https"} or not host or parsed.username is not None: + return False + # Accessing port also rejects malformed/out-of-range port numbers. + if parsed.port is not None and parsed.port == 0: + return False + try: + ip_address(host) + return True + except ValueError: + ascii_host = host.rstrip(".").encode("idna").decode("ascii") + return len(ascii_host) <= 253 and all( + _HOST_LABEL.fullmatch(label) for label in ascii_host.split(".") + ) + except (ValueError, UnicodeError): + return False + + +def _result_url(value: str) -> str: + if not _is_http_url(value): + return "" + parsed = urlsplit(value) + if (parsed.hostname or "").rstrip(".") == "r.search.yahoo.com" and "/RU=" in parsed.path: + target = parsed.path.split("/RU=", 1)[1] + value = unquote(re.split(r"/R[KS]=", target, maxsplit=1)[0]) + if not _is_http_url(value): + return "" + parsed = urlsplit(value) + if ( + (parsed.hostname or "").rstrip(".") in {"bing.com", "www.bing.com"} + and parsed.path.rstrip("/") == "/aclick" + ): + return "" + return value + + +def _url_title(title: str, href: str) -> bool: + def display_key(value: str) -> str: + value = re.sub(r"^https?://", "", unquote(value), flags=re.I) + value = re.sub(r"^www\.", "", value, flags=re.I) + return value.rstrip("/") + + if display_key(title) == display_key(href): + return True + # Only whole URL-like labels, not headings such as "example.com の使い方". + candidate = title if re.match(r"^https?://", title, re.I) else "https://" + title + if _is_http_url(candidate): + host = urlsplit(candidate).hostname or "" + tld = host.rstrip(".").rsplit(".", 1)[-1].encode("idna").decode("ascii") + return bool(re.match(r"^https?://", title, re.I)) or ( + "." in host and bool(re.fullmatch(r"[a-z]{2,}|xn--[a-z0-9-]+", tld, re.I)) + ) + return False + def web_search(query: str, max_results: int | None = None) -> list[dict[str, str]]: - """Search the web using DuckDuckGo and return compact result summaries. + """Search the web with ddgs auto and return compact result summaries. + + Search region is selected internally from the query or server settings. Args: query: The search query string. @@ -20,7 +98,9 @@ def web_search(query: str, max_results: int | None = None) -> list[dict[str, str Returns: A list of dictionaries with keys 'title', 'href' and 'body' for each - result. Returns an empty list if no results are found. Provider failures + result. A missing or URL-only title is an empty string; use the body + for context. Invalid URLs and known ad links are omitted, so fewer + results may be returned. Returns an empty list for empty results. Provider failures are logged and propagated to the tool execution layer. """ if not query or not query.strip(): @@ -31,20 +111,25 @@ def web_search(query: str, max_results: int | None = None) -> list[dict[str, str ) results: list[dict[str, str]] = [] + region = _resolve_search_region(query) try: with DDGS() as ddgs: - for item in ddgs.text(query, max_results=limit): + for item in ddgs.text(query, max_results=limit, region=region, backend="auto"): + href = _result_url(str(item.get("href") or item.get("url") or "").strip()) + if not href: + continue + title = str(item.get("title") or "").strip() results.append( { - "title": str(item.get("title", "")).strip(), - "href": str(item.get("href") or item.get("url") or "").strip(), + "title": "" if _url_title(title, href) else title, + "href": href, "body": str(item.get("body") or item.get("snippet") or "").strip(), } ) except Exception: # noqa: BLE001 # Preserve the provider exception for the tool layer while recording # the traceback without logging the potentially sensitive query. - logger.exception("DuckDuckGo web search failed") + logger.exception("ddgs web search failed") raise return results[:limit] -- cgit v1.2.3