Improve web search region selection and normalize provider resultsHEADmain

Update ddgs to 9.16.0 and keep backend="auto" while preserving the
LLM-facing query/max_results inputs and title/href/body result schema.

Implementation:
- Add OLLAMA_WEB_SEARCH_REGION with auto/us-en/jp-jp values, normalized
  configuration input, and a clear startup error for unsupported values.
- In auto mode, detect kana after NFKC normalization and exclusion of URLs
  and site:/filetype: tokens. Select jp-jp for kana and us-en otherwise,
  without modifying the original query or sharing per-call region state.
- Preserve href/url and body/snippet fallbacks and normalize None values.
- Reject malformed HTTP(S) URLs without DNS lookups or extra requests.
- Unwrap known Yahoo redirect URLs before excluding Bing aclick ads.
- Represent missing and URL-only titles as empty strings rather than
  inventing titles from snippets. Preserve remaining provider order and
  allow fewer results after filtering, without additional searches.
- Preserve provider exception propagation and identify ddgs in error logs.

Coverage and documentation:
- Add 72 regression cases for configuration, region routing, concurrency,
  public schemas, title/URL/ad boundaries, limits, and search_and_fetch
  continuation. Update existing provider mocks and log assertions.
- Document the region environment setting and result semantics in both
  READMEs and tools-schema.md; update the changelog.
- Keep only a short backend availability caveat in tools-schema.md and
  remove the temporary backend comparison, design, and validation docs.

Validation:
- pytest: 222 passed; ruff check . and mypy src/ passed.
- uv lock --check and git diff --check passed.
- Initial live acceptance: 56 searches plus 3 registry smoke checks;
  default-region auto succeeded 8/18 times and real ad filtering removed
  two ads. Stable relevance/title quality targets remain unmet because
  provider acquisition is unreliable.
- Follow-up backend study: 234 comparative searches plus 3 Yahoo response
  diagnostics. Yahoo led individual backends at 9/18; restricted multi-
  backend combinations did not outperform the contemporaneous auto
  control. Authentication, access restrictions, temporary upstream
  failures, and Yahoo HTML/parser mismatches were observed.

Do not claim search reliability is fixed or add backend configuration,
custom retries, ranking changes, or challenge bypasses. Running app
processes were not restarted; restart is needed to load the changes.

2 files changed, 100 insertions, 7 deletions

diff --git a/src/ollama_web/config.py b/src/ollama_web/config.py
index fa07384..484038a 100644
--- a/src/ollama_web/config.py
+++ b/src/ollama_web/config.py
@@ -12,6 +12,13 @@ def _csv_env(name: str) -> list[str]:
     return [part.strip() for part in raw.split(",") if part.strip()]
 
 
+def _search_region_env() -> str:
+    region = os.environ.get("OLLAMA_WEB_SEARCH_REGION", "auto").strip().lower()
+    if region not in {"auto", "us-en", "jp-jp"}:
+        raise ValueError("OLLAMA_WEB_SEARCH_REGION must be one of: auto, us-en, jp-jp")
+    return region
+
+
 @dataclass
 class Settings:
     """Runtime settings resolved from environment variables."""
@@ -48,6 +55,7 @@ class Settings:
     scrape_timeout: float = 15.0
     scrape_max_chars: int = 4000
     search_max_results: int = 5
+    search_region: str = field(default_factory=_search_region_env)
     fetch_max_results: int = 3
     fetch_max_chars: int = 8000
     max_fetch_mb: int = 10
diff --git a/src/ollama_web/tools/search.py b/src/ollama_web/tools/search.py
index 773f7f3..c3127df 100644
--- a/src/ollama_web/tools/search.py
+++ b/src/ollama_web/tools/search.py
@@ -1,8 +1,12 @@
-"""Web search tool backed by DuckDuckGo (ddgs)."""
+"""Web metasearch with internal region selection and compact summaries."""
 
 from __future__ import annotations
 
 import logging
+import re
+import unicodedata
+from ipaddress import ip_address
+from urllib.parse import unquote, urlsplit
 
 from ddgs import DDGS
 
@@ -10,9 +14,83 @@ from ..config import settings
 
 logger = logging.getLogger(__name__)
 
+_QUERY_METADATA = re.compile(r"(?:https?://|www\.)\S+|(?<!\S)-?(?:site|filetype):\S+", re.I)
+_KANA = re.compile(r"[\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fd-\u30ff]")
+_HOST_LABEL = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", re.I)
+
+
+def _resolve_search_region(query: str) -> str:
+    if settings.search_region != "auto":
+        return settings.search_region
+    text = _QUERY_METADATA.sub("", unicodedata.normalize("NFKC", query))
+    return "jp-jp" if _KANA.search(text) else "us-en"
+
+
+def _is_http_url(value: str) -> bool:
+    """Validate syntax only; search summaries do not resolve or fetch URLs."""
+    if not value or any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in value):
+        return False
+    try:
+        parsed = urlsplit(value)
+        host = parsed.hostname
+        if parsed.scheme not in {"http", "https"} or not host or parsed.username is not None:
+            return False
+        # Accessing port also rejects malformed/out-of-range port numbers.
+        if parsed.port is not None and parsed.port == 0:
+            return False
+        try:
+            ip_address(host)
+            return True
+        except ValueError:
+            ascii_host = host.rstrip(".").encode("idna").decode("ascii")
+            return len(ascii_host) <= 253 and all(
+                _HOST_LABEL.fullmatch(label) for label in ascii_host.split(".")
+            )
+    except (ValueError, UnicodeError):
+        return False
+
+
+def _result_url(value: str) -> str:
+    if not _is_http_url(value):
+        return ""
+    parsed = urlsplit(value)
+    if (parsed.hostname or "").rstrip(".") == "r.search.yahoo.com" and "/RU=" in parsed.path:
+        target = parsed.path.split("/RU=", 1)[1]
+        value = unquote(re.split(r"/R[KS]=", target, maxsplit=1)[0])
+        if not _is_http_url(value):
+            return ""
+        parsed = urlsplit(value)
+    if (
+        (parsed.hostname or "").rstrip(".") in {"bing.com", "www.bing.com"}
+        and parsed.path.rstrip("/") == "/aclick"
+    ):
+        return ""
+    return value
+
+
+def _url_title(title: str, href: str) -> bool:
+    def display_key(value: str) -> str:
+        value = re.sub(r"^https?://", "", unquote(value), flags=re.I)
+        value = re.sub(r"^www\.", "", value, flags=re.I)
+        return value.rstrip("/")
+
+    if display_key(title) == display_key(href):
+        return True
+    # Only whole URL-like labels, not headings such as "example.com の使い方".
+    candidate = title if re.match(r"^https?://", title, re.I) else "https://" + title
+    if _is_http_url(candidate):
+        host = urlsplit(candidate).hostname or ""
+        tld = host.rstrip(".").rsplit(".", 1)[-1].encode("idna").decode("ascii")
+        return bool(re.match(r"^https?://", title, re.I)) or (
+            "." in host and bool(re.fullmatch(r"[a-z]{2,}|xn--[a-z0-9-]+", tld, re.I))
+        )
+    return False
+
 
 def web_search(query: str, max_results: int | None = None) -> list[dict[str, str]]:
-    """Search the web using DuckDuckGo and return compact result summaries.
+    """Search the web with ddgs auto and return compact result summaries.
+
+    Search region is selected internally from the query or server settings.
 
     Args:
       query: The search query string.
@@ -20,7 +98,9 @@ def web_search(query: str, max_results: int | None = None) -> list[dict[str, str
 
     Returns:
       A list of dictionaries with keys 'title', 'href' and 'body' for each
-      result. Returns an empty list if no results are found. Provider failures
+      result. A missing or URL-only title is an empty string; use the body
+      for context. Invalid URLs and known ad links are omitted, so fewer
+      results may be returned. Returns an empty list for empty results. Provider failures
       are logged and propagated to the tool execution layer.
     """
     if not query or not query.strip():
@@ -31,20 +111,25 @@ def web_search(query: str, max_results: int | None = None) -> list[dict[str, str
     )
 
     results: list[dict[str, str]] = []
+    region = _resolve_search_region(query)
     try:
         with DDGS() as ddgs:
-            for item in ddgs.text(query, max_results=limit):
+            for item in ddgs.text(query, max_results=limit, region=region, backend="auto"):
+                href = _result_url(str(item.get("href") or item.get("url") or "").strip())
+                if not href:
+                    continue
+                title = str(item.get("title") or "").strip()
                 results.append(
                     {
-                        "title": str(item.get("title", "")).strip(),
-                        "href": str(item.get("href") or item.get("url") or "").strip(),
+                        "title": "" if _url_title(title, href) else title,
+                        "href": href,
                         "body": str(item.get("body") or item.get("snippet") or "").strip(),
                     }
                 )
     except Exception:  # noqa: BLE001
         # Preserve the provider exception for the tool layer while recording
         # the traceback without logging the potentially sensitive query.
-        logger.exception("DuckDuckGo web search failed")
+        logger.exception("ddgs web search failed")
         raise
 
     return results[:limit]