refactor: streamline chat flow and improve diagnostics

Store server.log inside OLLAMA_WEB_DATA_DIR and manage the ollama-web file handler through the Starlette lifespan. Close and detach the handler on shutdown to prevent duplicate records and Windows file locks when applications are recreated.

Stop converting DuckDuckGo provider failures into empty search results. Record a query-safe traceback, re-raise the original exception, and let ToolRegistry expose the exception type and message as a tool error without adding or spoofing Referer headers.

Remove the unused non-streaming chat_with_tools implementation and its private helpers. Move capability, options, standard tool, MCP tool, and final tools=[] regression coverage onto the production streaming path.

Add integration coverage for data/server.log placement, handler cleanup, repeated lifespans, and provider 403 diagnostics. Update the Unreleased changelog entry.

Validated with 118 tests, Ruff, mypy, diff checks, real DuckDuckGo searches, successful result-page fetches, and observable 401/403 responses from sites that reject automated retrieval.

3 files changed, 34 insertions, 194 deletions

diff --git a/src/ollama_web/app.py b/src/ollama_web/app.py
index eeb3bae..9792c7b 100644
--- a/src/ollama_web/app.py
+++ b/src/ollama_web/app.py
@@ -3,6 +3,8 @@
 from __future__ import annotations
 
 import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
 from pathlib import Path
 from typing import Any
 
@@ -42,17 +44,29 @@ class NoCacheStaticFiles(StaticFiles):
 _STATIC_DIR = Path(__file__).resolve().parent / "static"
 
 
+@asynccontextmanager
+async def _lifespan(app: Starlette) -> AsyncIterator[None]:
+    """Keep the application log handler scoped to the ASGI lifespan."""
+    log_dir = Path(app.state.settings.data_dir).resolve()
+    log_dir.mkdir(parents=True, exist_ok=True)
+    handler = logging.FileHandler(str(log_dir / "server.log"), encoding="utf-8")
+    handler.setFormatter(
+        logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    )
+    logger = logging.getLogger("ollama_web")
+    previous_level = logger.level
+    logger.setLevel(logging.INFO)
+    logger.addHandler(handler)
+    try:
+        yield
+    finally:
+        logger.removeHandler(handler)
+        handler.close()
+        logger.setLevel(previous_level)
+
+
 def create_app() -> Starlette:
     """Build and return the configured Starlette application."""
-    # Ensure application logs are written alongside uvicorn access logs.
-    log_path = Path(settings.data_dir).resolve().parent / "server.log"
-    log_path.parent.mkdir(parents=True, exist_ok=True)
-    handler = logging.FileHandler(str(log_path), encoding="utf-8")
-    handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
-    root = logging.getLogger()
-    root.setLevel(logging.INFO)
-    root.addHandler(handler)
-
     routes = [
         Route("/login", pages_route.login, name="login_page"),
         Route("/api/auth/login", auth_route.login, methods=["POST"], name="auth_login"),
@@ -139,7 +153,7 @@ def create_app() -> Starlette:
             )
         )
 
-    app = Starlette(routes=routes, middleware=middleware)
+    app = Starlette(routes=routes, middleware=middleware, lifespan=_lifespan)
     app.state.settings = settings
     app.state.session_store = SessionStore(settings.data_dir)
     app.state.tool_names = ["web_search", "scrape_url", "search_and_fetch", "pdf_to_text"]
diff --git a/src/ollama_web/llm.py b/src/ollama_web/llm.py
index 4951b36..187b5e7 100644
--- a/src/ollama_web/llm.py
+++ b/src/ollama_web/llm.py
@@ -132,14 +132,6 @@ def _embed_images_in_content(message: Message) -> Message:
     return Message(**kwargs)
 
 
-def _tool_calls(response: Any) -> list[dict[str, Any]]:
-    """Extract tool calls from an ollama response in a tolerant manner."""
-    msg = getattr(response, "message", None) or (
-        response.get("message") if isinstance(response, dict) else None
-    )
-    return _tool_calls_from_message(msg)
-
-
 def _tool_calls_from_message(msg: Any) -> list[dict[str, Any]]:
     """Extract tool calls from a message object/dict."""
     if msg is None:
@@ -166,18 +158,6 @@ def _tool_calls_from_message(msg: Any) -> list[dict[str, Any]]:
     return out
 
 
-def _assistant_message(response: Any) -> Message:
-    """Convert a non-streaming response into a Message for re-submission."""
-    msg = getattr(response, "message", None) or (
-        response.get("message") if isinstance(response, dict) else None
-    )
-    if isinstance(msg, dict):
-        return Message(**msg)
-    if msg is not None:
-        return cast(Message, msg)
-    return Message(role="assistant", content="")
-
-
 def _token_counts(response: Any) -> dict[str, int] | None:
     """Extract prompt_eval_count and eval_count from an ollama response."""
     if response is None:
@@ -193,161 +173,6 @@ def _token_counts(response: Any) -> dict[str, int] | None:
     return {"prompt_eval_count": int(prompt_eval), "eval_count": int(eval_count)}
 
 
-def chat_with_tools(
-    messages: list[Message],
-    model: str,
-    *,
-    think: bool | str | None = None,
-    registry: ToolRegistry | None = None,
-    host: str | None = None,
-    capabilities: set[str] | None = None,
-    options: dict[str, Any] | None = None,
-    language: str | None = None,
-) -> Iterator[dict[str, Any]]:
-    """Run a multi-round tool-calling chat and yield progress events.
-
-    Events are dicts with a ``type`` key:
-
-    - ``{"type": "tool_start", "name": str, "arguments": dict}``
-    - ``{"type": "tool_end", "name": str, "result": str}``
-    - ``{"type": "delta", "content": str}``
-    - ``{"type": "thinking", "content": str}``
-    - ``{"type": "done", "prompt_eval_count": int, "eval_count": int}``
-    - ``{"type": "error", "message": str}``
-
-    The final assistant message is streamed token-by-token via ``delta`` events.
-    """
-    import logging
-
-    logger = logging.getLogger("ollama_web.llm")
-
-    caps = capabilities or set()
-    raw_client = get_client(host)
-    reg = registry or default_registry()
-    tools: list[ToolCallable] = [cast(Any, t) for t in reg.ollama_tools]
-    effective_tools = tools if "tools" in caps else []
-    effective_think = think if "thinking" in caps else None
-
-    # Non-vision models cannot use the ``images`` field; embed them as base64.
-    if "vision" not in caps:
-        messages = [_embed_images_in_content(m) if _is_image_message(m) else m for m in messages]
-
-    cumulative_tool_chars = 0
-
-    rounds = 0
-    while rounds < MAX_TOOL_ROUNDS:
-        rounds += 1
-        logger.info(
-            "tool round %d/%d start messages=%d model=%s cumulative_tool_chars=%d",
-            rounds,
-            MAX_TOOL_ROUNDS,
-            len(messages),
-            model,
-            cumulative_tool_chars,
-        )
-        try:
-            resp = cast(Any, raw_client).chat(
-                model=model,
-                messages=messages,
-                tools=effective_tools,
-                think=effective_think,
-                options=options or None,
-                stream=False,
-            )
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("ollama chat call failed: %s", exc)
-            yield {"type": "error", "message": str(exc)}
-            return
-
-        calls = _tool_calls(resp)
-        tokens = _token_counts(resp)
-        if not calls:
-            # No tool calls: stream the final answer for UI effect.
-            content = _assistant_content(resp)
-            thinking = _assistant_thinking(resp)
-            if thinking:
-                yield {"type": "thinking", "content": thinking}
-            if content:
-                yield {"type": "delta", "content": content}
-            done_event: dict[str, Any] = {"type": "done"}
-            if tokens:
-                done_event.update(tokens)
-            yield done_event
-            return
-
-        # Append the assistant message that requested tool calls.
-        messages.append(_assistant_message(resp))
-
-        # Execute each requested tool and append tool-role messages.
-        tool_results: list[tuple[str, str]] = []
-        for call in calls:
-            name = call.get("name", "")
-            args = call.get("arguments", {})
-
-            yield {"type": "tool_start", "name": name, "arguments": _safe_tool_arguments(args)}
-            result = reg.execute(name, args if isinstance(args, dict) else json.dumps(args))
-            result = _safe_tool_result(_trim_tool_result(result, settings.max_tool_result_chars))
-            yield {"type": "tool_end", "name": name, "result": result}
-            tool_results.append((name, result))
-
-        if sum(len(r) for _, r in tool_results) > MAX_TOOL_CONTEXT_CHARS:
-            budget = MAX_TOOL_CONTEXT_CHARS - len(tool_results) * 200
-            max_per_tool = budget // max(1, len(tool_results))
-            logger.warning(
-                "tool results too large (%d chars), truncating to %d chars per tool",
-                sum(len(r) for _, r in tool_results),
-                max_per_tool,
-            )
-            tool_results = [(n, _trim_tool_result(r, max_per_tool)) for n, r in tool_results]
-
-        cumulative_tool_chars += sum(len(r) for _, r in tool_results)
-        for name, result in tool_results:
-            messages.append(Message(role="tool", tool_name=name, content=result))
-
-    # Exceeded tool rounds: ask for a final answer without tools.
-    logger.info(
-        (
-            "max tool rounds exceeded, requesting final answer without tools "
-            "(cumulative_tool_chars=%d)"
-        ),
-        cumulative_tool_chars,
-    )
-    yield {
-        "type": "status",
-        "message": t("status.tool_limit_reached", lang=language),
-    }
-    messages.append(
-        Message(
-            role="user",
-            content=get_prompt("tool_limit_instruction", lang=language),
-        )
-    )
-    try:
-        resp = cast(Any, raw_client).chat(
-            model=model,
-            messages=messages,
-            tools=[],
-            think=effective_think,
-            options=options or None,
-            stream=False,
-        )
-    except Exception as exc:  # noqa: BLE001
-        logger.warning("final ollama chat call failed: %s", exc)
-        yield {"type": "error", "message": str(exc)}
-        return
-    tokens = _token_counts(resp)
-    content = _assistant_content(resp)
-    thinking = _assistant_thinking(resp)
-    if thinking:
-        yield {"type": "thinking", "content": thinking}
-    if content:
-        yield {"type": "delta", "content": content}
-    done_event_final: dict[str, Any] = {"type": "done"}
-    if tokens:
-        done_event_final.update(tokens)
-    yield done_event_final
-
-
 def _trim_tool_result(result: str, limit: int) -> str:
     """Truncate ``result`` to ``limit`` chars while keeping it readable.
 
@@ -684,12 +509,6 @@ def _assistant_thinking(response: Any) -> str:
     return str(getattr(msg, "thinking", "") or "")
 
 
-def _chunk_text(text: str, size: int) -> Iterator[str]:
-    """Yield successive chunks of ``text`` up to ``size`` characters."""
-    for i in range(0, len(text), size):
-        yield text[i : i + size]
-
-
 def _assistant_message_chunk(full_message: dict[str, Any]) -> Message | None:
     """Build an assistant ``Message`` from the accumulated streamed message.
 
diff --git a/src/ollama_web/tools/search.py b/src/ollama_web/tools/search.py
index 761f182..773f7f3 100644
--- a/src/ollama_web/tools/search.py
+++ b/src/ollama_web/tools/search.py
@@ -2,10 +2,14 @@
 
 from __future__ import annotations
 
+import logging
+
 from ddgs import DDGS
 
 from ..config import settings
 
+logger = logging.getLogger(__name__)
+
 
 def web_search(query: str, max_results: int | None = None) -> list[dict[str, str]]:
     """Search the web using DuckDuckGo and return compact result summaries.
@@ -16,7 +20,8 @@ def web_search(query: str, max_results: int | None = None) -> list[dict[str, str
 
     Returns:
       A list of dictionaries with keys 'title', 'href' and 'body' for each
-      result. Returns an empty list if the search fails or no results are found.
+      result. Returns an empty list if no results are found. Provider failures
+      are logged and propagated to the tool execution layer.
     """
     if not query or not query.strip():
         return []
@@ -37,7 +42,9 @@ def web_search(query: str, max_results: int | None = None) -> list[dict[str, str
                     }
                 )
     except Exception:  # noqa: BLE001
-        # Swallow network/runtime errors so the chat loop can continue.
-        return []
+        # Preserve the provider exception for the tool layer while recording
+        # the traceback without logging the potentially sensitive query.
+        logger.exception("DuckDuckGo web search failed")
+        raise
 
     return results[:limit]