web-connector: Added comprehensive logging and HTTP‑client centralization.

.gitignore

@@ -11,3 +11,7 @@ build/
 .DS_Store
 Thumbs.db
 uv.lock
+
+.idea/
+
+.github/

docs/guide.md

@@ -723,6 +723,7 @@ oikb sync --dry-run                 Preview without uploading
 oikb sync --max-file-size 50mb      Skip large files
 oikb sync --concurrency 4           Parallel uploads
 oikb sync --scan-secrets            Block files with credentials
+oikb sync --log_level DEBUG         Set log level (currently only supported for the web-connector)
 oikb watch <dir> --kb-id ID         Auto-sync on file change
 oikb daemon                         Start scheduled daemon
 oikb daemon --log-format json       JSON logging
@@ -779,3 +780,7 @@ oikb validate --deep # Verify API + KB connectivity
 
 Open WebUI → Knowledge → click a KB → the ID is in the URL:
 `http://localhost:3000/knowledge/8f3a2b1c-1234-5678-9abc-def012345678`
+
+### Issues for self-signed certificates
+Use the environment variable `OIKB_INSECURE_SSL=true` to disable SSL verification.  
+This one is currently only supported for the `web`-connector.

src/oikb/cli.py

@@ -924,13 +924,15 @@ def validate(config_file: str | None, deep: bool):
 @click.option("--no-server", is_flag=True, help="Run scheduler only, no HTTP server.")
 @click.option("--config", "config_file", default=None, type=click.Path(), help="Path to .oikb.yaml (default: ./.oikb.yaml).")
 @click.option("--log-format", default=None, type=click.Choice(["text", "json"]), help="Log output format (default: text, env: LOG_FORMAT).")
-def daemon(port: int, no_server: bool, config_file: str | None, log_format: str | None):
+@click.option("--log-level", default=None, type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"], case_sensitive=False), help="Log level (default: INFO, env: LOG_LEVEL).")
+def daemon(port: int, no_server: bool, config_file: str | None, log_format: str | None, log_level: str | None):
     """Run as a long-lived daemon with scheduled sync.
 
     Reads .oikb.yaml and syncs each source on its configured interval.
     Exposes /health, /history, and /sync endpoints.
     """
     log_format = log_format or os.environ.get("LOG_FORMAT", "text")
+    log_level = log_level or os.environ.get("LOG_LEVEL", "INFO")
 
     if config_file:
         import yaml
@@ -955,7 +957,7 @@ def daemon(port: int, no_server: bool, config_file: str | None, log_format: str
             sys.exit(1)
 
     from oikb.daemon import start_daemon
-    start_daemon(entries=entries, port=port, no_server=no_server, log_format=log_format)
+    start_daemon(entries=entries, port=port, no_server=no_server, log_format=log_format, log_level=log_level)
 
 
 # ── history ─────────────────────────────────────────────────────

src/oikb/client.py

@@ -7,6 +7,8 @@
 
 import httpx
 
+from oikb.http import make_http_client
+
 
 class OikbClient:
     """Stateless HTTP client for the Open WebUI KB API.
@@ -16,7 +18,7 @@ class OikbClient:
 
     def __init__(self, base_url: str, token: str, timeout: float = 120.0):
         self._base_url = base_url.rstrip("/")
-        self._http = httpx.Client(
+        self._http = make_http_client(
             base_url=f"{self._base_url}/api/v1",
             headers={"Authorization": f"Bearer {token}"},
             timeout=timeout,

src/oikb/connectors/web.py

@@ -7,6 +7,7 @@
 from __future__ import annotations
 
 import hashlib
+import logging
 import re
 import time
 import xml.etree.ElementTree as ET
@@ -16,6 +17,9 @@
 import httpx
 
 from oikb.connectors import BaseConnector, ManifestEntry
+from oikb.http import make_http_client
+
+log = logging.getLogger(__name__)
 
 
 def _html_to_text(html: str) -> str:
@@ -24,11 +28,14 @@ def _html_to_text(html: str) -> str:
         from bs4 import BeautifulSoup
 
         soup = BeautifulSoup(html, "html.parser")
+
         # Remove script and style elements.
         for tag in soup(["script", "style", "nav", "footer", "header"]):
             tag.decompose()
         return soup.get_text(separator="\n", strip=True)
     except ImportError:
+        log.debug("BeautifulSoup not available, falling back to regex-based HTML stripping")
+
         # Fallback: regex strip.
         text = re.sub(r"<[^>]+>", " ", html)
         return re.sub(r"\s+", " ", text).strip()
@@ -55,7 +62,7 @@ def __init__(
         self._parsed = urlparse(self.url)
         self._domain = f"{self._parsed.scheme}://{self._parsed.netloc}"
 
-        self._http = httpx.Client(
+        self._http = make_http_client(
             timeout=30.0,
             follow_redirects=True,
             headers={"User-Agent": "oikb/0.1 (+https://github.com/open-webui/oikb)"},
@@ -64,15 +71,32 @@ def __init__(
         # Cache: url -> text content.
         self._cache: dict[str, str] = {}
 
+        log.debug(f"WebConnector initialised: url={self.url}, delay={self.delay}s, max_pages={self.max_pages}")
+
     def build_manifest(self) -> list[ManifestEntry]:
         """Discover pages via sitemap or crawling, then build manifest."""
+        log.info(f"Starting manifest build for {self.url}")
+
         urls = self._discover_urls()
+        pageCount = len(urls)
+        log.info(f"Discovered {pageCount} URL(s) for {self.url}")
+
+        if pageCount > self.max_pages:
+            log.warning(
+                f"Discovered {pageCount} URLs but max_pages={self.max_pages} — "
+                f"truncating to {self.max_pages} pages"
+            )
+
         entries: list[ManifestEntry] = []
+        skippedEmptyCount = 0
+        errorCount = 0
 
         for url in urls[:self.max_pages]:
             try:
                 text = self._fetch_page(url)
                 if not text.strip():
+                    log.debug(f"Skipping {url} — page yielded no text content")
+                    skippedEmptyCount += 1
                     continue
 
                 self._cache[url] = text
@@ -103,35 +127,52 @@ def build_manifest(self) -> list[ManifestEntry]:
                         size=len(text.encode()),
                     )
                 )
+                log.debug(f"Added manifest entry: {dir_path}/{name} ({len(text.encode())} bytes, checksum={checksum})")
 
                 if self.delay > 0:
                     time.sleep(self.delay)
 
-            except Exception:
+            except Exception as exc:
+                log.warning(f"Failed to process page {url}: {exc}")
+                errorCount += 1
                 continue
 
         entries.sort(key=lambda e: e.display_path)
+
+        log.info(
+            f"Manifest build complete for {self.url}: "
+            f"{len(entries)} entries added, {skippedEmptyCount} skipped (empty), {errorCount} failed"
+        )
         return entries
 
     def _discover_urls(self) -> list[str]:
         """Discover URLs from sitemap.xml or by crawling links."""
         # Try sitemap first.
         if self.url.endswith(".xml"):
+            log.debug(f"URL ends with .xml, parsing directly as sitemap: {self.url}")
             return self._parse_sitemap(self.url)
 
         sitemap_url = f"{self._domain}/sitemap.xml"
+        log.debug(f"Trying sitemap at {sitemap_url}")
+
         try:
             urls = self._parse_sitemap(sitemap_url)
             if urls:
+                log.info(f"Using sitemap {sitemap_url}: found {len(urls)} URL(s)")
                 return urls
-        except Exception:
-            pass
+
+            log.debug(f"Sitemap at {sitemap_url} returned no URLs, falling back to link crawl")
+        except Exception as exc:
+            log.debug(f"Sitemap not available at {sitemap_url} ({exc}), falling back to link crawl")
 
         # Fall back to crawling.
+        log.info(f"Discovering URLs via link crawl starting from {self.url}")
         return self._crawl_links()
 
     def _parse_sitemap(self, url: str) -> list[str]:
         """Parse a sitemap.xml and return all URLs."""
+        log.debug(f"Fetching sitemap: {url}")
+
         resp = self._http.get(url)
         resp.raise_for_status()
 
@@ -141,75 +182,108 @@ def _parse_sitemap(self, url: str) -> list[str]:
         urls: list[str] = []
 
         # Handle sitemap index.
-        for sitemap in root.findall(".//sm:sitemap/sm:loc", ns):
+        sitemapIndexEntries = root.findall(".//sm:sitemap/sm:loc", ns)
+        if sitemapIndexEntries:
+            log.debug(f"Sitemap index detected at {url}: processing {len(sitemapIndexEntries)} sub-sitemap(s)")
+
+        for sitemap in sitemapIndexEntries:
             if sitemap.text:
-                urls.extend(self._parse_sitemap(sitemap.text))
+                subUrls = self._parse_sitemap(sitemap.text)
+                log.debug(f"Sub-sitemap {sitemap.text} yielded {len(subUrls)} URL(s)")
+                urls.extend(subUrls)
 
         # Handle regular sitemap.
         for loc in root.findall(".//sm:url/sm:loc", ns):
             if loc.text:
                 urls.append(loc.text)
 
+        log.debug(f"Parsed sitemap {url}: {len(urls)} URL(s) total")
         return urls
 
     def _crawl_links(self) -> list[str]:
         """Crawl same-domain links starting from the root URL."""
         visited: set[str] = set()
         queue = [self.url]
         urls: list[str] = []
+        errorCount = 0
 
         while queue and len(urls) < self.max_pages:
             url = queue.pop(0)
             if url in visited:
                 continue
+
             visited.add(url)
             urls.append(url)
+            log.debug(f"Crawling page {len(urls)}/{self.max_pages}: {url}")
 
             try:
                 resp = self._http.get(url)
                 resp.raise_for_status()
 
                 # Extract same-domain links.
+                newLinkCount = 0
                 for match in re.finditer(r'href=["\']([^"\']+)["\']', resp.text):
                     link = urljoin(url, match.group(1))
                     parsed = urlparse(link)
+
                     # Same domain, no fragments, no query params.
                     clean = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
                     if parsed.netloc == self._parsed.netloc and clean not in visited:
                         queue.append(clean)
+                        newLinkCount += 1
+
+                log.debug(f"Found {newLinkCount} new link(s) on {url}")
 
                 if self.delay > 0:
                     time.sleep(self.delay)
-            except Exception:
+
+            except Exception as exc:
+                log.warning(f"Failed to crawl {url}: {exc}")
+                errorCount += 1
                 continue
 
+        if len(urls) >= self.max_pages:
+            log.warning(f"Link crawl reached max_pages limit ({self.max_pages}) — some pages may be omitted")
+
+        log.info(f"Link crawl finished: {len(urls)} URL(s) collected, {errorCount} error(s), domain={self._domain}")
         return urls
 
     def _fetch_page(self, url: str) -> str:
         """Fetch a page and extract text."""
         if url in self._cache:
+            log.debug(f"Cache hit for {url}")
             return self._cache[url]
 
+        log.debug(f"Fetching page: {url}")
         resp = self._http.get(url)
         resp.raise_for_status()
-        return _html_to_text(resp.text)
+
+        text = _html_to_text(resp.text)
+        log.debug(f"Fetched {url}: HTTP {resp.status_code}, {len(text)} chars extracted")
+        return text
 
     def read_file(self, path: str, filename: str) -> bytes:
         """Return cached page content."""
         # Find matching URL from cache.
         target = f"{path}/{filename}" if path else filename
         target = target.removesuffix(".txt")
 
+        log.debug(f"read_file: looking up cache for target={target}")
+
         for url, text in self._cache.items():
             url_path = urlparse(url).path.strip("/")
             if not url_path:
                 url_path = "index"
             if url_path == target or url_path.endswith(target):
+                log.debug(f"read_file: resolved {target} -> {url}")
                 return text.encode("utf-8")
 
+        log.error(f"read_file: page not found in cache: {target} (cache size={len(self._cache)})")
         raise FileNotFoundError(f"Page not in cache: {target}")
 
     def close(self) -> None:
+        """Close the underlying HTTP client."""
+        log.debug("Closing WebConnector HTTP client")
         self._http.close()
 
 

src/oikb/daemon.py

@@ -464,12 +464,13 @@ def start_daemon(
     port: int = 8080,
     no_server: bool = False,
     log_format: str = "text",
+    log_level: str = "INFO",
 ) -> None:
     """Start the daemon with scheduler + optional HTTP server."""
     global _history, _entries
 
     from oikb.logging import configure_logging
-    configure_logging(log_format=log_format)
+    configure_logging(log_format=log_format, log_level=log_level)
 
     _entries = entries
     _history = SyncHistory()

src/oikb/http.py

@@ -0,0 +1,61 @@
+"""Shared HTTP client factory for oikb.
+
+Centralises httpx.Client creation and applies project-wide defaults,
+including the OIKB_INSECURE_SSL environment variable that disables
+TLS certificate verification for self-signed certificates.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+
+import httpx
+
+log = logging.getLogger(__name__)
+
+# Environment variable name used to disable TLS certificate verification.
+_ENV_INSECURE_SSL = "OIKB_INSECURE_SSL"
+
+
+def is_ssl_verification_disabled() -> bool:
+    """Return True if TLS certificate verification is disabled via environment variable.
+
+    <p>Reads the {@code OIKB_INSECURE_SSL} environment variable.
+    Any value other than an empty string or {@code "0"} is treated as truthy.</p>
+
+    @return True if SSL verification should be skipped, False otherwise.
+    """
+    raw = os.environ.get(_ENV_INSECURE_SSL, "").strip()
+    return raw not in ("", "0", "false", "False", "FALSE", "no", "No", "NO")
+
+
+def make_http_client(**kwargs: Any) -> httpx.Client:
+    """Create an {@link httpx.Client} with project-wide defaults applied.
+
+    <p>If the environment variable {@code OIKB_INSECURE_SSL} is set to a truthy
+    value (anything other than empty string, {@code "0"}, {@code "false"}, or
+    {@code "no"}), TLS certificate verification is disabled.
+    A {@code WARNING} is logged in that case.</p>
+
+    <p>Caller-supplied keyword arguments are merged over the defaults, so
+    individual connectors can still customise {@code base_url}, {@code headers},
+    {@code timeout}, etc.</p>
+
+    @param kwargs: Additional keyword arguments forwarded to {@link httpx.Client}.
+    @return A configured {@link httpx.Client} instance.
+    """
+    isInsecure = is_ssl_verification_disabled()
+
+    if isInsecure:
+        log.warning(
+            "TLS certificate verification is DISABLED (OIKB_INSECURE_SSL is set). "
+            "Do not use this setting in production environments."
+        )
+
+    # Apply verify=False only if not already explicitly set by the caller.
+    if isInsecure and "verify" not in kwargs:
+        kwargs["verify"] = False
+
+    return httpx.Client(**kwargs)