sentibot.py

import os
import asyncio
import json
import requests
from datetime import datetime
from dotenv import load_dotenv
import feedparser
from newspaper import Article
import torch
import numpy as np
from sklearn.cluster import AgglomerativeClustering
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, BitsAndBytesConfig
import yfinance as yf
import discord
from config import (logger, DISCORD_MAX_LEN, RSS_FEEDS, RELEVANT_TOPICS, MIN_RELEVANT_MASS, CANDIDATE_LABELS, REPORTS_DIR, LOG_PATH)
from utils import ( topic_namer_prompt_builder,facts_and_tickers_extractor_prompt_builder,topic_summariser_prompt_builder,group_summary_prompt_builder)
from system_monitor import SystemMonitor

load_dotenv()

"""
PYTORCH_CUDA_ALLOC_CONF allows CUDA memory segments to grow in place instead of allocating new contiguous blocks,
which in turn reduces fragmentation from variable-sized attention matrices.
"""
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" # prevent CUDA memory fragmentation during large attention matrix allocations

class Sentibot:

    def __init__(self):
        logger.info("Initialising Sentibot...")

        logger.info(f"CUDA available: {torch.cuda.is_available()}")
        if torch.cuda.is_available():
            logger.info(f"GPU: {torch.cuda.get_device_name(0)}")

        logger.info("Loading FinBERT sentiment model...")
        self.sentiment_model = pipeline("text-classification", model="ProsusAI/finbert", device=0)

        logger.info("Loading sentence embedder all-MiniLM-L6-v2 ...")
        self.embedder = SentenceTransformer("all-MiniLM-L6-v2")

        logger.info("Loading MiniLM topic classifier (CPU)...")
        self.topic_classifier = pipeline( "zero-shot-classification", model="cross-encoder/nli-MiniLM2-L6-H768", device=-1 )

        logger.info("Loading Qwen 1.5B (4-bit quantised)...")

        quant_config = BitsAndBytesConfig( 
            load_in_4bit=True, # stores weights in 4-bit instead of 16/32-bit, 
            bnb_4bit_compute_dtype=torch.float16, # math still runs in float16 even though weights are stored in 4-bit
            bnb_4bit_use_double_quant=True, # quantizes the quantization constants too, for a bit more memory savings
            bnb_4bit_quant_type="nf4" # a 4-bit format tuned for normally-distributed weights, more accurate than plain int4
        )

        self.tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-1.5B-Instruct")
        self.model = AutoModelForCausalLM.from_pretrained( 
            "Qwen/Qwen2.5-1.5B-Instruct", 
            quantization_config=quant_config, 
            device_map="auto", # split layers across GPU/CPU automatically 
            low_cpu_mem_usage=True, # loads weights straight into place instead of spiking CPU RAM with a full-precision copy first
            max_memory={0: "3GiB", "cpu": "16GiB"} # caps GPU/CPU usage, so anything over 3GiB spills to CPU.
        )

        logger.info("All models loaded.")


    async def _discord_run(self, coro_body):
        """Spin up a short-lived Discord client, run coro_body(client), then close. Used to send multiple messages in sequence when the final report exceeds 2000 chars"""
        client = discord.Client(intents=discord.Intents.default())

        @client.event
        async def on_ready():
            try:
                await coro_body(client)
            finally:
                await client.close()

        await client.start(os.getenv("DISCORD_TOKEN"))


    def chunk_markdown_report(self, sections: list[str], max_len: int = DISCORD_MAX_LEN) -> list[str]:
        """
        Pack a list of self-contained markdown "sections" (each itself possibly multi-line) into Discord messages of at most max_len characters.
        A section is never split mid-sentence/line by this function. Sections are expected to already be pre-wrapped into safe chunks by the caller if they
        risk exceeding max_len on their own. This function just bins whole sections together so consecutive sections share a message when they fit.
        """
        messages = []
        current = ""

        for section in sections:
            candidate = f"{current}\n\n{section}" if current else section

            if len(candidate) <= max_len:
                current = candidate
                continue

            # Doesn't fit alongside current -> flush current, start fresh
            if current:
                messages.append(current)
                current = ""

            if len(section) <= max_len:
                current = section
            else:
                # Section itself too large: split on line boundaries
                for piece in self._split_long_section(section, max_len):
                    if current and len(current) + 2 + len(piece) <= max_len:
                        current = f"{current}\n\n{piece}"
                    else:
                        if current:
                            messages.append(current)
                        current = piece

        if current:
            messages.append(current)

        return messages


    def _split_long_section(self, section: str, max_len: int) -> list[str]:
        """Split an oversized section into chunks on line breaks, never mid-line."""
        lines = section.split("\n")
        chunks = []
        current = ""

        for line in lines:
            candidate = f"{current}\n{line}" if current else line

            if len(candidate) <= max_len:
                current = candidate
                continue

            if current:
                chunks.append(current)

            if len(line) <= max_len:
                current = line
            else:
                # Single line too long (rare): hard-split on whitespace
                words = line.split(" ")
                current = ""
                for word in words:
                    w_candidate = f"{current} {word}" if current else word
                    if len(w_candidate) <= max_len:
                        current = w_candidate
                    else:
                        if current:
                            chunks.append(current)
                        current = word

        if current:
            chunks.append(current)

        return chunks


    def _generate(self, messages: list[dict], max_new_tokens: int = 256, temperature: float = 0.7) -> str:
        """Shared helper method that tokenises, generates LLM response, and decodes it """
        encoded = self.tokenizer.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, return_tensors="pt", return_dict=True )
        input_ids = encoded["input_ids"].to(self.model.device)
        attention_mask = encoded["attention_mask"].to(self.model.device)
        output = self.model.generate( input_ids, attention_mask=attention_mask, max_new_tokens=max_new_tokens, temperature=temperature, do_sample=True, pad_token_id=self.tokenizer.eos_token_id )
        new_tokens = output[0][input_ids.shape[-1]:]
        return self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()


    def _classify_relevance(self, text: str) -> tuple[str, float, bool]:
        """
        Run the zero-shot topic classifier on a piece of text (typically a
        short LLM-generated cluster name) and report whether it's relevant.

        Uses the same summed relevant-mass approach as filter_relevant rather
        than single top-label gating, because cluster names like "Stock Market
        Analysis" split probability across finance/business/economics and no
        single label clears MIN_TOPIC_SCORE even though all three are relevant.

        Returns (top_label, top_score, is_relevant).
        """
        result = self.topic_classifier(text, CANDIDATE_LABELS)
        top_label = result["labels"][0]
        top_score = result["scores"][0]
        relevant_mass = sum( score for label, score in zip(result["labels"], result["scores"]) if label in RELEVANT_TOPICS )
        is_relevant = relevant_mass >= MIN_RELEVANT_MASS
        return top_label, top_score, is_relevant
    

    def _parse_article(self, url: str, fallback_title: str = "", fallback_text: str = "") -> dict | None:
        """Download and parse a single article URL. Falls back to RSS article title and summary on failure."""
        try:
            article = Article(url)
            article.download()
            article.parse()
            title = article.title or fallback_title
            text = article.text
            if not text or len(text) < 100:
                text = fallback_text
                logger.warning(f"Fell back to RSS snippet: {url}")
            if title and text:
                return {"title": title, "text": text, "url": url}
        except Exception as e:
            logger.warning(f"Failed to parse {url}: {e}")
        return None
    

    def ingest(self) -> tuple[list[dict], list[dict]]:
        """
        Fetch every RSS feed and parse each entry into a standardised article dictionary
        Returns: corpus - a list of parsed article dicts, feed_stats - list of per-feed dicts with ingestion counts, used for the report's ingestion summary section
        """
        logger.info(f"Ingesting from {len(RSS_FEEDS)} RSS feeds.")
        corpus = []
        feed_stats = []

        for feed_name, feed_url in RSS_FEEDS.items():
            logger.info(f"Fetching '{feed_name}'")
            feed = feedparser.parse(feed_url)
            logger.info(f"  {len(feed.entries)} entries returned.")

            ingested_count, skipped_count = 0, 0

            for entry in feed.entries:
                url = entry.link
                result = self._parse_article( url, fallback_title=entry.get("title", ""), fallback_text=entry.get("summary", "") )
                if result:
                    corpus.append({ "title": result["title"], "text": result["text"], "url": result["url"], "feed": feed_name, "ingested_at": datetime.now().isoformat(), "char_count": len(result["text"]) })
                    ingested_count += 1
                    logger.info(f"  Ingested [{feed_name}]: '{result['title']}'")
                else:
                    skipped_count += 1
                    logger.warning(f"  Skipped unparseable entry: {url}")

            feed_stats.append({ "feed": feed_name, "total_entries": len(feed.entries), "ingested": ingested_count, "skipped": skipped_count })

        logger.info(f"Ingestion complete. {len(corpus)} articles collected.")
        return corpus, feed_stats


    def deduplicate(self, articles: list[dict], threshold: float = 0.85) -> list[dict]:
        """ Embed titles with all-MiniLM-L6-v2 and discard near-duplicates using cosine similarity. threshold=0.85 catches reworded version of the same story across different feeds. """
        logger.info(f"Deduplicating {len(articles)} articles (threshold={threshold}).")
        if not articles:
            return []

        titles = [a["title"] for a in articles]
        embeddings = self.embedder.encode(titles)
        similarity = cosine_similarity(embeddings)

        seen, unique = set(), []

        for i, article in enumerate(articles):
            if i in seen:
                continue
            unique.append(article)
            for j in range(i + 1, len(articles)):
                if similarity[i][j] >= threshold:
                    seen.add(j)
                    logger.info(
                        f"  DUPLICATE: '{articles[j]['title']}' ({articles[j]['feed']}) "
                        f"~ '{article['title']}' ({article['feed']}) [{similarity[i][j]:.2%}]"
                    )

        logger.info(f"Deduplication: {len(articles)} -> {len(unique)} ({len(articles) - len(unique)} removed).")
        return unique


    def filter_relevant(self, articles: list[dict]) -> list[dict]:
        """
        Use MiniLM zero-shot classification to keep only articles relevant to
        finance, business, geopolitics, technology, or economics.

        Rather than requiring a single label to clear a high bar, we sum the
        scores across all RELEVANT_TOPICS labels (see _relevant_mass) and
        keep anything whose combined relevant mass clears MIN_RELEVANT_MASS.
        This avoids discarding stories that the model finds plausible under
        several relevant labels at once but doesn't strongly commit to any
        single one.
        """
        logger.info(f"Filtering {len(articles)} articles (min_relevant_mass={MIN_RELEVANT_MASS}).")
        if not articles:
            return []

        titles = [a["title"] for a in articles]
        results = self.topic_classifier(titles, CANDIDATE_LABELS, batch_size=16)

        kept = []
        discarded = 0

        for article, result in zip(articles, results):
            top_label = result["labels"][0]
            top_score = result["scores"][0]
            relevant_mass = sum( score for label, score in zip(result["labels"], result["scores"]) if label in RELEVANT_TOPICS )

            if relevant_mass >= MIN_RELEVANT_MASS:
                article["topic_label"] = top_label
                article["topic_score"] = top_score
                article["relevant_mass"] = relevant_mass
                kept.append(article)
                logger.info(f"  KEPT [{top_label} {top_score:.2%}, relevant_mass={relevant_mass:.2%}] '{article['title']}'")
            else:
                discarded += 1
                logger.info(f"  DISCARDED [{top_label} {top_score:.2%}, relevant_mass={relevant_mass:.2%}] '{article['title']}'")

        logger.info(f"Filter: {len(articles)} -> {len(kept)} ({discarded} discarded).")
        return kept


    def group_by_topic(self, articles: list[dict], max_groups: int = 7) -> list[dict]:
        """
        Cluster articles into topic groups using agglomerative clustering on
        title embeddings. The number of clusters is determined dynamically
        (capped at max_groups). Each cluster is then named by passing a sample
        of its titles to the LLM, and the resulting name is re-checked against
        the topic classifier - clusters whose name doesn't score above
        MIN_TOPIC_SCORE on a relevant label are discarded before any further
        (expensive) LLM processing.

        Returns a list of group dicts:
            {
                name: str, # LLM-derived topic name
                articles: list[dict]
            }
        """
        logger.info(f"Grouping {len(articles)} articles into topics (max_groups={max_groups}).")

        if len(articles) <= 1:
            logger.warning("Too few articles to cluster. Returning single group.")
            return [{"name": "General News", "articles": articles}]

        titles = [a["title"] for a in articles]
        embeddings = self.embedder.encode(titles)

        # Determine number of clusters dynamically using sqrt(n = number of articles/2), which is a common heuristic, capped at max_groups
        n_clusters = min(max_groups, max(2, int(np.sqrt(len(articles) / 2))))
        logger.info(f"Using {n_clusters} clusters for {len(articles)} articles.")

        clustering = AgglomerativeClustering(n_clusters=n_clusters, metric="cosine", linkage="average")
        labels = clustering.fit_predict(embeddings)

        # Organise articles by cluster label
        clusters: dict[int, list[dict]] = {}
        for article, label in zip(articles, labels):
            clusters.setdefault(label, []).append(article)

        # Name each cluster using the LLM, then re-check relevance
        groups = []
        for cluster_id, cluster_articles in clusters.items():
            logger.info(f"Naming cluster {cluster_id} ({len(cluster_articles)} articles).")

            # Pass up to 5 representative titles to the LLM for naming
            sample_titles = "\n".join( f"- {a['title']}" for a in cluster_articles[:5] )
            messages = [{ "role": "user", "content": topic_namer_prompt_builder(sample_titles) }]
            name = self._generate(messages, max_new_tokens=50, temperature=0.3)
            name = name.strip().strip('"').strip("'")

            # Second relevance pass: check the LLM-derived name itself
            top_label, top_score, is_relevant = self._classify_relevance(name)

            if not is_relevant:
                logger.info(f"  Cluster {cluster_id} named: '{name}' ({len(cluster_articles)} articles) -> DISCARDED [{top_label} {top_score:.2%}] (name failed relevance check)")
                continue

            logger.info(f"  Cluster {cluster_id} named: '{name}' ({len(cluster_articles)} articles) -> KEPT [{top_label} {top_score:.2%}]")
            groups.append({"name": name, "articles": cluster_articles})

        logger.info(f"Grouping complete. {len(groups)} topic group(s) kept after relevance re-check.")
        return groups


    def _average_sentiment(self, articles: list[dict]) -> dict:
        """Compute group-level sentiment by running FinBERT on all titles and averaging the scores. Returns the dominant label and mean confidence."""
        titles = [a["title"] for a in articles]
        results = self.sentiment_model(titles, batch_size=16)

        label_map = {"positive": 1, "neutral": 0, "negative": -1}
        scores = [label_map.get(r["label"], 0) * r["score"] for r in results]
        mean_score = np.mean(scores)

        if mean_score > 0.1:
            label = "positive"
        elif mean_score < -0.1:
            label = "negative"
        else:
            label = "neutral"

        mean_confidence = np.mean([r["score"] for r in results])
        return {"label": label, "confidence": f"{mean_confidence:.2%}", "raw_score": mean_score}


    def extract_facts_and_tickers(self, title: str, text: str) -> tuple[str, list[str]]:
        """
        Single LLM call per article that returns both key facts and any stock tickers mentioned. Combining these into one call halves the number of
        inference steps compared to running them separately. Returns:
        facts   - bullet-point string of key facts
        tickers - list of ticker symbols (may be empty)
        """
        messages = [{ "role": "user", "content": facts_and_tickers_extractor_prompt_builder(title, text) }]
        raw = self._generate(messages, max_new_tokens=200, temperature=0.7)
        logger.info(f"  Raw facts + tickers for '{title}': {raw[:120]}...")

        # Parse FACTS and TICKERS sections from the structured response
        facts = ""
        tickers = []

        if "FACTS:" in raw and "TICKERS:" in raw:
            parts = raw.split("TICKERS:")
            facts_block = parts[0].replace("FACTS:", "").strip()
            ticker_block = parts[1].strip()
            facts = facts_block
            if ticker_block.upper() != "NONE" and ticker_block:
                tickers = [t.strip().upper() for t in ticker_block.split(",") if t.strip()]
        else:
            # Fallback: treat entire response as facts if model ignores the format
            facts = raw

        """
        Extract ticker from parentheses in title (e.g. Yahoo Finance's "Is Ballard Power Systems Inc. (BLDP) A Good Stock To Buy Now?").
        Done outside the NONE/format guard so it fires even when the model returns TICKERS: NONE or ignores the format entirely.
        """
        if "(" in title and ")" in title:
            potential_ticker = title.split("(", 1)[1].split(")", 1)[0].strip().upper()
            if potential_ticker and potential_ticker not in tickers:
                tickers.append(potential_ticker)

        logger.info(f"  Tickers from '{title}': {tickers}")
        return facts, tickers


    def _summarise_chunk(self, group_name: str, chunk: list[dict]) -> str:
        """
        Summarise a single chunk of articles (map step). Uses extracted facts rather than raw article text - facts are already
        distilled and concise, so the prompt is smaller and more focused.
        """
        corpus = "\n".join( f"- {a['title']}:\n{a.get('facts', a['text'][:300])}" for a in chunk )
        messages = [{ "role": "user", "content": topic_summariser_prompt_builder(group_name, corpus) }]
        return self._generate(messages, max_new_tokens=200, temperature=0.7)


    def _summarise_group(self, group_name: str, articles: list[dict], chunk_size: int = 5) -> str:
        """
        Map-reduce summarisation for a topic group.

        Map:    Split articles into chunks of chunk_size. Summarise each chunk
                independently - each LLM call stays well within the context window.
        Reduce: If there is more than one chunk summary, pass all chunk summaries
                to the LLM for a final synthesis. This removes the hard corpus cap
                and handles arbitrarily large groups.

        For small groups (<=chunk_size articles) only a single LLM call is made.
        """
        if not articles:
            return ""

        # Split into chunks
        chunks = [articles[i:i + chunk_size] for i in range(0, len(articles), chunk_size)]
        logger.info(f"  Map-reduce: {len(articles)} articles -> {len(chunks)} chunk(s) for '{group_name}'.")

        # Map: summarise each chunk
        chunk_summaries = []
        for idx, chunk in enumerate(chunks):
            logger.info(f"    Summarising chunk {idx + 1}/{len(chunks)} ({len(chunk)} articles).")
            summary = self._summarise_chunk(group_name, chunk)
            chunk_summaries.append(summary)
            logger.info(f"    Chunk {idx + 1} summary: {len(summary)} chars.")

        # Reduce: if only one chunk, return directly - no second call needed
        if len(chunk_summaries) == 1:
            logger.info(f"  Single chunk - no reduce step needed for '{group_name}'.")
            return chunk_summaries[0]

        # Reduce: synthesise chunk summaries into a final summary
        logger.info(f"  Reducing {len(chunk_summaries)} chunk summaries for '{group_name}'.")
        combined = "\n\n".join( f"Part {i + 1}: {s}" for i, s in enumerate(chunk_summaries) )

        messages = [{ "role": "user", "content": group_summary_prompt_builder(group_name, combined) }]
        final = self._generate(messages, max_new_tokens=250, temperature=0.7)
        logger.info(f"  Final summary for '{group_name}': {len(final)} chars.")
        return final


    def summarise_groups(self, groups: list[dict]) -> tuple[list[dict], list[str]]:
        """
        For each topic group:
            - Per article: extract facts AND tickers in a single LLM call
            - Accumulate tickers into a global store across all groups
            - Compute group-level sentiment by averaging FinBERT scores
            - Summarise the group using extracted facts via map-reduce

        Returns:
            enriched_groups - list of enriched group dicts
            global_tickers  - deduplicated list of all tickers found today
        """
        logger.info(f"Summarising {len(groups)} topic group(s).")
        enriched = []
        global_tickers = set()  # accumulates tickers across all groups

        for group in groups:
            name = group["name"]
            articles = group["articles"]
            group_tickers = set()
            logger.info(f"Processing group '{name}' ({len(articles)} articles).")

            # Per-article: extract facts and tickers in one LLM call
            for i, article in enumerate(articles):
                logger.info(f"  Facts+tickers [{i + 1}/{len(articles)}]: '{article['title']}'")
                facts, tickers = self.extract_facts_and_tickers(article["title"], article["text"])
                article["facts"] = facts
                group_tickers.update(tickers)
                global_tickers.update(tickers)

            logger.info(f"  Group tickers found: {list(group_tickers)}")

            # Group-level sentiment (batched FinBERT call)
            sentiment = self._average_sentiment(articles)

            # Map-reduce summarisation using extracted facts as corpus
            summary = self._summarise_group(name, articles)

            enriched.append({ "name": name, "articles": articles, "sentiment": sentiment, "tickers": list(group_tickers), "summary": summary })
            logger.info(f"Group '{name}' done. Sentiment: {sentiment['label']} ({sentiment['confidence']}). Tickers: {list(group_tickers)}.")

        logger.info(f"All groups processed. Global ticker store: {list(global_tickers)}")
        return enriched, list(global_tickers)


    def validate_tickers(self, tickers: list[str]) -> list[str]:
        """Filter hallucinated tickers against yfinance."""
        valid = []
        for ticker in set(tickers):
            try:
                info = yf.Ticker(ticker).info
                if info.get("currentPrice") or info.get("regularMarketPrice"):
                    valid.append(ticker)
                    logger.info(f"  Ticker valid: {ticker}")
                else:
                    logger.warning(f"  Ticker invalid (no price): {ticker}")
            except Exception:
                logger.warning(f"  Ticker validation failed: {ticker}")
        return valid


    def get_stock_metrics(self, tickers: list[str]) -> dict:
        """Fetch live price, day change, and fundamentals for each ticker."""
        logger.info(f"Fetching metrics for {len(tickers)} ticker(s): {tickers}")
        metrics = {}

        for ticker in tickers:
            try:
                stock = yf.Ticker(ticker)
                info = stock.info
                hist = stock.history(period="5d")

                if hist.empty:
                    logger.warning(f"  No price history for {ticker}.")
                    continue

                latest = hist["Close"].iloc[-1]
                prev = hist["Close"].iloc[-2] if len(hist) > 1 else latest
                change = ((latest - prev) / prev) * 100

                metrics[ticker] = { 
                    "name": info.get("longName", ticker), "price": f"${latest:.2f}", "day_change": f"{change:+.2f}%", "pe_ratio": info.get("trailingPE", "N/A"), "52w_high": info.get("fiftyTwoWeekHigh", "N/A"),
                    "52w_low": info.get("fiftyTwoWeekLow", "N/A"), "analyst_target": info.get("targetMeanPrice", "N/A"), "sector": info.get("sector", "N/A"), "market_cap": info.get("marketCap", "N/A")
                }
                logger.info(f"  {ticker}: {metrics[ticker]['price']} ({metrics[ticker]['day_change']})")

            except Exception as e:
                logger.warning(f"  Failed to fetch {ticker}: {e}")

        return metrics


    def get_fear_and_greed(self) -> dict:
        """
        Fetch the CNN Fear & Greed Index. Returns score (0-100) and label (e.g. 'Fear', 'Greed').
        """
        try:
            url = "https://production.dataviz.cnn.io/index/fearandgreed/graphdata"
            # send realistic headers to bypass CNN's bot protection
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
                "Accept": "application/json, text/plain, */*",
                "Accept-Language": "en-US,en;q=0.9",
                "Referer": "https://edition.cnn.com/markets/fear-and-greed",
            }
            resp = requests.get(url, timeout=10, headers=headers)
            resp.raise_for_status()
            data = resp.json()
            score = data["fear_and_greed"]["score"]
            label = data["fear_and_greed"]["rating"].replace("_", " ").title()
            logger.info(f"Fear & Greed: {score:.1f} ({label})")
            return {"score": round(score, 1), "label": label}
        except Exception as e:
            logger.warning(f"Failed to fetch Fear & Greed index: {e}")
            return {"score": None, "label": "Unavailable"}


    @staticmethod
    def _fmt_number(value, decimals: int = 2) -> str:
        """Round a numeric metric to `decimals` places; pass through non-numeric values (e.g. 'N/A')."""
        try:
            return f"{float(value):.{decimals}f}"
        except (TypeError, ValueError):
            return str(value)


    def build_markdown_sections(self, groups: list[dict], metrics: dict, fear_greed: dict, feed_stats: list[dict], dedupe_removed: int) -> list[str]:
        """
        Render the daily briefing as a list of self-contained markdown "sections":
            1. Header (title + execution date)
            2. Ingestion summary (per-feed counts + dedup count)
            3. One section per topic group (merged overview + sector analysis, sentiment line with score)
            4. Market outlook (Fear & Greed, then one sub-section per ticker with metrics + sentiment score)

        Each section is logically complete so chunk_markdown_report can pack them into Discord messages without breaking mid-thought.
        """
        sections = []

        # --- Header -------------------------------------------------------
        sections.append(f"# Daily News Briefing\n **{datetime.now().strftime('%A %d %B %Y, %H:%M')}**")

        # --- Ingestion summary --------------------------------------------
        ingestion_lines = ["## Ingestion Summary", ""]
        for fs in feed_stats:
            ingestion_lines.append(
                f"- **{fs['feed']}**: {fs['ingested']} ingested, "
                f"{fs['skipped']} skipped (of {fs['total_entries']} entries)"
            )
        ingestion_lines.append(f"- **Deduplicated:** {dedupe_removed} article(s) removed")
        sections.append("\n".join(ingestion_lines))

        # --- One section per topic group ---------------------------------
        for group in groups:
            sent_label = group["sentiment"]["label"]
            sent_conf  = group["sentiment"]["confidence"]
            sent_score = f"{group['sentiment']['raw_score']:+.3f}"
            lines = [f"## {group['name']}", ""]

            lines.append("**Overview & Sector Analysis:**")
            lines.append(group["summary"].strip())
            lines.append("")
            lines.append(
                f"**Sentiment:** {sent_label.title()} "
                f"(score {sent_score}, {sent_conf} confidence) "
                f"\u2022 {len(group['articles'])} article(s)"
            )

            sections.append("\n".join(lines))

        # --- Market outlook with per-ticker blocks ------------------------
        fg_text = (
            f"**CNN Fear & Greed Index:** {fear_greed['score']} - {fear_greed['label']}"
            if fear_greed["score"] is not None
            else "**CNN Fear & Greed Index:** Unavailable"
        )

        # Reverse map: ticker -> full group sentiment dict (label + conf + score)
        ticker_sentiment: dict[str, dict] = {}
        for group in groups:
            for ticker in group["tickers"]:
                ticker_sentiment[ticker] = group["sentiment"]

        sections.append("\n".join(["## Market Outlook", fg_text]))

        if not metrics:
            sections.append("*No tickers identified today.*")
        else:
            for ticker, m in metrics.items():
                t_sent     = ticker_sentiment.get(ticker, {})
                sent_label = t_sent.get("label", "neutral")
                sent_conf  = t_sent.get("confidence", "N/A")
                sent_score = f"{t_sent['raw_score']:+.3f}" if "raw_score" in t_sent else "N/A"

                ticker_lines = [
                    f"### {ticker} — {m['name']}",
                    f"- **Price:** ${self._fmt_number(m['price'].lstrip('$'))} ({m['day_change']})"
                    f" \u2022 **Sector:** {m['sector']}",
                    f"- **P/E:** {self._fmt_number(m['pe_ratio'])}"
                    f" \u2022 **52w:** ${self._fmt_number(m['52w_low'])}-${self._fmt_number(m['52w_high'])}"
                    f" \u2022 **Target:** ${self._fmt_number(m['analyst_target'])}",
                    f"- **Sentiment:** {sent_label.title()} (score {sent_score}, {sent_conf} confidence)",
                ]

                sections.append("\n".join(ticker_lines))

        return sections


    def save_daily_json(self, groups: list[dict], fear_greed: dict, output_path: str):
        """Save a minimal JSON snapshot of today's report locally for recordkeeping."""

        payload = {
            "date": datetime.now().strftime("%Y-%m-%d"),
            "fear_and_greed": fear_greed,
            "groups": [
                {
                    "name": g["name"],
                    "sentiment": g["sentiment"],
                    "article_count": len(g["articles"]),
                    "tickers": g["tickers"],
                    "summary": g["summary"],
                }
                for g in groups
            ],
        }

        with open(output_path, "w", encoding="utf-8") as f:
            json.dump(payload, f, indent=2, ensure_ascii=False)

        logger.info(f"Daily JSON saved: {output_path} ({len(groups)} groups).")


    async def send_markdown_report(self, sections: list[str]) -> None:
        """Send the report as a sequence of markdown messages to the summary channel."""
        channel_id = os.getenv("DISCORD_SUMMARY_CHANNEL_ID")
        if not channel_id:
            logger.error("DISCORD_SUMMARY_CHANNEL_ID is not set. Skipping markdown report.")
            return

        messages = self.chunk_markdown_report(sections)
        logger.info(f"Sending markdown report as {len(messages)} message(s).")

        async def body(client: discord.Client):
            channel = client.get_channel(int(channel_id))
            if not channel:
                logger.error("Summary channel not found. Check DISCORD_SUMMARY_CHANNEL_ID.")
                return
            for msg in messages:
                await channel.send(msg)
            logger.info("Markdown report sent.")

        await self._discord_run(body)


    async def send_observability_update(self, caption: str, files: list[str]) -> None:
        """Send a status message plus any attachments (logs, benchmark PDF) to the observability channel."""
        channel_id = os.getenv("DISCORD_OBSERVABILITY_CHANNEL_ID")
        if not channel_id:
            logger.error("DISCORD_OBSERVABILITY_CHANNEL_ID is not set. Skipping observability update.")
            return

        logger.info(f"Sending observability update with {len(files)} file(s).")

        async def body(client: discord.Client):
            channel = client.get_channel(int(channel_id))
            if not channel:
                logger.error("Observability channel not found. Check DISCORD_OBSERVABILITY_CHANNEL_ID.")
                return

            attachments = [discord.File(f) for f in files if f and os.path.exists(f)]
            if attachments:
                await channel.send(content=caption, files=attachments)
            else:
                await channel.send(content=caption)
            logger.info("Observability update sent.")

        await self._discord_run(body)


    async def main(self) -> None:
        logger.info("=" * 60 + "\nSentibot daily job started.")

        monitor = SystemMonitor()
        monitor.start()

        date_str = datetime.now().strftime("%Y-%m-%d")
        benchmark_path = os.path.join(REPORTS_DIR, f"Sentibot_{date_str}_benchmark.pdf")
        report_sections = None

        try:
            monitor.mark_stage("Ingest")
            articles, feed_stats = self.ingest()
            if not articles:
                logger.warning("No articles ingested. Exiting.")
                return

            monitor.mark_stage("Deduplicate")
            pre_dedupe_count = len(articles)
            articles = self.deduplicate(articles)
            dedupe_removed = pre_dedupe_count - len(articles)
            if not articles:
                logger.warning("No articles after deduplication. Exiting.")
                return

            monitor.mark_stage("Filter")
            articles = self.filter_relevant(articles)
            if not articles:
                logger.warning("No articles after filtering. Exiting.")
                return

            monitor.mark_stage("Group")
            groups = self.group_by_topic(articles)
            if not groups:
                logger.warning("No topic groups survived the relevance re-check. Exiting.")
                return

            monitor.mark_stage("Summarise")
            groups, global_tickers = self.summarise_groups(groups)

            monitor.mark_stage("Stock Metrics")
            valid_tickers = self.validate_tickers(global_tickers)
            metrics = self.get_stock_metrics(valid_tickers)

            monitor.mark_stage("Fear & Greed")
            fear_greed = self.get_fear_and_greed()

            monitor.mark_stage("Build Report")
            report_sections = self.build_markdown_sections( groups, metrics, fear_greed, feed_stats, dedupe_removed )

            json_path = os.path.join(REPORTS_DIR, f"Sentibot_{date_str}.json")
            self.save_daily_json(groups, fear_greed, json_path)

            logger.info("Sentibot daily job completed successfully.")

        except Exception as e:
            logger.error(f"Sentibot job failed: {e}", exc_info=True)
            raise

        finally:
            monitor.mark_stage("Deliver")
            monitor.stop()

            monitor.save_benchmark_pdf(benchmark_path)

            if report_sections:
                await self.send_markdown_report(report_sections)
                obs_caption = "**Sentibot Daily Briefing** - run completed successfully."
            else:
                obs_caption = ( "**Sentibot Daily Briefing**\n\n Pipeline failed before the report could be generated" )

            obs_files = []
            if os.path.exists(LOG_PATH):
                obs_files.append(LOG_PATH)
            if os.path.exists(benchmark_path):
                obs_files.append(benchmark_path)

            await self.send_observability_update(obs_caption, obs_files)


if __name__ == "__main__":
    bot = Sentibot()
    asyncio.run(bot.main())