summarizer.js

// summarizer.js
process.env.ORT_LOG_LEVEL = "error";

const { pipeline } = require('@xenova/transformers');

// CONFIG
const MODEL = 'Xenova/t5-small';
const MAX_CHUNK = 800; // characters per chunk

// STOPWORDS
const STOPWORDS = new Set([
"a","an","the","and","or","but","if","while","of","at","by","for","with",
"about","against","between","into","through","during","before","after",
"above","below","to","from","up","down","in","out","on","off","over",
"under","again","further","then","once","here","there","when","where",
"why","how","all","any","both","each","few","more","most","other","some",
"such","no","nor","not","only","own","same","so","than","too","very"
]);

// TEXT CLEANING
function stripHtml(html = "") {
  return html.replace(/<[^>]*>/g, '');
}

function normalizeText(text) {
  return stripHtml(text)
    .replace(/\s+/g, ' ')
    .trim();
}

// CHUNK LONG TEXT
function chunkText(text) {
  const chunks = [];
  let i = 0;

  while (i < text.length) {
    chunks.push(text.slice(i, i + MAX_CHUNK));
    i += MAX_CHUNK;
  }

  return chunks;
}

// TITLE GENERATION
function generateTitle(summary) {

  const words = summary
    .toLowerCase()
    .split(/\W+/)
    .filter(w => w.length > 3 && !STOPWORDS.has(w));

  const freq = {};

  for (let w of words)
    freq[w] = (freq[w] || 0) + 1;

  const top = Object.entries(freq)
    .sort((a,b)=>b[1]-a[1])
    .slice(0,4)
    .map(([w]) => w.charAt(0).toUpperCase()+w.slice(1));

  return top.join(" ");
}

// SUMMARIZER CLASS
class Summarizer {

  constructor(){
    this.model = null;
    this.loading = null;
  }

  async loadModel(){

    if(this.model) return this.model;

    if(!this.loading){
      console.log("Loading news summarization model...");
      this.loading = pipeline('summarization', MODEL);
    }

    this.model = await this.loading;

    console.log("Summarizer ready.");

    return this.model;
  }

  releaseModel(){
    this.model = null;
    this.loading = null;
  }

  // MAIN SUMMARIZE FUNCTION
  async summarizeText(text){

    await this.loadModel();

    const clean = normalizeText(text);

    // Skip model summary generation if text already short
    if(clean.length < 250){
      return {
        title: generateTitle(clean),
        summary: clean
      };
    }

    const chunks = chunkText(clean);

    const partial = [];

    for(const chunk of chunks){

      const result = await this.model(chunk,{
        max_length:80,
        min_length:25
      });

      partial.push(result[0].summary_text);
    }

    const combined = partial.join(" ");

    const final = await this.model(combined,{
      max_length:90,
      min_length:35
    });

    const summary = final[0].summary_text;

    return {
      title: generateTitle(summary),
      summary
    };
  }

}

module.exports = Summarizer;