index.js

const axios = require('axios');
const cheerio = require('cheerio');

const BASE_URL = 'https://www.linkedin.com';
const HEADERS = {
  'authority': 'www.linkedin.com',
  'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
  'accept-language': 'en-US,en;q=0.9',
  'cache-control': 'max-age=0',
  'upgrade-insecure-requests': '1',
  'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
};

const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));

const parseJobType = (text) => {
  if (!text) return [];
  const types = [];
  const lower = text.toLowerCase();
  if (lower.includes('full-time')) types.push('FULL_TIME');
  if (lower.includes('part-time')) types.push('PART_TIME');
  if (lower.includes('contract')) types.push('CONTRACT');
  if (lower.includes('temporary')) types.push('TEMPORARY');
  if (lower.includes('internship')) types.push('INTERNSHIP');
  return types;
};

async function scrapeJobs(options) {
  const {
    searchTerm,
    location,
    resultsWanted = 10,
    hoursOld = null
  } = options;

  let jobList = [];
  let seenIds = new Set();
  let start = 0;
  let requestCount = 0;

  console.log(`Starting scrape for "${searchTerm}" in "${location}"...`);

  while (jobList.length < resultsWanted) {
    requestCount++;
    console.log(`Fetching page ${requestCount} (Jobs found: ${jobList.length})...`);

    const params = {
      keywords: searchTerm,
      location: location,
      start: start,
      f_WT: 2,
      pageNum: 0,
    };

    if (hoursOld) {
      params.f_TPR = `r${hoursOld * 3600}`;
    }

    try {
      const response = await axios.get(`${BASE_URL}/jobs-guest/jobs/api/seeMoreJobPostings/search`, {
        params,
        headers: HEADERS,
        timeout: 10000,
        validateStatus: status => status >= 200 && status < 500
      });

      if (response.status === 429) {
        console.warn('Rate limited. Waiting before retrying...');
        await sleep(5000);
        continue;
      }

      if (response.status !== 200) {
        console.error(`Error: Received status ${response.status}`);
        break;
      }

      const $ = cheerio.load(response.data);
      const jobCards = $('.base-search-card');

      if (jobCards.length === 0) {
        console.log('No more jobs found.');
        break;
      }

      let i = 0;
      while (i < jobCards.length) {
        if (jobList.length >= resultsWanted) break;

        const card = $(jobCards[i]);
        const linkTag = card.find('a.base-card__full-link');
        const href = linkTag.attr('href');

        if (!href) continue;

        const jobId = href.split('?')[0].split('-').pop();

        if (seenIds.has(jobId)) continue;
        seenIds.add(jobId);

        const title = card.find('.base-search-card__title').text().trim();
        const company = card.find('.base-search-card__subtitle').text().trim();
        const locationText = card.find('.job-search-card__location').text().trim();
        const dateTag = card.find('time.job-search-card__listdate');
        const datePosted = dateTag.attr('datetime') || null;

        try {
          await sleep(1000 + Math.random() * 1000);
          const jobUrl = `${BASE_URL}/jobs-guest/jobs/api/jobPosting/${jobId}`;
          const detailResponse = await axios.get(jobUrl, {
            headers: HEADERS,
            timeout: 5000,
            validateStatus: status => status >= 200 && status < 500
          });

          if (detailResponse.status === 200) {
            const $detail = cheerio.load(detailResponse.data);
            let description = $detail('.description__text').text().trim();

            description = description
              .replace(/See more/g, '')
              .replace(/See less/g, '')
              .replace(/<br>/g, '\n')
              .replace(/\\n/g, '\n')
              .trim();

            const criteriaList = $detail('.description__job-criteria-list');
            let jobTypeFound = 'N/A';
            let salaryFound = 'N/A';

            criteriaList.find('.description__job-criteria-item').each((_, el) => {
              const header = $detail(el).find('.description__job-criteria-subheader').text().trim();
              const content = $detail(el).find('.description__job-criteria-text').text().trim();

              if (header.includes('Employment type')) {
                jobTypeFound = parseJobType(content).join(', ') || content;
              }
            });

            const salaryElement = $detail('.salary-compensation__text');
            if (salaryElement.length > 0) {
              salaryFound = salaryElement.text().trim();
            }

            const jobPost = {
              id: jobId,
              title,
              company,
              location: locationText,
              date: datePosted,
              jobUrl: href.split('?')[0],
              jobType: jobTypeFound,
              salary: salaryFound,
              description: description || 'N/A'
            };

            jobList.push(jobPost);
            console.log(`Saved job: ${title} at ${company}`);

          } else {
            console.warn(`Failed to fetch details for job ${jobId}: Status ${detailResponse.status}`);
          }

        } catch (err) {
          console.error(`Error fetching details for job ${jobId}:`, err.message);
        }
        i++;
      }

      start += jobCards.length;
      await sleep(2000);

    } catch (error) {
      console.error('Error fetching jobs:', error.message);
      break;
    }
  }

  console.log(`Scraping complete. Found ${jobList.length} jobs.`);

  if (jobList.length > 0) {
    console.log(JSON.stringify(jobList, null, 2));
  }
}

const parseArgs = () => {
  const args = process.argv.slice(2);
  const options = {
    searchTerm: null,
    location: null,
    resultsWanted: 10,
    hoursOld: null
  };

  let i = 0;
  while (i < args.length) {
    const arg = args[i];
    const nextArg = args[i + 1];

    if (arg === '-t' && nextArg) {
      const titleParts = [];
      let j = i + 1;
      while (j < args.length && !args[j].startsWith('-')) {
        titleParts.push(args[j]);
        j++;
      }
      options.searchTerm = titleParts.join(' ');
      i = j;
    } else if (arg === '-l' && nextArg) {
      const locationParts = [];
      let j = i + 1;
      while (j < args.length && !args[j].startsWith('-')) {
        locationParts.push(args[j]);
        j++;
      }
      options.location = locationParts.join(' ');
      i = j;
    } else if (arg === '-r' && nextArg) {
      options.resultsWanted = parseInt(nextArg, 10);
      i += 2;
    } else if (arg === '-h' && nextArg) {
      options.hoursOld = parseInt(nextArg, 10);
      i += 2;
    } else {
      i++;
    }
  }

  return options;
};

const options = parseArgs();

if (!options.searchTerm || !options.location) {
  console.error('Error: Both -t (title) and -l (location) are required.');
  console.error('Usage: node index.js -t "job title" -l "location" [-r results] [-h hours]');
  process.exit(1);
}

scrapeJobs(options);