scrape.js

#!/usr/bin/env node
const { chromium } = require('playwright-extra');
const stealth = require('puppeteer-extra-plugin-stealth')();
chromium.use(stealth);

const registernummer = process.argv[2] || 'D-21RP-R1O5O-37';

(async () => {
  const browser = await chromium.launch({
    headless: true,
    args: ['--disable-blink-features=AutomationControlled', '--no-sandbox']
  });
  const context = await browser.newContext({
    userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    viewport: { width: 1920, height: 1080 }
  });
  const page = await context.newPage();

  const apiResponses = {};
  page.on('response', async (response) => {
    const url = response.url();
    if (url.includes('/suche/') && !url.includes('ip-test') && !url.includes('puzzle') && response.status() === 200) {
      try {
        const ct = response.headers()['content-type'] || '';
        if (ct.includes('json')) {
          const data = await response.json();
          const endpoint = url.split('/suche/')[1].split('?')[0];
          apiResponses[endpoint] = data;
          console.error(`Captured: ${endpoint}`);
        }
      } catch (e) {}
    }
  });

  try {
    await page.goto('https://www.vermittlerregister.info/recherche', {
      waitUntil: 'networkidle', timeout: 30000
    });

    await page.evaluate(() => {
      const k = document.getElementById('klaro'); if (k) k.remove();
      const w = document.getElementById('frc-widget'); if (w) w.classList.remove('hidden');
    });

    await page.fill('#registernummer', registernummer);

    await page.evaluate(() => {
      const btn = document.querySelector('#frc-widget .frc-button');
      if (btn) btn.click();
    });

    const start = Date.now();
    try {
      await page.waitForFunction(() => {
        const el = document.querySelector('[name="frc-captcha-solution"]');
        return el && el.value && el.value.length > 20 && !el.value.startsWith('.');
      }, { timeout: 30000 });
      console.error(`Captcha: ${((Date.now()-start)/1000).toFixed(1)}s`);
    } catch (e) {
      console.error('Captcha failed');
      process.exit(1);
    }

    await page.click('button[type="submit"]', { force: true });

    // Wait for results
    try {
      await page.waitForSelector('.result-single', { timeout: 15000 });
    } catch (e) {
      await page.waitForTimeout(5000);
    }

    // Also extract from DOM as backup
    const domData = await page.evaluate(() => {
      const result = {};
      document.querySelectorAll('.result-single dl > div').forEach(div => {
        const dt = div.querySelector('dt');
        const dd = div.querySelector('dd');
        if (dt && dd) result[dt.textContent.trim()] = dd.textContent.trim();
      });
      return result;
    });

    const output = {
      registernummer,
      api: apiResponses,
      dom: Object.keys(domData).length > 0 ? domData : undefined
    };

    console.log(JSON.stringify(output, null, 2));

  } catch (err) {
    console.error('Error:', err.message);
    process.exit(1);
  } finally {
    await browser.close();
  }
})();