researchChat/lib/tools.js

const path = require('path');
const fs = require('fs');
const { chromium } = require('playwright');

const DASHBOARD_DIR = path.join(__dirname, '..', 'dashboards');

let browser = null;

const USER_AGENT =
  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36';

/** DDG-Redirect-URLs (duckduckgo.com/l/?uddg=...) auf die echte URL auflösen. */
function realUrl(href) {
  try {
    const u = new URL(href);
    if (u.hostname === 'duckduckgo.com' && u.pathname.startsWith('/l/') && u.searchParams.get('uddg')) {
      return u.searchParams.get('uddg');
    }
  } catch {}
  return href;
}

async function getBrowser() {
  if (!browser || !browser.isConnected()) {
    browser = await chromium.launch({
      channel: 'chromium',
      headless: true,
      args: ['--disable-blink-features=AutomationControlled', '--no-sandbox'],
    });
  }
  return browser;
}

async function withPage(url, fn) {
  const b = await getBrowser();
  const page = await b.newPage({ userAgent: USER_AGENT, viewport: { width: 1280, height: 900 } });
  try {
    return await fn(page);
  } finally {
    await page.close();
  }
}

async function goto(page, url, timeout = 30000) {
  let resp;
  try {
    resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
  } catch (e) {
    return { ok: false, error: `Navigation fehlgeschlagen: ${e.message.slice(0, 200)}` };
  }
  await page.waitForTimeout(1200);
  return {
    ok: resp ? resp.status() < 400 : true,
    status: resp ? resp.status() : 0,
    finalUrl: page.url(),
  };
}

/** DuckDuckGo HTML-Suche: Ergebnisse parsen. */
async function search(query, log) {
  const url = `https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
  const results = await withPage(url, async (page) => {
    const nav = await goto(page, url);
    if (!nav.ok) return { nav, results: [], note: 'Suchseite nicht erreichbar' };

    await page.waitForTimeout(2500);
    const links = await page
      .locator('a.result__a')
      .evaluateAll((els) =>
        els.slice(0, 10).map((a) => ({
          title: (a.textContent || '').trim(),
          url: a.href,
        }))
      );
    for (const l of links) l.url = realUrl(l.url);

    const snippets = await page
      .locator('.result__snippet')
      .evaluateAll((els) => els.slice(0, 10).map((e) => (e.textContent || '').trim()));

    const blocked =
      (await page.title().catch(() => '')) === 'CAPTCHA' ||
      (await page.locator('text=CAPTCHA').count().catch(() => 0)) > 0;

    return {
      nav,
      blocked,
      results: links.map((l, i) => ({ ...l, snippet: snippets[i] || '' })),
    };
  });
  log?.(`Suche "${query}" -> ${results.results.length} Ergebnisse`);
  return results;
}

/** Seite besuchen und lesbaren Text extrahieren. */
async function browse(url, log) {
  return withPage(url, async (page) => {
    const nav = await goto(page, url);
    if (!nav.ok) return { nav, error: 'Seite konnte nicht geladen werden' };

    const data = await page.evaluate(() => {
      const title = document.title || '';
      const text = (document.body ? document.body.innerText : '').replace(/\s+\n/g, '\n');
      return { title, text: text.slice(0, 15000) };
    });
    log?.(`Besuche ${url} -> ${data.title}`);
    return { nav, title: data.title, text: data.text };
  });
}

/** Link prüfen: lädt die Seite erfolgreich und hat sie echten Inhalt? */
async function verify(url, log) {
  return withPage(url, async (page) => {
    const nav = await goto(page, url, 25000);
    if (!nav.ok) {
      log?.(`Verifizierung FEHLGESCHLAGEN (${nav.status || 'Timeout'}): ${url}`);
      return { url, ok: false, reason: `Seite lädt nicht (${nav.status || 'Timeout'})` };
    }
    const title = await page.title().catch(() => '');
    const bodyLen = await page.evaluate(() => document.body?.innerText?.length || 0).catch(() => 0);
    const badTitle = /404|not found|seite nicht gefunden|page not found/i.test(title);
    const ok = !badTitle && bodyLen > 200;
    log?.(`${ok ? 'VERIFIZIERT' : 'FEHLGESCHLAGEN'}: ${url}`);
    return {
      url,
      ok,
      reason: ok ? '' : badTitle ? 'Fehlerseite' : 'Kein nennenswerter Inhalt',
      title,
    };
  });
}

/** Dashboard als HTML-Datei speichern. Gibt den URL-Pfad zurück. */
function saveDashboard(sessionId, filename, html, log) {
  fs.mkdirSync(DASHBOARD_DIR, { recursive: true });
  const safeName = (filename || 'dashboard')
    .toLowerCase()
    .replace(/[^a-z0-9äöü-]+/g, '-')
    .replace(/^-+|-+$/g, '')
    .slice(0, 80) || 'dashboard';
  const file = `${sessionId}-${safeName}.html`;
  fs.writeFileSync(path.join(DASHBOARD_DIR, file), html, 'utf8');
  log?.(`Dashboard gespeichert: ${file}`);
  return { file, url: `/dashboards/${file}` };
}

async function closeBrowser() {
  if (browser) {
    await browser.close().catch(() => {});
    browser = null;
  }
}

module.exports = { search, browse, verify, saveDashboard, closeBrowser };