Research Chat: Web-Chatoberfläche für den Research-Agent (DeepSeek v4 Flash + Playwright, Docker/Podman)

lib/tools.js

@@ -0,0 +1,154 @@
+const path = require('path');
+const fs = require('fs');
+const { chromium } = require('playwright');
+
+const DASHBOARD_DIR = path.join(__dirname, '..', 'dashboards');
+
+let browser = null;
+
+const USER_AGENT =
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36';
+
+/** DDG-Redirect-URLs (duckduckgo.com/l/?uddg=...) auf die echte URL auflösen. */
+function realUrl(href) {
+  try {
+    const u = new URL(href);
+    if (u.hostname === 'duckduckgo.com' && u.pathname.startsWith('/l/') && u.searchParams.get('uddg')) {
+      return u.searchParams.get('uddg');
+    }
+  } catch {}
+  return href;
+}
+
+async function getBrowser() {
+  if (!browser || !browser.isConnected()) {
+    browser = await chromium.launch({
+      channel: 'chromium',
+      headless: true,
+      args: ['--disable-blink-features=AutomationControlled', '--no-sandbox'],
+    });
+  }
+  return browser;
+}
+
+async function withPage(url, fn) {
+  const b = await getBrowser();
+  const page = await b.newPage({ userAgent: USER_AGENT, viewport: { width: 1280, height: 900 } });
+  try {
+    return await fn(page);
+  } finally {
+    await page.close();
+  }
+}
+
+async function goto(page, url, timeout = 30000) {
+  let resp;
+  try {
+    resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
+  } catch (e) {
+    return { ok: false, error: `Navigation fehlgeschlagen: ${e.message.slice(0, 200)}` };
+  }
+  await page.waitForTimeout(1200);
+  return {
+    ok: resp ? resp.status() < 400 : true,
+    status: resp ? resp.status() : 0,
+    finalUrl: page.url(),
+  };
+}
+
+/** DuckDuckGo HTML-Suche: Ergebnisse parsen. */
+async function search(query, log) {
+  const url = `https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
+  const results = await withPage(url, async (page) => {
+    const nav = await goto(page, url);
+    if (!nav.ok) return { nav, results: [], note: 'Suchseite nicht erreichbar' };
+
+    await page.waitForTimeout(2500);
+    const links = await page
+      .locator('a.result__a')
+      .evaluateAll((els) =>
+        els.slice(0, 10).map((a) => ({
+          title: (a.textContent || '').trim(),
+          url: a.href,
+        }))
+      );
+    for (const l of links) l.url = realUrl(l.url);
+
+    const snippets = await page
+      .locator('.result__snippet')
+      .evaluateAll((els) => els.slice(0, 10).map((e) => (e.textContent || '').trim()));
+
+    const blocked =
+      (await page.title().catch(() => '')) === 'CAPTCHA' ||
+      (await page.locator('text=CAPTCHA').count().catch(() => 0)) > 0;
+
+    return {
+      nav,
+      blocked,
+      results: links.map((l, i) => ({ ...l, snippet: snippets[i] || '' })),
+    };
+  });
+  log?.(`Suche "${query}" -> ${results.results.length} Ergebnisse`);
+  return results;
+}
+
+/** Seite besuchen und lesbaren Text extrahieren. */
+async function browse(url, log) {
+  return withPage(url, async (page) => {
+    const nav = await goto(page, url);
+    if (!nav.ok) return { nav, error: 'Seite konnte nicht geladen werden' };
+
+    const data = await page.evaluate(() => {
+      const title = document.title || '';
+      const text = (document.body ? document.body.innerText : '').replace(/\s+\n/g, '\n');
+      return { title, text: text.slice(0, 15000) };
+    });
+    log?.(`Besuche ${url} -> ${data.title}`);
+    return { nav, title: data.title, text: data.text };
+  });
+}
+
+/** Link prüfen: lädt die Seite erfolgreich und hat sie echten Inhalt? */
+async function verify(url, log) {
+  return withPage(url, async (page) => {
+    const nav = await goto(page, url, 25000);
+    if (!nav.ok) {
+      log?.(`Verifizierung FEHLGESCHLAGEN (${nav.status || 'Timeout'}): ${url}`);
+      return { url, ok: false, reason: `Seite lädt nicht (${nav.status || 'Timeout'})` };
+    }
+    const title = await page.title().catch(() => '');
+    const bodyLen = await page.evaluate(() => document.body?.innerText?.length || 0).catch(() => 0);
+    const badTitle = /404|not found|seite nicht gefunden|page not found/i.test(title);
+    const ok = !badTitle && bodyLen > 200;
+    log?.(`${ok ? 'VERIFIZIERT' : 'FEHLGESCHLAGEN'}: ${url}`);
+    return {
+      url,
+      ok,
+      reason: ok ? '' : badTitle ? 'Fehlerseite' : 'Kein nennenswerter Inhalt',
+      title,
+    };
+  });
+}
+
+/** Dashboard als HTML-Datei speichern. Gibt den URL-Pfad zurück. */
+function saveDashboard(sessionId, filename, html, log) {
+  fs.mkdirSync(DASHBOARD_DIR, { recursive: true });
+  const safeName = (filename || 'dashboard')
+    .toLowerCase()
+    .replace(/[^a-z0-9äöü-]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 80) || 'dashboard';
+  const file = `${sessionId}-${safeName}.html`;
+  fs.writeFileSync(path.join(DASHBOARD_DIR, file), html, 'utf8');
+  log?.(`Dashboard gespeichert: ${file}`);
+  return { file, url: `/dashboards/${file}` };
+}
+
+async function closeBrowser() {
+  if (browser) {
+    await browser.close().catch(() => {});
+    browser = null;
+  }
+}
+
+module.exports = { search, browse, verify, saveDashboard, closeBrowser };