const path = require('path'); const fs = require('fs'); const { chromium } = require('playwright'); const DASHBOARD_DIR = path.join(__dirname, '..', 'dashboards'); let browser = null; const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36'; /** DDG-Redirect-URLs (duckduckgo.com/l/?uddg=...) auf die echte URL auflösen. */ function realUrl(href) { try { const u = new URL(href); if (u.hostname === 'duckduckgo.com' && u.pathname.startsWith('/l/') && u.searchParams.get('uddg')) { return u.searchParams.get('uddg'); } } catch {} return href; } async function getBrowser() { if (!browser || !browser.isConnected()) { browser = await chromium.launch({ channel: 'chromium', headless: true, args: ['--disable-blink-features=AutomationControlled', '--no-sandbox'], }); } return browser; } async function withPage(url, fn) { const b = await getBrowser(); const page = await b.newPage({ userAgent: USER_AGENT, viewport: { width: 1280, height: 900 } }); try { return await fn(page); } finally { await page.close(); } } async function goto(page, url, timeout = 30000) { let resp; try { resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout }); } catch (e) { return { ok: false, error: `Navigation fehlgeschlagen: ${e.message.slice(0, 200)}` }; } await page.waitForTimeout(1200); return { ok: resp ? resp.status() < 400 : true, status: resp ? resp.status() : 0, finalUrl: page.url(), }; } /** DuckDuckGo HTML-Suche: Ergebnisse parsen. */ async function search(query, log) { const url = `https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`; const results = await withPage(url, async (page) => { const nav = await goto(page, url); if (!nav.ok) return { nav, results: [], note: 'Suchseite nicht erreichbar' }; await page.waitForTimeout(2500); const links = await page .locator('a.result__a') .evaluateAll((els) => els.slice(0, 10).map((a) => ({ title: (a.textContent || '').trim(), url: a.href, })) ); for (const l of links) l.url = realUrl(l.url); const snippets = await page .locator('.result__snippet') .evaluateAll((els) => els.slice(0, 10).map((e) => (e.textContent || '').trim())); const blocked = (await page.title().catch(() => '')) === 'CAPTCHA' || (await page.locator('text=CAPTCHA').count().catch(() => 0)) > 0; return { nav, blocked, results: links.map((l, i) => ({ ...l, snippet: snippets[i] || '' })), }; }); log?.(`Suche "${query}" -> ${results.results.length} Ergebnisse`); return results; } /** Seite besuchen und lesbaren Text extrahieren. */ async function browse(url, log) { return withPage(url, async (page) => { const nav = await goto(page, url); if (!nav.ok) return { nav, error: 'Seite konnte nicht geladen werden' }; const data = await page.evaluate(() => { const title = document.title || ''; const text = (document.body ? document.body.innerText : '').replace(/\s+\n/g, '\n'); return { title, text: text.slice(0, 15000) }; }); log?.(`Besuche ${url} -> ${data.title}`); return { nav, title: data.title, text: data.text }; }); } /** Link prüfen: lädt die Seite erfolgreich und hat sie echten Inhalt? */ async function verify(url, log) { return withPage(url, async (page) => { const nav = await goto(page, url, 25000); if (!nav.ok) { log?.(`Verifizierung FEHLGESCHLAGEN (${nav.status || 'Timeout'}): ${url}`); return { url, ok: false, reason: `Seite lädt nicht (${nav.status || 'Timeout'})` }; } const title = await page.title().catch(() => ''); const bodyLen = await page.evaluate(() => document.body?.innerText?.length || 0).catch(() => 0); const badTitle = /404|not found|seite nicht gefunden|page not found/i.test(title); const ok = !badTitle && bodyLen > 200; log?.(`${ok ? 'VERIFIZIERT' : 'FEHLGESCHLAGEN'}: ${url}`); return { url, ok, reason: ok ? '' : badTitle ? 'Fehlerseite' : 'Kein nennenswerter Inhalt', title, }; }); } /** Dashboard als HTML-Datei speichern. Gibt den URL-Pfad zurück. */ function saveDashboard(sessionId, filename, html, log) { fs.mkdirSync(DASHBOARD_DIR, { recursive: true }); const safeName = (filename || 'dashboard') .toLowerCase() .replace(/[^a-z0-9äöü-]+/g, '-') .replace(/^-+|-+$/g, '') .slice(0, 80) || 'dashboard'; const file = `${sessionId}-${safeName}.html`; fs.writeFileSync(path.join(DASHBOARD_DIR, file), html, 'utf8'); log?.(`Dashboard gespeichert: ${file}`); return { file, url: `/dashboards/${file}` }; } async function closeBrowser() { if (browser) { await browser.close().catch(() => {}); browser = null; } } module.exports = { search, browse, verify, saveDashboard, closeBrowser };