const path = require('path');
const fs = require('fs');
const { chromium } = require('playwright');
const DASHBOARD_DIR = path.join(__dirname, '..', 'dashboards');
let browser = null;
const USER_AGENT =
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36';
/** DDG-Redirect-URLs (duckduckgo.com/l/?uddg=...) auf die echte URL auflösen. */
function realUrl(href) {
try {
const u = new URL(href);
if (u.hostname === 'duckduckgo.com' && u.pathname.startsWith('/l/') && u.searchParams.get('uddg')) {
return u.searchParams.get('uddg');
}
} catch {}
return href;
}
async function getBrowser() {
if (!browser || !browser.isConnected()) {
browser = await chromium.launch({
channel: 'chromium',
headless: true,
args: ['--disable-blink-features=AutomationControlled', '--no-sandbox'],
});
}
return browser;
}
async function withPage(url, fn) {
const b = await getBrowser();
const page = await b.newPage({ userAgent: USER_AGENT, viewport: { width: 1280, height: 900 } });
try {
return await fn(page);
} finally {
await page.close();
}
}
async function goto(page, url, timeout = 30000) {
let resp;
try {
resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
} catch (e) {
return { ok: false, error: `Navigation fehlgeschlagen: ${e.message.slice(0, 200)}` };
}
await page.waitForTimeout(1200);
return {
ok: resp ? resp.status() < 400 : true,
status: resp ? resp.status() : 0,
finalUrl: page.url(),
};
}
/** DuckDuckGo HTML-Suche: Ergebnisse parsen. */
async function search(query, log) {
const url = `https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
const results = await withPage(url, async (page) => {
const nav = await goto(page, url);
if (!nav.ok) return { nav, results: [], note: 'Suchseite nicht erreichbar' };
await page.waitForTimeout(2500);
const links = await page
.locator('a.result__a')
.evaluateAll((els) =>
els.slice(0, 10).map((a) => ({
title: (a.textContent || '').trim(),
url: a.href,
}))
);
for (const l of links) l.url = realUrl(l.url);
const snippets = await page
.locator('.result__snippet')
.evaluateAll((els) => els.slice(0, 10).map((e) => (e.textContent || '').trim()));
const blocked =
(await page.title().catch(() => '')) === 'CAPTCHA' ||
(await page.locator('text=CAPTCHA').count().catch(() => 0)) > 0;
return {
nav,
blocked,
results: links.map((l, i) => ({ ...l, snippet: snippets[i] || '' })),
};
});
log?.(`Suche "${query}" -> ${results.results.length} Ergebnisse`);
return results;
}
/** Seite besuchen und lesbaren Text extrahieren. */
async function browse(url, log) {
return withPage(url, async (page) => {
const nav = await goto(page, url);
if (!nav.ok) return { nav, error: 'Seite konnte nicht geladen werden' };
const data = await page.evaluate(() => {
const title = document.title || '';
const text = (document.body ? document.body.innerText : '').replace(/\s+\n/g, '\n');
return { title, text: text.slice(0, 15000) };
});
log?.(`Besuche ${url} -> ${data.title}`);
return { nav, title: data.title, text: data.text };
});
}
/** Link prüfen: lädt die Seite erfolgreich und hat sie echten Inhalt? */
async function verify(url, log) {
return withPage(url, async (page) => {
const nav = await goto(page, url, 25000);
if (!nav.ok) {
log?.(`Verifizierung FEHLGESCHLAGEN (${nav.status || 'Timeout'}): ${url}`);
return { url, ok: false, reason: `Seite lädt nicht (${nav.status || 'Timeout'})` };
}
const title = await page.title().catch(() => '');
const bodyLen = await page.evaluate(() => document.body?.innerText?.length || 0).catch(() => 0);
const badTitle = /404|not found|seite nicht gefunden|page not found/i.test(title);
const ok = !badTitle && bodyLen > 200;
log?.(`${ok ? 'VERIFIZIERT' : 'FEHLGESCHLAGEN'}: ${url}`);
return {
url,
ok,
reason: ok ? '' : badTitle ? 'Fehlerseite' : 'Kein nennenswerter Inhalt',
title,
};
});
}
/** Dashboard als HTML-Datei speichern. Gibt den URL-Pfad zurück. */
function saveDashboard(sessionId, filename, html, log) {
fs.mkdirSync(DASHBOARD_DIR, { recursive: true });
const safeName = (filename || 'dashboard')
.toLowerCase()
.replace(/[^a-z0-9äöü-]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 80) || 'dashboard';
const file = `${sessionId}-${safeName}.html`;
fs.writeFileSync(path.join(DASHBOARD_DIR, file), html, 'utf8');
log?.(`Dashboard gespeichert: ${file}`);
return { file, url: `/dashboards/${file}` };
}
/** Screenshot einer Seite (JPEG, als Data-URL) – für Vision-Modelle. */
async function screenshot(url, fullPage = false, log) {
return withPage(url, async (page) => {
const nav = await goto(page, url);
if (!nav.ok) return { error: `Seite konnte nicht geladen werden (${nav.status || 'Timeout'})` };
const title = await page.title().catch(() => '');
let useFull = !!fullPage;
if (useFull) {
const h = await page.evaluate(() => document.body?.scrollHeight || 0).catch(() => 0);
if (h > 6000) useFull = false;
}
const buf = await page.screenshot({ fullPage: useFull, type: 'jpeg', quality: 70 });
log?.(`Screenshot: ${url}${useFull ? ' (voll)' : ''} – ${title || ''}`);
return {
title: title || nav.finalUrl || url,
url: nav.finalUrl || url,
image: `data:image/jpeg;base64,${buf.toString('base64')}`,
};
});
}
// ---------------------------------------------------------------------------
// Tour-Feature: Geocoding (Nominatim) + Routing (OSRM) + GPX-Export
// ---------------------------------------------------------------------------
const NOMINATIM_UA = 'ResearchChat/1.0 (Kontakt: research@localhost)';
let lastGeocodeTime = 0;
/** Koordinaten-Paare in OSRM-Format: lon,lat (ACHTUNG Reihenfolge). */
const toOsrm = (stops) => stops.map((s) => `${s.lon},${s.lat}`).join(';');
/** GPX-/XML-sichere Zeichen escapen. */
function xmlEscape(s) {
return String(s ?? '')
.replace(/&/g, '&')
.replace(//g, '>')
.replace(/"/g, '"')
.replace(/'/g, ''');
}
/** Koordinaten-Angabe erkennen: "52.5163, 13.3777" oder "52.5163 13.3777". */
function parseCoords(query) {
const m = String(query).match(/(-?\d{1,3}(?:\.\d+)?)\s*[,;\s]\s*(-?\d{1,3}(?:\.\d+)?)/);
if (!m) return null;
const lat = parseFloat(m[1]);
const lon = parseFloat(m[2]);
if (Math.abs(lat) <= 90 && Math.abs(lon) <= 180) return { lat, lon };
return null;
}
/**
* Ort über Nominatim (OpenStreetMap) in Koordinaten umwandeln.
* Kostenlos, Fair-Use: max. 1 Request/Sekunde.
*/
async function geocode(query, config, log) {
const coords = parseCoords(query);
if (coords) {
log?.(`Geocode "${query}" -> Koordinaten direkt übernommen`);
return { ...coords, name: query, displayName: query, direct: true };
}
const wait = Math.max(0, lastGeocodeTime + 1100 - Date.now());
if (wait > 0) await new Promise((r) => setTimeout(r, wait));
lastGeocodeTime = Date.now();
const url = new URL('https://nominatim.openstreetmap.org/search');
url.searchParams.set('q', query);
url.searchParams.set('format', 'json');
url.searchParams.set('limit', '1');
url.searchParams.set('accept-language', 'de');
const headers = {
'User-Agent': config?.nominatimEmail ? `ResearchChat/1.0 (${config.nominatimEmail})` : NOMINATIM_UA,
};
try {
const resp = await fetch(url, { headers, timeout: 15000 });
if (!resp.ok) return { error: `Nominatim Fehler ${resp.status}` };
const results = await resp.json();
if (!results.length) return { error: `Kein Ort gefunden: "${query}"` };
const r = results[0];
log?.(`Geocode "${query}" -> ${r.lat}, ${r.lon}`);
return {
name: r.name || query,
lat: parseFloat(r.lat),
lon: parseFloat(r.lon),
displayName: r.display_name,
};
} catch (e) {
return { error: `Nominatim nicht erreichbar: ${e.message.slice(0, 120)}` };
}
}
/** Geometrie entschlacken (max. 500 Punkte für das Dashboard). */
function decimate(coords, max = 500) {
if (coords.length <= max) return coords;
const step = coords.length / max;
const out = [];
for (let i = 0; i < coords.length; i += step) out.push(coords[Math.floor(i)]);
out.push(coords[coords.length - 1]);
return out;
}
/**
* Tour speichern: OSRM-Fuß-Route durch alle Stopps, GPX-Datei für OsmAnd,
* tour.json für die Dashboard-Karte, Google-Maps-URLs.
*/
async function saveTour({ stops, title, filename, sessionId }, config, log) {
if (!Array.isArray(stops) || stops.length < 2) {
return { error: 'Mindestens 2 Stopps nötig (Name, lat, lon)' };
}
const clean = stops
.filter((s) => s && s.name && Number.isFinite(Number(s.lat)) && Number.isFinite(Number(s.lon)))
.map((s) => ({
name: String(s.name).slice(0, 100),
lat: Number(s.lat),
lon: Number(s.lon),
description: String(s.description || '').slice(0, 300),
}));
if (clean.length < 2) return { error: 'Stopps ungültig (Name + lat + lon erforderlich)' };
// OSRM-Fuß-Route (Fallback: Luftlinie)
let route = { coordinates: clean.map((s) => [s.lon, s.lat]), distanceKm: 0, durationMin: 0, source: 'line' };
if (clean.length <= 25) {
try {
const osrmUrl = `${config?.osrmUrl || 'https://router.project-osrm.org'}/route/v1/foot/${toOsrm(clean)}?overview=full&geometries=geojson`;
const resp = await fetch(osrmUrl, { timeout: 20000 });
if (resp.ok) {
const data = await resp.json();
const r = data.routes?.[0];
if (r?.geometry?.coordinates?.length) {
route = {
coordinates: decimate(r.geometry.coordinates), // für die Karte im Dashboard
fullCoordinates: r.geometry.coordinates, // für die GPX-Datei (OsmAnd)
distanceKm: Math.round((r.distance / 1000) * 10) / 10,
durationMin: Math.round(r.duration / 60),
source: 'osrm',
};
}
} else {
log?.(`OSRM antwortet ${resp.status} – Fallback: Luftlinie`);
}
} catch (e) {
log?.(`OSRM nicht erreichbar – Fallback: Luftlinie (${e.message.slice(0, 80)})`);
}
}
// GPX 1.1 bauen
const wpts = clean
.map(
(s, i) =>
` \n` +
` ${xmlEscape(i + 1)}. ${xmlEscape(s.name)}\n` +
(s.description ? ` ${xmlEscape(s.description)}\n` : '') +
` `
)
.join('\n');
const rtepts = clean
.map((s) => ` ${xmlEscape(s.name)}`)
.join('\n');
const trkpts = (route.fullCoordinates || route.coordinates)
.map(([lon, lat]) => ` `)
.join('\n');
const gpx = `
${xmlEscape(title || 'Tour')}
${wpts}
${xmlEscape(title || 'Tour')}
${rtepts}
${xmlEscape(title || 'Tour')} (Route)
${trkpts}
`;
const safeName =
(filename || title || 'tour').toLowerCase().replace(/[^a-z0-9äöü-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 60) ||
'tour';
const base = `${sessionId}-${safeName.replace(/-tour$/, '')}-tour`;
fs.mkdirSync(DASHBOARD_DIR, { recursive: true });
fs.writeFileSync(path.join(DASHBOARD_DIR, `${base}.gpx`), gpx, 'utf8');
const { fullCoordinates, ...routeForJson } = route;
fs.writeFileSync(
path.join(DASHBOARD_DIR, `${base}.tour.json`),
JSON.stringify({
title: title || 'Tour',
stops: clean,
route: routeForJson,
}),
'utf8'
);
const fmt = (lat, lon) => `${lat.toFixed(6)},${lon.toFixed(6)}`;
const googleMapsUrl = `https://www.google.com/maps/dir/${clean.map((s) => fmt(s.lat, s.lon)).join('/')}/`;
const stopsWithNav = clean.map((s) => ({
...s,
navUrl: `https://www.google.com/maps/dir/?api=1&destination=${fmt(s.lat, s.lon)}&travelmode=walking`,
}));
log?.(
`Tour gespeichert: ${base} (${clean.length} Stopps, ${route.distanceKm} km, ${route.durationMin} min, ${route.source})`
);
return {
ok: true,
title: title || 'Tour',
gpxUrl: `/dashboards/${base}.gpx`,
tourJsonUrl: `/dashboards/${base}.tour.json`,
googleMapsUrl,
stops: stopsWithNav,
distanceKm: route.distanceKm,
durationMin: route.durationMin,
routeSource: route.source,
osmAndHint: 'GPX-Datei herunterladen und in OsmAnd importieren (Menü → Meine Orte → Importieren).',
};
}
async function closeBrowser() {
if (browser) {
await browser.close().catch(() => {});
browser = null;
}
}
module.exports = { search, browse, verify, saveDashboard, screenshot, geocode, saveTour, closeBrowser };