nico
b740ce1316
- deepseek-flash als einziges Modell mit Vision, Preise aktualisiert - OpenRouter/GLM samt Code, Config und Env-Variablen entfernt - Legacy-Modell-IDs in Sessions werden automatisch migriert - browse lädt Produktbilder lokal nach dashboards/ (Dedupe + Cleanup) - Vision-Prompt: Workflow für visuelle Produktsuche, max. ~5 Screenshots
466 lines
16 KiB
JavaScript
466 lines
16 KiB
JavaScript
const path = require('path');
|
||
const fs = require('fs');
|
||
const crypto = require('crypto');
|
||
const { chromium } = require('playwright');
|
||
|
||
const DASHBOARD_DIR = path.join(__dirname, '..', 'dashboards');
|
||
|
||
let browser = null;
|
||
|
||
const USER_AGENT =
|
||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36';
|
||
|
||
/** DDG-Redirect-URLs (duckduckgo.com/l/?uddg=...) auf die echte URL auflösen. */
|
||
function realUrl(href) {
|
||
try {
|
||
const u = new URL(href);
|
||
if (u.hostname === 'duckduckgo.com' && u.pathname.startsWith('/l/') && u.searchParams.get('uddg')) {
|
||
return u.searchParams.get('uddg');
|
||
}
|
||
} catch {}
|
||
return href;
|
||
}
|
||
|
||
async function getBrowser() {
|
||
if (!browser || !browser.isConnected()) {
|
||
browser = await chromium.launch({
|
||
channel: 'chromium',
|
||
headless: true,
|
||
args: ['--disable-blink-features=AutomationControlled', '--no-sandbox'],
|
||
});
|
||
}
|
||
return browser;
|
||
}
|
||
|
||
async function withPage(url, fn) {
|
||
const b = await getBrowser();
|
||
const page = await b.newPage({ userAgent: USER_AGENT, viewport: { width: 1280, height: 900 } });
|
||
try {
|
||
return await fn(page);
|
||
} finally {
|
||
await page.close();
|
||
}
|
||
}
|
||
|
||
async function goto(page, url, timeout = 30000) {
|
||
let resp;
|
||
try {
|
||
resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
|
||
} catch (e) {
|
||
return { ok: false, error: `Navigation fehlgeschlagen: ${e.message.slice(0, 200)}` };
|
||
}
|
||
await page.waitForTimeout(1200);
|
||
return {
|
||
ok: resp ? resp.status() < 400 : true,
|
||
status: resp ? resp.status() : 0,
|
||
finalUrl: page.url(),
|
||
};
|
||
}
|
||
|
||
/** DuckDuckGo HTML-Suche: Ergebnisse parsen. */
|
||
async function search(query, log) {
|
||
const url = `https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
||
const results = await withPage(url, async (page) => {
|
||
const nav = await goto(page, url);
|
||
if (!nav.ok) return { nav, results: [], note: 'Suchseite nicht erreichbar' };
|
||
|
||
await page.waitForTimeout(2500);
|
||
const links = await page
|
||
.locator('a.result__a')
|
||
.evaluateAll((els) =>
|
||
els.slice(0, 10).map((a) => ({
|
||
title: (a.textContent || '').trim(),
|
||
url: a.href,
|
||
}))
|
||
);
|
||
for (const l of links) l.url = realUrl(l.url);
|
||
|
||
const snippets = await page
|
||
.locator('.result__snippet')
|
||
.evaluateAll((els) => els.slice(0, 10).map((e) => (e.textContent || '').trim()));
|
||
|
||
const blocked =
|
||
(await page.title().catch(() => '')) === 'CAPTCHA' ||
|
||
(await page.locator('text=CAPTCHA').count().catch(() => 0)) > 0;
|
||
|
||
return {
|
||
nav,
|
||
blocked,
|
||
results: links.map((l, i) => ({ ...l, snippet: snippets[i] || '' })),
|
||
};
|
||
});
|
||
log?.(`Suche "${query}" -> ${results.results.length} Ergebnisse`);
|
||
return results;
|
||
}
|
||
|
||
const IMAGE_TYPES = { 'image/jpeg': 'jpg', 'image/png': 'png', 'image/webp': 'webp', 'image/gif': 'gif' };
|
||
const MAX_IMAGE_BYTES = 4 * 1024 * 1024;
|
||
const MAX_TOTAL_IMAGE_BYTES = 12 * 1024 * 1024;
|
||
const MAX_IMAGES = 8;
|
||
|
||
/**
|
||
* Extrahiert Produktbild-URLs (nur absolute http(s), ausreichend groß, dedupliziert).
|
||
* Läuft im Browser-Kontext der besuchten Seite.
|
||
*/
|
||
function collectImages() {
|
||
const images = [];
|
||
const seen = new Set();
|
||
for (const img of document.querySelectorAll('img')) {
|
||
const src = (img.currentSrc || img.src || '').split('#')[0];
|
||
if (!/^https?:/i.test(src) || seen.has(src)) continue;
|
||
const nw = img.naturalWidth || 0;
|
||
const nh = img.naturalHeight || 0;
|
||
const rect = img.getBoundingClientRect();
|
||
const w = nw || rect.width || img.width || 0;
|
||
const h = nh || rect.height || img.height || 0;
|
||
if (w < 120 || h < 120) continue;
|
||
seen.add(src);
|
||
images.push({ url: src, alt: (img.alt || '').trim().slice(0, 120) });
|
||
if (images.length >= 12) break;
|
||
}
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* Lädt extrahierte Bilder lokal in den Dashboards-Ordner, damit das Dashboard
|
||
* sie unabhängig von Hotlink-Schutz und ablaufenden CDN-URLs einbetten kann.
|
||
* Dateiname: <sessionId>-img-<hash>.<ext> – wird beim Session-Löschen mit entfernt.
|
||
*/
|
||
async function cacheImages(page, images, sessionId, log) {
|
||
if (!sessionId) return images;
|
||
fs.mkdirSync(DASHBOARD_DIR, { recursive: true });
|
||
const existingFiles = new Set(fs.readdirSync(DASHBOARD_DIR));
|
||
const out = [];
|
||
let total = 0;
|
||
for (const img of images) {
|
||
if (out.length >= MAX_IMAGES || total >= MAX_TOTAL_IMAGE_BYTES) break;
|
||
try {
|
||
const hash = crypto.createHash('sha1').update(img.url).digest('hex').slice(0, 12);
|
||
const known = [...existingFiles].find((f) => f.startsWith(`${sessionId}-img-${hash}.`));
|
||
if (known) {
|
||
out.push({ url: `/dashboards/${known}`, alt: img.alt });
|
||
continue;
|
||
}
|
||
const resp = await page.request.get(img.url, {
|
||
timeout: 15000,
|
||
headers: { Referer: page.url() },
|
||
});
|
||
const type = String(resp.headers()['content-type'] || '').split(';')[0].trim().toLowerCase();
|
||
const ext = IMAGE_TYPES[type];
|
||
if (!ext || !resp.ok()) continue;
|
||
const buf = await resp.body();
|
||
if (buf.length < 512 || buf.length > MAX_IMAGE_BYTES || total + buf.length > MAX_TOTAL_IMAGE_BYTES) continue;
|
||
const file = `${sessionId}-img-${hash}.${ext}`;
|
||
fs.writeFileSync(path.join(DASHBOARD_DIR, file), buf);
|
||
existingFiles.add(file);
|
||
total += buf.length;
|
||
out.push({ url: `/dashboards/${file}`, alt: img.alt });
|
||
} catch {}
|
||
}
|
||
log?.(`Produktbilder lokal gespeichert: ${out.length}`);
|
||
return out;
|
||
}
|
||
|
||
/** Seite besuchen und lesbaren Text + Produktbilder (lokal gespeichert) extrahieren. */
|
||
async function browse(url, log, opts = {}) {
|
||
return withPage(url, async (page) => {
|
||
const nav = await goto(page, url);
|
||
if (!nav.ok) return { nav, error: 'Seite konnte nicht geladen werden' };
|
||
|
||
const data = await page.evaluate(() => {
|
||
const title = document.title || '';
|
||
const text = (document.body ? document.body.innerText : '').replace(/\s+\n/g, '\n');
|
||
return { title, text: text.slice(0, 6000) };
|
||
});
|
||
const extracted = await page.evaluate(collectImages);
|
||
const images = await cacheImages(page, extracted, opts.sessionId, log);
|
||
log?.(`Besuche ${url} -> ${data.title} (${images.length} Bilder lokal)`);
|
||
return { nav, title: data.title, images, text: data.text };
|
||
});
|
||
}
|
||
|
||
/** Link prüfen: lädt die Seite erfolgreich und hat sie echten Inhalt? */
|
||
async function verify(url, log) {
|
||
return withPage(url, async (page) => {
|
||
const nav = await goto(page, url, 25000);
|
||
if (!nav.ok) {
|
||
log?.(`Verifizierung FEHLGESCHLAGEN (${nav.status || 'Timeout'}): ${url}`);
|
||
return { url, ok: false, reason: `Seite lädt nicht (${nav.status || 'Timeout'})` };
|
||
}
|
||
const title = await page.title().catch(() => '');
|
||
const bodyLen = await page.evaluate(() => document.body?.innerText?.length || 0).catch(() => 0);
|
||
const badTitle = /404|not found|seite nicht gefunden|page not found/i.test(title);
|
||
const ok = !badTitle && bodyLen > 200;
|
||
log?.(`${ok ? 'VERIFIZIERT' : 'FEHLGESCHLAGEN'}: ${url}`);
|
||
return {
|
||
url,
|
||
ok,
|
||
reason: ok ? '' : badTitle ? 'Fehlerseite' : 'Kein nennenswerter Inhalt',
|
||
title,
|
||
};
|
||
});
|
||
}
|
||
|
||
/** Dashboard als HTML-Datei speichern. Gibt den URL-Pfad zurück. */
|
||
function saveDashboard(sessionId, filename, html, log) {
|
||
fs.mkdirSync(DASHBOARD_DIR, { recursive: true });
|
||
const safeName = (filename || 'dashboard')
|
||
.toLowerCase()
|
||
.replace(/[^a-z0-9äöü-]+/g, '-')
|
||
.replace(/^-+|-+$/g, '')
|
||
.slice(0, 80) || 'dashboard';
|
||
const file = `${sessionId}-${safeName}.html`;
|
||
fs.writeFileSync(path.join(DASHBOARD_DIR, file), html, 'utf8');
|
||
log?.(`Dashboard gespeichert: ${file}`);
|
||
return { file, url: `/dashboards/${file}` };
|
||
}
|
||
|
||
/** Alle Dashboard-Dateien einer Session (HTML/GPX/tour.json) löschen. */
|
||
function deleteSessionFiles(sessionId) {
|
||
let entries;
|
||
try {
|
||
entries = fs.readdirSync(DASHBOARD_DIR);
|
||
} catch {
|
||
return;
|
||
}
|
||
for (const f of entries) {
|
||
if (f.startsWith(`${sessionId}-`)) {
|
||
try {
|
||
fs.unlinkSync(path.join(DASHBOARD_DIR, f));
|
||
} catch {}
|
||
}
|
||
}
|
||
}
|
||
|
||
/** Screenshot einer Seite (JPEG, als Data-URL) – für Vision-Modelle. */
|
||
async function screenshot(url, fullPage = false, log) {
|
||
return withPage(url, async (page) => {
|
||
const nav = await goto(page, url);
|
||
if (!nav.ok) return { error: `Seite konnte nicht geladen werden (${nav.status || 'Timeout'})` };
|
||
const title = await page.title().catch(() => '');
|
||
let useFull = !!fullPage;
|
||
if (useFull) {
|
||
const h = await page.evaluate(() => document.body?.scrollHeight || 0).catch(() => 0);
|
||
if (h > 6000) useFull = false;
|
||
}
|
||
const buf = await page.screenshot({ fullPage: useFull, type: 'jpeg', quality: 70 });
|
||
log?.(`Screenshot: ${url}${useFull ? ' (voll)' : ''} – ${title || ''}`);
|
||
return {
|
||
title: title || nav.finalUrl || url,
|
||
url: nav.finalUrl || url,
|
||
image: `data:image/jpeg;base64,${buf.toString('base64')}`,
|
||
};
|
||
});
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Tour-Feature: Geocoding (Nominatim) + Routing (OSRM) + GPX-Export
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const NOMINATIM_UA = 'ResearchChat/1.0 (Kontakt: research@localhost)';
|
||
let lastGeocodeTime = 0;
|
||
|
||
/** Koordinaten-Paare in OSRM-Format: lon,lat (ACHTUNG Reihenfolge). */
|
||
const toOsrm = (stops) => stops.map((s) => `${s.lon},${s.lat}`).join(';');
|
||
|
||
/** GPX-/XML-sichere Zeichen escapen. */
|
||
function xmlEscape(s) {
|
||
return String(s ?? '')
|
||
.replace(/&/g, '&')
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, ''');
|
||
}
|
||
|
||
/** Koordinaten-Angabe erkennen: "52.5163, 13.3777" oder "52.5163 13.3777". */
|
||
function parseCoords(query) {
|
||
const m = String(query).match(/(-?\d{1,3}(?:\.\d+)?)\s*[,;\s]\s*(-?\d{1,3}(?:\.\d+)?)/);
|
||
if (!m) return null;
|
||
const lat = parseFloat(m[1]);
|
||
const lon = parseFloat(m[2]);
|
||
if (Math.abs(lat) <= 90 && Math.abs(lon) <= 180) return { lat, lon };
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* Ort über Nominatim (OpenStreetMap) in Koordinaten umwandeln.
|
||
* Kostenlos, Fair-Use: max. 1 Request/Sekunde.
|
||
*/
|
||
async function geocode(query, config, log) {
|
||
const coords = parseCoords(query);
|
||
if (coords) {
|
||
log?.(`Geocode "${query}" -> Koordinaten direkt übernommen`);
|
||
return { ...coords, name: query, displayName: query, direct: true };
|
||
}
|
||
|
||
const wait = Math.max(0, lastGeocodeTime + 1100 - Date.now());
|
||
if (wait > 0) await new Promise((r) => setTimeout(r, wait));
|
||
lastGeocodeTime = Date.now();
|
||
|
||
const url = new URL('https://nominatim.openstreetmap.org/search');
|
||
url.searchParams.set('q', query);
|
||
url.searchParams.set('format', 'json');
|
||
url.searchParams.set('limit', '1');
|
||
url.searchParams.set('accept-language', 'de');
|
||
const headers = {
|
||
'User-Agent': config?.nominatimEmail ? `ResearchChat/1.0 (${config.nominatimEmail})` : NOMINATIM_UA,
|
||
};
|
||
|
||
try {
|
||
const resp = await fetch(url, { headers, timeout: 15000 });
|
||
if (!resp.ok) return { error: `Nominatim Fehler ${resp.status}` };
|
||
const results = await resp.json();
|
||
if (!results.length) return { error: `Kein Ort gefunden: "${query}"` };
|
||
const r = results[0];
|
||
log?.(`Geocode "${query}" -> ${r.lat}, ${r.lon}`);
|
||
return {
|
||
name: r.name || query,
|
||
lat: parseFloat(r.lat),
|
||
lon: parseFloat(r.lon),
|
||
displayName: r.display_name,
|
||
};
|
||
} catch (e) {
|
||
return { error: `Nominatim nicht erreichbar: ${e.message.slice(0, 120)}` };
|
||
}
|
||
}
|
||
|
||
/** Geometrie entschlacken (max. 500 Punkte für das Dashboard). */
|
||
function decimate(coords, max = 500) {
|
||
if (coords.length <= max) return coords;
|
||
const step = coords.length / max;
|
||
const out = [];
|
||
for (let i = 0; i < coords.length; i += step) out.push(coords[Math.floor(i)]);
|
||
out.push(coords[coords.length - 1]);
|
||
return out;
|
||
}
|
||
|
||
/**
|
||
* Tour speichern: OSRM-Fuß-Route durch alle Stopps, GPX-Datei für OsmAnd,
|
||
* tour.json für die Dashboard-Karte, Google-Maps-URLs.
|
||
*/
|
||
async function saveTour({ stops, title, filename, sessionId }, config, log) {
|
||
if (!Array.isArray(stops) || stops.length < 2) {
|
||
return { error: 'Mindestens 2 Stopps nötig (Name, lat, lon)' };
|
||
}
|
||
const clean = stops
|
||
.filter((s) => s && s.name && Number.isFinite(Number(s.lat)) && Number.isFinite(Number(s.lon)))
|
||
.map((s) => ({
|
||
name: String(s.name).slice(0, 100),
|
||
lat: Number(s.lat),
|
||
lon: Number(s.lon),
|
||
description: String(s.description || '').slice(0, 300),
|
||
}));
|
||
if (clean.length < 2) return { error: 'Stopps ungültig (Name + lat + lon erforderlich)' };
|
||
|
||
// OSRM-Fuß-Route (Fallback: Luftlinie)
|
||
let route = { coordinates: clean.map((s) => [s.lon, s.lat]), distanceKm: 0, durationMin: 0, source: 'line' };
|
||
if (clean.length <= 25) {
|
||
try {
|
||
const osrmUrl = `${config?.osrmUrl || 'https://router.project-osrm.org'}/route/v1/foot/${toOsrm(clean)}?overview=full&geometries=geojson`;
|
||
const resp = await fetch(osrmUrl, { timeout: 20000 });
|
||
if (resp.ok) {
|
||
const data = await resp.json();
|
||
const r = data.routes?.[0];
|
||
if (r?.geometry?.coordinates?.length) {
|
||
route = {
|
||
coordinates: decimate(r.geometry.coordinates), // für die Karte im Dashboard
|
||
fullCoordinates: r.geometry.coordinates, // für die GPX-Datei (OsmAnd)
|
||
distanceKm: Math.round((r.distance / 1000) * 10) / 10,
|
||
durationMin: Math.round(r.duration / 60),
|
||
source: 'osrm',
|
||
};
|
||
}
|
||
} else {
|
||
log?.(`OSRM antwortet ${resp.status} – Fallback: Luftlinie`);
|
||
}
|
||
} catch (e) {
|
||
log?.(`OSRM nicht erreichbar – Fallback: Luftlinie (${e.message.slice(0, 80)})`);
|
||
}
|
||
}
|
||
|
||
// GPX 1.1 bauen
|
||
const wpts = clean
|
||
.map(
|
||
(s, i) =>
|
||
` <wpt lat="${s.lat.toFixed(6)}" lon="${s.lon.toFixed(6)}">\n` +
|
||
` <name>${xmlEscape(i + 1)}. ${xmlEscape(s.name)}</name>\n` +
|
||
(s.description ? ` <desc>${xmlEscape(s.description)}</desc>\n` : '') +
|
||
` </wpt>`
|
||
)
|
||
.join('\n');
|
||
const rtepts = clean
|
||
.map((s) => ` <rtept lat="${s.lat.toFixed(6)}" lon="${s.lon.toFixed(6)}"><name>${xmlEscape(s.name)}</name></rtept>`)
|
||
.join('\n');
|
||
const trkpts = (route.fullCoordinates || route.coordinates)
|
||
.map(([lon, lat]) => ` <trkpt lat="${lat.toFixed(6)}" lon="${lon.toFixed(6)}"></trkpt>`)
|
||
.join('\n');
|
||
const gpx = `<?xml version="1.0" encoding="UTF-8"?>
|
||
<gpx version="1.1" creator="ResearchChat" xmlns="http://www.topografix.com/GPX/1/1">
|
||
<metadata>
|
||
<name>${xmlEscape(title || 'Tour')}</name>
|
||
</metadata>
|
||
${wpts}
|
||
<rte>
|
||
<name>${xmlEscape(title || 'Tour')}</name>
|
||
${rtepts}
|
||
</rte>
|
||
<trk>
|
||
<name>${xmlEscape(title || 'Tour')} (Route)</name>
|
||
<trkseg>
|
||
${trkpts}
|
||
</trkseg>
|
||
</trk>
|
||
</gpx>
|
||
`;
|
||
|
||
const safeName =
|
||
(filename || title || 'tour').toLowerCase().replace(/[^a-z0-9äöü-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 60) ||
|
||
'tour';
|
||
const base = `${sessionId}-${safeName.replace(/-tour$/, '')}-tour`;
|
||
fs.mkdirSync(DASHBOARD_DIR, { recursive: true });
|
||
fs.writeFileSync(path.join(DASHBOARD_DIR, `${base}.gpx`), gpx, 'utf8');
|
||
const { fullCoordinates, ...routeForJson } = route;
|
||
fs.writeFileSync(
|
||
path.join(DASHBOARD_DIR, `${base}.tour.json`),
|
||
JSON.stringify({
|
||
title: title || 'Tour',
|
||
stops: clean,
|
||
route: routeForJson,
|
||
}),
|
||
'utf8'
|
||
);
|
||
|
||
const fmt = (lat, lon) => `${lat.toFixed(6)},${lon.toFixed(6)}`;
|
||
const googleMapsUrl = `https://www.google.com/maps/dir/${clean.map((s) => fmt(s.lat, s.lon)).join('/')}/`;
|
||
const stopsWithNav = clean.map((s) => ({
|
||
...s,
|
||
navUrl: `https://www.google.com/maps/dir/?api=1&destination=${fmt(s.lat, s.lon)}&travelmode=walking`,
|
||
}));
|
||
|
||
log?.(
|
||
`Tour gespeichert: ${base} (${clean.length} Stopps, ${route.distanceKm} km, ${route.durationMin} min, ${route.source})`
|
||
);
|
||
return {
|
||
ok: true,
|
||
title: title || 'Tour',
|
||
gpxUrl: `/dashboards/${base}.gpx`,
|
||
tourJsonUrl: `/dashboards/${base}.tour.json`,
|
||
googleMapsUrl,
|
||
stops: stopsWithNav,
|
||
distanceKm: route.distanceKm,
|
||
durationMin: route.durationMin,
|
||
routeSource: route.source,
|
||
osmAndHint: 'GPX-Datei herunterladen und in OsmAnd importieren (Menü → Meine Orte → Importieren).',
|
||
};
|
||
}
|
||
|
||
async function closeBrowser() {
|
||
if (browser) {
|
||
await browser.close().catch(() => {});
|
||
browser = null;
|
||
}
|
||
}
|
||
|
||
module.exports = { search, browse, verify, saveDashboard, screenshot, geocode, saveTour, deleteSessionFiles, closeBrowser };
|