From 400d497fcab559b17e09cb8e90a3dc201d0bf441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Tue, 1 Jul 2025 16:25:49 +0200 Subject: [PATCH] feat(scrapeURL): ask user to increase timeout if there's a DOM.getDocument or queryAXTree error (#1739) * feat(scrapeURL): ask user to increase timeout if there's a DOM.getDocument or queryAXTree error * fix: move result tracking to meta --- .../engines/fire-engine/checkStatus.ts | 9 +++- .../scrapeURL/engines/fire-engine/index.ts | 4 +- apps/api/src/scraper/scrapeURL/error.ts | 6 +++ apps/api/src/scraper/scrapeURL/index.ts | 42 +++++++++++-------- 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts index 68d107fc..0b43f3e8 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts @@ -9,7 +9,8 @@ import { SiteError, SSLError, UnsupportedFileError, - DNSResolutionError + DNSResolutionError, + FEPageLoadFailed } from "../../error"; import { MockState } from "../../lib/mock"; import { fireEngineURL } from "./scrape"; @@ -202,6 +203,12 @@ export async function fireEngineCheckStatus( throw new UnsupportedFileError( "File size exceeds " + status.error.split("File size exceeds ")[1], ); + } else if ( + typeof status.error === "string" && + status.error.includes("failed to finish without timing out") + ) { + logger.warn("CDP timed out while loading the page", { status, jobId }); + throw new FEPageLoadFailed(); } else if ( typeof status.error === "string" && // TODO: improve this later diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts index a3c0c156..66349fbc 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts @@ -21,6 +21,7 @@ import { SSLError, TimeoutError, UnsupportedFileError, + FEPageLoadFailed, } from "../../error"; import * as Sentry from "@sentry/node"; import { specialtyScrapeCheck } from "../utils/specialtyHandler"; @@ -102,7 +103,8 @@ async function performFireEngineScrape< error instanceof SSLError || error instanceof DNSResolutionError || error instanceof ActionError || - error instanceof UnsupportedFileError + error instanceof UnsupportedFileError || + error instanceof FEPageLoadFailed ) { fireEngineDelete( logger.child({ diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts index df3dd05e..5afcca69 100644 --- a/apps/api/src/scraper/scrapeURL/error.ts +++ b/apps/api/src/scraper/scrapeURL/error.ts @@ -115,3 +115,9 @@ export class PDFPrefetchFailed extends Error { super("Failed to prefetch PDF that is protected by anti-bot. Please contact help@firecrawl.com"); } } + +export class FEPageLoadFailed extends Error { + constructor() { + super("The page failed to load with the specified timeout. Please increase the timeout parameter in your request."); + } +} diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index e0f74ef0..3493c015 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -26,6 +26,7 @@ import { IndexMissError, DNSResolutionError, PDFPrefetchFailed, + FEPageLoadFailed, } from "./error"; import { executeTransformers } from "./transformers"; import { LLMRefusalError } from "./transformers/llmExtract"; @@ -65,6 +66,7 @@ export type Meta = { } | null | undefined; // undefined: no prefetch yet, null: prefetch came back empty costTracking: CostTracking; winnerEngine?: Engine; + results: EngineResultsTracker; }; function buildFeatureFlags( @@ -193,6 +195,7 @@ async function buildMetaObject( : null, pdfPrefetch: undefined, costTracking, + results: {}, }; } @@ -261,7 +264,6 @@ async function scrapeURLLoop(meta: Meta): Promise { const fallbackList = buildFallbackList(meta); - const results: EngineResultsTracker = {}; let result: EngineScrapeResultWithContext | null = null; const timeToRun = @@ -293,7 +295,7 @@ async function scrapeURLLoop(meta: Meta): Promise { const hasNoPageError = engineResult.error === undefined; const isLikelyProxyError = [403, 429].includes(engineResult.statusCode); - results[engine] = { + meta.results[engine] = { state: "success", result: engineResult, factors: { isLongEnough, isGoodStatusCode, hasNoPageError, isLikelyProxyError }, @@ -327,7 +329,7 @@ async function scrapeURLLoop(meta: Meta): Promise { meta.logger.warn("Engine " + engine + " could not scrape the page.", { error, }); - results[engine] = { + meta.results[engine] = { state: "error", error: safeguardCircularError(error), unexpected: false, @@ -338,7 +340,7 @@ async function scrapeURLLoop(meta: Meta): Promise { meta.logger.info("Engine " + engine + " could not find the page in the index.", { error, }); - results[engine] = { + meta.results[engine] = { state: "error", error: safeguardCircularError(error), unexpected: false, @@ -349,7 +351,7 @@ async function scrapeURLLoop(meta: Meta): Promise { meta.logger.info("Engine " + engine + " timed out while scraping.", { error, }); - results[engine] = { + meta.results[engine] = { state: "timeout", startedAt, finishedAt: Date.now(), @@ -360,14 +362,14 @@ async function scrapeURLLoop(meta: Meta): Promise { ) { throw error; } else if (error instanceof LLMRefusalError) { - results[engine] = { + meta.results[engine] = { state: "error", error: safeguardCircularError(error), unexpected: true, startedAt, finishedAt: Date.now(), }; - error.results = results; + error.results = meta.results; meta.logger.warn("LLM refusal encountered", { error }); throw error; } else if (error instanceof SiteError) { @@ -386,13 +388,21 @@ async function scrapeURLLoop(meta: Meta): Promise { throw error; } else if (error instanceof PDFInsufficientTimeError) { throw error; + } else if (error instanceof FEPageLoadFailed) { + meta.results[engine] = { + state: "error", + error, + unexpected: false, + startedAt, + finishedAt: Date.now(), + }; } else { Sentry.captureException(error); meta.logger.warn( "An unexpected error happened while scraping with " + engine + ".", { error }, ); - results[engine] = { + meta.results[engine] = { state: "error", error: safeguardCircularError(error), unexpected: true, @@ -406,7 +416,7 @@ async function scrapeURLLoop(meta: Meta): Promise { if (result === null) { throw new NoEnginesLeftError( fallbackList.map((x) => x.engine), - results, + meta.results, ); } @@ -423,7 +433,7 @@ async function scrapeURLLoop(meta: Meta): Promise { numPages: result.result.numPages, contentType: result.result.contentType, proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic", - ...((results["index"] || results["index;documents"]) ? ( + ...((meta.results["index"] || meta.results["index;documents"]) ? ( result.result.cacheInfo ? { cacheState: "hit", cachedAt: result.result.cacheInfo.created_at.toISOString(), @@ -452,7 +462,7 @@ async function scrapeURLLoop(meta: Meta): Promise { success: true, document, logs: meta.logs, - engines: results, + engines: meta.results, }; } @@ -524,14 +534,12 @@ export async function scrapeURL( } } } catch (error) { - let results: EngineResultsTracker = {}; - - if (error instanceof NoEnginesLeftError) { + if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) { + throw new FEPageLoadFailed(); + } else if (error instanceof NoEnginesLeftError) { meta.logger.warn("scrapeURL: All scraping engines failed!", { error }); - results = error.results; } else if (error instanceof LLMRefusalError) { meta.logger.warn("scrapeURL: LLM refused to extract content", { error }); - results = error.results!; } else if ( error instanceof Error && error.message.includes("Invalid schema for response_format") @@ -565,7 +573,7 @@ export async function scrapeURL( success: false, error, logs: meta.logs, - engines: results, + engines: meta.results, }; } }