From 13f012c583cbe58a595ac82f8baacc67a812971f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Mon, 30 Jun 2025 17:37:31 +0200
Subject: [PATCH] add pdf prefetch log for debugging (ENG-2542) (#1734)

* feat

* feat: pdf prefetch anti-loop error
---
 apps/api/src/scraper/scrapeURL/engines/pdf/index.ts | 13 +++++++++++--
 .../scrapeURL/engines/utils/specialtyHandler.ts     |  5 +++--
 apps/api/src/scraper/scrapeURL/error.ts             |  6 ++++++
 apps/api/src/scraper/scrapeURL/index.ts             |  3 +++
 4 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 1135264f..7d14bd0d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -10,6 +10,7 @@ import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";
 import {
   PDFAntibotError,
   PDFInsufficientTimeError,
+  PDFPrefetchFailed,
   RemoveFeatureError,
   TimeoutError,
 } from "../../error";
@@ -194,7 +195,11 @@ export async function scrapePDF(
       const ct = file.response.headers.get("Content-Type");
       if (ct && !ct.includes("application/pdf")) {
         // if downloaded file wasn't a PDF
-        throw new PDFAntibotError();
+        if (meta.pdfPrefetch === undefined) {
+          throw new PDFAntibotError();
+        } else {
+          throw new PDFPrefetchFailed();
+        }
       }
 
       const content = file.buffer.toString("base64");
@@ -223,7 +228,11 @@ export async function scrapePDF(
     const ct = r.headers.get("Content-Type");
     if (ct && !ct.includes("application/pdf")) {
       // if downloaded file wasn't a PDF
-      throw new PDFAntibotError();
+      if (meta.pdfPrefetch === undefined) {
+        throw new PDFAntibotError();
+      } else {
+        throw new PDFPrefetchFailed();
+      }
     }
   }
 
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index 02d04aca..c67581cd 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -6,8 +6,9 @@ import os from "os";
 import { writeFile } from "fs/promises";
 import { Meta } from "../..";
 
-async function feResToPdfPrefetch(feRes: FireEngineCheckStatusSuccess | undefined): Promise<Meta["pdfPrefetch"]> {
+async function feResToPdfPrefetch(logger: Logger, feRes: FireEngineCheckStatusSuccess | undefined): Promise<Meta["pdfPrefetch"]> {
   if (!feRes?.file) {
+    logger.warn("No file in pdf prefetch");
     return null;
   }
 
@@ -40,7 +41,7 @@ export async function specialtyScrapeCheck(
     contentType.startsWith("application/pdf;")
   ) {
     // .pdf
-    throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(feRes));
+    throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));
   } else if (
     contentType ===
       "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index f139cfbd..df3dd05e 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -109,3 +109,9 @@ export class IndexMissError extends Error {
     super("Index doesn't have the page we're looking for");
   }
 }
+
+export class PDFPrefetchFailed extends Error {
+  constructor() {
+    super("Failed to prefetch PDF that is protected by anti-bot. Please contact help@firecrawl.com");
+  }
+}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 31f65617..e0f74ef0 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -25,6 +25,7 @@ import {
   PDFInsufficientTimeError,
   IndexMissError,
   DNSResolutionError,
+  PDFPrefetchFailed,
 } from "./error";
 import { executeTransformers } from "./transformers";
 import { LLMRefusalError } from "./transformers/llmExtract";
@@ -550,6 +551,8 @@ export async function scrapeURL(
       });
     } else if (error instanceof PDFInsufficientTimeError) {
       meta.logger.warn("scrapeURL: Insufficient time to process PDF", { error });
+    } else if (error instanceof PDFPrefetchFailed) {
+      meta.logger.warn("scrapeURL: Failed to prefetch PDF that is protected by anti-bot", { error });
     } else if (error instanceof TimeoutSignal) {
       throw error;
     } else {