diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 1135264f..7d14bd0d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -10,6 +10,7 @@ import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";
import {
PDFAntibotError,
PDFInsufficientTimeError,
+ PDFPrefetchFailed,
RemoveFeatureError,
TimeoutError,
} from "../../error";
@@ -194,7 +195,11 @@ export async function scrapePDF(
const ct = file.response.headers.get("Content-Type");
if (ct && !ct.includes("application/pdf")) {
// if downloaded file wasn't a PDF
- throw new PDFAntibotError();
+ if (meta.pdfPrefetch === undefined) {
+ throw new PDFAntibotError();
+ } else {
+ throw new PDFPrefetchFailed();
+ }
}
const content = file.buffer.toString("base64");
@@ -223,7 +228,11 @@ export async function scrapePDF(
const ct = r.headers.get("Content-Type");
if (ct && !ct.includes("application/pdf")) {
// if downloaded file wasn't a PDF
- throw new PDFAntibotError();
+ if (meta.pdfPrefetch === undefined) {
+ throw new PDFAntibotError();
+ } else {
+ throw new PDFPrefetchFailed();
+ }
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index 02d04aca..c67581cd 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -6,8 +6,9 @@ import os from "os";
import { writeFile } from "fs/promises";
import { Meta } from "../..";
-async function feResToPdfPrefetch(feRes: FireEngineCheckStatusSuccess | undefined): Promise {
+async function feResToPdfPrefetch(logger: Logger, feRes: FireEngineCheckStatusSuccess | undefined): Promise {
if (!feRes?.file) {
+ logger.warn("No file in pdf prefetch");
return null;
}
@@ -40,7 +41,7 @@ export async function specialtyScrapeCheck(
contentType.startsWith("application/pdf;")
) {
// .pdf
- throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(feRes));
+ throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));
} else if (
contentType ===
"application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index f139cfbd..df3dd05e 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -109,3 +109,9 @@ export class IndexMissError extends Error {
super("Index doesn't have the page we're looking for");
}
}
+
+export class PDFPrefetchFailed extends Error {
+ constructor() {
+ super("Failed to prefetch PDF that is protected by anti-bot. Please contact help@firecrawl.com");
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 31f65617..e0f74ef0 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -25,6 +25,7 @@ import {
PDFInsufficientTimeError,
IndexMissError,
DNSResolutionError,
+ PDFPrefetchFailed,
} from "./error";
import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
@@ -550,6 +551,8 @@ export async function scrapeURL(
});
} else if (error instanceof PDFInsufficientTimeError) {
meta.logger.warn("scrapeURL: Insufficient time to process PDF", { error });
+ } else if (error instanceof PDFPrefetchFailed) {
+ meta.logger.warn("scrapeURL: Failed to prefetch PDF that is protected by anti-bot", { error });
} else if (error instanceof TimeoutSignal) {
throw error;
} else {