From 916295274431fc23237c2f7542e7a8601e92ac51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Mon, 30 Jun 2025 17:37:19 +0200
Subject: [PATCH] proxy used improvement (#1727)

---
 apps/api/src/scraper/scrapeURL/engines/docx/index.ts        | 2 ++
 apps/api/src/scraper/scrapeURL/engines/fetch/index.ts       | 2 ++
 .../scraper/scrapeURL/engines/fire-engine/checkStatus.ts    | 2 ++
 apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts | 6 ++++++
 apps/api/src/scraper/scrapeURL/engines/index.ts             | 2 ++
 apps/api/src/scraper/scrapeURL/engines/index/index.ts       | 4 +++-
 apps/api/src/scraper/scrapeURL/engines/pdf/index.ts         | 6 ++++++
 apps/api/src/scraper/scrapeURL/engines/playwright/index.ts  | 2 ++
 .../src/scraper/scrapeURL/engines/utils/specialtyHandler.ts | 1 +
 apps/api/src/scraper/scrapeURL/index.ts                     | 1 +
 10 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
index 416e77fc..11c7304a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
@@ -14,5 +14,7 @@ export async function scrapeDOCX(meta: Meta, timeToRun: number | undefined): Pro
     statusCode: response.status,
 
     html: (await mammoth.convertToHtml({ path: tempFilePath })).value,
+
+    proxyUsed: "basic",
   };
 }
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
index c4e487b8..1e4897ce 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
@@ -120,5 +120,7 @@ export async function scrapeURLWithFetch(
     contentType: (response.headers.find(
       (x) => x[0].toLowerCase() === "content-type",
     ) ?? [])[1] ?? undefined,
+
+    proxyUsed: "basic",
   };
 }
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 4a26d876..68d107fc 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -94,6 +94,8 @@ const successSchema = z.object({
     .or(z.null()),
 
   docUrl: z.string().optional(),
+
+  usedMobileProxy: z.boolean().optional(),
 });
 
 export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 27498958..a3c0c156 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -291,6 +291,8 @@ export async function scrapeURLWithFireEngineChromeCDP(
           },
         }
       : {}),
+
+    proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
   };
 }
 
@@ -354,6 +356,8 @@ export async function scrapeURLWithFireEnginePlaywright(
           screenshot: response.screenshots[0],
         }
       : {}),
+
+    proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
   };
 }
 
@@ -409,5 +413,7 @@ export async function scrapeURLWithFireEngineTLSClient(
     contentType: (Object.entries(response.responseHeaders ?? {}).find(
       (x) => x[0].toLowerCase() === "content-type",
     ) ?? [])[1] ?? undefined,
+
+    proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
   };
 }
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 5b254259..5879b70b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -119,6 +119,8 @@ export type EngineScrapeResult = {
   };
   
   contentType?: string;
+
+  proxyUsed: "basic" | "stealth";
 };
 
 const engineHandlers: {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index fb0c765d..1f423f42 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -198,6 +198,8 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult
 
         cacheInfo: {
             created_at: new Date(data[0].created_at),
-        }
+        },
+
+        proxyUsed: doc.proxyUsed ?? "basic",
     };
 }
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 8bc96813..1135264f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -183,6 +183,8 @@ export async function scrapePDF(
 
         html: content,
         markdown: content,
+
+        proxyUsed: meta.pdfPrefetch.proxyUsed,
       };
     } else {
       const file = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {
@@ -202,6 +204,8 @@ export async function scrapePDF(
 
         html: content,
         markdown: content,
+
+        proxyUsed: "basic",
       };
     }
   }
@@ -300,5 +304,7 @@ export async function scrapePDF(
     html: result?.html ?? "",
     markdown: result?.markdown ?? "",
     numPages: pageCount,
+
+    proxyUsed: "basic",
   };
 }
diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
index 13f47fdf..c38bc045 100644
--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
@@ -53,5 +53,7 @@ export async function scrapeURLWithPlaywright(
     statusCode: response.pageStatusCode,
     error: response.pageError,
     contentType: response.contentType,
+
+    proxyUsed: "basic",
   };
 }
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index f762cfb1..02d04aca 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -18,6 +18,7 @@ async function feResToPdfPrefetch(feRes: FireEngineCheckStatusSuccess | undefine
     status: feRes.pageStatusCode,
     url: feRes.url,
     filePath,
+    proxyUsed: feRes.usedMobileProxy ? "stealth" : "basic",
   };
 }
 
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 5e0e9780..31f65617 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -60,6 +60,7 @@ export type Meta = {
     filePath: string;
     url?: string;
     status: number;
+    proxyUsed: "basic" | "stealth";
   } | null | undefined; // undefined: no prefetch yet, null: prefetch came back empty
   costTracking: CostTracking;
   winnerEngine?: Engine;