diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts index 049a4b57..5444c3cf 100644 --- a/apps/api/src/controllers/v1/scrape.ts +++ b/apps/api/src/controllers/v1/scrape.ts @@ -81,9 +81,9 @@ export async function scrapeController( let doc: Document; try { - doc = await waitForJob(jobId, timeout + totalWait, logger.child({ module: "queue-jobs", method: "waitForJob" })); + doc = await waitForJob(jobId, timeout + totalWait); } catch (e) { - logger.error(`Error in scrapeController: ${e}`, { + logger.error(`Error in scrapeController`, { startTime, }); @@ -129,12 +129,8 @@ export async function scrapeController( } } - logger.info("Scrape completed"); - await getScrapeQueue().remove(jobId); - logger.info("Job removed from queue"); - if (!req.body.formats.includes("rawHtml")) { if (doc && doc.rawHtml) { delete doc.rawHtml; diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts index 9b4390f8..01ef9d31 100644 --- a/apps/api/src/lib/gcs-jobs.ts +++ b/apps/api/src/lib/gcs-jobs.ts @@ -87,15 +87,8 @@ export async function getJobFromGCS(jobId: string): Promise { const storage = new Storage({ credentials }); const bucket = storage.bucket(process.env.GCS_BUCKET_NAME); const blob = bucket.file(`${jobId}.json`); - const time1 = Date.now(); const [content] = await blob.download(); - const time2 = Date.now(); const x = JSON.parse(content.toString()); - logger.debug("getJobFromGCS time insights", { - downloadTook: time2 - time1, - jobId, - scrapeId: jobId, - }); return x; } catch (error) { if (error instanceof ApiError && error.code === 404 && error.message.includes("No such object:")) { diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index 48c807fb..451d31f4 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -121,14 +121,10 @@ export async function scrapeURLWithIndex(meta: Meta): Promise { return new Promise((resolve, reject) => { const start = Date.now(); @@ -360,7 +359,6 @@ export function waitForJob( } else { const state = await getScrapeQueue().getJobState(jobId); if (state === "completed") { - logger?.info("Job completed, getting document"); clearInterval(int); let doc: Document; doc = (await getScrapeQueue().getJob(jobId))!.returnvalue; @@ -373,11 +371,8 @@ export function waitForJob( doc = docs[0]; } - logger?.info("Document retrieved"); - resolve(doc); } else if (state === "failed") { - // console.log("failed", (await getScrapeQueue().getJob(jobId)).failedReason); const job = await getScrapeQueue().getJob(jobId); if (job && job.failedReason !== "Concurrency limit hit") { clearInterval(int);