diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts index c08ea93b..5016e2ab 100644 --- a/apps/api/src/controllers/v1/batch-scrape.ts +++ b/apps/api/src/controllers/v1/batch-scrape.ts @@ -85,7 +85,6 @@ export async function batchScrapeController( appendToId: req.body.appendToId, account: req.account, }); - logger.debug("[batch-scrape - batchScrapeController] Integration: " + req.body.integration); if (!req.body.appendToId) { await logCrawl(id, req.auth.team_id); diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts index fb178bbf..81234b2d 100644 --- a/apps/api/src/controllers/v1/crawl.ts +++ b/apps/api/src/controllers/v1/crawl.ts @@ -31,7 +31,6 @@ export async function crawlController( originalRequest: preNormalizedBody, account: req.account, }); - logger.debug("[crawl - crawlController] Integration: " + req.body.integration); await logCrawl(id, req.auth.team_id); diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts index 19cdc9fa..4ffaddfe 100644 --- a/apps/api/src/controllers/v1/extract.ts +++ b/apps/api/src/controllers/v1/extract.ts @@ -83,7 +83,6 @@ export async function extractController( subId: req.acuc?.sub_id, extractId, }); - _logger.debug("[extract - extractController] Integration: " + req.body.integration); const jobData = { request: req.body, diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts index 8697b7dd..332fade6 100644 --- a/apps/api/src/controllers/v1/map.ts +++ b/apps/api/src/controllers/v1/map.ts @@ -331,7 +331,6 @@ export async function mapController( originalRequest, teamId: req.auth.team_id, }); - logger.debug("[map - mapController] Integration: " + req.body.integration); let result: Awaited>; const abort = new AbortController(); diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts index 9967043c..fb68f7ff 100644 --- a/apps/api/src/controllers/v1/scrape.ts +++ b/apps/api/src/controllers/v1/scrape.ts @@ -29,7 +29,6 @@ export async function scrapeController( teamId: req.auth.team_id, account: req.account, }); - logger.debug("[scrape - scrapeController] Integration: " + req.body.integration); req.body = scrapeRequestSchema.parse(req.body); diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts index f45e5eeb..c94480db 100644 --- a/apps/api/src/controllers/v1/search.ts +++ b/apps/api/src/controllers/v1/search.ts @@ -181,7 +181,6 @@ export async function searchController( query: req.body.query, origin: req.body.origin, }); - logger.debug("[search - searchController] Integration: " + req.body.integration); let limit = req.body.limit; diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 06768ffb..d8a5fefc 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -23,7 +23,6 @@ export enum IntegrationEnum { MAKE = "make", FLOWISE = "flowise", METAGPT = "metagpt", - UNKNOWN = "unknown", } export type Format = @@ -485,7 +484,7 @@ export const extractV1Options = z enableWebSearch: z.boolean().default(false), scrapeOptions: baseScrapeOptions.default({ onlyMainContent: false }).optional(), origin: z.string().optional().default("api"), - integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN), + integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null), urlTrace: z.boolean().default(false), timeout: z.number().int().positive().finite().safe().default(60000), __experimental_streamSteps: z.boolean().default(false), @@ -544,7 +543,7 @@ export const scrapeRequestSchema = baseScrapeOptions extract: extractOptionsWithAgent.optional(), jsonOptions: extractOptionsWithAgent.optional(), origin: z.string().optional().default("api"), - integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN), + integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null), timeout: z.number().int().positive().finite().safe().default(30000), }) .strict(strictMessage) @@ -579,7 +578,7 @@ export const batchScrapeRequestSchema = baseScrapeOptions .extend({ urls: url.array(), origin: z.string().optional().default("api"), - integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN), + integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null), webhook: webhookSchema.optional(), appendToId: z.string().uuid().optional(), ignoreInvalidURLs: z.boolean().default(false), @@ -593,7 +592,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions .extend({ urls: z.string().array(), origin: z.string().optional().default("api"), - integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN), + integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null), webhook: webhookSchema.optional(), appendToId: z.string().uuid().optional(), ignoreInvalidURLs: z.boolean().default(false), @@ -641,7 +640,7 @@ export const crawlRequestSchema = crawlerOptions .extend({ url, origin: z.string().optional().default("api"), - integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN), + integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null), scrapeOptions: baseScrapeOptions.default({}), webhook: webhookSchema.optional(), limit: z.number().default(10000), @@ -673,7 +672,7 @@ export const mapRequestSchema = crawlerOptions .extend({ url, origin: z.string().optional().default("api"), - integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN), + integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null), includeSubdomains: z.boolean().default(true), search: z.string().optional(), ignoreSitemap: z.boolean().default(false), @@ -1194,7 +1193,7 @@ export const searchRequestSchema = z country: z.string().optional().default("us"), location: z.string().optional(), origin: z.string().optional().default("api"), - integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN), + integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null), timeout: z.number().int().positive().finite().safe().default(60000), ignoreInvalidURLs: z.boolean().optional().default(false), __searchPreviewToken: z.string().optional(), diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index 3167e788..eed92da5 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -44,7 +44,7 @@ export interface WebScraperOptions { sitemapped?: boolean; webhook?: z.infer; v1?: boolean; - integration?: string; + integration?: string | null; /** * Disables billing on the worker side. @@ -95,7 +95,7 @@ export interface FirecrawlJob { crawlerOptions?: any; scrapeOptions?: any; origin: string; - integration?: string; + integration?: string | null; num_tokens?: number; retry?: boolean; crawl_id?: string;