AI/FireCrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Fix LLMs.txt cache bug with subdomains and add bypass option (#1557)

Browse Source 
* Fix LLMs.txt cache bug with subdomains and add bypass option (#1519)

Co-Authored-By: hello@sideguide.dev <hello+firecrawl@sideguide.dev>

* Nick:

* Update LLMs.txt test file to use helper functions and concurrent tests

Co-Authored-By: hello@sideguide.dev <hello+firecrawl@sideguide.dev>

* Remove LLMs.txt test file as requested

Co-Authored-By: hello@sideguide.dev <hello+firecrawl@sideguide.dev>

* Change parameter name to 'cache' and keep 7-day expiration

Co-Authored-By: hello@sideguide.dev <hello+firecrawl@sideguide.dev>

* Update generate-llmstxt-supabase.ts

* Update JS and Python SDKs to include cache parameter

Co-Authored-By: hello@sideguide.dev <hello+firecrawl@sideguide.dev>

* Fix LLMs.txt cache implementation to use normalizeUrl and exact matching

Co-Authored-By: hello@sideguide.dev <hello+firecrawl@sideguide.dev>

* Revert "Fix LLMs.txt cache implementation to use normalizeUrl and exact matching"

This reverts commit d05b9964677b7b2384453329d2ac99d841467053.

* Nick:

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: hello@sideguide.dev <hello+firecrawl@sideguide.dev>
Co-authored-by: Nicolas <nicolascamara29@gmail.com>
This commit is contained in:
devin-ai-integration[bot]
2025-05-16 16:29:09 -03:00
committed by GitHub
co-authored by hello@sideguide.dev <hello+firecrawl@sideguide.dev> Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Nicolas
parent ab30c8e4ac
commit 7ccbbec488
12 changed files with 47 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/python-sdk/firecrawl/__init__.py
+1 -1
View File
@@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa
__version__ = "2.5.4"
__version__ = "2.6.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
apps/python-sdk/firecrawl/firecrawl.py
+12
View File
@@ -347,6 +347,7 @@ class GenerateLLMsTextParams(pydantic.BaseModel):
"""
maxUrls: Optional[int] = 10
showFullText: Optional[bool] = False
cache: Optional[bool] = True
__experimental_stream: Optional[bool] = None
class DeepResearchParams(pydantic.BaseModel):
@@ -1870,6 +1871,7 @@ class FirecrawlApp:
*,
max_urls: Optional[int] = None,
show_full_text: Optional[bool] = None,
cache: Optional[bool] = None,
experimental_stream: Optional[bool] = None) -> GenerateLLMsTextStatusResponse:
"""
Generate LLMs.txt for a given URL and poll until completion.
@@ -1878,6 +1880,7 @@ class FirecrawlApp:
url (str): Target URL to generate LLMs.txt from
max_urls (Optional[int]): Maximum URLs to process (default: 10)
show_full_text (Optional[bool]): Include full text in output (default: False)
cache (Optional[bool]): Whether to use cached content if available (default: True)
experimental_stream (Optional[bool]): Enable experimental streaming
Returns:
@@ -1893,6 +1896,7 @@ class FirecrawlApp:
params = GenerateLLMsTextParams(
maxUrls=max_urls,
showFullText=show_full_text,
cache=cache,
__experimental_stream=experimental_stream
)
@@ -1900,6 +1904,7 @@ class FirecrawlApp:
url,
max_urls=max_urls,
show_full_text=show_full_text,
cache=cache,
experimental_stream=experimental_stream
)
@@ -1935,6 +1940,7 @@ class FirecrawlApp:
*,
max_urls: Optional[int] = None,
show_full_text: Optional[bool] = None,
cache: Optional[bool] = None,
experimental_stream: Optional[bool] = None) -> GenerateLLMsTextResponse:
"""
Initiate an asynchronous LLMs.txt generation operation.
@@ -1943,6 +1949,7 @@ class FirecrawlApp:
url (str): The target URL to generate LLMs.txt from. Must be a valid HTTP/HTTPS URL.
max_urls (Optional[int]): Maximum URLs to process (default: 10)
show_full_text (Optional[bool]): Include full text in output (default: False)
cache (Optional[bool]): Whether to use cached content if available (default: True)
experimental_stream (Optional[bool]): Enable experimental streaming
Returns:
@@ -1957,6 +1964,7 @@ class FirecrawlApp:
params = GenerateLLMsTextParams(
maxUrls=max_urls,
showFullText=show_full_text,
cache=cache,
__experimental_stream=experimental_stream
)
@@ -4001,6 +4009,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
url,
max_urls=max_urls,
show_full_text=show_full_text,
cache=cache,
experimental_stream=experimental_stream
)
if not response.get('success') or 'id' not in response:
@@ -4027,6 +4036,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
*,
max_urls: Optional[int] = None,
show_full_text: Optional[bool] = None,
cache: Optional[bool] = None,
experimental_stream: Optional[bool] = None) -> GenerateLLMsTextResponse:
"""
Initiate an asynchronous LLMs.txt generation job without waiting for completion.
@@ -4035,6 +4045,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
url (str): Target URL to generate LLMs.txt from
max_urls (Optional[int]): Maximum URLs to process (default: 10)
show_full_text (Optional[bool]): Include full text in output (default: False)
cache (Optional[bool]): Whether to use cached content if available (default: True)
experimental_stream (Optional[bool]): Enable experimental streaming
Returns:
@@ -4057,6 +4068,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
params = GenerateLLMsTextParams(
maxUrls=max_urls,
showFullText=show_full_text,
cache=cache,
__experimental_stream=experimental_stream
)