|
|
|
@@ -22,7 +22,6 @@ class FirecrawlApp: |
|
|
|
"formats": ["markdown"], |
|
|
|
"onlyMainContent": True, |
|
|
|
"timeout": 30000, |
|
|
|
"integration": "dify", |
|
|
|
} |
|
|
|
if params: |
|
|
|
json_data.update(params) |
|
|
|
@@ -40,7 +39,7 @@ class FirecrawlApp: |
|
|
|
def crawl_url(self, url, params=None) -> str: |
|
|
|
# Documentation: https://docs.firecrawl.dev/api-reference/endpoint/crawl-post |
|
|
|
headers = self._prepare_headers() |
|
|
|
json_data = {"url": url, "integration": "dify"} |
|
|
|
json_data = {"url": url} |
|
|
|
if params: |
|
|
|
json_data.update(params) |
|
|
|
response = self._post_request(f"{self.base_url}/v1/crawl", json_data, headers) |
|
|
|
@@ -138,7 +137,6 @@ class FirecrawlApp: |
|
|
|
"timeout": 60000, |
|
|
|
"ignoreInvalidURLs": False, |
|
|
|
"scrapeOptions": {}, |
|
|
|
"integration": "dify", |
|
|
|
} |
|
|
|
if params: |
|
|
|
json_data.update(params) |