浏览代码

transform document

tags/2.0.0-beta.1
jyong 3 个月前
父节点
当前提交
8e96b9ed77
共有 2 个文件被更改,包括 8 次插入8 次删除
  1. 0
    1
      api/controllers/console/datasets/website.py
  2. 8
    7
      api/services/website_service.py

+ 0
- 1
api/controllers/console/datasets/website.py 查看文件

parser.add_argument( parser.add_argument(
"provider", type=str, choices=["firecrawl", "watercrawl", "jinareader"], required=True, location="args" "provider", type=str, choices=["firecrawl", "watercrawl", "jinareader"], required=True, location="args"
) )
parser.add_argument("credential_id", type=str, required=True, nullable=True, location="args")
args = parser.parse_args() args = parser.parse_args()


# Create typed request and validate # Create typed request and validate

+ 8
- 7
api/services/website_service.py 查看文件



provider: str provider: str
job_id: str job_id: str
credential_id: Optional[str] = None

@classmethod @classmethod
def from_args(cls, args: dict, job_id: str) -> "WebsiteCrawlStatusApiRequest": def from_args(cls, args: dict, job_id: str) -> "WebsiteCrawlStatusApiRequest":
"""Create from Flask-RESTful parsed arguments.""" """Create from Flask-RESTful parsed arguments."""
provider = args.get("provider") provider = args.get("provider")
credential_id = args.get("credential_id")

if not provider: if not provider:
raise ValueError("Provider is required") raise ValueError("Provider is required")
if not job_id: if not job_id:
raise ValueError("Job ID is required") raise ValueError("Job ID is required")


return cls(provider=provider, job_id=job_id, credential_id=credential_id)
return cls(provider=provider, job_id=job_id)




class WebsiteService: class WebsiteService:
provider=provider, provider=provider,
plugin_id=plugin_id, plugin_id=plugin_id,
) )
return credential.get("api_key"), credential
if provider == "firecrawl":
return credential.get("firecrawl_api_key"), credential
elif provider in {"watercrawl", "jinareader"}:
return credential.get("api_key"), credential
else:
raise ValueError("Invalid provider")


@classmethod @classmethod
def _get_decrypted_api_key(cls, tenant_id: str, config: dict) -> str: def _get_decrypted_api_key(cls, tenant_id: str, config: dict) -> str:
headers={"Accept": "application/json", "Authorization": f"Bearer {api_key}"}, headers={"Accept": "application/json", "Authorization": f"Bearer {api_key}"},
) )
if response.json().get("code") != 200: if response.json().get("code") != 200:
raise ValueError("Failed to crawl")
raise ValueError(f"Failed to crawl:")
return {"status": "active", "data": response.json().get("data")} return {"status": "active", "data": response.json().get("data")}
else: else:
response = requests.post( response = requests.post(

正在加载...
取消
保存