Browse Source

r2

tags/2.0.0-beta.1
jyong 5 months ago
parent
commit
42fcda3dc8

+ 3
- 3
api/controllers/console/datasets/rag_pipeline/rag_pipeline.py View File

@login_required @login_required
@account_initialization_required @account_initialization_required
@enterprise_license_required @enterprise_license_required
def get(self, pipeline_id: str):
pipeline_template = RagPipelineService.get_pipeline_template_detail(pipeline_id)
def get(self, template_id: str):
pipeline_template = RagPipelineService.get_pipeline_template_detail(template_id)
return pipeline_template, 200 return pipeline_template, 200




) )
api.add_resource( api.add_resource(
PipelineTemplateDetailApi, PipelineTemplateDetailApi,
"/rag/pipeline/templates/<string:pipeline_id>",
"/rag/pipeline/templates/<string:template_id>",
) )
api.add_resource( api.add_resource(
CustomizedPipelineTemplateApi, CustomizedPipelineTemplateApi,

+ 20
- 14
api/core/datasource/entities/datasource_entities.py View File



from pydantic import BaseModel, Field, ValidationInfo, field_validator from pydantic import BaseModel, Field, ValidationInfo, field_validator


from core.entities.provider_entities import ProviderConfig
from core.plugin.entities.oauth import OAuthSchema
from core.plugin.entities.parameters import ( from core.plugin.entities.parameters import (
PluginParameter, PluginParameter,
PluginParameterOption, PluginParameterOption,
init_frontend_parameter, init_frontend_parameter,
) )
from core.tools.entities.common_entities import I18nObject from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_entities import ToolProviderEntity
from core.tools.entities.tool_entities import ToolLabelEnum, ToolProviderEntity




class DatasourceProviderType(enum.StrEnum): class DatasourceProviderType(enum.StrEnum):
icon: Optional[str] = None icon: Optional[str] = None




class DatasourceDescription(BaseModel):
human: I18nObject = Field(..., description="The description presented to the user")
llm: str = Field(..., description="The description presented to the LLM")


class DatasourceEntity(BaseModel): class DatasourceEntity(BaseModel):
identity: DatasourceIdentity identity: DatasourceIdentity
parameters: list[DatasourceParameter] = Field(default_factory=list) parameters: list[DatasourceParameter] = Field(default_factory=list)
description: Optional[DatasourceDescription] = None
description: I18nObject = Field(..., description="The label of the datasource")
output_schema: Optional[dict] = None output_schema: Optional[dict] = None
has_runtime_parameters: bool = Field(default=False, description="Whether the tool has runtime parameters")


@field_validator("parameters", mode="before") @field_validator("parameters", mode="before")
@classmethod @classmethod
def set_parameters(cls, v, validation_info: ValidationInfo) -> list[DatasourceParameter]: def set_parameters(cls, v, validation_info: ValidationInfo) -> list[DatasourceParameter]:
return v or [] return v or []


class DatasourceProviderIdentity(BaseModel):
author: str = Field(..., description="The author of the tool")
name: str = Field(..., description="The name of the tool")
description: I18nObject = Field(..., description="The description of the tool")
icon: str = Field(..., description="The icon of the tool")
label: I18nObject = Field(..., description="The label of the tool")
tags: Optional[list[ToolLabelEnum]] = Field(
default=[],
description="The tags of the tool",
)


class DatasourceProviderEntity(ToolProviderEntity):
class DatasourceProviderEntity(BaseModel):
""" """
Datasource provider entity Datasource provider entity
""" """

identity: DatasourceProviderIdentity
credentials_schema: list[ProviderConfig] = Field(default_factory=list)
oauth_schema: Optional[OAuthSchema] = None
provider_type: DatasourceProviderType provider_type: DatasourceProviderType




Get online document pages request Get online document pages request
""" """


tenant_id: str = Field(..., description="The tenant id")




class OnlineDocumentPageIcon(BaseModel): class OnlineDocumentPageIcon(BaseModel):
""" """
Get website crawl request Get website crawl request
""" """

url: str = Field(..., description="The url of the website")
crawl_parameters: dict = Field(..., description="The crawl parameters") crawl_parameters: dict = Field(..., description="The crawl parameters")




Get website crawl response Get website crawl response
""" """


result: WebSiteInfo
result: list[WebSiteInfo]

+ 2
- 1
api/core/datasource/online_document/online_document_plugin.py View File

from typing import Any, Mapping
from core.datasource.__base.datasource_plugin import DatasourcePlugin from core.datasource.__base.datasource_plugin import DatasourcePlugin
from core.datasource.__base.datasource_runtime import DatasourceRuntime from core.datasource.__base.datasource_runtime import DatasourceRuntime
from core.datasource.entities.datasource_entities import ( from core.datasource.entities.datasource_entities import (
def _get_online_document_pages( def _get_online_document_pages(
self, self,
user_id: str, user_id: str,
datasource_parameters: GetOnlineDocumentPagesRequest,
datasource_parameters: Mapping[str, Any],
provider_type: str, provider_type: str,
) -> GetOnlineDocumentPagesResponse: ) -> GetOnlineDocumentPagesResponse:
manager = PluginDatasourceManager() manager = PluginDatasourceManager()

+ 2
- 1
api/core/datasource/website_crawl/website_crawl_plugin.py View File

from typing import Any, Mapping
from core.datasource.__base.datasource_plugin import DatasourcePlugin from core.datasource.__base.datasource_plugin import DatasourcePlugin
from core.datasource.__base.datasource_runtime import DatasourceRuntime from core.datasource.__base.datasource_runtime import DatasourceRuntime
from core.datasource.entities.datasource_entities import ( from core.datasource.entities.datasource_entities import (
def _get_website_crawl( def _get_website_crawl(
self, self,
user_id: str, user_id: str,
datasource_parameters: GetWebsiteCrawlRequest,
datasource_parameters: Mapping[str, Any],
provider_type: str, provider_type: str,
) -> GetWebsiteCrawlResponse: ) -> GetWebsiteCrawlResponse:
manager = PluginDatasourceManager() manager = PluginDatasourceManager()

+ 0
- 1
api/core/plugin/entities/plugin_daemon.py View File

provider: str provider: str
plugin_unique_identifier: str plugin_unique_identifier: str
plugin_id: str plugin_id: str
author: str
declaration: DatasourceProviderEntityWithPlugin declaration: DatasourceProviderEntityWithPlugin





+ 3
- 5
api/core/plugin/impl/datasource.py View File

from typing import Any
from typing import Any, Mapping


from core.datasource.entities.api_entities import DatasourceProviderApiEntity from core.datasource.entities.api_entities import DatasourceProviderApiEntity
from core.datasource.entities.datasource_entities import ( from core.datasource.entities.datasource_entities import (
GetOnlineDocumentPageContentRequest, GetOnlineDocumentPageContentRequest,
GetOnlineDocumentPageContentResponse, GetOnlineDocumentPageContentResponse,
GetOnlineDocumentPagesRequest,
GetOnlineDocumentPagesResponse, GetOnlineDocumentPagesResponse,
GetWebsiteCrawlRequest,
GetWebsiteCrawlResponse, GetWebsiteCrawlResponse,
) )
from core.plugin.entities.plugin import GenericProviderID, ToolProviderID from core.plugin.entities.plugin import GenericProviderID, ToolProviderID
datasource_provider: str, datasource_provider: str,
datasource_name: str, datasource_name: str,
credentials: dict[str, Any], credentials: dict[str, Any],
datasource_parameters: GetWebsiteCrawlRequest,
datasource_parameters: Mapping[str, Any],
provider_type: str, provider_type: str,
) -> GetWebsiteCrawlResponse: ) -> GetWebsiteCrawlResponse:
""" """
datasource_provider: str, datasource_provider: str,
datasource_name: str, datasource_name: str,
credentials: dict[str, Any], credentials: dict[str, Any],
datasource_parameters: GetOnlineDocumentPagesRequest,
datasource_parameters: Mapping[str, Any],
provider_type: str, provider_type: str,
) -> GetOnlineDocumentPagesResponse: ) -> GetOnlineDocumentPagesResponse:
""" """

+ 7
- 7
api/services/rag_pipeline/rag_pipeline.py View File

return result.get("pipeline_templates") return result.get("pipeline_templates")


@classmethod @classmethod
def get_pipeline_template_detail(cls, pipeline_id: str) -> Optional[dict]:
def get_pipeline_template_detail(cls, template_id: str) -> Optional[dict]:
""" """
Get pipeline template detail. Get pipeline template detail.
:param pipeline_id: pipeline id
:param template_id: template id
:return: :return:
""" """
mode = dify_config.HOSTED_FETCH_PIPELINE_TEMPLATES_MODE mode = dify_config.HOSTED_FETCH_PIPELINE_TEMPLATES_MODE
retrieval_instance = PipelineTemplateRetrievalFactory.get_pipeline_template_factory(mode)
result: Optional[dict] = retrieval_instance.get_pipeline_template_detail(pipeline_id)
retrieval_instance = PipelineTemplateRetrievalFactory.get_pipeline_template_factory(mode)()
result: Optional[dict] = retrieval_instance.get_pipeline_template_detail(template_id)
return result return result


@classmethod @classmethod
online_document_result: GetOnlineDocumentPagesResponse = ( online_document_result: GetOnlineDocumentPagesResponse = (
datasource_runtime._get_online_document_pages( datasource_runtime._get_online_document_pages(
user_id=account.id, user_id=account.id,
datasource_parameters=GetOnlineDocumentPagesRequest(tenant_id=pipeline.tenant_id),
datasource_parameters=user_inputs,
provider_type=datasource_runtime.datasource_provider_type(), provider_type=datasource_runtime.datasource_provider_type(),
) )
) )
datasource_runtime = cast(WebsiteCrawlDatasourcePlugin, datasource_runtime) datasource_runtime = cast(WebsiteCrawlDatasourcePlugin, datasource_runtime)
website_crawl_result: GetWebsiteCrawlResponse = datasource_runtime._get_website_crawl( website_crawl_result: GetWebsiteCrawlResponse = datasource_runtime._get_website_crawl(
user_id=account.id, user_id=account.id,
datasource_parameters=GetWebsiteCrawlRequest(**user_inputs),
datasource_parameters=user_inputs,
provider_type=datasource_runtime.datasource_provider_type(), provider_type=datasource_runtime.datasource_provider_type(),
) )
return { return {
"result": website_crawl_result.result.model_dump(),
"result": [result.model_dump() for result in website_crawl_result.result],
"provider_type": datasource_node_data.get("provider_type"), "provider_type": datasource_node_data.get("provider_type"),
} }
else: else:

Loading…
Cancel
Save