### What problem does this PR solve? Add license statement. ### Type of change - [x] Refactoring Signed-off-by: Jin Hai <haijin.chn@gmail.com>tags/v0.16.0
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from beartype.claw import beartype_this_package | from beartype.claw import beartype_this_package | ||||
| beartype_this_package() | beartype_this_package() |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import importlib | import importlib | ||||
| from .begin import Begin, BeginParam | from .begin import Begin, BeginParam | ||||
| from .generate import Generate, GenerateParam | from .generate import Generate, GenerateParam |
| # | # | ||||
| # Copyright 2019 The FATE Authors. All Rights Reserved. | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | # | ||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from beartype.claw import beartype_this_package | from beartype.claw import beartype_this_package | ||||
| beartype_this_package() | beartype_this_package() |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import base64 | import base64 | ||||
| import os | import os | ||||
| import sys | import sys |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import re | import re | ||||
| import socket | import socket | ||||
| from urllib.parse import urlparse | from urllib.parse import urlparse |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from beartype.claw import beartype_this_package | from beartype.claw import beartype_this_package | ||||
| beartype_this_package() | beartype_this_package() |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # -*- coding: utf-8 -*- | # -*- coding: utf-8 -*- | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| from rag.nlp import find_codec | from rag.nlp import find_codec | ||||
| import readability | import readability | ||||
| import html_text | import html_text |
| # -*- coding: utf-8 -*- | # -*- coding: utf-8 -*- | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| # The following documents are mainly referenced, and only adaptation modifications have been made | # The following documents are mainly referenced, and only adaptation modifications have been made | ||||
| # from https://github.com/langchain-ai/langchain/blob/master/libs/text-splitters/langchain_text_splitters/json.py | # from https://github.com/langchain-ai/langchain/blob/master/libs/text-splitters/langchain_text_splitters/json.py | ||||
| # -*- coding: utf-8 -*- | # -*- coding: utf-8 -*- | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import re | import re | ||||
| class RAGFlowMarkdownParser: | class RAGFlowMarkdownParser: |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| from io import BytesIO | from io import BytesIO | ||||
| from pptx import Presentation | from pptx import Presentation |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import re | import re | ||||
| TBL = { | TBL = { |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| import re | import re | ||||
| import copy | import copy |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import re | import re | ||||
| from deepdoc.parser.utils import get_text | from deepdoc.parser.utils import get_text |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import os | import os | ||||
| import re | import re | ||||
| from collections import Counter | from collections import Counter |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| import os | import os | ||||
| import sys | import sys |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from beartype.claw import beartype_this_package | from beartype.claw import beartype_this_package | ||||
| beartype_this_package() | beartype_this_package() | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import logging | import logging | ||||
| import requests | import requests | ||||
| from bridge.context import ContextType # Import Context, ContextType | from bridge.context import ContextType # Import Context, ContextType |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from beartype.claw import beartype_this_package | from beartype.claw import beartype_this_package | ||||
| beartype_this_package() | beartype_this_package() |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import re | import re | ||||
| from api.db import LLMType | from api.db import LLMType |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| from tika import parser | from tika import parser | ||||
| import re | import re |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import re | import re | ||||
| from graphrag.index import build_knowledge_graph_chunks | from graphrag.index import build_knowledge_graph_chunks |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| from tika import parser | from tika import parser | ||||
| import re | import re |
| # | # | ||||
| # Copyright 2024 The InfiniFlow Authors. All Rights Reserved. | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | # | ||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| from tika import parser | from tika import parser | ||||
| from io import BytesIO | from io import BytesIO |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| from tika import parser | from tika import parser | ||||
| from io import BytesIO | from io import BytesIO |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| import copy | import copy | ||||
| import re | import re |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import io | import io | ||||
| import numpy as np | import numpy as np |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import copy | import copy | ||||
| import re | import re | ||||
| from io import BytesIO | from io import BytesIO |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| import re | import re | ||||
| import csv | import csv |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import logging | import logging | ||||
| import base64 | import base64 | ||||
| import datetime | import datetime |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import copy | import copy | ||||
| import re | import re | ||||
| from io import BytesIO | from io import BytesIO |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | # Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| # you may not use this file except in compliance with the License. | # you may not use this file except in compliance with the License. | ||||
| # You may obtain a copy of the License at | # You may obtain a copy of the License at | ||||
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| import re | import re | ||||
| import csv | import csv | ||||
| from copy import deepcopy | from copy import deepcopy |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from jina import Deployment | from jina import Deployment | ||||
| from docarray import BaseDoc | from docarray import BaseDoc | ||||
| from jina import Executor, requests | from jina import Executor, requests |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import logging | import logging | ||||
| import os | import os | ||||
| import time | import time |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import logging | import logging | ||||
| import os | import os | ||||
| import time | import time |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from abc import ABC, abstractmethod | from abc import ABC, abstractmethod | ||||
| from dataclasses import dataclass | from dataclasses import dataclass | ||||
| import numpy as np | import numpy as np |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import logging | import logging | ||||
| import re | import re | ||||
| import json | import json |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import logging | import logging | ||||
| import os | import os | ||||
| import re | import re |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import logging | import logging | ||||
| import time | import time | ||||
| from minio import Minio | from minio import Minio |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import logging | import logging | ||||
| import json | import json | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import logging | import logging | ||||
| import boto3 | import boto3 | ||||
| from botocore.exceptions import ClientError | from botocore.exceptions import ClientError |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import os | import os | ||||
| from enum import Enum | from enum import Enum | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import ragflow_sdk | import ragflow_sdk | ||||
| print(ragflow_sdk.__version__) | |||||
| print(ragflow_sdk.__version__) |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from beartype.claw import beartype_this_package | from beartype.claw import beartype_this_package | ||||
| beartype_this_package() | beartype_this_package() | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from .base import Base | from .base import Base | ||||
| from .session import Session | from .session import Session | ||||
| import requests | import requests |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| class Base(object): | class Base(object): | ||||
| def __init__(self, rag, res_dict): | def __init__(self, rag, res_dict): | ||||
| self.rag = rag | self.rag = rag |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from .base import Base | from .base import Base | ||||
| from .session import Session | from .session import Session | ||||
| def update(self, update_message: dict): | def update(self, update_message: dict): | ||||
| res = self.put(f'/chats/{self.id}', | res = self.put(f'/chats/{self.id}', | ||||
| update_message) | |||||
| update_message) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") != 0: | if res.get("code") != 0: | ||||
| raise Exception(res["message"]) | raise Exception(res["message"]) | ||||
| def create_session(self, name: str = "New session") -> Session: | def create_session(self, name: str = "New session") -> Session: | ||||
| res = self.post(f"/chats/{self.id}/sessions", {"name": name}) | res = self.post(f"/chats/{self.id}/sessions", {"name": name}) | ||||
| res = res.json() | res = res.json() | ||||
| return Session(self.rag, res['data']) | return Session(self.rag, res['data']) | ||||
| raise Exception(res["message"]) | raise Exception(res["message"]) | ||||
| def list_sessions(self,page: int = 1, page_size: int = 30, orderby: str = "create_time", desc: bool = True, | |||||
| def list_sessions(self, page: int = 1, page_size: int = 30, orderby: str = "create_time", desc: bool = True, | |||||
| id: str = None, name: str = None) -> list[Session]: | id: str = None, name: str = None) -> list[Session]: | ||||
| res = self.get(f'/chats/{self.id}/sessions',{"page": page, "page_size": page_size, "orderby": orderby, "desc": desc, "id": id, "name": name} ) | |||||
| res = self.get(f'/chats/{self.id}/sessions', | |||||
| {"page": page, "page_size": page_size, "orderby": orderby, "desc": desc, "id": id, "name": name}) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") == 0: | if res.get("code") == 0: | ||||
| result_list = [] | result_list = [] | ||||
| return result_list | return result_list | ||||
| raise Exception(res["message"]) | raise Exception(res["message"]) | ||||
| def delete_sessions(self,ids: list[str] | None = None): | |||||
| def delete_sessions(self, ids: list[str] | None = None): | |||||
| res = self.rm(f"/chats/{self.id}/sessions", {"ids": ids}) | res = self.rm(f"/chats/{self.id}/sessions", {"ids": ids}) | ||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") != 0: | if res.get("code") != 0: |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from .base import Base | from .base import Base | ||||
| res_dict.pop(k) | res_dict.pop(k) | ||||
| super().__init__(rag, res_dict) | super().__init__(rag, res_dict) | ||||
| def update(self,update_message:dict): | |||||
| res = self.put(f"/datasets/{self.dataset_id}/documents/{self.document_id}/chunks/{self.id}",update_message) | |||||
| def update(self, update_message: dict): | |||||
| res = self.put(f"/datasets/{self.dataset_id}/documents/{self.document_id}/chunks/{self.id}", update_message) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") != 0 : | |||||
| if res.get("code") != 0: | |||||
| raise Exception(res["message"]) | raise Exception(res["message"]) | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from .document import Document | from .document import Document | ||||
| from .base import Base | from .base import Base | ||||
| def update(self, update_message: dict): | def update(self, update_message: dict): | ||||
| res = self.put(f'/datasets/{self.id}', | res = self.put(f'/datasets/{self.id}', | ||||
| update_message) | |||||
| update_message) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") != 0: | if res.get("code") != 0: | ||||
| raise Exception(res["message"]) | raise Exception(res["message"]) | ||||
| def upload_documents(self,document_list: list[dict]): | |||||
| def upload_documents(self, document_list: list[dict]): | |||||
| url = f"/datasets/{self.id}/documents" | url = f"/datasets/{self.id}/documents" | ||||
| files = [("file",(ele["display_name"],ele["blob"])) for ele in document_list] | |||||
| res = self.post(path=url,json=None,files=files) | |||||
| files = [("file", (ele["display_name"], ele["blob"])) for ele in document_list] | |||||
| res = self.post(path=url, json=None, files=files) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") == 0: | if res.get("code") == 0: | ||||
| doc_list=[] | |||||
| doc_list = [] | |||||
| for doc in res["data"]: | for doc in res["data"]: | ||||
| document = Document(self.rag,doc) | |||||
| document = Document(self.rag, doc) | |||||
| doc_list.append(document) | doc_list.append(document) | ||||
| return doc_list | return doc_list | ||||
| raise Exception(res.get("message")) | raise Exception(res.get("message")) | ||||
| def list_documents(self, id: str | None = None, keywords: str | None = None, page: int = 1, page_size: int = 30, orderby: str = "create_time", desc: bool = True): | |||||
| res = self.get(f"/datasets/{self.id}/documents",params={"id": id,"keywords": keywords,"page": page,"page_size": page_size,"orderby": orderby,"desc": desc}) | |||||
| def list_documents(self, id: str | None = None, keywords: str | None = None, page: int = 1, page_size: int = 30, | |||||
| orderby: str = "create_time", desc: bool = True): | |||||
| res = self.get(f"/datasets/{self.id}/documents", | |||||
| params={"id": id, "keywords": keywords, "page": page, "page_size": page_size, "orderby": orderby, | |||||
| "desc": desc}) | |||||
| res = res.json() | res = res.json() | ||||
| documents = [] | documents = [] | ||||
| if res.get("code") == 0: | if res.get("code") == 0: | ||||
| for document in res["data"].get("docs"): | for document in res["data"].get("docs"): | ||||
| documents.append(Document(self.rag,document)) | |||||
| documents.append(Document(self.rag, document)) | |||||
| return documents | return documents | ||||
| raise Exception(res["message"]) | raise Exception(res["message"]) | ||||
| def delete_documents(self,ids: list[str] | None = None): | |||||
| res = self.rm(f"/datasets/{self.id}/documents",{"ids":ids}) | |||||
| def delete_documents(self, ids: list[str] | None = None): | |||||
| res = self.rm(f"/datasets/{self.id}/documents", {"ids": ids}) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") != 0: | if res.get("code") != 0: | ||||
| raise Exception(res["message"]) | raise Exception(res["message"]) | ||||
| def async_parse_documents(self,document_ids): | |||||
| res = self.post(f"/datasets/{self.id}/chunks",{"document_ids":document_ids}) | |||||
| def async_parse_documents(self, document_ids): | |||||
| res = self.post(f"/datasets/{self.id}/chunks", {"document_ids": document_ids}) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") != 0: | if res.get("code") != 0: | ||||
| raise Exception(res.get("message")) | raise Exception(res.get("message")) | ||||
| def async_cancel_parse_documents(self,document_ids): | |||||
| res = self.rm(f"/datasets/{self.id}/chunks",{"document_ids":document_ids}) | |||||
| def async_cancel_parse_documents(self, document_ids): | |||||
| res = self.rm(f"/datasets/{self.id}/chunks", {"document_ids": document_ids}) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") != 0: | if res.get("code") != 0: | ||||
| raise Exception(res.get("message")) | raise Exception(res.get("message")) |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import json | import json | ||||
| from .base import Base | from .base import Base | ||||
| from .chunk import Chunk | from .chunk import Chunk | ||||
| res_dict.pop(k) | res_dict.pop(k) | ||||
| super().__init__(rag, res_dict) | super().__init__(rag, res_dict) | ||||
| def update(self, update_message: dict): | def update(self, update_message: dict): | ||||
| res = self.put(f'/datasets/{self.dataset_id}/documents/{self.id}', | res = self.put(f'/datasets/{self.dataset_id}/documents/{self.id}', | ||||
| update_message) | update_message) | ||||
| except json.JSONDecodeError: | except json.JSONDecodeError: | ||||
| return res.content | return res.content | ||||
| def list_chunks(self,page=1, page_size=30, keywords=""): | |||||
| data={"keywords": keywords,"page":page,"page_size":page_size} | |||||
| def list_chunks(self, page=1, page_size=30, keywords=""): | |||||
| data = {"keywords": keywords, "page": page, "page_size": page_size} | |||||
| res = self.get(f'/datasets/{self.dataset_id}/documents/{self.id}/chunks', data) | res = self.get(f'/datasets/{self.dataset_id}/documents/{self.id}/chunks', data) | ||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") == 0: | if res.get("code") == 0: | ||||
| chunks=[] | |||||
| chunks = [] | |||||
| for data in res["data"].get("chunks"): | for data in res["data"].get("chunks"): | ||||
| chunk = Chunk(self.rag,data) | |||||
| chunk = Chunk(self.rag, data) | |||||
| chunks.append(chunk) | chunks.append(chunk) | ||||
| return chunks | return chunks | ||||
| raise Exception(res.get("message")) | raise Exception(res.get("message")) | ||||
| def add_chunk(self, content: str, important_keywords: list[str] = [], questions: list[str] = []): | def add_chunk(self, content: str, important_keywords: list[str] = [], questions: list[str] = []): | ||||
| res = self.post(f'/datasets/{self.dataset_id}/documents/{self.id}/chunks', | res = self.post(f'/datasets/{self.dataset_id}/documents/{self.id}/chunks', | ||||
| {"content":content,"important_keywords":important_keywords, "questions": questions}) | |||||
| {"content": content, "important_keywords": important_keywords, "questions": questions}) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") == 0: | if res.get("code") == 0: | ||||
| return Chunk(self.rag,res["data"].get("chunk")) | |||||
| return Chunk(self.rag, res["data"].get("chunk")) | |||||
| raise Exception(res.get("message")) | raise Exception(res.get("message")) | ||||
| def delete_chunks(self,ids:list[str] | None = None): | |||||
| res = self.rm(f"/datasets/{self.dataset_id}/documents/{self.id}/chunks",{"chunk_ids":ids}) | |||||
| def delete_chunks(self, ids: list[str] | None = None): | |||||
| res = self.rm(f"/datasets/{self.dataset_id}/documents/{self.id}/chunks", {"chunk_ids": ids}) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code")!=0: | |||||
| raise Exception(res.get("message")) | |||||
| if res.get("code") != 0: | |||||
| raise Exception(res.get("message")) |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import json | import json | ||||
| from .base import Base | from .base import Base | ||||
| self.id = None | self.id = None | ||||
| self.name = "New session" | self.name = "New session" | ||||
| self.messages = [{"role": "assistant", "content": "Hi! I am your assistant,can I help you?"}] | self.messages = [{"role": "assistant", "content": "Hi! I am your assistant,can I help you?"}] | ||||
| for key,value in res_dict.items(): | |||||
| if key =="chat_id" and value is not None: | |||||
| for key, value in res_dict.items(): | |||||
| if key == "chat_id" and value is not None: | |||||
| self.chat_id = None | self.chat_id = None | ||||
| self.__session_type = "chat" | self.__session_type = "chat" | ||||
| if key == "agent_id" and value is not None: | if key == "agent_id" and value is not None: | ||||
| self.__session_type = "agent" | self.__session_type = "agent" | ||||
| super().__init__(rag, res_dict) | super().__init__(rag, res_dict) | ||||
| def ask(self, question="",stream=True,**kwargs): | |||||
| def ask(self, question="", stream=True, **kwargs): | |||||
| if self.__session_type == "agent": | if self.__session_type == "agent": | ||||
| res=self._ask_agent(question,stream) | |||||
| res = self._ask_agent(question, stream) | |||||
| elif self.__session_type == "chat": | elif self.__session_type == "chat": | ||||
| res=self._ask_chat(question,stream,**kwargs) | |||||
| res = self._ask_chat(question, stream, **kwargs) | |||||
| for line in res.iter_lines(): | for line in res.iter_lines(): | ||||
| line = line.decode("utf-8") | line = line.decode("utf-8") | ||||
| if line.startswith("{"): | if line.startswith("{"): | ||||
| message = Message(self.rag, temp_dict) | message = Message(self.rag, temp_dict) | ||||
| yield message | yield message | ||||
| def _ask_chat(self, question: str, stream: bool,**kwargs): | |||||
| json_data={"question": question, "stream": True,"session_id":self.id} | |||||
| def _ask_chat(self, question: str, stream: bool, **kwargs): | |||||
| json_data = {"question": question, "stream": True, "session_id": self.id} | |||||
| json_data.update(kwargs) | json_data.update(kwargs) | ||||
| res = self.post(f"/chats/{self.chat_id}/completions", | res = self.post(f"/chats/{self.chat_id}/completions", | ||||
| json_data, stream=stream) | json_data, stream=stream) | ||||
| return res | return res | ||||
| def _ask_agent(self,question:str,stream:bool): | |||||
| def _ask_agent(self, question: str, stream: bool): | |||||
| res = self.post(f"/agents/{self.agent_id}/completions", | res = self.post(f"/agents/{self.agent_id}/completions", | ||||
| {"question": question, "stream": True,"session_id":self.id}, stream=stream) | |||||
| {"question": question, "stream": True, "session_id": self.id}, stream=stream) | |||||
| return res | return res | ||||
| def update(self,update_message): | |||||
| def update(self, update_message): | |||||
| res = self.put(f"/chats/{self.chat_id}/sessions/{self.id}", | res = self.put(f"/chats/{self.chat_id}/sessions/{self.id}", | ||||
| update_message) | |||||
| update_message) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") != 0: | if res.get("code") != 0: | ||||
| raise Exception(res.get("message")) | raise Exception(res.get("message")) | ||||
| class Message(Base): | class Message(Base): | ||||
| def __init__(self, rag, res_dict): | def __init__(self, rag, res_dict): | ||||
| self.content = "Hi! I am your assistant,can I help you?" | self.content = "Hi! I am your assistant,can I help you?" |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import os | import os | ||||
| import pytest | import pytest | ||||
| import requests | import requests | ||||
| HOST_ADDRESS = os.getenv('HOST_ADDRESS', 'http://127.0.0.1:9380') | HOST_ADDRESS = os.getenv('HOST_ADDRESS', 'http://127.0.0.1:9380') | ||||
| # def generate_random_email(): | # def generate_random_email(): | ||||
| # return 'user_' + ''.join(random.choices(string.ascii_lowercase + string.digits, k=8))+'@1.com' | # return 'user_' + ''.join(random.choices(string.ascii_lowercase + string.digits, k=8))+'@1.com' | ||||
| def generate_email(): | def generate_email(): | ||||
| return 'user_123@1.com' | return 'user_123@1.com' | ||||
| EMAIL = generate_email() | EMAIL = generate_email() | ||||
| # password is "123" | # password is "123" | ||||
| PASSWORD='''ctAseGvejiaSWWZ88T/m4FQVOpQyUvP+x7sXtdv3feqZACiQleuewkUi35E16wSd5C5QcnkkcV9cYc8TKPTRZlxappDuirxghxoOvFcJxFU4ixLsD | |||||
| PASSWORD = '''ctAseGvejiaSWWZ88T/m4FQVOpQyUvP+x7sXtdv3feqZACiQleuewkUi35E16wSd5C5QcnkkcV9cYc8TKPTRZlxappDuirxghxoOvFcJxFU4ixLsD | |||||
| fN33jCHRoDUW81IH9zjij/vaw8IbVyb6vuwg6MX6inOEBRRzVbRYxXOu1wkWY6SsI8X70oF9aeLFp/PzQpjoe/YbSqpTq8qqrmHzn9vO+yvyYyvmDsphXe | fN33jCHRoDUW81IH9zjij/vaw8IbVyb6vuwg6MX6inOEBRRzVbRYxXOu1wkWY6SsI8X70oF9aeLFp/PzQpjoe/YbSqpTq8qqrmHzn9vO+yvyYyvmDsphXe | ||||
| X8f7fp9c7vUsfOCkM+gHY3PadG+QHa7KI7mzTKgUTZImK6BZtfRBATDTthEUbbaTewY4H0MnWiCeeDhcbeQao6cFy1To8pE3RpmxnGnS8BsBn8w==''' | X8f7fp9c7vUsfOCkM+gHY3PadG+QHa7KI7mzTKgUTZImK6BZtfRBATDTthEUbbaTewY4H0MnWiCeeDhcbeQao6cFy1To8pE3RpmxnGnS8BsBn8w==''' | ||||
| def register(): | def register(): | ||||
| url = HOST_ADDRESS + "/v1/user/register" | url = HOST_ADDRESS + "/v1/user/register" | ||||
| name = "user" | name = "user" | ||||
| register_data = {"email":EMAIL,"nickname":name,"password":PASSWORD} | |||||
| res = requests.post(url=url,json=register_data) | |||||
| register_data = {"email": EMAIL, "nickname": name, "password": PASSWORD} | |||||
| res = requests.post(url=url, json=register_data) | |||||
| res = res.json() | res = res.json() | ||||
| if res.get("code") != 0: | if res.get("code") != 0: | ||||
| raise Exception(res.get("message")) | raise Exception(res.get("message")) | ||||
| def login(): | def login(): | ||||
| url = HOST_ADDRESS + "/v1/user/login" | url = HOST_ADDRESS + "/v1/user/login" | ||||
| login_data = {"email":EMAIL,"password":PASSWORD} | |||||
| response=requests.post(url=url,json=login_data) | |||||
| login_data = {"email": EMAIL, "password": PASSWORD} | |||||
| response = requests.post(url=url, json=login_data) | |||||
| res = response.json() | res = response.json() | ||||
| if res.get("code")!=0: | |||||
| if res.get("code") != 0: | |||||
| raise Exception(res.get("message")) | raise Exception(res.get("message")) | ||||
| auth = response.headers["Authorization"] | auth = response.headers["Authorization"] | ||||
| return auth | return auth | ||||
| @pytest.fixture(scope="session") | @pytest.fixture(scope="session") | ||||
| def get_api_key_fixture(): | def get_api_key_fixture(): | ||||
| try: | try: | ||||
| auth = login() | auth = login() | ||||
| url = HOST_ADDRESS + "/v1/system/new_token" | url = HOST_ADDRESS + "/v1/system/new_token" | ||||
| auth = {"Authorization": auth} | auth = {"Authorization": auth} | ||||
| response = requests.post(url=url,headers=auth) | |||||
| response = requests.post(url=url, headers=auth) | |||||
| res = response.json() | res = response.json() | ||||
| if res.get("code") != 0: | if res.get("code") != 0: | ||||
| raise Exception(res.get("message")) | raise Exception(res.get("message")) | ||||
| return res["data"].get("token") | return res["data"].get("token") | ||||
| @pytest.fixture(scope="session") | @pytest.fixture(scope="session") | ||||
| def get_auth(): | def get_auth(): | ||||
| try: | try: | ||||
| auth = login() | auth = login() | ||||
| return auth | return auth | ||||
| @pytest.fixture(scope="session") | @pytest.fixture(scope="session") | ||||
| def get_email(): | def get_email(): | ||||
| return EMAIL | return EMAIL |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import os | import os | ||||
| import requests | import requests | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| def test_get_email(get_email): | def test_get_email(get_email): | ||||
| print("\nEmail account:",flush=True) | print("\nEmail account:",flush=True) | ||||
| print(f"{get_email}\n",flush=True) | print(f"{get_email}\n",flush=True) |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from common import create_dataset, list_dataset, rm_dataset, update_dataset, DATASET_NAME_LIMIT | from common import create_dataset, list_dataset, rm_dataset, update_dataset, DATASET_NAME_LIMIT | ||||
| import re | import re | ||||
| import random | import random | ||||
| print(f"found {len(dataset_list)} datasets") | print(f"found {len(dataset_list)} datasets") | ||||
| dataset_id = dataset_list[0] | dataset_id = dataset_list[0] | ||||
| json_req = {"kb_id": dataset_id, "name": "test_update_dataset", "description": "test", "permission": "me", "parser_id": "presentation", | |||||
| json_req = {"kb_id": dataset_id, "name": "test_update_dataset", "description": "test", "permission": "me", | |||||
| "parser_id": "presentation", | |||||
| "language": "spanish"} | "language": "spanish"} | ||||
| res = update_dataset(get_auth, json_req) | res = update_dataset(get_auth, json_req) | ||||
| assert res.get("code") == 0, f"{res.get('message')}" | assert res.get("code") == 0, f"{res.get('message')}" | ||||
| res = rm_dataset(get_auth, dataset_id) | res = rm_dataset(get_auth, dataset_id) | ||||
| assert res.get("code") == 0, f"{res.get('message')}" | assert res.get("code") == 0, f"{res.get('message')}" | ||||
| print(f"{len(dataset_list)} datasets are deleted") | print(f"{len(dataset_list)} datasets are deleted") | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| import os | import os | ||||
| HOST_ADDRESS=os.getenv('HOST_ADDRESS', 'http://127.0.0.1:9380') | |||||
| HOST_ADDRESS = os.getenv('HOST_ADDRESS', 'http://127.0.0.1:9380') |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| def test_get_email(get_email): | def test_get_email(get_email): | ||||
| print("\nEmail account:",flush=True) | |||||
| print(f"{get_email}\n",flush=True) | |||||
| print("\nEmail account:", flush=True) | |||||
| print(f"{get_email}\n", flush=True) |
| from ragflow_sdk import RAGFlow,Agent | |||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from ragflow_sdk import RAGFlow, Agent | |||||
| from common import HOST_ADDRESS | from common import HOST_ADDRESS | ||||
| import pytest | import pytest | ||||
| @pytest.mark.skip(reason="") | @pytest.mark.skip(reason="") | ||||
| def test_list_agents_with_success(get_api_key_fixture): | def test_list_agents_with_success(get_api_key_fixture): | ||||
| API_KEY=get_api_key_fixture | |||||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||||
| API_KEY = get_api_key_fixture | |||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||||
| rag.list_agents() | rag.list_agents() | ||||
| def test_converse_with_agent_with_success(get_api_key_fixture): | def test_converse_with_agent_with_success(get_api_key_fixture): | ||||
| API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | ||||
| agent_id = "ebfada2eb2bc11ef968a0242ac120006" | agent_id = "ebfada2eb2bc11ef968a0242ac120006" | ||||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||||
| lang = "Chinese" | lang = "Chinese" | ||||
| file = "How is the weather tomorrow?" | file = "How is the weather tomorrow?" | ||||
| Agent.ask(agent_id=agent_id,rag=rag,lang=lang,file=file) | |||||
| Agent.ask(agent_id=agent_id, rag=rag, lang=lang, file=file) |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from ragflow_sdk import RAGFlow | from ragflow_sdk import RAGFlow | ||||
| from common import HOST_ADDRESS | from common import HOST_ADDRESS | ||||
| def test_create_chat_with_name(get_api_key_fixture): | def test_create_chat_with_name(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| display_name = "ragflow.txt" | display_name = "ragflow.txt" | ||||
| with open("test_data/ragflow.txt", "rb") as file: | with open("test_data/ragflow.txt", "rb") as file: | ||||
| blob = file.read() | blob = file.read() | ||||
| document = {"display_name":display_name,"blob":blob} | |||||
| document = {"display_name": display_name, "blob": blob} | |||||
| documents = [] | documents = [] | ||||
| documents.append(document) | documents.append(document) | ||||
| docs= kb.upload_documents(documents) | |||||
| docs = kb.upload_documents(documents) | |||||
| for doc in docs: | for doc in docs: | ||||
| doc.add_chunk("This is a test to add chunk") | doc.add_chunk("This is a test to add chunk") | ||||
| rag.create_chat("test_create_chat", dataset_ids=[kb.id]) | rag.create_chat("test_create_chat", dataset_ids=[kb.id]) | ||||
| chat = rag.create_chat("test_delete_chat", dataset_ids=[kb.id]) | chat = rag.create_chat("test_delete_chat", dataset_ids=[kb.id]) | ||||
| rag.delete_chats(ids=[chat.id]) | rag.delete_chats(ids=[chat.id]) | ||||
| def test_list_chats_with_success(get_api_key_fixture): | def test_list_chats_with_success(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| rag.create_chat("test_list_1", dataset_ids=[kb.id]) | rag.create_chat("test_list_1", dataset_ids=[kb.id]) | ||||
| rag.create_chat("test_list_2", dataset_ids=[kb.id]) | rag.create_chat("test_list_2", dataset_ids=[kb.id]) | ||||
| rag.list_chats() | rag.list_chats() | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from ragflow_sdk import RAGFlow | from ragflow_sdk import RAGFlow | ||||
| from common import HOST_ADDRESS | from common import HOST_ADDRESS | ||||
| from time import sleep | from time import sleep | ||||
| def test_parse_document_with_txt(get_api_key_fixture): | def test_parse_document_with_txt(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_parse_document") | ds = rag.create_dataset(name="test_parse_document") | ||||
| name = 'ragflow_test.txt' | name = 'ragflow_test.txt' | ||||
| with open("test_data/ragflow_test.txt", "rb") as file : | |||||
| with open("test_data/ragflow_test.txt", "rb") as file: | |||||
| blob = file.read() | blob = file.read() | ||||
| docs = ds.upload_documents([{"display_name": name, "blob": blob}]) | docs = ds.upload_documents([{"display_name": name, "blob": blob}]) | ||||
| doc = docs[0] | doc = docs[0] | ||||
| raise Exception("Run time ERROR: Document parsing did not complete in time.") | raise Exception("Run time ERROR: Document parsing did not complete in time.") | ||||
| ''' | ''' | ||||
| def test_parse_and_cancel_document(get_api_key_fixture): | def test_parse_and_cancel_document(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_parse_and_cancel_document") | ds = rag.create_dataset(name="test_parse_and_cancel_document") | ||||
| name = 'ragflow_test.txt' | name = 'ragflow_test.txt' | ||||
| with open("test_data/ragflow_test.txt", "rb") as file : | |||||
| with open("test_data/ragflow_test.txt", "rb") as file: | |||||
| blob = file.read() | blob = file.read() | ||||
| docs=ds.upload_documents([{"display_name": name, "blob": blob}]) | |||||
| docs = ds.upload_documents([{"display_name": name, "blob": blob}]) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents(document_ids=[doc.id]) | ds.async_parse_documents(document_ids=[doc.id]) | ||||
| sleep(1) | sleep(1) | ||||
| raise Exception("Run time ERROR: Bulk document parsing did not complete in time.") | raise Exception("Run time ERROR: Bulk document parsing did not complete in time.") | ||||
| ''' | ''' | ||||
| def test_list_chunks_with_success(get_api_key_fixture): | def test_list_chunks_with_success(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | ||||
| ] | ] | ||||
| ''' | ''' | ||||
| documents =[{"display_name":"test_list_chunks_with_success.txt","blob":blob}] | |||||
| documents = [{"display_name": "test_list_chunks_with_success.txt", "blob": blob}] | |||||
| docs = ds.upload_documents(documents) | docs = ds.upload_documents(documents) | ||||
| ids = [doc.id for doc in docs] | ids = [doc.id for doc in docs] | ||||
| ds.async_parse_documents(ids) | ds.async_parse_documents(ids) | ||||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | ||||
| ] | ] | ||||
| ''' | ''' | ||||
| documents =[{"display_name":"test_list_chunks_with_success.txt","blob":blob}] | |||||
| documents = [{"display_name": "test_list_chunks_with_success.txt", "blob": blob}] | |||||
| docs = ds.upload_documents(documents) | docs = ds.upload_documents(documents) | ||||
| doc = docs[0] | doc = docs[0] | ||||
| doc.add_chunk(content="This is a chunk addition test") | doc.add_chunk(content="This is a chunk addition test") | ||||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | ||||
| ] | ] | ||||
| ''' | ''' | ||||
| documents =[{"display_name":"test_delete_chunk_with_success.txt","blob":blob}] | |||||
| documents = [{"display_name": "test_delete_chunk_with_success.txt", "blob": blob}] | |||||
| docs = ds.upload_documents(documents) | docs = ds.upload_documents(documents) | ||||
| doc = docs[0] | doc = docs[0] | ||||
| chunk = doc.add_chunk(content="This is a chunk addition test") | chunk = doc.add_chunk(content="This is a chunk addition test") | ||||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | ||||
| ] | ] | ||||
| ''' | ''' | ||||
| documents =[{"display_name":"test_update_chunk_content_with_success.txt","blob":blob}] | |||||
| documents = [{"display_name": "test_update_chunk_content_with_success.txt", "blob": blob}] | |||||
| docs = ds.upload_documents(documents) | docs = ds.upload_documents(documents) | ||||
| doc = docs[0] | doc = docs[0] | ||||
| chunk = doc.add_chunk(content="This is a chunk addition test") | chunk = doc.add_chunk(content="This is a chunk addition test") | ||||
| # For Elasticsearch, the chunk is not searchable in shot time (~2s). | # For Elasticsearch, the chunk is not searchable in shot time (~2s). | ||||
| sleep(3) | sleep(3) | ||||
| chunk.update({"content":"This is a updated content"}) | |||||
| chunk.update({"content": "This is a updated content"}) | |||||
| def test_update_chunk_available(get_api_key_fixture): | def test_update_chunk_available(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | ||||
| ] | ] | ||||
| ''' | ''' | ||||
| documents =[{"display_name":"test_update_chunk_available_with_success.txt","blob":blob}] | |||||
| documents = [{"display_name": "test_update_chunk_available_with_success.txt", "blob": blob}] | |||||
| docs = ds.upload_documents(documents) | docs = ds.upload_documents(documents) | ||||
| doc = docs[0] | doc = docs[0] | ||||
| chunk = doc.add_chunk(content="This is a chunk addition test") | chunk = doc.add_chunk(content="This is a chunk addition test") | ||||
| # For Elasticsearch, the chunk is not searchable in shot time (~2s). | # For Elasticsearch, the chunk is not searchable in shot time (~2s). | ||||
| sleep(3) | sleep(3) | ||||
| chunk.update({"available":0}) | |||||
| chunk.update({"available": 0}) | |||||
| def test_retrieve_chunks(get_api_key_fixture): | def test_retrieve_chunks(get_api_key_fixture): | ||||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | ||||
| ] | ] | ||||
| ''' | ''' | ||||
| documents =[{"display_name":"test_retrieve_chunks.txt","blob":blob}] | |||||
| documents = [{"display_name": "test_retrieve_chunks.txt", "blob": blob}] | |||||
| docs = ds.upload_documents(documents) | docs = ds.upload_documents(documents) | ||||
| doc = docs[0] | doc = docs[0] | ||||
| doc.add_chunk(content="This is a chunk addition test") | doc.add_chunk(content="This is a chunk addition test") | ||||
| rag.retrieve(dataset_ids=[ds.id],document_ids=[doc.id]) | |||||
| rag.retrieve(dataset_ids=[ds.id], document_ids=[doc.id]) | |||||
| rag.delete_datasets(ids=[ds.id]) | rag.delete_datasets(ids=[ds.id]) | ||||
| # test different parameters for the retrieval | # test different parameters for the retrieval |
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from ragflow_sdk import RAGFlow | from ragflow_sdk import RAGFlow | ||||
| import random | import random | ||||
| import pytest | import pytest | ||||
| from common import HOST_ADDRESS | from common import HOST_ADDRESS | ||||
| def test_create_dataset_with_name(get_api_key_fixture): | def test_create_dataset_with_name(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| rag.create_dataset("test_create_dataset_with_name") | rag.create_dataset("test_create_dataset_with_name") | ||||
| def test_create_dataset_with_duplicated_name(get_api_key_fixture): | def test_create_dataset_with_duplicated_name(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| rag.create_dataset("test_create_dataset_with_duplicated_name") | rag.create_dataset("test_create_dataset_with_duplicated_name") | ||||
| assert str(exc_info.value) == "Duplicated dataset name in creating dataset." | assert str(exc_info.value) == "Duplicated dataset name in creating dataset." | ||||
| def test_create_dataset_with_random_chunk_method(get_api_key_fixture): | def test_create_dataset_with_random_chunk_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| valid_chunk_methods = ["naive","manual","qa","table","paper","book","laws","presentation","picture","one","knowledge_graph","email"] | |||||
| valid_chunk_methods = ["naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one", | |||||
| "knowledge_graph", "email"] | |||||
| random_chunk_method = random.choice(valid_chunk_methods) | random_chunk_method = random.choice(valid_chunk_methods) | ||||
| rag.create_dataset("test_create_dataset_with_random_chunk_method",chunk_method=random_chunk_method) | |||||
| rag.create_dataset("test_create_dataset_with_random_chunk_method", chunk_method=random_chunk_method) | |||||
| def test_create_dataset_with_invalid_parameter(get_api_key_fixture): | def test_create_dataset_with_invalid_parameter(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| "knowledge_graph", "email", "tag"] | "knowledge_graph", "email", "tag"] | ||||
| chunk_method = "invalid_chunk_method" | chunk_method = "invalid_chunk_method" | ||||
| with pytest.raises(Exception) as exc_info: | with pytest.raises(Exception) as exc_info: | ||||
| rag.create_dataset("test_create_dataset_with_invalid_chunk_method",chunk_method=chunk_method) | |||||
| rag.create_dataset("test_create_dataset_with_invalid_chunk_method", chunk_method=chunk_method) | |||||
| assert str(exc_info.value) == f"'{chunk_method}' is not in {valid_chunk_methods}" | assert str(exc_info.value) == f"'{chunk_method}' is not in {valid_chunk_methods}" | ||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from ragflow_sdk import RAGFlow | from ragflow_sdk import RAGFlow | ||||
| from common import HOST_ADDRESS | from common import HOST_ADDRESS | ||||
| import pytest | import pytest | ||||
| def test_upload_document_with_success(get_api_key_fixture): | def test_upload_document_with_success(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_upload_document") | ds = rag.create_dataset(name="test_upload_document") | ||||
| blob = b"Sample document content for test." | blob = b"Sample document content for test." | ||||
| with open("test_data/ragflow.txt", "rb") as file: | with open("test_data/ragflow.txt", "rb") as file: | ||||
| blob_2=file.read() | |||||
| blob_2 = file.read() | |||||
| document_infos = [] | document_infos = [] | ||||
| document_infos.append({"display_name": "test_1.txt","blob": blob}) | |||||
| document_infos.append({"display_name": "test_2.txt","blob": blob_2}) | |||||
| document_infos.append({"display_name": "test_1.txt", "blob": blob}) | |||||
| document_infos.append({"display_name": "test_2.txt", "blob": blob_2}) | |||||
| ds.upload_documents(document_infos) | ds.upload_documents(document_infos) | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_update_document") | ds = rag.create_dataset(name="test_update_document") | ||||
| blob = b"Sample document content for test." | blob = b"Sample document content for test." | ||||
| document_infos=[{"display_name":"test.txt","blob":blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| document_infos = [{"display_name": "test.txt", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| doc.update({"chunk_method": "manual", "name": "manual.txt"}) | doc.update({"chunk_method": "manual", "name": "manual.txt"}) | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_download_document") | ds = rag.create_dataset(name="test_download_document") | ||||
| blob = b"Sample document content for test." | blob = b"Sample document content for test." | ||||
| document_infos=[{"display_name": "test_1.txt","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| document_infos = [{"display_name": "test_1.txt", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| with open("test_download.txt","wb+") as file: | |||||
| with open("test_download.txt", "wb+") as file: | |||||
| file.write(doc.download()) | file.write(doc.download()) | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_list_documents") | ds = rag.create_dataset(name="test_list_documents") | ||||
| blob = b"Sample document content for test." | blob = b"Sample document content for test." | ||||
| document_infos = [{"display_name": "test.txt","blob":blob}] | |||||
| document_infos = [{"display_name": "test.txt", "blob": blob}] | |||||
| ds.upload_documents(document_infos) | ds.upload_documents(document_infos) | ||||
| ds.list_documents(keywords="test", page=1, page_size=12) | ds.list_documents(keywords="test", page=1, page_size=12) | ||||
| ds = rag.create_dataset(name="test_delete_documents") | ds = rag.create_dataset(name="test_delete_documents") | ||||
| name = "test_delete_documents.txt" | name = "test_delete_documents.txt" | ||||
| blob = b"Sample document content for test." | blob = b"Sample document content for test." | ||||
| document_infos=[{"display_name": name, "blob": blob}] | |||||
| document_infos = [{"display_name": name, "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | docs = ds.upload_documents(document_infos) | ||||
| ds.delete_documents([docs[0].id]) | ds.delete_documents([docs[0].id]) | ||||
| # upload and parse the document with different in different parse method. | # upload and parse the document with different in different parse method. | ||||
| def test_upload_and_parse_pdf_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_pdf_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_pdf_document") | ds = rag.create_dataset(name="test_pdf_document") | ||||
| with open("test_data/test.pdf", "rb") as file: | with open("test_data/test.pdf", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.pdf","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.pdf", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | ds.async_parse_documents([doc.id]) | ||||
| def test_upload_and_parse_docx_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_docx_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_docx_document") | ds = rag.create_dataset(name="test_docx_document") | ||||
| with open("test_data/test.docx", "rb") as file: | with open("test_data/test.docx", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.docx","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.docx", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | ds.async_parse_documents([doc.id]) | ||||
| def test_upload_and_parse_excel_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_excel_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_excel_document") | ds = rag.create_dataset(name="test_excel_document") | ||||
| with open("test_data/test.xlsx", "rb") as file: | with open("test_data/test.xlsx", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.xlsx","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.xlsx", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | ds.async_parse_documents([doc.id]) | ||||
| def test_upload_and_parse_ppt_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_ppt_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_ppt_document") | ds = rag.create_dataset(name="test_ppt_document") | ||||
| with open("test_data/test.ppt", "rb") as file: | with open("test_data/test.ppt", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.ppt","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.ppt", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | ds.async_parse_documents([doc.id]) | ||||
| def test_upload_and_parse_image_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_image_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_image_document") | ds = rag.create_dataset(name="test_image_document") | ||||
| with open("test_data/test.jpg", "rb") as file: | with open("test_data/test.jpg", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.jpg","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.jpg", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | ds.async_parse_documents([doc.id]) | ||||
| def test_upload_and_parse_txt_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_txt_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_txt_document") | ds = rag.create_dataset(name="test_txt_document") | ||||
| with open("test_data/test.txt", "rb") as file: | with open("test_data/test.txt", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.txt","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.txt", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | ds.async_parse_documents([doc.id]) | ||||
| def test_upload_and_parse_md_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_md_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_md_document") | ds = rag.create_dataset(name="test_md_document") | ||||
| with open("test_data/test.md", "rb") as file: | with open("test_data/test.md", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.md","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.md", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | ds.async_parse_documents([doc.id]) | ||||
| def test_upload_and_parse_json_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_json_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_json_document") | ds = rag.create_dataset(name="test_json_document") | ||||
| with open("test_data/test.json", "rb") as file: | with open("test_data/test.json", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.json","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.json", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | ds.async_parse_documents([doc.id]) | ||||
| @pytest.mark.skip(reason="") | @pytest.mark.skip(reason="") | ||||
| def test_upload_and_parse_eml_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_eml_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_eml_document") | ds = rag.create_dataset(name="test_eml_document") | ||||
| with open("test_data/test.eml", "rb") as file: | with open("test_data/test.eml", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.eml","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.eml", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | ds.async_parse_documents([doc.id]) | ||||
| def test_upload_and_parse_html_documents_with_general_parse_method(get_api_key_fixture): | def test_upload_and_parse_html_documents_with_general_parse_method(get_api_key_fixture): | ||||
| API_KEY = get_api_key_fixture | API_KEY = get_api_key_fixture | ||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | rag = RAGFlow(API_KEY, HOST_ADDRESS) | ||||
| ds = rag.create_dataset(name="test_html_document") | ds = rag.create_dataset(name="test_html_document") | ||||
| with open("test_data/test.html", "rb") as file: | with open("test_data/test.html", "rb") as file: | ||||
| blob=file.read() | |||||
| document_infos = [{"display_name": "test.html","blob": blob}] | |||||
| docs=ds.upload_documents(document_infos) | |||||
| blob = file.read() | |||||
| document_infos = [{"display_name": "test.html", "blob": blob}] | |||||
| docs = ds.upload_documents(document_infos) | |||||
| doc = docs[0] | doc = docs[0] | ||||
| ds.async_parse_documents([doc.id]) | |||||
| ds.async_parse_documents([doc.id]) |
| from ragflow_sdk import RAGFlow,Agent | |||||
| # | |||||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||||
| # | |||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from ragflow_sdk import RAGFlow, Agent | |||||
| from common import HOST_ADDRESS | from common import HOST_ADDRESS | ||||
| import pytest | import pytest | ||||
| display_name = "ragflow.txt" | display_name = "ragflow.txt" | ||||
| with open("test_data/ragflow.txt", "rb") as file: | with open("test_data/ragflow.txt", "rb") as file: | ||||
| blob = file.read() | blob = file.read() | ||||
| document = {"display_name":display_name,"blob":blob} | |||||
| document = {"display_name": display_name, "blob": blob} | |||||
| documents = [] | documents = [] | ||||
| documents.append(document) | documents.append(document) | ||||
| docs= kb.upload_documents(documents) | |||||
| docs = kb.upload_documents(documents) | |||||
| for doc in docs: | for doc in docs: | ||||
| doc.add_chunk("This is a test to add chunk") | doc.add_chunk("This is a test to add chunk") | ||||
| assistant=rag.create_chat("test_create_session", dataset_ids=[kb.id]) | |||||
| assistant = rag.create_chat("test_create_session", dataset_ids=[kb.id]) | |||||
| assistant.create_session() | assistant.create_session() | ||||
| question = "What is AI" | question = "What is AI" | ||||
| for ans in session.ask(question): | for ans in session.ask(question): | ||||
| pass | pass | ||||
| # assert not ans.content.startswith("**ERROR**"), "Please check this error." | # assert not ans.content.startswith("**ERROR**"), "Please check this error." | ||||
| display_name = "ragflow.txt" | display_name = "ragflow.txt" | ||||
| with open("test_data/ragflow.txt", "rb") as file: | with open("test_data/ragflow.txt", "rb") as file: | ||||
| blob = file.read() | blob = file.read() | ||||
| document = {"display_name":display_name,"blob":blob} | |||||
| document = {"display_name": display_name, "blob": blob} | |||||
| documents = [] | documents = [] | ||||
| documents.append(document) | documents.append(document) | ||||
| docs= kb.upload_documents(documents) | |||||
| docs = kb.upload_documents(documents) | |||||
| for doc in docs: | for doc in docs: | ||||
| doc.add_chunk("This is a test to add chunk") | doc.add_chunk("This is a test to add chunk") | ||||
| assistant=rag.create_chat("test_delete_session", dataset_ids=[kb.id]) | |||||
| assistant = rag.create_chat("test_delete_session", dataset_ids=[kb.id]) | |||||
| session = assistant.create_session() | session = assistant.create_session() | ||||
| assistant.delete_sessions(ids=[session.id]) | assistant.delete_sessions(ids=[session.id]) | ||||
| display_name = "ragflow.txt" | display_name = "ragflow.txt" | ||||
| with open("test_data/ragflow.txt", "rb") as file: | with open("test_data/ragflow.txt", "rb") as file: | ||||
| blob = file.read() | blob = file.read() | ||||
| document = {"display_name":display_name,"blob":blob} | |||||
| document = {"display_name": display_name, "blob": blob} | |||||
| documents = [] | documents = [] | ||||
| documents.append(document) | documents.append(document) | ||||
| docs= kb.upload_documents(documents) | |||||
| docs = kb.upload_documents(documents) | |||||
| for doc in docs: | for doc in docs: | ||||
| doc.add_chunk("This is a test to add chunk") | doc.add_chunk("This is a test to add chunk") | ||||
| assistant=rag.create_chat("test_list_session", dataset_ids=[kb.id]) | |||||
| assistant = rag.create_chat("test_list_session", dataset_ids=[kb.id]) | |||||
| assistant.create_session("test_1") | assistant.create_session("test_1") | ||||
| assistant.create_session("test_2") | assistant.create_session("test_2") | ||||
| assistant.list_sessions() | assistant.list_sessions() | ||||
| @pytest.mark.skip(reason="") | @pytest.mark.skip(reason="") | ||||
| def test_create_agent_session_with_success(get_api_key_fixture): | def test_create_agent_session_with_success(get_api_key_fixture): | ||||
| API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | ||||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||||
| Agent.create_session("2e45b5209c1011efa3e90242ac120006", rag) | Agent.create_session("2e45b5209c1011efa3e90242ac120006", rag) | ||||
| @pytest.mark.skip(reason="") | @pytest.mark.skip(reason="") | ||||
| def test_create_agent_conversation_with_success(get_api_key_fixture): | def test_create_agent_conversation_with_success(get_api_key_fixture): | ||||
| API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | ||||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||||
| session = Agent.create_session("2e45b5209c1011efa3e90242ac120006", rag) | session = Agent.create_session("2e45b5209c1011efa3e90242ac120006", rag) | ||||
| session.ask("What is this job") | session.ask("What is this job") | ||||
| @pytest.mark.skip(reason="") | @pytest.mark.skip(reason="") | ||||
| def test_list_agent_sessions_with_success(get_api_key_fixture): | def test_list_agent_sessions_with_success(get_api_key_fixture): | ||||
| API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | ||||
| agent_id = "2710f2269b4611ef8fdf0242ac120006" | agent_id = "2710f2269b4611ef8fdf0242ac120006" | ||||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||||
| Agent.list_sessions(agent_id,rag) | |||||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||||
| Agent.list_sessions(agent_id, rag) |