### What problem does this PR solve? Add license statement. ### Type of change - [x] Refactoring Signed-off-by: Jin Hai <haijin.chn@gmail.com>tags/v0.16.0
| @@ -1,2 +1,18 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from beartype.claw import beartype_this_package | |||
| beartype_this_package() | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import importlib | |||
| from .begin import Begin, BeginParam | |||
| from .generate import Generate, GenerateParam | |||
| @@ -1,5 +1,5 @@ | |||
| # | |||
| # Copyright 2019 The FATE Authors. All Rights Reserved. | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| @@ -1,2 +1,18 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from beartype.claw import beartype_this_package | |||
| beartype_this_package() | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import base64 | |||
| import os | |||
| import sys | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import re | |||
| import socket | |||
| from urllib.parse import urlparse | |||
| @@ -1,2 +1,18 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from beartype.claw import beartype_this_package | |||
| beartype_this_package() | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,4 +1,7 @@ | |||
| # -*- coding: utf-8 -*- | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -11,6 +14,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from rag.nlp import find_codec | |||
| import readability | |||
| import html_text | |||
| @@ -1,4 +1,20 @@ | |||
| # -*- coding: utf-8 -*- | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| # The following documents are mainly referenced, and only adaptation modifications have been made | |||
| # from https://github.com/langchain-ai/langchain/blob/master/libs/text-splitters/langchain_text_splitters/json.py | |||
| @@ -1,4 +1,7 @@ | |||
| # -*- coding: utf-8 -*- | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -11,6 +14,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import re | |||
| class RAGFlowMarkdownParser: | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| from io import BytesIO | |||
| from pptx import Presentation | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -0,0 +1,15 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import re | |||
| TBL = { | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import re | |||
| import copy | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import re | |||
| from deepdoc.parser.utils import get_text | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import os | |||
| import re | |||
| from collections import Counter | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import os | |||
| import sys | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from beartype.claw import beartype_this_package | |||
| beartype_this_package() | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import requests | |||
| from bridge.context import ContextType # Import Context, ContextType | |||
| @@ -1,2 +1,18 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from beartype.claw import beartype_this_package | |||
| beartype_this_package() | |||
| @@ -0,0 +1,15 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import re | |||
| from api.db import LLMType | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| from tika import parser | |||
| import re | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import re | |||
| from graphrag.index import build_knowledge_graph_chunks | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| from tika import parser | |||
| import re | |||
| @@ -1,5 +1,5 @@ | |||
| # | |||
| # Copyright 2024 The InfiniFlow Authors. All Rights Reserved. | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| from tika import parser | |||
| from io import BytesIO | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| from tika import parser | |||
| from io import BytesIO | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import copy | |||
| import re | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import io | |||
| import numpy as np | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import copy | |||
| import re | |||
| from io import BytesIO | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import re | |||
| import csv | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import base64 | |||
| import datetime | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import copy | |||
| import re | |||
| from io import BytesIO | |||
| @@ -1,3 +1,6 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| @@ -10,6 +13,7 @@ | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import re | |||
| import csv | |||
| from copy import deepcopy | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from jina import Deployment | |||
| from docarray import BaseDoc | |||
| from jina import Executor, requests | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import os | |||
| import time | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import os | |||
| import time | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from abc import ABC, abstractmethod | |||
| from dataclasses import dataclass | |||
| import numpy as np | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import re | |||
| import json | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import os | |||
| import re | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import time | |||
| from minio import Minio | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import json | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import logging | |||
| import boto3 | |||
| from botocore.exceptions import ClientError | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import os | |||
| from enum import Enum | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import ragflow_sdk | |||
| print(ragflow_sdk.__version__) | |||
| print(ragflow_sdk.__version__) | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from beartype.claw import beartype_this_package | |||
| beartype_this_package() | |||
| @@ -0,0 +1,15 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from .base import Base | |||
| from .session import Session | |||
| import requests | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| class Base(object): | |||
| def __init__(self, rag, res_dict): | |||
| self.rag = rag | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from .base import Base | |||
| from .session import Session | |||
| @@ -44,12 +60,11 @@ class Chat(Base): | |||
| def update(self, update_message: dict): | |||
| res = self.put(f'/chats/{self.id}', | |||
| update_message) | |||
| update_message) | |||
| res = res.json() | |||
| if res.get("code") != 0: | |||
| raise Exception(res["message"]) | |||
| def create_session(self, name: str = "New session") -> Session: | |||
| res = self.post(f"/chats/{self.id}/sessions", {"name": name}) | |||
| res = res.json() | |||
| @@ -57,9 +72,10 @@ class Chat(Base): | |||
| return Session(self.rag, res['data']) | |||
| raise Exception(res["message"]) | |||
| def list_sessions(self,page: int = 1, page_size: int = 30, orderby: str = "create_time", desc: bool = True, | |||
| def list_sessions(self, page: int = 1, page_size: int = 30, orderby: str = "create_time", desc: bool = True, | |||
| id: str = None, name: str = None) -> list[Session]: | |||
| res = self.get(f'/chats/{self.id}/sessions',{"page": page, "page_size": page_size, "orderby": orderby, "desc": desc, "id": id, "name": name} ) | |||
| res = self.get(f'/chats/{self.id}/sessions', | |||
| {"page": page, "page_size": page_size, "orderby": orderby, "desc": desc, "id": id, "name": name}) | |||
| res = res.json() | |||
| if res.get("code") == 0: | |||
| result_list = [] | |||
| @@ -68,7 +84,7 @@ class Chat(Base): | |||
| return result_list | |||
| raise Exception(res["message"]) | |||
| def delete_sessions(self,ids: list[str] | None = None): | |||
| def delete_sessions(self, ids: list[str] | None = None): | |||
| res = self.rm(f"/chats/{self.id}/sessions", {"ids": ids}) | |||
| res = res.json() | |||
| if res.get("code") != 0: | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from .base import Base | |||
| @@ -18,11 +34,8 @@ class Chunk(Base): | |||
| res_dict.pop(k) | |||
| super().__init__(rag, res_dict) | |||
| def update(self,update_message:dict): | |||
| res = self.put(f"/datasets/{self.dataset_id}/documents/{self.document_id}/chunks/{self.id}",update_message) | |||
| def update(self, update_message: dict): | |||
| res = self.put(f"/datasets/{self.dataset_id}/documents/{self.document_id}/chunks/{self.id}", update_message) | |||
| res = res.json() | |||
| if res.get("code") != 0 : | |||
| if res.get("code") != 0: | |||
| raise Exception(res["message"]) | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from .document import Document | |||
| from .base import Base | |||
| @@ -29,48 +45,51 @@ class DataSet(Base): | |||
| def update(self, update_message: dict): | |||
| res = self.put(f'/datasets/{self.id}', | |||
| update_message) | |||
| update_message) | |||
| res = res.json() | |||
| if res.get("code") != 0: | |||
| raise Exception(res["message"]) | |||
| def upload_documents(self,document_list: list[dict]): | |||
| def upload_documents(self, document_list: list[dict]): | |||
| url = f"/datasets/{self.id}/documents" | |||
| files = [("file",(ele["display_name"],ele["blob"])) for ele in document_list] | |||
| res = self.post(path=url,json=None,files=files) | |||
| files = [("file", (ele["display_name"], ele["blob"])) for ele in document_list] | |||
| res = self.post(path=url, json=None, files=files) | |||
| res = res.json() | |||
| if res.get("code") == 0: | |||
| doc_list=[] | |||
| doc_list = [] | |||
| for doc in res["data"]: | |||
| document = Document(self.rag,doc) | |||
| document = Document(self.rag, doc) | |||
| doc_list.append(document) | |||
| return doc_list | |||
| raise Exception(res.get("message")) | |||
| def list_documents(self, id: str | None = None, keywords: str | None = None, page: int = 1, page_size: int = 30, orderby: str = "create_time", desc: bool = True): | |||
| res = self.get(f"/datasets/{self.id}/documents",params={"id": id,"keywords": keywords,"page": page,"page_size": page_size,"orderby": orderby,"desc": desc}) | |||
| def list_documents(self, id: str | None = None, keywords: str | None = None, page: int = 1, page_size: int = 30, | |||
| orderby: str = "create_time", desc: bool = True): | |||
| res = self.get(f"/datasets/{self.id}/documents", | |||
| params={"id": id, "keywords": keywords, "page": page, "page_size": page_size, "orderby": orderby, | |||
| "desc": desc}) | |||
| res = res.json() | |||
| documents = [] | |||
| if res.get("code") == 0: | |||
| for document in res["data"].get("docs"): | |||
| documents.append(Document(self.rag,document)) | |||
| documents.append(Document(self.rag, document)) | |||
| return documents | |||
| raise Exception(res["message"]) | |||
| def delete_documents(self,ids: list[str] | None = None): | |||
| res = self.rm(f"/datasets/{self.id}/documents",{"ids":ids}) | |||
| def delete_documents(self, ids: list[str] | None = None): | |||
| res = self.rm(f"/datasets/{self.id}/documents", {"ids": ids}) | |||
| res = res.json() | |||
| if res.get("code") != 0: | |||
| raise Exception(res["message"]) | |||
| def async_parse_documents(self,document_ids): | |||
| res = self.post(f"/datasets/{self.id}/chunks",{"document_ids":document_ids}) | |||
| def async_parse_documents(self, document_ids): | |||
| res = self.post(f"/datasets/{self.id}/chunks", {"document_ids": document_ids}) | |||
| res = res.json() | |||
| if res.get("code") != 0: | |||
| raise Exception(res.get("message")) | |||
| def async_cancel_parse_documents(self,document_ids): | |||
| res = self.rm(f"/datasets/{self.id}/chunks",{"document_ids":document_ids}) | |||
| def async_cancel_parse_documents(self, document_ids): | |||
| res = self.rm(f"/datasets/{self.id}/chunks", {"document_ids": document_ids}) | |||
| res = res.json() | |||
| if res.get("code") != 0: | |||
| raise Exception(res.get("message")) | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import json | |||
| from .base import Base | |||
| from .chunk import Chunk | |||
| @@ -32,7 +48,6 @@ class Document(Base): | |||
| res_dict.pop(k) | |||
| super().__init__(rag, res_dict) | |||
| def update(self, update_message: dict): | |||
| res = self.put(f'/datasets/{self.dataset_id}/documents/{self.id}', | |||
| update_message) | |||
| @@ -48,29 +63,28 @@ class Document(Base): | |||
| except json.JSONDecodeError: | |||
| return res.content | |||
| def list_chunks(self,page=1, page_size=30, keywords=""): | |||
| data={"keywords": keywords,"page":page,"page_size":page_size} | |||
| def list_chunks(self, page=1, page_size=30, keywords=""): | |||
| data = {"keywords": keywords, "page": page, "page_size": page_size} | |||
| res = self.get(f'/datasets/{self.dataset_id}/documents/{self.id}/chunks', data) | |||
| res = res.json() | |||
| if res.get("code") == 0: | |||
| chunks=[] | |||
| chunks = [] | |||
| for data in res["data"].get("chunks"): | |||
| chunk = Chunk(self.rag,data) | |||
| chunk = Chunk(self.rag, data) | |||
| chunks.append(chunk) | |||
| return chunks | |||
| raise Exception(res.get("message")) | |||
| def add_chunk(self, content: str, important_keywords: list[str] = [], questions: list[str] = []): | |||
| res = self.post(f'/datasets/{self.dataset_id}/documents/{self.id}/chunks', | |||
| {"content":content,"important_keywords":important_keywords, "questions": questions}) | |||
| {"content": content, "important_keywords": important_keywords, "questions": questions}) | |||
| res = res.json() | |||
| if res.get("code") == 0: | |||
| return Chunk(self.rag,res["data"].get("chunk")) | |||
| return Chunk(self.rag, res["data"].get("chunk")) | |||
| raise Exception(res.get("message")) | |||
| def delete_chunks(self,ids:list[str] | None = None): | |||
| res = self.rm(f"/datasets/{self.dataset_id}/documents/{self.id}/chunks",{"chunk_ids":ids}) | |||
| def delete_chunks(self, ids: list[str] | None = None): | |||
| res = self.rm(f"/datasets/{self.dataset_id}/documents/{self.id}/chunks", {"chunk_ids": ids}) | |||
| res = res.json() | |||
| if res.get("code")!=0: | |||
| raise Exception(res.get("message")) | |||
| if res.get("code") != 0: | |||
| raise Exception(res.get("message")) | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import json | |||
| from .base import Base | |||
| @@ -8,8 +24,8 @@ class Session(Base): | |||
| self.id = None | |||
| self.name = "New session" | |||
| self.messages = [{"role": "assistant", "content": "Hi! I am your assistant,can I help you?"}] | |||
| for key,value in res_dict.items(): | |||
| if key =="chat_id" and value is not None: | |||
| for key, value in res_dict.items(): | |||
| if key == "chat_id" and value is not None: | |||
| self.chat_id = None | |||
| self.__session_type = "chat" | |||
| if key == "agent_id" and value is not None: | |||
| @@ -17,11 +33,11 @@ class Session(Base): | |||
| self.__session_type = "agent" | |||
| super().__init__(rag, res_dict) | |||
| def ask(self, question="",stream=True,**kwargs): | |||
| def ask(self, question="", stream=True, **kwargs): | |||
| if self.__session_type == "agent": | |||
| res=self._ask_agent(question,stream) | |||
| res = self._ask_agent(question, stream) | |||
| elif self.__session_type == "chat": | |||
| res=self._ask_chat(question,stream,**kwargs) | |||
| res = self._ask_chat(question, stream, **kwargs) | |||
| for line in res.iter_lines(): | |||
| line = line.decode("utf-8") | |||
| if line.startswith("{"): | |||
| @@ -44,25 +60,26 @@ class Session(Base): | |||
| message = Message(self.rag, temp_dict) | |||
| yield message | |||
| def _ask_chat(self, question: str, stream: bool,**kwargs): | |||
| json_data={"question": question, "stream": True,"session_id":self.id} | |||
| def _ask_chat(self, question: str, stream: bool, **kwargs): | |||
| json_data = {"question": question, "stream": True, "session_id": self.id} | |||
| json_data.update(kwargs) | |||
| res = self.post(f"/chats/{self.chat_id}/completions", | |||
| json_data, stream=stream) | |||
| return res | |||
| def _ask_agent(self,question:str,stream:bool): | |||
| def _ask_agent(self, question: str, stream: bool): | |||
| res = self.post(f"/agents/{self.agent_id}/completions", | |||
| {"question": question, "stream": True,"session_id":self.id}, stream=stream) | |||
| {"question": question, "stream": True, "session_id": self.id}, stream=stream) | |||
| return res | |||
| def update(self,update_message): | |||
| def update(self, update_message): | |||
| res = self.put(f"/chats/{self.chat_id}/sessions/{self.id}", | |||
| update_message) | |||
| update_message) | |||
| res = res.json() | |||
| if res.get("code") != 0: | |||
| raise Exception(res.get("message")) | |||
| class Message(Base): | |||
| def __init__(self, rag, res_dict): | |||
| self.content = "Hi! I am your assistant,can I help you?" | |||
| @@ -1,41 +1,61 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import os | |||
| import pytest | |||
| import requests | |||
| HOST_ADDRESS = os.getenv('HOST_ADDRESS', 'http://127.0.0.1:9380') | |||
| # def generate_random_email(): | |||
| # return 'user_' + ''.join(random.choices(string.ascii_lowercase + string.digits, k=8))+'@1.com' | |||
| def generate_email(): | |||
| return 'user_123@1.com' | |||
| EMAIL = generate_email() | |||
| # password is "123" | |||
| PASSWORD='''ctAseGvejiaSWWZ88T/m4FQVOpQyUvP+x7sXtdv3feqZACiQleuewkUi35E16wSd5C5QcnkkcV9cYc8TKPTRZlxappDuirxghxoOvFcJxFU4ixLsD | |||
| PASSWORD = '''ctAseGvejiaSWWZ88T/m4FQVOpQyUvP+x7sXtdv3feqZACiQleuewkUi35E16wSd5C5QcnkkcV9cYc8TKPTRZlxappDuirxghxoOvFcJxFU4ixLsD | |||
| fN33jCHRoDUW81IH9zjij/vaw8IbVyb6vuwg6MX6inOEBRRzVbRYxXOu1wkWY6SsI8X70oF9aeLFp/PzQpjoe/YbSqpTq8qqrmHzn9vO+yvyYyvmDsphXe | |||
| X8f7fp9c7vUsfOCkM+gHY3PadG+QHa7KI7mzTKgUTZImK6BZtfRBATDTthEUbbaTewY4H0MnWiCeeDhcbeQao6cFy1To8pE3RpmxnGnS8BsBn8w==''' | |||
| def register(): | |||
| url = HOST_ADDRESS + "/v1/user/register" | |||
| name = "user" | |||
| register_data = {"email":EMAIL,"nickname":name,"password":PASSWORD} | |||
| res = requests.post(url=url,json=register_data) | |||
| register_data = {"email": EMAIL, "nickname": name, "password": PASSWORD} | |||
| res = requests.post(url=url, json=register_data) | |||
| res = res.json() | |||
| if res.get("code") != 0: | |||
| raise Exception(res.get("message")) | |||
| def login(): | |||
| url = HOST_ADDRESS + "/v1/user/login" | |||
| login_data = {"email":EMAIL,"password":PASSWORD} | |||
| response=requests.post(url=url,json=login_data) | |||
| login_data = {"email": EMAIL, "password": PASSWORD} | |||
| response = requests.post(url=url, json=login_data) | |||
| res = response.json() | |||
| if res.get("code")!=0: | |||
| if res.get("code") != 0: | |||
| raise Exception(res.get("message")) | |||
| auth = response.headers["Authorization"] | |||
| return auth | |||
| @pytest.fixture(scope="session") | |||
| def get_api_key_fixture(): | |||
| try: | |||
| @@ -45,12 +65,13 @@ def get_api_key_fixture(): | |||
| auth = login() | |||
| url = HOST_ADDRESS + "/v1/system/new_token" | |||
| auth = {"Authorization": auth} | |||
| response = requests.post(url=url,headers=auth) | |||
| response = requests.post(url=url, headers=auth) | |||
| res = response.json() | |||
| if res.get("code") != 0: | |||
| raise Exception(res.get("message")) | |||
| return res["data"].get("token") | |||
| @pytest.fixture(scope="session") | |||
| def get_auth(): | |||
| try: | |||
| @@ -60,6 +81,7 @@ def get_auth(): | |||
| auth = login() | |||
| return auth | |||
| @pytest.fixture(scope="session") | |||
| def get_email(): | |||
| return EMAIL | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import os | |||
| import requests | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| def test_get_email(get_email): | |||
| print("\nEmail account:",flush=True) | |||
| print(f"{get_email}\n",flush=True) | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from common import create_dataset, list_dataset, rm_dataset, update_dataset, DATASET_NAME_LIMIT | |||
| import re | |||
| import random | |||
| @@ -121,7 +137,8 @@ def test_update_different_params_dataset_success(get_auth): | |||
| print(f"found {len(dataset_list)} datasets") | |||
| dataset_id = dataset_list[0] | |||
| json_req = {"kb_id": dataset_id, "name": "test_update_dataset", "description": "test", "permission": "me", "parser_id": "presentation", | |||
| json_req = {"kb_id": dataset_id, "name": "test_update_dataset", "description": "test", "permission": "me", | |||
| "parser_id": "presentation", | |||
| "language": "spanish"} | |||
| res = update_dataset(get_auth, json_req) | |||
| assert res.get("code") == 0, f"{res.get('message')}" | |||
| @@ -164,4 +181,3 @@ def test_update_different_params_dataset_fail(get_auth): | |||
| res = rm_dataset(get_auth, dataset_id) | |||
| assert res.get("code") == 0, f"{res.get('message')}" | |||
| print(f"{len(dataset_list)} datasets are deleted") | |||
| @@ -1,2 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| import os | |||
| HOST_ADDRESS=os.getenv('HOST_ADDRESS', 'http://127.0.0.1:9380') | |||
| HOST_ADDRESS = os.getenv('HOST_ADDRESS', 'http://127.0.0.1:9380') | |||
| @@ -1,3 +1,19 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| def test_get_email(get_email): | |||
| print("\nEmail account:",flush=True) | |||
| print(f"{get_email}\n",flush=True) | |||
| print("\nEmail account:", flush=True) | |||
| print(f"{get_email}\n", flush=True) | |||
| @@ -1,11 +1,28 @@ | |||
| from ragflow_sdk import RAGFlow,Agent | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from ragflow_sdk import RAGFlow, Agent | |||
| from common import HOST_ADDRESS | |||
| import pytest | |||
| @pytest.mark.skip(reason="") | |||
| def test_list_agents_with_success(get_api_key_fixture): | |||
| API_KEY=get_api_key_fixture | |||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| rag.list_agents() | |||
| @@ -13,7 +30,7 @@ def test_list_agents_with_success(get_api_key_fixture): | |||
| def test_converse_with_agent_with_success(get_api_key_fixture): | |||
| API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | |||
| agent_id = "ebfada2eb2bc11ef968a0242ac120006" | |||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| lang = "Chinese" | |||
| file = "How is the weather tomorrow?" | |||
| Agent.ask(agent_id=agent_id,rag=rag,lang=lang,file=file) | |||
| Agent.ask(agent_id=agent_id, rag=rag, lang=lang, file=file) | |||
| @@ -1,6 +1,23 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from ragflow_sdk import RAGFlow | |||
| from common import HOST_ADDRESS | |||
| def test_create_chat_with_name(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| @@ -8,10 +25,10 @@ def test_create_chat_with_name(get_api_key_fixture): | |||
| display_name = "ragflow.txt" | |||
| with open("test_data/ragflow.txt", "rb") as file: | |||
| blob = file.read() | |||
| document = {"display_name":display_name,"blob":blob} | |||
| document = {"display_name": display_name, "blob": blob} | |||
| documents = [] | |||
| documents.append(document) | |||
| docs= kb.upload_documents(documents) | |||
| docs = kb.upload_documents(documents) | |||
| for doc in docs: | |||
| doc.add_chunk("This is a test to add chunk") | |||
| rag.create_chat("test_create_chat", dataset_ids=[kb.id]) | |||
| @@ -50,6 +67,7 @@ def test_delete_chats_with_success(get_api_key_fixture): | |||
| chat = rag.create_chat("test_delete_chat", dataset_ids=[kb.id]) | |||
| rag.delete_chats(ids=[chat.id]) | |||
| def test_list_chats_with_success(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| @@ -66,5 +84,3 @@ def test_list_chats_with_success(get_api_key_fixture): | |||
| rag.create_chat("test_list_1", dataset_ids=[kb.id]) | |||
| rag.create_chat("test_list_2", dataset_ids=[kb.id]) | |||
| rag.list_chats() | |||
| @@ -1,13 +1,30 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from ragflow_sdk import RAGFlow | |||
| from common import HOST_ADDRESS | |||
| from time import sleep | |||
| def test_parse_document_with_txt(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_parse_document") | |||
| name = 'ragflow_test.txt' | |||
| with open("test_data/ragflow_test.txt", "rb") as file : | |||
| with open("test_data/ragflow_test.txt", "rb") as file: | |||
| blob = file.read() | |||
| docs = ds.upload_documents([{"display_name": name, "blob": blob}]) | |||
| doc = docs[0] | |||
| @@ -21,14 +38,15 @@ def test_parse_document_with_txt(get_api_key_fixture): | |||
| raise Exception("Run time ERROR: Document parsing did not complete in time.") | |||
| ''' | |||
| def test_parse_and_cancel_document(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_parse_and_cancel_document") | |||
| name = 'ragflow_test.txt' | |||
| with open("test_data/ragflow_test.txt", "rb") as file : | |||
| with open("test_data/ragflow_test.txt", "rb") as file: | |||
| blob = file.read() | |||
| docs=ds.upload_documents([{"display_name": name, "blob": blob}]) | |||
| docs = ds.upload_documents([{"display_name": name, "blob": blob}]) | |||
| doc = docs[0] | |||
| ds.async_parse_documents(document_ids=[doc.id]) | |||
| sleep(1) | |||
| @@ -60,6 +78,7 @@ def test_bulk_parse_documents(get_api_key_fixture): | |||
| raise Exception("Run time ERROR: Bulk document parsing did not complete in time.") | |||
| ''' | |||
| def test_list_chunks_with_success(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| @@ -73,7 +92,7 @@ def test_list_chunks_with_success(get_api_key_fixture): | |||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | |||
| ] | |||
| ''' | |||
| documents =[{"display_name":"test_list_chunks_with_success.txt","blob":blob}] | |||
| documents = [{"display_name": "test_list_chunks_with_success.txt", "blob": blob}] | |||
| docs = ds.upload_documents(documents) | |||
| ids = [doc.id for doc in docs] | |||
| ds.async_parse_documents(ids) | |||
| @@ -103,7 +122,7 @@ def test_add_chunk_with_success(get_api_key_fixture): | |||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | |||
| ] | |||
| ''' | |||
| documents =[{"display_name":"test_list_chunks_with_success.txt","blob":blob}] | |||
| documents = [{"display_name": "test_list_chunks_with_success.txt", "blob": blob}] | |||
| docs = ds.upload_documents(documents) | |||
| doc = docs[0] | |||
| doc.add_chunk(content="This is a chunk addition test") | |||
| @@ -122,7 +141,7 @@ def test_delete_chunk_with_success(get_api_key_fixture): | |||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | |||
| ] | |||
| ''' | |||
| documents =[{"display_name":"test_delete_chunk_with_success.txt","blob":blob}] | |||
| documents = [{"display_name": "test_delete_chunk_with_success.txt", "blob": blob}] | |||
| docs = ds.upload_documents(documents) | |||
| doc = docs[0] | |||
| chunk = doc.add_chunk(content="This is a chunk addition test") | |||
| @@ -143,13 +162,14 @@ def test_update_chunk_content(get_api_key_fixture): | |||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | |||
| ] | |||
| ''' | |||
| documents =[{"display_name":"test_update_chunk_content_with_success.txt","blob":blob}] | |||
| documents = [{"display_name": "test_update_chunk_content_with_success.txt", "blob": blob}] | |||
| docs = ds.upload_documents(documents) | |||
| doc = docs[0] | |||
| chunk = doc.add_chunk(content="This is a chunk addition test") | |||
| # For Elasticsearch, the chunk is not searchable in shot time (~2s). | |||
| sleep(3) | |||
| chunk.update({"content":"This is a updated content"}) | |||
| chunk.update({"content": "This is a updated content"}) | |||
| def test_update_chunk_available(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| @@ -164,13 +184,13 @@ def test_update_chunk_available(get_api_key_fixture): | |||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | |||
| ] | |||
| ''' | |||
| documents =[{"display_name":"test_update_chunk_available_with_success.txt","blob":blob}] | |||
| documents = [{"display_name": "test_update_chunk_available_with_success.txt", "blob": blob}] | |||
| docs = ds.upload_documents(documents) | |||
| doc = docs[0] | |||
| chunk = doc.add_chunk(content="This is a chunk addition test") | |||
| # For Elasticsearch, the chunk is not searchable in shot time (~2s). | |||
| sleep(3) | |||
| chunk.update({"available":0}) | |||
| chunk.update({"available": 0}) | |||
| def test_retrieve_chunks(get_api_key_fixture): | |||
| @@ -186,11 +206,11 @@ def test_retrieve_chunks(get_api_key_fixture): | |||
| {'display_name': f'chunk_{i}.txt', 'blob': chunk} for i, chunk in enumerate(chunks) | |||
| ] | |||
| ''' | |||
| documents =[{"display_name":"test_retrieve_chunks.txt","blob":blob}] | |||
| documents = [{"display_name": "test_retrieve_chunks.txt", "blob": blob}] | |||
| docs = ds.upload_documents(documents) | |||
| doc = docs[0] | |||
| doc.add_chunk(content="This is a chunk addition test") | |||
| rag.retrieve(dataset_ids=[ds.id],document_ids=[doc.id]) | |||
| rag.retrieve(dataset_ids=[ds.id], document_ids=[doc.id]) | |||
| rag.delete_datasets(ids=[ds.id]) | |||
| # test different parameters for the retrieval | |||
| @@ -1,13 +1,31 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from ragflow_sdk import RAGFlow | |||
| import random | |||
| import pytest | |||
| from common import HOST_ADDRESS | |||
| def test_create_dataset_with_name(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| rag.create_dataset("test_create_dataset_with_name") | |||
| def test_create_dataset_with_duplicated_name(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| @@ -16,12 +34,15 @@ def test_create_dataset_with_duplicated_name(get_api_key_fixture): | |||
| rag.create_dataset("test_create_dataset_with_duplicated_name") | |||
| assert str(exc_info.value) == "Duplicated dataset name in creating dataset." | |||
| def test_create_dataset_with_random_chunk_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| valid_chunk_methods = ["naive","manual","qa","table","paper","book","laws","presentation","picture","one","knowledge_graph","email"] | |||
| valid_chunk_methods = ["naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one", | |||
| "knowledge_graph", "email"] | |||
| random_chunk_method = random.choice(valid_chunk_methods) | |||
| rag.create_dataset("test_create_dataset_with_random_chunk_method",chunk_method=random_chunk_method) | |||
| rag.create_dataset("test_create_dataset_with_random_chunk_method", chunk_method=random_chunk_method) | |||
| def test_create_dataset_with_invalid_parameter(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| @@ -30,7 +51,7 @@ def test_create_dataset_with_invalid_parameter(get_api_key_fixture): | |||
| "knowledge_graph", "email", "tag"] | |||
| chunk_method = "invalid_chunk_method" | |||
| with pytest.raises(Exception) as exc_info: | |||
| rag.create_dataset("test_create_dataset_with_invalid_chunk_method",chunk_method=chunk_method) | |||
| rag.create_dataset("test_create_dataset_with_invalid_chunk_method", chunk_method=chunk_method) | |||
| assert str(exc_info.value) == f"'{chunk_method}' is not in {valid_chunk_methods}" | |||
| @@ -1,17 +1,34 @@ | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from ragflow_sdk import RAGFlow | |||
| from common import HOST_ADDRESS | |||
| import pytest | |||
| def test_upload_document_with_success(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_upload_document") | |||
| blob = b"Sample document content for test." | |||
| with open("test_data/ragflow.txt", "rb") as file: | |||
| blob_2=file.read() | |||
| blob_2 = file.read() | |||
| document_infos = [] | |||
| document_infos.append({"display_name": "test_1.txt","blob": blob}) | |||
| document_infos.append({"display_name": "test_2.txt","blob": blob_2}) | |||
| document_infos.append({"display_name": "test_1.txt", "blob": blob}) | |||
| document_infos.append({"display_name": "test_2.txt", "blob": blob_2}) | |||
| ds.upload_documents(document_infos) | |||
| @@ -20,8 +37,8 @@ def test_update_document_with_success(get_api_key_fixture): | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_update_document") | |||
| blob = b"Sample document content for test." | |||
| document_infos=[{"display_name":"test.txt","blob":blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| document_infos = [{"display_name": "test.txt", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| doc.update({"chunk_method": "manual", "name": "manual.txt"}) | |||
| @@ -31,10 +48,10 @@ def test_download_document_with_success(get_api_key_fixture): | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_download_document") | |||
| blob = b"Sample document content for test." | |||
| document_infos=[{"display_name": "test_1.txt","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| document_infos = [{"display_name": "test_1.txt", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| with open("test_download.txt","wb+") as file: | |||
| with open("test_download.txt", "wb+") as file: | |||
| file.write(doc.download()) | |||
| @@ -43,7 +60,7 @@ def test_list_documents_in_dataset_with_success(get_api_key_fixture): | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_list_documents") | |||
| blob = b"Sample document content for test." | |||
| document_infos = [{"display_name": "test.txt","blob":blob}] | |||
| document_infos = [{"display_name": "test.txt", "blob": blob}] | |||
| ds.upload_documents(document_infos) | |||
| ds.list_documents(keywords="test", page=1, page_size=12) | |||
| @@ -54,113 +71,128 @@ def test_delete_documents_in_dataset_with_success(get_api_key_fixture): | |||
| ds = rag.create_dataset(name="test_delete_documents") | |||
| name = "test_delete_documents.txt" | |||
| blob = b"Sample document content for test." | |||
| document_infos=[{"display_name": name, "blob": blob}] | |||
| document_infos = [{"display_name": name, "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| ds.delete_documents([docs[0].id]) | |||
| # upload and parse the document with different in different parse method. | |||
| def test_upload_and_parse_pdf_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_pdf_document") | |||
| with open("test_data/test.pdf", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.pdf","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.pdf", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| def test_upload_and_parse_docx_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_docx_document") | |||
| with open("test_data/test.docx", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.docx","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.docx", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| def test_upload_and_parse_excel_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_excel_document") | |||
| with open("test_data/test.xlsx", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.xlsx","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.xlsx", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| def test_upload_and_parse_ppt_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_ppt_document") | |||
| with open("test_data/test.ppt", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.ppt","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.ppt", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| def test_upload_and_parse_image_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_image_document") | |||
| with open("test_data/test.jpg", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.jpg","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.jpg", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| def test_upload_and_parse_txt_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_txt_document") | |||
| with open("test_data/test.txt", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.txt","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.txt", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| def test_upload_and_parse_md_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_md_document") | |||
| with open("test_data/test.md", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.md","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.md", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| def test_upload_and_parse_json_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_json_document") | |||
| with open("test_data/test.json", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.json","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.json", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| @pytest.mark.skip(reason="") | |||
| def test_upload_and_parse_eml_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_eml_document") | |||
| with open("test_data/test.eml", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.eml","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.eml", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| def test_upload_and_parse_html_documents_with_general_parse_method(get_api_key_fixture): | |||
| API_KEY = get_api_key_fixture | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| ds = rag.create_dataset(name="test_html_document") | |||
| with open("test_data/test.html", "rb") as file: | |||
| blob=file.read() | |||
| document_infos = [{"display_name": "test.html","blob": blob}] | |||
| docs=ds.upload_documents(document_infos) | |||
| blob = file.read() | |||
| document_infos = [{"display_name": "test.html", "blob": blob}] | |||
| docs = ds.upload_documents(document_infos) | |||
| doc = docs[0] | |||
| ds.async_parse_documents([doc.id]) | |||
| ds.async_parse_documents([doc.id]) | |||
| @@ -1,4 +1,20 @@ | |||
| from ragflow_sdk import RAGFlow,Agent | |||
| # | |||
| # Copyright 2025 The InfiniFlow Authors. All Rights Reserved. | |||
| # | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||
| # you may not use this file except in compliance with the License. | |||
| # You may obtain a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| # See the License for the specific language governing permissions and | |||
| # limitations under the License. | |||
| # | |||
| from ragflow_sdk import RAGFlow, Agent | |||
| from common import HOST_ADDRESS | |||
| import pytest | |||
| @@ -10,13 +26,13 @@ def test_create_session_with_success(get_api_key_fixture): | |||
| display_name = "ragflow.txt" | |||
| with open("test_data/ragflow.txt", "rb") as file: | |||
| blob = file.read() | |||
| document = {"display_name":display_name,"blob":blob} | |||
| document = {"display_name": display_name, "blob": blob} | |||
| documents = [] | |||
| documents.append(document) | |||
| docs= kb.upload_documents(documents) | |||
| docs = kb.upload_documents(documents) | |||
| for doc in docs: | |||
| doc.add_chunk("This is a test to add chunk") | |||
| assistant=rag.create_chat("test_create_session", dataset_ids=[kb.id]) | |||
| assistant = rag.create_chat("test_create_session", dataset_ids=[kb.id]) | |||
| assistant.create_session() | |||
| @@ -38,7 +54,7 @@ def test_create_conversation_with_success(get_api_key_fixture): | |||
| question = "What is AI" | |||
| for ans in session.ask(question): | |||
| pass | |||
| # assert not ans.content.startswith("**ERROR**"), "Please check this error." | |||
| @@ -49,13 +65,13 @@ def test_delete_sessions_with_success(get_api_key_fixture): | |||
| display_name = "ragflow.txt" | |||
| with open("test_data/ragflow.txt", "rb") as file: | |||
| blob = file.read() | |||
| document = {"display_name":display_name,"blob":blob} | |||
| document = {"display_name": display_name, "blob": blob} | |||
| documents = [] | |||
| documents.append(document) | |||
| docs= kb.upload_documents(documents) | |||
| docs = kb.upload_documents(documents) | |||
| for doc in docs: | |||
| doc.add_chunk("This is a test to add chunk") | |||
| assistant=rag.create_chat("test_delete_session", dataset_ids=[kb.id]) | |||
| assistant = rag.create_chat("test_delete_session", dataset_ids=[kb.id]) | |||
| session = assistant.create_session() | |||
| assistant.delete_sessions(ids=[session.id]) | |||
| @@ -85,33 +101,36 @@ def test_list_sessions_with_success(get_api_key_fixture): | |||
| display_name = "ragflow.txt" | |||
| with open("test_data/ragflow.txt", "rb") as file: | |||
| blob = file.read() | |||
| document = {"display_name":display_name,"blob":blob} | |||
| document = {"display_name": display_name, "blob": blob} | |||
| documents = [] | |||
| documents.append(document) | |||
| docs= kb.upload_documents(documents) | |||
| docs = kb.upload_documents(documents) | |||
| for doc in docs: | |||
| doc.add_chunk("This is a test to add chunk") | |||
| assistant=rag.create_chat("test_list_session", dataset_ids=[kb.id]) | |||
| assistant = rag.create_chat("test_list_session", dataset_ids=[kb.id]) | |||
| assistant.create_session("test_1") | |||
| assistant.create_session("test_2") | |||
| assistant.list_sessions() | |||
| @pytest.mark.skip(reason="") | |||
| def test_create_agent_session_with_success(get_api_key_fixture): | |||
| API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | |||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| Agent.create_session("2e45b5209c1011efa3e90242ac120006", rag) | |||
| @pytest.mark.skip(reason="") | |||
| def test_create_agent_conversation_with_success(get_api_key_fixture): | |||
| API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | |||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| session = Agent.create_session("2e45b5209c1011efa3e90242ac120006", rag) | |||
| session.ask("What is this job") | |||
| @pytest.mark.skip(reason="") | |||
| def test_list_agent_sessions_with_success(get_api_key_fixture): | |||
| API_KEY = "ragflow-BkOGNhYjIyN2JiODExZWY5MzVhMDI0Mm" | |||
| agent_id = "2710f2269b4611ef8fdf0242ac120006" | |||
| rag = RAGFlow(API_KEY,HOST_ADDRESS) | |||
| Agent.list_sessions(agent_id,rag) | |||
| rag = RAGFlow(API_KEY, HOST_ADDRESS) | |||
| Agent.list_sessions(agent_id, rag) | |||