### What problem does this PR solve? ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)tags/v0.7.0
| if not rerank_id: req["rerank_id"] = "" | if not rerank_id: req["rerank_id"] = "" | ||||
| similarity_threshold = req.get("similarity_threshold", 0.1) | similarity_threshold = req.get("similarity_threshold", 0.1) | ||||
| vector_similarity_weight = req.get("vector_similarity_weight", 0.3) | vector_similarity_weight = req.get("vector_similarity_weight", 0.3) | ||||
| if vector_similarity_weight is None: vector_similarity_weight = 0.3 | |||||
| llm_setting = req.get("llm_setting", {}) | llm_setting = req.get("llm_setting", {}) | ||||
| default_prompt = { | default_prompt = { | ||||
| "system": """你是一个智能助手,请总结知识库的内容来回答问题,请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时,你的回答必须包括“知识库中未找到您要的答案!”这句话。回答需要考虑聊天历史。 | "system": """你是一个智能助手,请总结知识库的内容来回答问题,请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时,你的回答必须包括“知识库中未找到您要的答案!”这句话。回答需要考虑聊天历史。 |
| return FileType.PDF.value | return FileType.PDF.value | ||||
| if re.match( | if re.match( | ||||
| r".*\.(doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|html)$", filename): | |||||
| r".*\.(doc|docx|ppt|pptx|yml|xml|htm|json|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|html)$", filename): | |||||
| return FileType.DOC.value | return FileType.DOC.value | ||||
| if re.match( | if re.match( |
| # -*- coding: utf-8 -*- | # -*- coding: utf-8 -*- | ||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| # you may not use this file except in compliance with the License. | |||||
| # You may obtain a copy of the License at | |||||
| # | |||||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||||
| # | |||||
| # Unless required by applicable law or agreed to in writing, software | |||||
| # distributed under the License is distributed on an "AS IS" BASIS, | |||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| # See the License for the specific language governing permissions and | |||||
| # limitations under the License. | |||||
| # | |||||
| from rag.nlp import find_codec | from rag.nlp import find_codec | ||||
| import readability | import readability | ||||
| import html_text | import html_text | ||||
| with open(file,'rb') as f: | with open(file,'rb') as f: | ||||
| tmp = chardet.detect(f.read()) | tmp = chardet.detect(f.read()) | ||||
| return tmp['encoding'] | return tmp['encoding'] | ||||
| class RAGFlowHtmlParser: | class RAGFlowHtmlParser: | ||||
| def __call__(self, fnm, binary=None): | def __call__(self, fnm, binary=None): | ||||
| txt = "" | txt = "" | ||||
| else: | else: | ||||
| with open(fnm, "r",encoding=get_encoding(fnm)) as f: | with open(fnm, "r",encoding=get_encoding(fnm)) as f: | ||||
| txt = f.read() | txt = f.read() | ||||
| html_doc = readability.Document(txt) | html_doc = readability.Document(txt) | ||||
| title = html_doc.title() | title = html_doc.title() | ||||
| content = html_text.extract_text(html_doc.summary(html_partial=True)) | content = html_text.extract_text(html_doc.summary(html_partial=True)) |
| for ck in hierarchical_merge(bull, sections, 5)] | for ck in hierarchical_merge(bull, sections, 5)] | ||||
| else: | else: | ||||
| sections = [s.split("@") for s, _ in sections] | sections = [s.split("@") for s, _ in sections] | ||||
| sections = [(pr[0], "@" + pr[1]) for pr in sections if len(pr) == 2] | |||||
| sections = [(pr[0], "@" + pr[1]) if len(pr) == 2 else (pr[0], '') for pr in sections ] | |||||
| chunks = naive_merge( | chunks = naive_merge( | ||||
| sections, kwargs.get( | sections, kwargs.get( | ||||
| "chunk_token_num", 256), kwargs.get( | "chunk_token_num", 256), kwargs.get( |