Explorar el Código

Refa: remove default build-in rerank model. (#6682)

### What problem does this PR solve?

### Type of change

- [x] Refactoring
- [x] Performance Improvement
tags/v0.18.0
Kevin Hu hace 7 meses
padre
commit
7d9dd1e5d3
No account linked to committer's email address

+ 0
- 2
Dockerfile Ver fichero

if [ "$LIGHTEN" != "1" ]; then \ if [ "$LIGHTEN" != "1" ]; then \
(tar -cf - \ (tar -cf - \
/huggingface.co/BAAI/bge-large-zh-v1.5 \ /huggingface.co/BAAI/bge-large-zh-v1.5 \
/huggingface.co/BAAI/bge-reranker-v2-m3 \
/huggingface.co/maidalun1020/bce-embedding-base_v1 \ /huggingface.co/maidalun1020/bce-embedding-base_v1 \
/huggingface.co/maidalun1020/bce-reranker-base_v1 \
| tar -xf - --strip-components=2 -C /root/.ragflow) \ | tar -xf - --strip-components=2 -C /root/.ragflow) \
fi fi



+ 0
- 10
api/apps/sdk/dataset.py Ver fichero

else: else:
valid_embedding_models = [ valid_embedding_models = [
"BAAI/bge-large-zh-v1.5", "BAAI/bge-large-zh-v1.5",
"BAAI/bge-base-en-v1.5",
"BAAI/bge-large-en-v1.5",
"BAAI/bge-small-en-v1.5",
"BAAI/bge-small-zh-v1.5",
"jinaai/jina-embeddings-v2-base-en",
"jinaai/jina-embeddings-v2-small-en",
"nomic-ai/nomic-embed-text-v1.5",
"sentence-transformers/all-MiniLM-L6-v2",
"text-embedding-v2",
"text-embedding-v3",
"maidalun1020/bce-embedding-base_v1", "maidalun1020/bce-embedding-base_v1",
] ]
embd_model = LLMService.query( embd_model = LLMService.query(

+ 1
- 62
conf/llm_factories.json Ver fichero

"logo": "", "logo": "",
"tags": "TEXT EMBEDDING", "tags": "TEXT EMBEDDING",
"status": "1", "status": "1",
"llm": [
{
"llm_name": "BAAI/bge-small-en-v1.5",
"tags": "TEXT EMBEDDING,",
"max_tokens": 512,
"model_type": "embedding"
},
{
"llm_name": "BAAI/bge-small-zh-v1.5",
"tags": "TEXT EMBEDDING,",
"max_tokens": 512,
"model_type": "embedding"
},
{
"llm_name": "BAAI/bge-base-en-v1.5",
"tags": "TEXT EMBEDDING,",
"max_tokens": 512,
"model_type": "embedding"
},
{
"llm_name": "BAAI/bge-large-en-v1.5",
"tags": "TEXT EMBEDDING,",
"max_tokens": 512,
"model_type": "embedding"
},
{
"llm_name": "sentence-transformers/all-MiniLM-L6-v2",
"tags": "TEXT EMBEDDING,",
"max_tokens": 512,
"model_type": "embedding"
},
{
"llm_name": "nomic-ai/nomic-embed-text-v1.5",
"tags": "TEXT EMBEDDING,",
"max_tokens": 8192,
"model_type": "embedding"
},
{
"llm_name": "jinaai/jina-embeddings-v2-small-en",
"tags": "TEXT EMBEDDING,",
"max_tokens": 2147483648,
"model_type": "embedding"
},
{
"llm_name": "jinaai/jina-embeddings-v2-base-en",
"tags": "TEXT EMBEDDING,",
"max_tokens": 2147483648,
"model_type": "embedding"
}
]
"llm": []
}, },
{ {
"name": "Xinference", "name": "Xinference",
"tags": "TEXT EMBEDDING,", "tags": "TEXT EMBEDDING,",
"max_tokens": 512, "max_tokens": 512,
"model_type": "embedding" "model_type": "embedding"
},
{
"llm_name": "maidalun1020/bce-reranker-base_v1",
"tags": "RE-RANK, 512",
"max_tokens": 512,
"model_type": "rerank"
} }
] ]
}, },
"tags": "TEXT EMBEDDING,", "tags": "TEXT EMBEDDING,",
"max_tokens": 1024, "max_tokens": 1024,
"model_type": "embedding" "model_type": "embedding"
},
{
"llm_name": "BAAI/bge-reranker-v2-m3",
"tags": "RE-RANK,2k",
"max_tokens": 2048,
"model_type": "rerank"
} }
] ]
}, },

+ 1
- 11
docker/README.md Ver fichero

- `infiniflow/ragflow:v0.17.2`: The RAGFlow Docker image with embedding models including: - `infiniflow/ragflow:v0.17.2`: The RAGFlow Docker image with embedding models including:
- Built-in embedding models: - Built-in embedding models:
- `BAAI/bge-large-zh-v1.5` - `BAAI/bge-large-zh-v1.5`
- `BAAI/bge-reranker-v2-m3`
- `maidalun1020/bce-embedding-base_v1` - `maidalun1020/bce-embedding-base_v1`
- `maidalun1020/bce-reranker-base_v1`
- Embedding models that will be downloaded once you select them in the RAGFlow UI:
- `BAAI/bge-base-en-v1.5`
- `BAAI/bge-large-en-v1.5`
- `BAAI/bge-small-en-v1.5`
- `BAAI/bge-small-zh-v1.5`
- `jinaai/jina-embeddings-v2-base-en`
- `jinaai/jina-embeddings-v2-small-en`
- `nomic-ai/nomic-embed-text-v1.5`
- `sentence-transformers/all-MiniLM-L6-v2`

> [!TIP] > [!TIP]
> If you cannot download the RAGFlow Docker image, try the following mirrors. > If you cannot download the RAGFlow Docker image, try the following mirrors.

+ 2
- 12
docs/configurations.md Ver fichero

- `infiniflow/ragflow:v0.17.2`: The RAGFlow Docker image with embedding models including: - `infiniflow/ragflow:v0.17.2`: The RAGFlow Docker image with embedding models including:
- Built-in embedding models: - Built-in embedding models:
- `BAAI/bge-large-zh-v1.5` - `BAAI/bge-large-zh-v1.5`
- `BAAI/bge-reranker-v2-m3`
- `maidalun1020/bce-embedding-base_v1` - `maidalun1020/bce-embedding-base_v1`
- `maidalun1020/bce-reranker-base_v1`
- Embedding models that will be downloaded once you select them in the RAGFlow UI:
- `BAAI/bge-base-en-v1.5`
- `BAAI/bge-large-en-v1.5`
- `BAAI/bge-small-en-v1.5`
- `BAAI/bge-small-zh-v1.5`
- `jinaai/jina-embeddings-v2-base-en`
- `jinaai/jina-embeddings-v2-small-en`
- `nomic-ai/nomic-embed-text-v1.5`
- `sentence-transformers/all-MiniLM-L6-v2`


:::tip NOTE :::tip NOTE
If you cannot download the RAGFlow Docker image, try the following mirrors. If you cannot download the RAGFlow Docker image, try the following mirrors.



+ 0
- 2
download_deps.py Ver fichero

"InfiniFlow/deepdoc", "InfiniFlow/deepdoc",
"InfiniFlow/huqie", "InfiniFlow/huqie",
"BAAI/bge-large-zh-v1.5", "BAAI/bge-large-zh-v1.5",
"BAAI/bge-reranker-v2-m3",
"maidalun1020/bce-embedding-base_v1", "maidalun1020/bce-embedding-base_v1",
"maidalun1020/bce-reranker-base_v1",
] ]


def download_model(repo_id): def download_model(repo_id):

+ 1
- 0
sdk/python/test/test_http_api/test_chunk_management_within_dataset/test_retrieval_chunks.py Ver fichero

else: else:
assert expected_message in res["message"] assert expected_message in res["message"]


@pytest.mark.skip
@pytest.mark.parametrize( @pytest.mark.parametrize(
"payload, expected_code, expected_message", "payload, expected_code, expected_message",
[ [

+ 0
- 22
sdk/python/test/test_http_api/test_dataset_mangement/test_create_dataset.py Ver fichero

"name, embedding_model, expected_code", "name, embedding_model, expected_code",
[ [
("BAAI/bge-large-zh-v1.5", "BAAI/bge-large-zh-v1.5", 0), ("BAAI/bge-large-zh-v1.5", "BAAI/bge-large-zh-v1.5", 0),
("BAAI/bge-base-en-v1.5", "BAAI/bge-base-en-v1.5", 0),
("BAAI/bge-large-en-v1.5", "BAAI/bge-large-en-v1.5", 0),
("BAAI/bge-small-en-v1.5", "BAAI/bge-small-en-v1.5", 0),
("BAAI/bge-small-zh-v1.5", "BAAI/bge-small-zh-v1.5", 0),
(
"jinaai/jina-embeddings-v2-base-en",
"jinaai/jina-embeddings-v2-base-en",
0,
),
(
"jinaai/jina-embeddings-v2-small-en",
"jinaai/jina-embeddings-v2-small-en",
0,
),
("nomic-ai/nomic-embed-text-v1.5", "nomic-ai/nomic-embed-text-v1.5", 0),
(
"sentence-transformers/all-MiniLM-L6-v2",
"sentence-transformers/all-MiniLM-L6-v2",
0,
),
("text-embedding-v2", "text-embedding-v2", 0),
("text-embedding-v3", "text-embedding-v3", 0),
( (
"maidalun1020/bce-embedding-base_v1", "maidalun1020/bce-embedding-base_v1",
"maidalun1020/bce-embedding-base_v1", "maidalun1020/bce-embedding-base_v1",

+ 0
- 10
sdk/python/test/test_http_api/test_dataset_mangement/test_update_dataset.py Ver fichero

"embedding_model, expected_code, expected_message", "embedding_model, expected_code, expected_message",
[ [
("BAAI/bge-large-zh-v1.5", 0, ""), ("BAAI/bge-large-zh-v1.5", 0, ""),
("BAAI/bge-base-en-v1.5", 0, ""),
("BAAI/bge-large-en-v1.5", 0, ""),
("BAAI/bge-small-en-v1.5", 0, ""),
("BAAI/bge-small-zh-v1.5", 0, ""),
("jinaai/jina-embeddings-v2-base-en", 0, ""),
("jinaai/jina-embeddings-v2-small-en", 0, ""),
("nomic-ai/nomic-embed-text-v1.5", 0, ""),
("sentence-transformers/all-MiniLM-L6-v2", 0, ""),
("text-embedding-v2", 0, ""),
("text-embedding-v3", 0, ""),
("maidalun1020/bce-embedding-base_v1", 0, ""), ("maidalun1020/bce-embedding-base_v1", 0, ""),
( (
"other_embedding_model", "other_embedding_model",

Cargando…
Cancelar
Guardar