Feat: extend S3 storage compatibility and add knowledge base ID prefix (#6355)

### What problem does this PR solve?

- Added support for S3-compatible protocols.
- Enabled the use of knowledge base ID as a file prefix when storing
files in S3.
- Updated docker/README.md to include detailed S3 and OSS configuration
instructions.

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

docker/README.md

@@ -136,6 +136,24 @@ The [.env](./.env) file contains important environment variables for Docker.
   - `password`: The password for MinIO.
   - `host`: The MinIO serving IP *and* port inside the Docker container. Defaults to `minio:9000`.
 
+- `oss`
+  - `access_key`: The access key ID used to authenticate requests to the OSS service.
+  - `secret_key`: The secret access key used to authenticate requests to the OSS service.
+  - `endpoint_url`: The URL of the OSS service endpoint.
+  - `region`: The OSS region where the bucket is located.
+  - `bucket`: The name of the OSS bucket where files will be stored. When you want to store all files in a specified bucket, you need this configuration item.
+  - `prefix_path`: Optional. A prefix path to prepend to file names in the OSS bucket, which can help organize files within the bucket.
+
+- `s3`:
+  - `access_key`: The access key ID used to authenticate requests to the S3 service.
+  - `secret_key`: The secret access key used to authenticate requests to the S3 service.
+  - `endpoint_url`: The URL of the S3-compatible service endpoint. This is necessary when using an S3-compatible protocol instead of the default AWS S3 endpoint.
+  - `bucket`: The name of the S3 bucket where files will be stored. When you want to store all files in a specified bucket, you need this configuration item.
+  - `region`: The AWS region where the S3 bucket is located. This is important for directing requests to the correct data center.
+  - `signature_version`: Optional. The version of the signature to use for authenticating requests. Common versions include `v4`.
+  - `addressing_style`: Optional. The style of addressing to use for the S3 endpoint. This can be `path` or `virtual`.
+  - `prefix_path`: Optional. A prefix path to prepend to file names in the S3 bucket, which can help organize files within the bucket.
+
 - `oauth`  
   The OAuth configuration for signing up or signing in to RAGFlow using a third-party account.  It is disabled by default. To enable this feature, uncomment the corresponding lines in **service_conf.yaml.template**.
   - `github`: The GitHub authentication settings for your application. Visit the [Github Developer Settings page](https://github.com/settings/developers) to obtain your client_id and secret_key.
@@ -152,4 +170,4 @@ The [.env](./.env) file contains important environment variables for Docker.
   - `api_key`: The API key for the specified LLM. You will need to apply for your model API key online.
 
 > [!TIP]  
-> If you do not set the default LLM here, configure the default LLM on the **Settings** page in the RAGFlow UI.
+> If you do not set the default LLM here, configure the default LLM on the **Settings** page in the RAGFlow UI.

docker/docker-compose.yml

@@ -1,6 +1,7 @@
 include:
   - ./docker-compose-base.yml
 
+# To ensure that the container processes the locally modified `service_conf.yaml.template` instead of the one included in its image, you need to mount the local `service_conf.yaml.template` to the container.
 services:
   ragflow:
     depends_on:
@@ -20,6 +21,7 @@ services:
       - ./nginx/proxy.conf:/etc/nginx/proxy.conf
       - ./nginx/nginx.conf:/etc/nginx/nginx.conf
       - ../history_data_agent:/ragflow/history_data_agent
+      - ./service_conf.yaml.template:/ragflow/conf/service_conf.yaml.template
 
     env_file: .env
     environment:

docker/service_conf.yaml.template

@@ -37,6 +37,11 @@ redis:
 #   access_key: 'access_key'
 #   secret_key: 'secret_key'
 #   region: 'region'
+#   endpoint_url: 'endpoint_url'
+#   bucket: 'bucket'
+#   prefix_path: 'prefix_path'
+#   signature_version: 'v4'
+#   addressing_style: 'path'
 # oss:
 #   access_key: '${ACCESS_KEY}'
 #   secret_key: '${SECRET_KEY}'

rag/utils/s3_conn.py

@@ -17,6 +17,7 @@
 import logging
 import boto3
 from botocore.exceptions import ClientError
+from botocore.config import Config
 import time
 from io import BytesIO
 from rag.utils import singleton
@@ -30,8 +31,34 @@ class RAGFlowS3:
         self.access_key = self.s3_config.get('access_key', None)
         self.secret_key = self.s3_config.get('secret_key', None)
         self.region = self.s3_config.get('region', None)
+        self.endpoint_url = self.s3_config.get('endpoint_url', None)
+        self.signature_version = self.s3_config.get('signature_version', None)
+        self.addressing_style = self.s3_config.get('addressing_style', None)
+        self.bucket = self.s3_config.get('bucket', None)
+        self.prefix_path = self.s3_config.get('prefix_path', None)
         self.__open__()
 
+    @staticmethod
+    def use_default_bucket(method):
+        def wrapper(self, bucket, *args, **kwargs):
+            # If there is a default bucket, use the default bucket
+            actual_bucket = self.bucket if self.bucket else bucket
+            return method(self, actual_bucket, *args, **kwargs)
+        return wrapper
+
+    @staticmethod
+    def use_prefix_path(method):
+        def wrapper(self, bucket, fnm, *args, **kwargs):
+            # If the prefix path is set, use the prefix path.
+            # The bucket passed from the upstream call is 
+            # used as the file prefix. This is especially useful when you're using the default bucket
+            if self.prefix_path:
+                fnm = f"{self.prefix_path}/{bucket}/{fnm}"
+            else:
+                fnm = f"{bucket}/{fnm}"
+            return method(self, bucket, fnm, *args, **kwargs)
+        return wrapper
+
     def __open__(self):
         try:
             if self.conn:
@@ -40,19 +67,27 @@ class RAGFlowS3:
             pass
 
         try:
-            self.conn = boto3.client(
-                's3',
-                region_name=self.region,
-                aws_access_key_id=self.access_key,
-                aws_secret_access_key=self.secret_key
-            )
+            s3_params = {
+                'aws_access_key_id': self.access_key,
+                'aws_secret_access_key': self.secret_key,
+            }
+            if self.region in self.s3_config:
+                s3_params['region_name'] = self.region
+            if 'endpoint_url' in self.s3_config:
+                s3_params['endpoint_url'] = self.endpoint_url
+            if 'signature_version' in self.s3_config:
+                s3_params['config'] = Config(s3={"signature_version": self.signature_version})
+            if 'addressing_style' in self.s3_config:
+                s3_params['config'] = Config(s3={"addressing_style": self.addressing_style})
+            self.conn = boto3.client('s3', **s3_params)
         except Exception:
-            logging.exception(f"Fail to connect at region {self.region}")
+            logging.exception(f"Fail to connect at region {self.region} or endpoint {self.endpoint_url}")
 
     def __close__(self):
         del self.conn
         self.conn = None
 
+    @use_default_bucket
     def bucket_exists(self, bucket):
         try:
             logging.debug(f"head_bucket bucketname {bucket}")
@@ -64,8 +99,9 @@ class RAGFlowS3:
         return exists
 
     def health(self):
-        bucket, fnm, binary = "txtxtxtxt1", "txtxtxtxt1", b"_t@@@1"
-
+        bucket = self.bucket
+        fnm = "txtxtxtxt1"
+        fnm, binary = f"{self.prefix_path}/{fnm}" if self.prefix_path else fnm, b"_t@@@1"
         if not self.bucket_exists(bucket):
             self.conn.create_bucket(Bucket=bucket)
             logging.debug(f"create bucket {bucket} ********")
@@ -79,6 +115,8 @@ class RAGFlowS3:
     def list(self, bucket, dir, recursive=True):
         return []
 
+    @use_prefix_path
+    @use_default_bucket
     def put(self, bucket, fnm, binary):
         logging.debug(f"bucket name {bucket}; filename :{fnm}:")
         for _ in range(1):
@@ -94,12 +132,16 @@ class RAGFlowS3:
                 self.__open__()
                 time.sleep(1)
 
+    @use_prefix_path
+    @use_default_bucket
     def rm(self, bucket, fnm):
         try:
             self.conn.delete_object(Bucket=bucket, Key=fnm)
         except Exception:
             logging.exception(f"Fail rm {bucket}/{fnm}")
 
+    @use_prefix_path
+    @use_default_bucket
     def get(self, bucket, fnm):
         for _ in range(1):
             try:
@@ -112,18 +154,20 @@ class RAGFlowS3:
                 time.sleep(1)
         return
 
+    @use_prefix_path
+    @use_default_bucket
     def obj_exist(self, bucket, fnm):
         try:
-
             if self.conn.head_object(Bucket=bucket, Key=fnm):
                 return True
         except ClientError as e:
             if e.response['Error']['Code'] == '404':
-
                 return False
             else:
                 raise
 
+    @use_prefix_path
+    @use_default_bucket
     def get_presigned_url(self, bucket, fnm, expires):
         for _ in range(10):
             try: