Browse Source

Fix: enhance aliyun oss access with adding prefix path (#5475)

### What problem does this PR solve?

Enhance aliyun oss access with adding prefix path.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
tags/v0.17.0
The Wind 8 months ago
parent
commit
85924e898e
No account linked to committer's email address
2 changed files with 19 additions and 2 deletions
  1. 1
    0
      docker/service_conf.yaml.template
  2. 18
    2
      rag/utils/oss_conn.py

+ 1
- 0
docker/service_conf.yaml.template View File

@@ -43,6 +43,7 @@ redis:
# endpoint_url: '${ENDPOINT}'
# region: '${REGION}'
# bucket: '${BUCKET}'
# prefix_path: '${OSS_PREFIX_PATH}'
# azure:
# auth_type: 'sas'
# container_url: 'container_url'

+ 18
- 2
rag/utils/oss_conn.py View File

@@ -33,6 +33,7 @@ class RAGFlowOSS(object):
self.endpoint_url = self.oss_config.get('endpoint_url', None)
self.region = self.oss_config.get('region', None)
self.bucket = self.oss_config.get('bucket', None)
self.prefix_path = self.oss_config.get('prefix_path', None)
self.__open__()

@staticmethod
@@ -42,6 +43,14 @@ class RAGFlowOSS(object):
actual_bucket = self.bucket if self.bucket else bucket
return method(self, actual_bucket, *args, **kwargs)
return wrapper
@staticmethod
def use_prefix_path(method):
def wrapper(self, bucket, fnm, *args, **kwargs):
# If the prefix path is set, use the prefix path
fnm = f"{self.prefix_path}/{fnm}" if self.prefix_path else fnm
return method(self, bucket, fnm, *args, **kwargs)
return wrapper

def __open__(self):
try:
@@ -79,8 +88,9 @@ class RAGFlowOSS(object):
return exists

def health(self):
bucket, fnm, binary = "txtxtxtxt1", "txtxtxtxt1", b"_t@@@1"

bucket = self.bucket
fnm = "txtxtxtxt1"
fnm, binary = f"{self.prefix_path}/{fnm}" if self.prefix_path else fnm, b"_t@@@1"
if not self.bucket_exists(bucket):
self.conn.create_bucket(Bucket=bucket)
logging.debug(f"create bucket {bucket} ********")
@@ -94,6 +104,7 @@ class RAGFlowOSS(object):
def list(self, bucket, dir, recursive=True):
return []

@use_prefix_path
@use_default_bucket
def put(self, bucket, fnm, binary):
logging.debug(f"bucket name {bucket}; filename :{fnm}:")
@@ -110,6 +121,7 @@ class RAGFlowOSS(object):
self.__open__()
time.sleep(1)

@use_prefix_path
@use_default_bucket
def rm(self, bucket, fnm):
try:
@@ -117,6 +129,7 @@ class RAGFlowOSS(object):
except Exception:
logging.exception(f"Fail rm {bucket}/{fnm}")

@use_prefix_path
@use_default_bucket
def get(self, bucket, fnm):
for _ in range(1):
@@ -130,6 +143,7 @@ class RAGFlowOSS(object):
time.sleep(1)
return

@use_prefix_path
@use_default_bucket
def obj_exist(self, bucket, fnm):
try:
@@ -141,6 +155,7 @@ class RAGFlowOSS(object):
else:
raise

@use_prefix_path
@use_default_bucket
def get_presigned_url(self, bucket, fnm, expires):
for _ in range(10):
@@ -156,3 +171,4 @@ class RAGFlowOSS(object):
self.__open__()
time.sleep(1)
return


Loading…
Cancel
Save