Browse Source

Fix: enhance aliyun oss access with adding prefix path (#5475)

### What problem does this PR solve?

Enhance aliyun oss access with adding prefix path.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
tags/v0.17.0
The Wind 8 months ago
parent
commit
85924e898e
No account linked to committer's email address
2 changed files with 19 additions and 2 deletions
  1. 1
    0
      docker/service_conf.yaml.template
  2. 18
    2
      rag/utils/oss_conn.py

+ 1
- 0
docker/service_conf.yaml.template View File

# endpoint_url: '${ENDPOINT}' # endpoint_url: '${ENDPOINT}'
# region: '${REGION}' # region: '${REGION}'
# bucket: '${BUCKET}' # bucket: '${BUCKET}'
# prefix_path: '${OSS_PREFIX_PATH}'
# azure: # azure:
# auth_type: 'sas' # auth_type: 'sas'
# container_url: 'container_url' # container_url: 'container_url'

+ 18
- 2
rag/utils/oss_conn.py View File

self.endpoint_url = self.oss_config.get('endpoint_url', None) self.endpoint_url = self.oss_config.get('endpoint_url', None)
self.region = self.oss_config.get('region', None) self.region = self.oss_config.get('region', None)
self.bucket = self.oss_config.get('bucket', None) self.bucket = self.oss_config.get('bucket', None)
self.prefix_path = self.oss_config.get('prefix_path', None)
self.__open__() self.__open__()


@staticmethod @staticmethod
actual_bucket = self.bucket if self.bucket else bucket actual_bucket = self.bucket if self.bucket else bucket
return method(self, actual_bucket, *args, **kwargs) return method(self, actual_bucket, *args, **kwargs)
return wrapper return wrapper
@staticmethod
def use_prefix_path(method):
def wrapper(self, bucket, fnm, *args, **kwargs):
# If the prefix path is set, use the prefix path
fnm = f"{self.prefix_path}/{fnm}" if self.prefix_path else fnm
return method(self, bucket, fnm, *args, **kwargs)
return wrapper


def __open__(self): def __open__(self):
try: try:
return exists return exists


def health(self): def health(self):
bucket, fnm, binary = "txtxtxtxt1", "txtxtxtxt1", b"_t@@@1"

bucket = self.bucket
fnm = "txtxtxtxt1"
fnm, binary = f"{self.prefix_path}/{fnm}" if self.prefix_path else fnm, b"_t@@@1"
if not self.bucket_exists(bucket): if not self.bucket_exists(bucket):
self.conn.create_bucket(Bucket=bucket) self.conn.create_bucket(Bucket=bucket)
logging.debug(f"create bucket {bucket} ********") logging.debug(f"create bucket {bucket} ********")
def list(self, bucket, dir, recursive=True): def list(self, bucket, dir, recursive=True):
return [] return []


@use_prefix_path
@use_default_bucket @use_default_bucket
def put(self, bucket, fnm, binary): def put(self, bucket, fnm, binary):
logging.debug(f"bucket name {bucket}; filename :{fnm}:") logging.debug(f"bucket name {bucket}; filename :{fnm}:")
self.__open__() self.__open__()
time.sleep(1) time.sleep(1)


@use_prefix_path
@use_default_bucket @use_default_bucket
def rm(self, bucket, fnm): def rm(self, bucket, fnm):
try: try:
except Exception: except Exception:
logging.exception(f"Fail rm {bucket}/{fnm}") logging.exception(f"Fail rm {bucket}/{fnm}")


@use_prefix_path
@use_default_bucket @use_default_bucket
def get(self, bucket, fnm): def get(self, bucket, fnm):
for _ in range(1): for _ in range(1):
time.sleep(1) time.sleep(1)
return return


@use_prefix_path
@use_default_bucket @use_default_bucket
def obj_exist(self, bucket, fnm): def obj_exist(self, bucket, fnm):
try: try:
else: else:
raise raise


@use_prefix_path
@use_default_bucket @use_default_bucket
def get_presigned_url(self, bucket, fnm, expires): def get_presigned_url(self, bucket, fnm, expires):
for _ in range(10): for _ in range(10):
self.__open__() self.__open__()
time.sleep(1) time.sleep(1)
return return


Loading…
Cancel
Save