### What problem does this PR solve?
Print configs at the RAGFlow startup phase.
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
```
2024-11-14 21:27:53,090 INFO 962231 Current configs, from /home/weilongma/Documents/development/ragflow/conf/service_conf.yaml:
2024-11-14 21:27:53,090 INFO 962231 ragflow: {'host': '0.0.0.0', 'http_port': 9380}
2024-11-14 21:27:53,090 INFO 962231 mysql: {'name': 'rag_flow', 'user': 'root', 'password': 'infini_rag_flow', 'host': 'mysql', 'port': 5455, 'max_connections': 100, 'stale_timeout': 30}
2024-11-14 21:27:53,090 INFO 962231 minio: {'user': 'rag_flow', 'password': 'infini_rag_flow', 'host': 'minio:9000'}
2024-11-14 21:27:53,090 INFO 962231 es: {'hosts': 'http://es01:1200', 'username': 'elastic', 'password': 'infini_rag_flow'}
2024-11-14 21:27:53,090 INFO 962231 redis: {'db': 1, 'password': 'infini_rag_flow', 'host': 'redis:6379'}
```
Signed-off-by: jinhai <haijin.chn@gmail.com>
tags/v0.14.0
| NAME_LENGTH_LIMIT = 2 ** 10 | NAME_LENGTH_LIMIT = 2 ** 10 | ||||
| IMG_BASE64_PREFIX = 'data:image/png;base64,' | |||||
| IMG_BASE64_PREFIX = 'data:image/png;base64,' | |||||
| SERVICE_CONF = "service_conf.yaml" |
| import logging | import logging | ||||
| import inspect | import inspect | ||||
| from api.utils.log_utils import initRootLogger | from api.utils.log_utils import initRootLogger | ||||
| initRootLogger(inspect.getfile(inspect.currentframe())) | initRootLogger(inspect.getfile(inspect.currentframe())) | ||||
| for module in ["pdfminer"]: | for module in ["pdfminer"]: | ||||
| module_logger = logging.getLogger(module) | module_logger = logging.getLogger(module) | ||||
| from api.db.db_models import init_database_tables as init_web_db | from api.db.db_models import init_database_tables as init_web_db | ||||
| from api.db.init_data import init_web_data | from api.db.init_data import init_web_data | ||||
| from api.versions import get_ragflow_version | from api.versions import get_ragflow_version | ||||
| from api.utils import show_configs | |||||
| def update_progress(): | def update_progress(): | ||||
| logging.info( | logging.info( | ||||
| f'project base: {utils.file_utils.get_project_base_directory()}' | f'project base: {utils.file_utils.get_project_base_directory()}' | ||||
| ) | ) | ||||
| show_configs() | |||||
| # init db | # init db | ||||
| init_web_db() | init_web_db() | ||||
| parser = argparse.ArgumentParser() | parser = argparse.ArgumentParser() | ||||
| parser.add_argument( | parser.add_argument( | ||||
| "--version", default=False, help="rag flow version", action="store_true" | |||||
| "--version", default=False, help="RAGFlow version", action="store_true" | |||||
| ) | ) | ||||
| parser.add_argument( | parser.add_argument( | ||||
| "--debug", default=False, help="debug mode", action="store_true" | "--debug", default=False, help="debug mode", action="store_true" | ||||
| RuntimeConfig.init_env() | RuntimeConfig.init_env() | ||||
| RuntimeConfig.init_config(JOB_SERVER_HOST=HOST, HTTP_PORT=HTTP_PORT) | RuntimeConfig.init_config(JOB_SERVER_HOST=HOST, HTTP_PORT=HTTP_PORT) | ||||
| thr = ThreadPoolExecutor(max_workers=1) | |||||
| thr.submit(update_progress) | |||||
| thread = ThreadPoolExecutor(max_workers=1) | |||||
| thread.submit(update_progress) | |||||
| # start http server | # start http server | ||||
| try: | try: |
| import time | import time | ||||
| import uuid | import uuid | ||||
| import requests | import requests | ||||
| import logging | |||||
| from enum import Enum, IntEnum | from enum import Enum, IntEnum | ||||
| import importlib | import importlib | ||||
| from Cryptodome.PublicKey import RSA | from Cryptodome.PublicKey import RSA | ||||
| from Cryptodome.Cipher import PKCS1_v1_5 as Cipher_pkcs1_v1_5 | from Cryptodome.Cipher import PKCS1_v1_5 as Cipher_pkcs1_v1_5 | ||||
| from filelock import FileLock | from filelock import FileLock | ||||
| from api.constants import SERVICE_CONF | |||||
| from . import file_utils | from . import file_utils | ||||
| SERVICE_CONF = "service_conf.yaml" | |||||
| def conf_realpath(conf_name): | def conf_realpath(conf_name): | ||||
| conf_path = f"conf/{conf_name}" | conf_path = f"conf/{conf_name}" | ||||
| return os.path.join(file_utils.get_project_base_directory(), conf_path) | return os.path.join(file_utils.get_project_base_directory(), conf_path) | ||||
| def get_base_config(key, default=None, conf_name=SERVICE_CONF) -> dict: | |||||
| def read_config(conf_name=SERVICE_CONF): | |||||
| local_config = {} | local_config = {} | ||||
| local_path = conf_realpath(f'local.{conf_name}') | local_path = conf_realpath(f'local.{conf_name}') | ||||
| if default is None: | |||||
| default = os.environ.get(key.upper()) | |||||
| # load local config file | |||||
| if os.path.exists(local_path): | if os.path.exists(local_path): | ||||
| local_config = file_utils.load_yaml_conf(local_path) | local_config = file_utils.load_yaml_conf(local_path) | ||||
| if not isinstance(local_config, dict): | if not isinstance(local_config, dict): | ||||
| raise ValueError(f'Invalid config file: "{local_path}".') | raise ValueError(f'Invalid config file: "{local_path}".') | ||||
| if key is not None and key in local_config: | |||||
| return local_config[key] | |||||
| global_config_path = conf_realpath(conf_name) | |||||
| global_config = file_utils.load_yaml_conf(global_config_path) | |||||
| config_path = conf_realpath(conf_name) | |||||
| config = file_utils.load_yaml_conf(config_path) | |||||
| if not isinstance(global_config, dict): | |||||
| raise ValueError(f'Invalid config file: "{global_config_path}".') | |||||
| if not isinstance(config, dict): | |||||
| raise ValueError(f'Invalid config file: "{config_path}".') | |||||
| global_config.update(local_config) | |||||
| return global_config | |||||
| config.update(local_config) | |||||
| return config.get(key, default) if key is not None else config | |||||
| CONFIGS = read_config() | |||||
| use_deserialize_safe_module = get_base_config( | |||||
| 'use_deserialize_safe_module', False) | |||||
| def show_configs(): | |||||
| logging.info(f"Current configs, from {conf_realpath(SERVICE_CONF)}:") | |||||
| for k, v in CONFIGS.items(): | |||||
| logging.info(f"{k}: {v}") | |||||
| class CoordinationCommunicationProtocol(object): | |||||
| HTTP = "http" | |||||
| GRPC = "grpc" | |||||
| def get_base_config(key, default=None): | |||||
| if key is None: | |||||
| return None | |||||
| if default is None: | |||||
| default = os.environ.get(key.upper()) | |||||
| return CONFIGS.get(key, default) | |||||
| use_deserialize_safe_module = get_base_config( | |||||
| 'use_deserialize_safe_module', False) | |||||
| class BaseType: | class BaseType: | ||||
| data = obj | data = obj | ||||
| return {"type": obj.__class__.__name__, | return {"type": obj.__class__.__name__, | ||||
| "data": data, "module": module} | "data": data, "module": module} | ||||
| return _dict(self) | return _dict(self) | ||||
| return "" | return "" | ||||
| response = requests.get(url) | response = requests.get(url) | ||||
| return "data:" + \ | return "data:" + \ | ||||
| response.headers.get('Content-Type', 'image/jpg') + ";" + \ | |||||
| "base64," + base64.b64encode(response.content).decode("utf-8") | |||||
| response.headers.get('Content-Type', 'image/jpg') + ";" + \ | |||||
| "base64," + base64.b64encode(response.content).decode("utf-8") | |||||
| def delta_seconds(date_string: str): | def delta_seconds(date_string: str): |
| try: | try: | ||||
| from multiprocessing import Pool | from multiprocessing import Pool | ||||
| pool = Pool(processes=1) | pool = Pool(processes=1) | ||||
| thr = pool.apply_async(download_nltk_data) | |||||
| binary = thr.get(timeout=60) | |||||
| thread = pool.apply_async(download_nltk_data) | |||||
| binary = thread.get(timeout=60) | |||||
| except Exception as e: | except Exception as e: | ||||
| print('\x1b[6;37;41m WARNING \x1b[0m' + "Downloading NLTK data failure.", flush=True) | print('\x1b[6;37;41m WARNING \x1b[0m' + "Downloading NLTK data failure.", flush=True) |