Signed-off-by: -LAN- <laipz8200@outlook.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>tags/2.0.0-beta.2^2
| @@ -300,8 +300,7 @@ class DatasetQueueMonitorConfig(BaseSettings): | |||
| class MiddlewareConfig( | |||
| # place the configs in alphabet order | |||
| CeleryConfig, | |||
| DatabaseConfig, | |||
| CeleryConfig, # Note: CeleryConfig already inherits from DatabaseConfig | |||
| KeywordStoreConfig, | |||
| RedisConfig, | |||
| # configs of storage and storage providers | |||
| @@ -1,9 +1,10 @@ | |||
| from typing import Optional | |||
| from pydantic import BaseModel, Field | |||
| from pydantic import Field | |||
| from pydantic_settings import BaseSettings | |||
| class ClickzettaConfig(BaseModel): | |||
| class ClickzettaConfig(BaseSettings): | |||
| """ | |||
| Clickzetta Lakehouse vector database configuration | |||
| """ | |||
| @@ -1,7 +1,8 @@ | |||
| from pydantic import BaseModel, Field | |||
| from pydantic import Field | |||
| from pydantic_settings import BaseSettings | |||
| class MatrixoneConfig(BaseModel): | |||
| class MatrixoneConfig(BaseSettings): | |||
| """Matrixone vector database configuration.""" | |||
| MATRIXONE_HOST: str = Field(default="localhost", description="Host address of the Matrixone server") | |||
| @@ -1,6 +1,6 @@ | |||
| from pydantic import Field | |||
| from configs.packaging.pyproject import PyProjectConfig, PyProjectTomlConfig | |||
| from configs.packaging.pyproject import PyProjectTomlConfig | |||
| class PackagingInfo(PyProjectTomlConfig): | |||
| @@ -4,8 +4,9 @@ import logging | |||
| import os | |||
| import threading | |||
| import time | |||
| from collections.abc import Mapping | |||
| from collections.abc import Callable, Mapping | |||
| from pathlib import Path | |||
| from typing import Any | |||
| from .python_3x import http_request, makedirs_wrapper | |||
| from .utils import ( | |||
| @@ -25,13 +26,13 @@ logger = logging.getLogger(__name__) | |||
| class ApolloClient: | |||
| def __init__( | |||
| self, | |||
| config_url, | |||
| app_id, | |||
| cluster="default", | |||
| secret="", | |||
| start_hot_update=True, | |||
| change_listener=None, | |||
| _notification_map=None, | |||
| config_url: str, | |||
| app_id: str, | |||
| cluster: str = "default", | |||
| secret: str = "", | |||
| start_hot_update: bool = True, | |||
| change_listener: Callable[[str, str, str, Any], None] | None = None, | |||
| _notification_map: dict[str, int] | None = None, | |||
| ): | |||
| # Core routing parameters | |||
| self.config_url = config_url | |||
| @@ -47,17 +48,17 @@ class ApolloClient: | |||
| # Private control variables | |||
| self._cycle_time = 5 | |||
| self._stopping = False | |||
| self._cache = {} | |||
| self._no_key = {} | |||
| self._hash = {} | |||
| self._cache: dict[str, dict[str, Any]] = {} | |||
| self._no_key: dict[str, str] = {} | |||
| self._hash: dict[str, str] = {} | |||
| self._pull_timeout = 75 | |||
| self._cache_file_path = os.path.expanduser("~") + "/.dify/config/remote-settings/apollo/cache/" | |||
| self._long_poll_thread = None | |||
| self._long_poll_thread: threading.Thread | None = None | |||
| self._change_listener = change_listener # "add" "delete" "update" | |||
| if _notification_map is None: | |||
| _notification_map = {"application": -1} | |||
| self._notification_map = _notification_map | |||
| self.last_release_key = None | |||
| self.last_release_key: str | None = None | |||
| # Private startup method | |||
| self._path_checker() | |||
| if start_hot_update: | |||
| @@ -68,7 +69,7 @@ class ApolloClient: | |||
| heartbeat.daemon = True | |||
| heartbeat.start() | |||
| def get_json_from_net(self, namespace="application"): | |||
| def get_json_from_net(self, namespace: str = "application") -> dict[str, Any] | None: | |||
| url = "{}/configs/{}/{}/{}?releaseKey={}&ip={}".format( | |||
| self.config_url, self.app_id, self.cluster, namespace, "", self.ip | |||
| ) | |||
| @@ -88,7 +89,7 @@ class ApolloClient: | |||
| logger.exception("an error occurred in get_json_from_net") | |||
| return None | |||
| def get_value(self, key, default_val=None, namespace="application"): | |||
| def get_value(self, key: str, default_val: Any = None, namespace: str = "application") -> Any: | |||
| try: | |||
| # read memory configuration | |||
| namespace_cache = self._cache.get(namespace) | |||
| @@ -104,7 +105,8 @@ class ApolloClient: | |||
| namespace_data = self.get_json_from_net(namespace) | |||
| val = get_value_from_dict(namespace_data, key) | |||
| if val is not None: | |||
| self._update_cache_and_file(namespace_data, namespace) | |||
| if namespace_data is not None: | |||
| self._update_cache_and_file(namespace_data, namespace) | |||
| return val | |||
| # read the file configuration | |||
| @@ -126,23 +128,23 @@ class ApolloClient: | |||
| # to ensure the real-time correctness of the function call. | |||
| # If the user does not have the same default val twice | |||
| # and the default val is used here, there may be a problem. | |||
| def _set_local_cache_none(self, namespace, key): | |||
| def _set_local_cache_none(self, namespace: str, key: str) -> None: | |||
| no_key = no_key_cache_key(namespace, key) | |||
| self._no_key[no_key] = key | |||
| def _start_hot_update(self): | |||
| def _start_hot_update(self) -> None: | |||
| self._long_poll_thread = threading.Thread(target=self._listener) | |||
| # When the asynchronous thread is started, the daemon thread will automatically exit | |||
| # when the main thread is launched. | |||
| self._long_poll_thread.daemon = True | |||
| self._long_poll_thread.start() | |||
| def stop(self): | |||
| def stop(self) -> None: | |||
| self._stopping = True | |||
| logger.info("Stopping listener...") | |||
| # Call the set callback function, and if it is abnormal, try it out | |||
| def _call_listener(self, namespace, old_kv, new_kv): | |||
| def _call_listener(self, namespace: str, old_kv: dict[str, Any] | None, new_kv: dict[str, Any] | None) -> None: | |||
| if self._change_listener is None: | |||
| return | |||
| if old_kv is None: | |||
| @@ -168,12 +170,12 @@ class ApolloClient: | |||
| except BaseException as e: | |||
| logger.warning(str(e)) | |||
| def _path_checker(self): | |||
| def _path_checker(self) -> None: | |||
| if not os.path.isdir(self._cache_file_path): | |||
| makedirs_wrapper(self._cache_file_path) | |||
| # update the local cache and file cache | |||
| def _update_cache_and_file(self, namespace_data, namespace="application"): | |||
| def _update_cache_and_file(self, namespace_data: dict[str, Any], namespace: str = "application") -> None: | |||
| # update the local cache | |||
| self._cache[namespace] = namespace_data | |||
| # update the file cache | |||
| @@ -187,7 +189,7 @@ class ApolloClient: | |||
| self._hash[namespace] = new_hash | |||
| # get the configuration from the local file | |||
| def _get_local_cache(self, namespace="application"): | |||
| def _get_local_cache(self, namespace: str = "application") -> dict[str, Any]: | |||
| cache_file_path = os.path.join(self._cache_file_path, f"{self.app_id}_configuration_{namespace}.txt") | |||
| if os.path.isfile(cache_file_path): | |||
| with open(cache_file_path) as f: | |||
| @@ -195,8 +197,8 @@ class ApolloClient: | |||
| return result | |||
| return {} | |||
| def _long_poll(self): | |||
| notifications = [] | |||
| def _long_poll(self) -> None: | |||
| notifications: list[dict[str, Any]] = [] | |||
| for key in self._cache: | |||
| namespace_data = self._cache[key] | |||
| notification_id = -1 | |||
| @@ -236,7 +238,7 @@ class ApolloClient: | |||
| except Exception as e: | |||
| logger.warning(str(e)) | |||
| def _get_net_and_set_local(self, namespace, n_id, call_change=False): | |||
| def _get_net_and_set_local(self, namespace: str, n_id: int, call_change: bool = False) -> None: | |||
| namespace_data = self.get_json_from_net(namespace) | |||
| if not namespace_data: | |||
| return | |||
| @@ -248,7 +250,7 @@ class ApolloClient: | |||
| new_kv = namespace_data.get(CONFIGURATIONS) | |||
| self._call_listener(namespace, old_kv, new_kv) | |||
| def _listener(self): | |||
| def _listener(self) -> None: | |||
| logger.info("start long_poll") | |||
| while not self._stopping: | |||
| self._long_poll() | |||
| @@ -266,13 +268,13 @@ class ApolloClient: | |||
| headers["Timestamp"] = time_unix_now | |||
| return headers | |||
| def _heart_beat(self): | |||
| def _heart_beat(self) -> None: | |||
| while not self._stopping: | |||
| for namespace in self._notification_map: | |||
| self._do_heart_beat(namespace) | |||
| time.sleep(60 * 10) # 10 minutes | |||
| def _do_heart_beat(self, namespace): | |||
| def _do_heart_beat(self, namespace: str) -> None: | |||
| url = f"{self.config_url}/configs/{self.app_id}/{self.cluster}/{namespace}?ip={self.ip}" | |||
| try: | |||
| code, body = http_request(url, timeout=3, headers=self._sign_headers(url)) | |||
| @@ -292,7 +294,7 @@ class ApolloClient: | |||
| logger.exception("an error occurred in _do_heart_beat") | |||
| return None | |||
| def get_all_dicts(self, namespace): | |||
| def get_all_dicts(self, namespace: str) -> dict[str, Any] | None: | |||
| namespace_data = self._cache.get(namespace) | |||
| if namespace_data is None: | |||
| net_namespace_data = self.get_json_from_net(namespace) | |||
| @@ -2,6 +2,8 @@ import logging | |||
| import os | |||
| import ssl | |||
| import urllib.request | |||
| from collections.abc import Mapping | |||
| from typing import Any | |||
| from urllib import parse | |||
| from urllib.error import HTTPError | |||
| @@ -19,9 +21,9 @@ urllib.request.install_opener(opener) | |||
| logger = logging.getLogger(__name__) | |||
| def http_request(url, timeout, headers={}): | |||
| def http_request(url: str, timeout: int | float, headers: Mapping[str, str] = {}) -> tuple[int, str | None]: | |||
| try: | |||
| request = urllib.request.Request(url, headers=headers) | |||
| request = urllib.request.Request(url, headers=dict(headers)) | |||
| res = urllib.request.urlopen(request, timeout=timeout) | |||
| body = res.read().decode("utf-8") | |||
| return res.code, body | |||
| @@ -33,9 +35,9 @@ def http_request(url, timeout, headers={}): | |||
| raise e | |||
| def url_encode(params): | |||
| def url_encode(params: dict[str, Any]) -> str: | |||
| return parse.urlencode(params) | |||
| def makedirs_wrapper(path): | |||
| def makedirs_wrapper(path: str) -> None: | |||
| os.makedirs(path, exist_ok=True) | |||
| @@ -1,5 +1,6 @@ | |||
| import hashlib | |||
| import socket | |||
| from typing import Any | |||
| from .python_3x import url_encode | |||
| @@ -10,7 +11,7 @@ NAMESPACE_NAME = "namespaceName" | |||
| # add timestamps uris and keys | |||
| def signature(timestamp, uri, secret): | |||
| def signature(timestamp: str, uri: str, secret: str) -> str: | |||
| import base64 | |||
| import hmac | |||
| @@ -19,16 +20,16 @@ def signature(timestamp, uri, secret): | |||
| return base64.b64encode(hmac_code).decode() | |||
| def url_encode_wrapper(params): | |||
| def url_encode_wrapper(params: dict[str, Any]) -> str: | |||
| return url_encode(params) | |||
| def no_key_cache_key(namespace, key): | |||
| def no_key_cache_key(namespace: str, key: str) -> str: | |||
| return f"{namespace}{len(namespace)}{key}" | |||
| # Returns whether the obtained value is obtained, and None if it does not | |||
| def get_value_from_dict(namespace_cache, key): | |||
| def get_value_from_dict(namespace_cache: dict[str, Any] | None, key: str) -> Any | None: | |||
| if namespace_cache: | |||
| kv_data = namespace_cache.get(CONFIGURATIONS) | |||
| if kv_data is None: | |||
| @@ -38,7 +39,7 @@ def get_value_from_dict(namespace_cache, key): | |||
| return None | |||
| def init_ip(): | |||
| def init_ip() -> str: | |||
| ip = "" | |||
| s = None | |||
| try: | |||
| @@ -11,16 +11,16 @@ logger = logging.getLogger(__name__) | |||
| from configs.remote_settings_sources.base import RemoteSettingsSource | |||
| from .utils import _parse_config | |||
| from .utils import parse_config | |||
| class NacosSettingsSource(RemoteSettingsSource): | |||
| def __init__(self, configs: Mapping[str, Any]): | |||
| self.configs = configs | |||
| self.remote_configs: dict[str, Any] = {} | |||
| self.remote_configs: dict[str, str] = {} | |||
| self.async_init() | |||
| def async_init(self): | |||
| def async_init(self) -> None: | |||
| data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties") | |||
| group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify") | |||
| tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "") | |||
| @@ -33,18 +33,15 @@ class NacosSettingsSource(RemoteSettingsSource): | |||
| logger.exception("[get-access-token] exception occurred") | |||
| raise | |||
| def _parse_config(self, content: str): | |||
| def _parse_config(self, content: str) -> dict[str, str]: | |||
| if not content: | |||
| return {} | |||
| try: | |||
| return _parse_config(self, content) | |||
| return parse_config(content) | |||
| except Exception as e: | |||
| raise RuntimeError(f"Failed to parse config: {e}") | |||
| def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: | |||
| if not isinstance(self.remote_configs, dict): | |||
| raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}") | |||
| field_value = self.remote_configs.get(field_name) | |||
| if field_value is None: | |||
| return None, field_name, False | |||
| @@ -17,11 +17,17 @@ class NacosHttpClient: | |||
| self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY") | |||
| self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY") | |||
| self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848") | |||
| self.token = None | |||
| self.token: str | None = None | |||
| self.token_ttl = 18000 | |||
| self.token_expire_time: float = 0 | |||
| def http_request(self, url, method="GET", headers=None, params=None): | |||
| def http_request( | |||
| self, url: str, method: str = "GET", headers: dict[str, str] | None = None, params: dict[str, str] | None = None | |||
| ) -> str: | |||
| if headers is None: | |||
| headers = {} | |||
| if params is None: | |||
| params = {} | |||
| try: | |||
| self._inject_auth_info(headers, params) | |||
| response = requests.request(method, url="http://" + self.server + url, headers=headers, params=params) | |||
| @@ -30,7 +36,7 @@ class NacosHttpClient: | |||
| except requests.RequestException as e: | |||
| return f"Request to Nacos failed: {e}" | |||
| def _inject_auth_info(self, headers, params, module="config"): | |||
| def _inject_auth_info(self, headers: dict[str, str], params: dict[str, str], module: str = "config") -> None: | |||
| headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"}) | |||
| if module == "login": | |||
| @@ -45,16 +51,17 @@ class NacosHttpClient: | |||
| headers["timeStamp"] = ts | |||
| if self.username and self.password: | |||
| self.get_access_token(force_refresh=False) | |||
| params["accessToken"] = self.token | |||
| if self.token is not None: | |||
| params["accessToken"] = self.token | |||
| def __do_sign(self, sign_str, sk): | |||
| def __do_sign(self, sign_str: str, sk: str) -> str: | |||
| return ( | |||
| base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest()) | |||
| .decode() | |||
| .strip() | |||
| ) | |||
| def get_sign_str(self, group, tenant, ts): | |||
| def get_sign_str(self, group: str, tenant: str, ts: str) -> str: | |||
| sign_str = "" | |||
| if tenant: | |||
| sign_str = tenant + "+" | |||
| @@ -63,7 +70,7 @@ class NacosHttpClient: | |||
| sign_str += ts # Directly concatenate ts without conditional checks, because the nacos auth header forced it. | |||
| return sign_str | |||
| def get_access_token(self, force_refresh=False): | |||
| def get_access_token(self, force_refresh: bool = False) -> str | None: | |||
| current_time = time.time() | |||
| if self.token and not force_refresh and self.token_expire_time > current_time: | |||
| return self.token | |||
| @@ -77,6 +84,7 @@ class NacosHttpClient: | |||
| self.token = response_data.get("accessToken") | |||
| self.token_ttl = response_data.get("tokenTtl", 18000) | |||
| self.token_expire_time = current_time + self.token_ttl - 10 | |||
| return self.token | |||
| except Exception: | |||
| logger.exception("[get-access-token] exception occur") | |||
| raise | |||
| @@ -1,4 +1,4 @@ | |||
| def _parse_config(self, content: str) -> dict[str, str]: | |||
| def parse_config(content: str) -> dict[str, str]: | |||
| config: dict[str, str] = {} | |||
| if not content: | |||
| return config | |||
| @@ -1,5 +1,7 @@ | |||
| { | |||
| "include": ["."], | |||
| "include": [ | |||
| "." | |||
| ], | |||
| "exclude": [ | |||
| "tests/", | |||
| "migrations/", | |||
| @@ -19,10 +21,9 @@ | |||
| "events/", | |||
| "contexts/", | |||
| "constants/", | |||
| "configs/", | |||
| "commands.py" | |||
| ], | |||
| "typeCheckingMode": "strict", | |||
| "pythonVersion": "3.11", | |||
| "pythonPlatform": "All" | |||
| } | |||
| } | |||