| UNSTRUCTURED_API_URL= | UNSTRUCTURED_API_URL= | ||||
| UNSTRUCTURED_API_KEY= | UNSTRUCTURED_API_KEY= | ||||
| #ssrf | |||||
| SSRF_PROXY_HTTP_URL= | SSRF_PROXY_HTTP_URL= | ||||
| SSRF_PROXY_HTTPS_URL= | SSRF_PROXY_HTTPS_URL= | ||||
| SSRF_DEFAULT_MAX_RETRIES=3 | SSRF_DEFAULT_MAX_RETRIES=3 | ||||
| SSRF_DEFAULT_TIME_OUT= | |||||
| SSRF_DEFAULT_CONNECT_TIME_OUT= | |||||
| SSRF_DEFAULT_READ_TIME_OUT= | |||||
| SSRF_DEFAULT_WRITE_TIME_OUT= | |||||
| BATCH_UPLOAD_LIMIT=10 | BATCH_UPLOAD_LIMIT=10 | ||||
| KEYWORD_DATA_SOURCE_TYPE=database | KEYWORD_DATA_SOURCE_TYPE=database |
| default=None, | default=None, | ||||
| ) | ) | ||||
| SSRF_DEFAULT_TIME_OUT: PositiveFloat = Field( | |||||
| description="The default timeout period used for network requests (SSRF)", | |||||
| default=5, | |||||
| ) | |||||
| SSRF_DEFAULT_CONNECT_TIME_OUT: PositiveFloat = Field( | |||||
| description="The default connect timeout period used for network requests (SSRF)", | |||||
| default=5, | |||||
| ) | |||||
| SSRF_DEFAULT_READ_TIME_OUT: PositiveFloat = Field( | |||||
| description="The default read timeout period used for network requests (SSRF)", | |||||
| default=5, | |||||
| ) | |||||
| SSRF_DEFAULT_WRITE_TIME_OUT: PositiveFloat = Field( | |||||
| description="The default write timeout period used for network requests (SSRF)", | |||||
| default=5, | |||||
| ) | |||||
| RESPECT_XFORWARD_HEADERS_ENABLED: bool = Field( | RESPECT_XFORWARD_HEADERS_ENABLED: bool = Field( | ||||
| description="Enable or disable the X-Forwarded-For Proxy Fix middleware from Werkzeug" | description="Enable or disable the X-Forwarded-For Proxy Fix middleware from Werkzeug" | ||||
| " to respect X-* headers to redirect clients", | " to respect X-* headers to redirect clients", |
| SSRF_PROXY_HTTP_URL = os.getenv("SSRF_PROXY_HTTP_URL", "") | SSRF_PROXY_HTTP_URL = os.getenv("SSRF_PROXY_HTTP_URL", "") | ||||
| SSRF_PROXY_HTTPS_URL = os.getenv("SSRF_PROXY_HTTPS_URL", "") | SSRF_PROXY_HTTPS_URL = os.getenv("SSRF_PROXY_HTTPS_URL", "") | ||||
| SSRF_DEFAULT_MAX_RETRIES = int(os.getenv("SSRF_DEFAULT_MAX_RETRIES", "3")) | SSRF_DEFAULT_MAX_RETRIES = int(os.getenv("SSRF_DEFAULT_MAX_RETRIES", "3")) | ||||
| SSRF_DEFAULT_TIME_OUT = float(os.getenv("SSRF_DEFAULT_TIME_OUT", "5")) | |||||
| SSRF_DEFAULT_CONNECT_TIME_OUT = float(os.getenv("SSRF_DEFAULT_CONNECT_TIME_OUT", "5")) | |||||
| SSRF_DEFAULT_READ_TIME_OUT = float(os.getenv("SSRF_DEFAULT_READ_TIME_OUT", "5")) | |||||
| SSRF_DEFAULT_WRITE_TIME_OUT = float(os.getenv("SSRF_DEFAULT_WRITE_TIME_OUT", "5")) | |||||
| proxy_mounts = ( | proxy_mounts = ( | ||||
| { | { | ||||
| if "follow_redirects" not in kwargs: | if "follow_redirects" not in kwargs: | ||||
| kwargs["follow_redirects"] = allow_redirects | kwargs["follow_redirects"] = allow_redirects | ||||
| if "timeout" not in kwargs: | |||||
| kwargs["timeout"] = httpx.Timeout( | |||||
| SSRF_DEFAULT_TIME_OUT, | |||||
| connect=SSRF_DEFAULT_CONNECT_TIME_OUT, | |||||
| read=SSRF_DEFAULT_READ_TIME_OUT, | |||||
| write=SSRF_DEFAULT_WRITE_TIME_OUT, | |||||
| ) | |||||
| retries = 0 | retries = 0 | ||||
| while retries <= max_retries: | while retries <= max_retries: | ||||
| try: | try: |