|  |  | @@ -6,6 +6,7 @@ | 
		
	
		
			
			|  |  |  | # dependencies = [ | 
		
	
		
			
			|  |  |  | #   "huggingface-hub", | 
		
	
		
			
			|  |  |  | #   "nltk", | 
		
	
		
			
			|  |  |  | #   "argparse", | 
		
	
		
			
			|  |  |  | # ] | 
		
	
		
			
			|  |  |  | # /// | 
		
	
		
			
			|  |  |  | 
 | 
		
	
	
		
			
			|  |  | @@ -13,16 +14,29 @@ from huggingface_hub import snapshot_download | 
		
	
		
			
			|  |  |  | import nltk | 
		
	
		
			
			|  |  |  | import os | 
		
	
		
			
			|  |  |  | import urllib.request | 
		
	
		
			
			|  |  |  | import argparse | 
		
	
		
			
			|  |  |  | 
 | 
		
	
		
			
			|  |  |  | urls = [ | 
		
	
		
			
			|  |  |  | "http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", | 
		
	
		
			
			|  |  |  | "http://ports.ubuntu.com/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", | 
		
	
		
			
			|  |  |  | "https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar", | 
		
	
		
			
			|  |  |  | "https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar.md5", | 
		
	
		
			
			|  |  |  | "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", | 
		
	
		
			
			|  |  |  | "https://bit.ly/chrome-linux64-121-0-6167-85", | 
		
	
		
			
			|  |  |  | "https://bit.ly/chromedriver-linux64-121-0-6167-85", | 
		
	
		
			
			|  |  |  | ] | 
		
	
		
			
			|  |  |  | def get_urls(use_china_mirrors=False): | 
		
	
		
			
			|  |  |  | if use_china_mirrors: | 
		
	
		
			
			|  |  |  | return [ | 
		
	
		
			
			|  |  |  | "http://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", | 
		
	
		
			
			|  |  |  | "http://mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", | 
		
	
		
			
			|  |  |  | "https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar", | 
		
	
		
			
			|  |  |  | "https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar.md5", | 
		
	
		
			
			|  |  |  | "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", | 
		
	
		
			
			|  |  |  | "https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chrome-linux64.zip", | 
		
	
		
			
			|  |  |  | "https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip", | 
		
	
		
			
			|  |  |  | ] | 
		
	
		
			
			|  |  |  | else: | 
		
	
		
			
			|  |  |  | return [ | 
		
	
		
			
			|  |  |  | "http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", | 
		
	
		
			
			|  |  |  | "http://ports.ubuntu.com/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", | 
		
	
		
			
			|  |  |  | "https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar", | 
		
	
		
			
			|  |  |  | "https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar.md5", | 
		
	
		
			
			|  |  |  | "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", | 
		
	
		
			
			|  |  |  | "https://bit.ly/chrome-linux64-121-0-6167-85", | 
		
	
		
			
			|  |  |  | "https://bit.ly/chromedriver-linux64-121-0-6167-85", | 
		
	
		
			
			|  |  |  | ] | 
		
	
		
			
			|  |  |  | 
 | 
		
	
		
			
			|  |  |  | repos = [ | 
		
	
		
			
			|  |  |  | "InfiniFlow/text_concat_xgb_v1.0", | 
		
	
	
		
			
			|  |  | @@ -39,6 +53,12 @@ def download_model(repo_id): | 
		
	
		
			
			|  |  |  | 
 | 
		
	
		
			
			|  |  |  | 
 | 
		
	
		
			
			|  |  |  | if __name__ == "__main__": | 
		
	
		
			
			|  |  |  | parser = argparse.ArgumentParser(description='Download dependencies with optional China mirror support') | 
		
	
		
			
			|  |  |  | parser.add_argument('--china-mirrors', action='store_true', help='Use China-accessible mirrors for downloads') | 
		
	
		
			
			|  |  |  | args = parser.parse_args() | 
		
	
		
			
			|  |  |  |  | 
		
	
		
			
			|  |  |  | urls = get_urls(args.china_mirrors) | 
		
	
		
			
			|  |  |  |  | 
		
	
		
			
			|  |  |  | for url in urls: | 
		
	
		
			
			|  |  |  | filename = url.split("/")[-1] | 
		
	
		
			
			|  |  |  | print(f"Downloading {url}...") | 
		
	
	
		
			
			|  |  | @@ -52,4 +72,4 @@ if __name__ == "__main__": | 
		
	
		
			
			|  |  |  | 
 | 
		
	
		
			
			|  |  |  | for repo_id in repos: | 
		
	
		
			
			|  |  |  | print(f"Downloading huggingface repo {repo_id}...") | 
		
	
		
			
			|  |  |  | download_model(repo_id) | 
		
	
		
			
			|  |  |  | download_model(repo_id) |