|
|
|
@@ -6,6 +6,7 @@ |
|
|
|
# dependencies = [ |
|
|
|
# "huggingface-hub", |
|
|
|
# "nltk", |
|
|
|
# "argparse", |
|
|
|
# ] |
|
|
|
# /// |
|
|
|
|
|
|
|
@@ -13,16 +14,29 @@ from huggingface_hub import snapshot_download |
|
|
|
import nltk |
|
|
|
import os |
|
|
|
import urllib.request |
|
|
|
import argparse |
|
|
|
|
|
|
|
urls = [ |
|
|
|
"http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", |
|
|
|
"http://ports.ubuntu.com/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", |
|
|
|
"https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar", |
|
|
|
"https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar.md5", |
|
|
|
"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", |
|
|
|
"https://bit.ly/chrome-linux64-121-0-6167-85", |
|
|
|
"https://bit.ly/chromedriver-linux64-121-0-6167-85", |
|
|
|
] |
|
|
|
def get_urls(use_china_mirrors=False): |
|
|
|
if use_china_mirrors: |
|
|
|
return [ |
|
|
|
"http://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", |
|
|
|
"http://mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", |
|
|
|
"https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar", |
|
|
|
"https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar.md5", |
|
|
|
"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", |
|
|
|
"https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chrome-linux64.zip", |
|
|
|
"https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip", |
|
|
|
] |
|
|
|
else: |
|
|
|
return [ |
|
|
|
"http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", |
|
|
|
"http://ports.ubuntu.com/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", |
|
|
|
"https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar", |
|
|
|
"https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.0.0/tika-server-standard-3.0.0.jar.md5", |
|
|
|
"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", |
|
|
|
"https://bit.ly/chrome-linux64-121-0-6167-85", |
|
|
|
"https://bit.ly/chromedriver-linux64-121-0-6167-85", |
|
|
|
] |
|
|
|
|
|
|
|
repos = [ |
|
|
|
"InfiniFlow/text_concat_xgb_v1.0", |
|
|
|
@@ -39,6 +53,12 @@ def download_model(repo_id): |
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
|
parser = argparse.ArgumentParser(description='Download dependencies with optional China mirror support') |
|
|
|
parser.add_argument('--china-mirrors', action='store_true', help='Use China-accessible mirrors for downloads') |
|
|
|
args = parser.parse_args() |
|
|
|
|
|
|
|
urls = get_urls(args.china_mirrors) |
|
|
|
|
|
|
|
for url in urls: |
|
|
|
filename = url.split("/")[-1] |
|
|
|
print(f"Downloading {url}...") |
|
|
|
@@ -52,4 +72,4 @@ if __name__ == "__main__": |
|
|
|
|
|
|
|
for repo_id in repos: |
|
|
|
print(f"Downloading huggingface repo {repo_id}...") |
|
|
|
download_model(repo_id) |
|
|
|
download_model(repo_id) |