| @@ -1,6 +1,9 @@ | |||
| import logging | |||
| from threading import Lock | |||
| from typing import Any | |||
| logger = logging.getLogger(__name__) | |||
| _tokenizer: Any = None | |||
| _lock = Lock() | |||
| @@ -43,5 +46,6 @@ class GPT2Tokenizer: | |||
| base_path = abspath(__file__) | |||
| gpt2_tokenizer_path = join(dirname(base_path), "gpt2") | |||
| _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path) | |||
| logger.info("Fallback to Transformers' GPT-2 tokenizer from tiktoken") | |||
| return _tokenizer | |||