Signed-off-by: -LAN- <laipz8200@outlook.com>tags/0.15.6-alpha.1
| @@ -25,7 +25,8 @@ class GPT2Tokenizer: | |||
| # future = _executor.submit(GPT2Tokenizer._get_num_tokens_by_gpt2, text) | |||
| # result = future.result() | |||
| # return cast(int, result) | |||
| return GPT2Tokenizer._get_num_tokens_by_gpt2(text) | |||
| # return GPT2Tokenizer._get_num_tokens_by_gpt2(text) | |||
| return 0 | |||
| @staticmethod | |||
| def get_encoder() -> Any: | |||