""" vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES slow_tokenizer_class = HerbertTokenizer def _init_ ( self, vocab_file = None, merges_file = None, tokenizer_file = None, cls_token = "", unk_token = "", pad_token = "", mask_token = "", sep_token = "", ** kwargs ): super (). merges_file (:obj:`str`): Path to the merges file. Args: vocab_file (:obj:`str`): Path to the vocabulary file. Users should refer to the superclass for more information regarding methods. This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the methods. Each occurrence of a punctuation character will be treated separately.
Peculiarities: - uses BERT's pre-tokenizer: BertPreTokenizer splits tokens on spaces, and also on punctuation. class HerbertTokenizerFast ( PreTrainedTokenizerFast ): """ Construct a "Fast" BPE tokenizer for HerBERT (backed by HuggingFace's `tokenizers` library). get_logger ( _name_ ) VOCAB_FILES_NAMES = tokenization_herbert import HerbertTokenizer logger = logging. tokenization_utils_fast import PreTrainedTokenizerFast from. from typing import List, Optional, Tuple from. # See the License for the specific language governing permissions and # limitations under the License. # You may obtain a copy of the License at # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # Licensed under the Apache License, Version 2.0 (the "License") # you may not use this file except in compliance with the License. # coding=utf-8 # Copyright 2020 The Google AI Language Team Authors,, Facebook Inc. Performance and Scalability: How To Fit a Bigger Model and Train It Faster.
0 kommentar(er)
