Kimirock/PaddleNLP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add unittest for bert modeling and tokenizer (#2624)

Browse Source 
* add unittest for bert modeling

* add unittest for bert modeling

* update

* update bert tokenizer unttest

* update bert tokenizer unttest

* update

* fix relative import

* add set_input_embeddings

* update

* update

* fix vocab bug

* delete print statements

* fix get_offset_mapping bug
This commit is contained in:
yingyibiao
2022-07-21 10:35:55 +08:00
committed by GitHub
parent 5f2115e4c5
commit 8e2f1dced5
13 changed files with 3711 additions and 831 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
paddlenlp/data/vocab.py
View File

@@ -264,9 +264,14 @@ class Vocab(object):
def __getitem__(self, tokens):
if not isinstance(tokens, (list, tuple)):
return self._token_to_idx[tokens]
return self._token_to_idx[
tokens] if tokens in self._token_to_idx else self._token_to_idx[
self.unk_token]
else:
return [self._token_to_idx[token] for token in tokens]
return [
self._token_to_idx[token] if token in self._token_to_idx else
self._token_to_idx[self.unk_token] for token in tokens
]
def __len__(self):
return len(self._idx_to_token)

12
paddlenlp/transformers/bert/modeling.py
View File

@@ -445,9 +445,10 @@ class BertModel(BertPretrainedModel):
and refer to the Paddle documentation for all matter related to general usage and behavior.
Args:
vocab_size (int):
vocab_size (int, optional):
Vocabulary size of `inputs_ids` in `BertModel`. Also is the vocab size of token embedding matrix.
Defines the number of different tokens that can be represented by the `inputs_ids` passed when calling `BertModel`.
Defaults to `30522`.
hidden_size (int, optional):
Dimensionality of the embedding layer, encoder layer and pooler layer. Defaults to `768`.
num_hidden_layers (int, optional):
@@ -476,7 +477,6 @@ class BertModel(BertPretrainedModel):
type_vocab_size (int, optional):
The vocabulary size of `token_type_ids`.
Defaults to `16`.
initializer_range (float, optional):
The standard deviation of the normal initializer.
Defaults to 0.02.
@@ -496,7 +496,7 @@ class BertModel(BertPretrainedModel):
"""
def __init__(self,
vocab_size,
vocab_size=30522,
hidden_size=768,
num_hidden_layers=12,
num_attention_heads=12,
@@ -548,6 +548,12 @@ class BertModel(BertPretrainedModel):
self.pooler = BertPooler(hidden_size, pool_act)
self.apply(self.init_weights)
def get_input_embeddings(self):
return self.embeddings.word_embeddings
def set_input_embeddings(self, value):
self.embeddings.word_embeddings = value
def forward(self,
input_ids,
token_type_ids=None,

135
paddlenlp/transformers/bert/tokenizer.py
View File

@@ -13,11 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import os
import collections
import unicodedata
from .. import PretrainedTokenizer, AddedToken
from ..tokenizer_utils import PretrainedTokenizer
from ..tokenizer_utils import convert_to_unicode, whitespace_tokenize, _is_whitespace, _is_control, _is_punctuation
__all__ = [
@@ -33,22 +33,38 @@ class BasicTokenizer(object):
Args:
do_lower_case (bool):
Whether or not to lowercase the input when tokenizing.
Whether to lowercase the input when tokenizing.
Defaults to `True`.
never_split (Iterable):
Collection of tokens which will never be split during tokenization. Only has an effect when
`do_basic_tokenize=True`
tokenize_chinese_chars (bool):
Whether to tokenize Chinese characters.
strip_accents: (bool):
Whether to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
"""
def __init__(self, do_lower_case=True):
def __init__(self,
do_lower_case=True,
never_split=None,
tokenize_chinese_chars=True,
strip_accents=None):
"""Constructs a BasicTokenizer."""
if never_split is None:
never_split = []
self.do_lower_case = do_lower_case
self.never_split = set(never_split)
self.tokenize_chinese_chars = tokenize_chinese_chars
self.strip_accents = strip_accents
def tokenize(self, text):
def tokenize(self, text, never_split=None):
"""
Tokenizes a piece of text using basic tokenizer.
Args:
text (str): A piece of text.
never_split (List[str]): List of token not to split.
Returns:
list(str): A list of tokens.
@@ -62,22 +78,25 @@ class BasicTokenizer(object):
'''
['he', 'was', 'a', 'puppeteer']
'''
"""
text = convert_to_unicode(text)
never_split = self.never_split.union(
set(never_split)) if never_split else self.never_split
text = self._clean_text(text)
text = self._tokenize_chinese_chars(text)
if self.tokenize_chinese_chars:
text = self._tokenize_chinese_chars(text)
orig_tokens = whitespace_tokenize(text)
split_tokens = []
for token in orig_tokens:
if self.do_lower_case:
token = token.lower()
token = self._run_strip_accents(token)
else:
token = self._run_strip_accents(token)
split_tokens.extend(self._run_split_on_punc(token))
if token not in never_split:
if self.do_lower_case:
token = token.lower()
if self.strip_accents is not False:
token = self._run_strip_accents(token)
elif self.strip_accents:
token = self._run_strip_accents(token)
split_tokens.extend(self._run_split_on_punc(token, never_split))
output_tokens = whitespace_tokenize(" ".join(split_tokens))
return output_tokens
@@ -95,10 +114,12 @@ class BasicTokenizer(object):
output.append(char)
return "".join(output)
def _run_split_on_punc(self, text):
def _run_split_on_punc(self, text, never_split=None):
"""
Splits punctuation on a piece of text.
"""
if never_split is not None and text in never_split:
return [text]
chars = list(text)
i = 0
start_new_word = True
@@ -136,7 +157,6 @@ class BasicTokenizer(object):
"""
Checks whether CP is the codepoint of a CJK character.
"""
# This defines a "chinese character" as anything in the CJK Unicode block:
# https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
#
@@ -144,7 +164,7 @@ class BasicTokenizer(object):
# despite its name. The modern Korean Hangul alphabet is a different block,
# as is Japanese Hiragana and Katakana. Those alphabets are used to write
# space-separated words, so they are not treated specially and handled
# like the all of the other languages.
# like the all the other languages.
if ((cp >= 0x4E00 and cp <= 0x9FFF) or #
(cp >= 0x3400 and cp <= 0x4DBF) or #
(cp >= 0x20000 and cp <= 0x2A6DF) or #
@@ -221,7 +241,6 @@ class WordpieceTokenizer(object):
'''
["un", "##aff", "##able"]
'''
"""
output_tokens = []
@@ -268,26 +287,39 @@ class BertTokenizer(PretrainedTokenizer):
vocab_file (str):
The vocabulary file path (ends with '.txt') required to instantiate
a `WordpieceTokenizer`.
do_lower_case (bool):
Whether or not to lowercase the input when tokenizing.
Defaults to`True`.
unk_token (str):
do_lower_case (bool, optional):
Whether to lowercase the input when tokenizing.
Defaults to `True`.
do_basic_tokenize (bool, optional):
Whether to use a basic tokenizer before a WordPiece tokenizer.
Defaults to `True`.
never_split (Iterable, optional):
Collection of tokens which will never be split during tokenization. Only has an effect when
`do_basic_tokenize=True`. Defaults to `None`.
unk_token (str, optional):
A special token representing the *unknown (out-of-vocabulary)* token.
An unknown token is set to be `unk_token` inorder to be converted to an ID.
Defaults to "[UNK]".
sep_token (str):
sep_token (str, optional):
A special token separating two different sentences in the same input.
Defaults to "[SEP]".
pad_token (str):
pad_token (str, optional):
A special token used to make arrays of tokens the same size for batching purposes.
Defaults to "[PAD]".
cls_token (str):
cls_token (str, optional):
A special token used for sequence classification. It is the last token
of the sequence when built with special tokens. Defaults to "[CLS]".
mask_token (str):
mask_token (str, optional):
A special token representing a masked token. This is the token used
in the masked language modeling task which the model tries to predict the original unmasked ones.
Defaults to "[MASK]".
tokenize_chinese_chars (bool, optional):
Whether to tokenize Chinese characters.
Defaults to `True`.
strip_accents: (bool, optional):
Whether to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
Defaults to `None`.
Examples:
.. code-block::
@@ -301,7 +333,6 @@ class BertTokenizer(PretrainedTokenizer):
'''
{'input_ids': [101, 2002, 2001, 1037, 13997, 11510, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0]}
'''
"""
resource_files_names = {"vocab_file": "vocab.txt"} # for save_pretrained
pretrained_resource_files_map = {
@@ -400,16 +431,40 @@ class BertTokenizer(PretrainedTokenizer):
"do_lower_case": True
},
}
max_model_input_sizes = {
"bert-base-uncased": 512,
"bert-large-uncased": 512,
"bert-base-cased": 512,
"bert-large-cased": 512,
"bert-base-multilingual-uncased": 512,
"bert-base-multilingual-cased": 512,
"bert-base-chinese": 512,
"bert-wwm-chinese": 512,
"bert-wwm-ext-chinese": 512,
"macbert-large-chinese": 512,
"macbert-base-chinese": 512,
"simbert-base-chinese": 512,
"uer/chinese-roberta-base": 512,
"uer/chinese-roberta-medium": 512,
"uer/chinese-roberta-6l-768h": 512,
"uer/chinese-roberta-small": 512,
"uer/chinese-roberta-mini": 512,
"uer/chinese-roberta-tiny": 512,
}
padding_side = 'right'
def __init__(self,
vocab_file,
do_lower_case=True,
do_basic_tokenize=True,
never_split=None,
unk_token="[UNK]",
sep_token="[SEP]",
pad_token="[PAD]",
cls_token="[CLS]",
mask_token="[MASK]",
tokenize_chinese_chars=True,
strip_accents=None,
**kwargs):
if not os.path.isfile(vocab_file):
@@ -420,7 +475,14 @@ class BertTokenizer(PretrainedTokenizer):
.format(vocab_file))
self.do_lower_case = do_lower_case
self.vocab = self.load_vocabulary(vocab_file, unk_token=unk_token)
self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)
self.do_basic_tokenize = do_basic_tokenize
if do_basic_tokenize:
self.basic_tokenizer = BasicTokenizer(
do_lower_case=do_lower_case,
never_split=never_split,
tokenize_chinese_chars=tokenize_chinese_chars,
strip_accents=strip_accents,
)
self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab,
unk_token=unk_token)
@@ -446,9 +508,16 @@ class BertTokenizer(PretrainedTokenizer):
list: A list of string representing converted tokens.
"""
split_tokens = []
for token in self.basic_tokenizer.tokenize(text):
for sub_token in self.wordpiece_tokenizer.tokenize(token):
split_tokens.append(sub_token)
if self.do_basic_tokenize:
for token in self.basic_tokenizer.tokenize(
text, never_split=self.all_special_tokens):
# If the token is part of the never_split set
if token in self.basic_tokenizer.never_split:
split_tokens.append(token)
else:
split_tokens += self.wordpiece_tokenizer.tokenize(token)
else:
split_tokens = self.wordpiece_tokenizer.tokenize(text)
return split_tokens
def convert_tokens_to_string(self, tokens):

9
paddlenlp/transformers/model_utils.py
View File

@@ -156,6 +156,15 @@ class PretrainedModel(Layer, GenerationMixin):
f'model of {type(base_model)} has not implemented the `get_input_embedding`'
' or `set_input_embedding` method')
def set_input_embeddings(self, value):
base_model = getattr(self, self.base_model_prefix, self)
if base_model is not self:
return base_model.set_input_embeddings(value)
else:
raise NotImplementedError(
f'model of {type(base_model)} has not implemented the `get_input_embedding`'
' or `set_input_embedding` method')
def get_output_embeddings(self):
return None # Overwrite for models with output embeddings

33
paddlenlp/transformers/tokenizer_utils.py
View File

@@ -75,7 +75,7 @@ def whitespace_tokenize(text):
"""
Runs basic whitespace cleaning and splitting on a peice of text.
Args:
text (str): Text to be tokened.
text (str): Text to be tokenized.
Returns:
list(str): Token list.
"""
@@ -546,7 +546,6 @@ class PretrainedTokenizer(PretrainedTokenizerBase):
`__init__` whose name ends with `_token`) as attributes of the tokenizer
instance.
"""
init_dict = fn_args_to_dict(original_init, *((self, ) + args), **kwargs)
init_dict.pop('self', None)
super(PretrainedTokenizer, self).__init__(**init_dict)
@@ -1321,18 +1320,33 @@ class PretrainedTokenizer(PretrainedTokenizerBase):
if text is None:
return None
split_tokens = []
for token in self.basic_tokenizer.tokenize(text):
for sub_token in self.wordpiece_tokenizer.tokenize(token):
if self.do_basic_tokenize:
for token in self.basic_tokenizer.tokenize(
text, never_split=self.all_special_tokens):
# If the token is part of the never_split set
if token in self.basic_tokenizer.never_split:
split_tokens.append(token)
else:
for sub_token in self.wordpiece_tokenizer.tokenize(token):
split_tokens.append(
sub_token if sub_token != self.unk_token else token)
else:
for sub_token in self.wordpiece_tokenizer.tokenize(text):
split_tokens.append(
sub_token if sub_token != self.unk_token else token)
sub_token if sub_token != self.unk_token else text)
normalized_text, char_mapping = '', []
for i, ch in enumerate(text):
if hasattr(self, "do_lower_case") and self.do_lower_case:
ch = ch.lower()
ch = unicodedata.normalize('NFD', ch)
ch = ''.join([c for c in ch if unicodedata.category(c) != 'Mn'])
if self.basic_tokenizer.strip_accents is not False:
ch = unicodedata.normalize('NFD', ch)
ch = ''.join(
[c for c in ch if unicodedata.category(c) != 'Mn'])
elif self.basic_tokenizer.strip_accents:
ch = unicodedata.normalize('NFD', ch)
ch = ''.join([c for c in ch if unicodedata.category(c) != 'Mn'])
ch = ''.join([
c for c in ch
@@ -1344,10 +1358,11 @@ class PretrainedTokenizer(PretrainedTokenizerBase):
text, token_mapping, offset = normalized_text, [], 0
for token in split_tokens:
if token[:2] == '##':
token = token[2:]
if token in self.all_special_tokens:
token = token.lower() if hasattr(
self, "do_lower_case") and self.do_lower_case else token
start = text[offset:].index(token) + offset
end = start + len(token)

51
paddlenlp/transformers/tokenizer_utils_base.py
View File

@@ -2008,22 +2008,23 @@ class PretrainedTokenizerBase(SpecialTokensMixin):
return padding_strategy, truncation_strategy, max_length, kwargs
def __call__(self,
text,
text_pair=None,
text: Union[str, List[str], List[List[str]]],
text_pair: Optional[Union[str, List[str],
List[List[str]]]] = None,
max_length: Optional[int] = None,
stride=0,
is_split_into_words=False,
stride: int = 0,
is_split_into_words: bool = False,
padding: Union[bool, str, PaddingStrategy] = False,
truncation: Union[bool, str, TruncationStrategy] = False,
return_position_ids=False,
return_token_type_ids=True,
return_attention_mask=False,
return_length=False,
return_overflowing_tokens=False,
return_special_tokens_mask=False,
return_dict=True,
return_offsets_mapping=False,
add_special_tokens=True,
return_position_ids: bool = False,
return_token_type_ids: bool = True,
return_attention_mask: bool = False,
return_length: bool = False,
return_overflowing_tokens: bool = False,
return_special_tokens_mask: bool = False,
return_dict: bool = True,
return_offsets_mapping: bool = False,
add_special_tokens: bool = True,
pad_to_multiple_of: Optional[int] = None,
return_tensors: Optional[Union[str, TensorType]] = None,
verbose: bool = True,
@@ -2268,22 +2269,22 @@ class PretrainedTokenizerBase(SpecialTokensMixin):
def encode(self,
text,
text_pair=None,
max_length=None,
stride: int = 0,
is_split_into_words: bool = False,
add_special_tokens=True,
padding: Union[bool, str, PaddingStrategy] = False,
truncation: Union[bool, str, TruncationStrategy] = False,
return_position_ids=False,
return_token_type_ids=True,
return_attention_mask=False,
return_length=False,
return_overflowing_tokens=False,
return_special_tokens_mask=False,
return_offsets_mapping=False,
add_special_tokens=True,
max_length: Optional[int] = None,
stride: int = 0,
is_split_into_words: bool = False,
pad_to_multiple_of: Optional[int] = None,
return_tensors: Optional[Union[str, TensorType]] = None,
return_token_type_ids: Optional[bool] = None,
return_attention_mask: Optional[bool] = None,
return_overflowing_tokens: bool = False,
return_special_tokens_mask: bool = False,
return_offsets_mapping: bool = False,
return_length: bool = False,
verbose: bool = True,
return_position_ids=False,
**kwargs) -> BatchEncoding:
"""
Tokenize and prepare for the model a sequence or a pair of sequences.
@@ -3220,7 +3221,7 @@ class PretrainedTokenizerBase(SpecialTokensMixin):
if not self.deprecation_warnings.get(
"sequence-length-is-longer-than-the-specified-maximum",
False):
warnings.warn(
logger.warning(
"Token indices sequence length is longer than the specified maximum sequence length "
f"for this model ({len(ids)} > {self.model_max_length}). Running this sequence through the model "
"will result in indexing errors")

33
tests/sample_text.txt Normal file
View File

@@ -0,0 +1,33 @@
This text is included to make sure Unicode is handled properly: 力加勝北区ᴵᴺᵀᵃছজটডণত
Text should be one-sentence-per-line, with empty lines between documents.
This sample text is public domain and was randomly selected from Project Guttenberg.
The rain had only ceased with the gray streaks of morning at Blazing Star, and the settlement awoke to a moral sense of cleanliness, and the finding of forgotten knives, tin cups, and smaller camp utensils, where the heavy showers had washed away the debris and dust heaps before the cabin doors.
Indeed, it was recorded in Blazing Star that a fortunate early riser had once picked up on the highway a solid chunk of gold quartz which the rain had freed from its incumbering soil, and washed into immediate and glittering popularity.
Possibly this may have been the reason why early risers in that locality, during the rainy season, adopted a thoughtful habit of body, and seldom lifted their eyes to the rifted or india-ink washed skies above them.
"Cass" Beard had risen early that morning, but not with a view to discovery.
A leak in his cabin roof,--quite consistent with his careless, improvident habits,--had roused him at 4 A. M., with a flooded "bunk" and wet blankets.
The chips from his wood pile refused to kindle a fire to dry his bed-clothes, and he had recourse to a more provident neighbor's to supply the deficiency.
This was nearly opposite.
Mr. Cassius crossed the highway, and stopped suddenly.
Something glittered in the nearest red pool before him.
Gold, surely!
But, wonderful to relate, not an irregular, shapeless fragment of crude ore, fresh from Nature's crucible, but a bit of jeweler's handicraft in the form of a plain gold ring.
Looking at it more attentively, he saw that it bore the inscription, "May to Cass."
Like most of his fellow gold-seekers, Cass was superstitious.
The fountain of classic wisdom, Hypatia herself.
As the ancient sage--the name is unimportant to a monk--pumped water nightly that he might study by day, so I, the guardian of cloaks and parasols, at the sacred doors of her lecture-room, imbibe celestial knowledge.
From my youth I felt in me a soul above the matter-entangled herd.
She revealed to me the glorious fact, that I am a spark of Divinity itself.
A fallen star, I am, sir!' continued he, pensively, stroking his lean stomach--'a fallen star!--fallen, if the dignity of philosophy will allow of the simile, among the hogs of the lower world--indeed, even into the hog-bucket itself. Well, after all, I will show you the way to the Archbishop's.
There is a philosophic pleasure in opening one's treasures to the modest young.
Perhaps you will assist me by carrying this basket of fruit?' And the little man jumped up, put his basket on Philammon's head, and trotted off up a neighbouring street.
Philammon followed, half contemptuous, half wondering at what this philosophy might be, which could feed the self-conceit of anything so abject as his ragged little apish guide;
but the novel roar and whirl of the street, the perpetual stream of busy faces, the line of curricles, palanquins, laden asses, camels, elephants, which met and passed him, and squeezed him up steps and into doorways, as they threaded their way through the great Moon-gate into the ample street beyond, drove everything from his mind but wondering curiosity, and a vague, helpless dread of that great living wilderness, more terrible than any dead wilderness of sand which he had left behind.
Already he longed for the repose, the silence of the Laura--for faces which knew him and smiled upon him; but it was too late to turn back now.
His guide held on for more than a mile up the great main street, crossed in the centre of the city, at right angles, by one equally magnificent, at each end of which, miles away, appeared, dim and distant over the heads of the living stream of passengers, the yellow sand-hills of the desert;
while at the end of the vista in front of them gleamed the blue harbour, through a network of countless masts.
At last they reached the quay at the opposite end of the street;
and there burst on Philammon's astonished eyes a vast semicircle of blue sea, ringed with palaces and towers.
He stopped involuntarily; and his little guide stopped also, and looked askance at the young monk, to watch the effect which that grand panorama should produce on him.

24
tests/util.py → tests/testing_utils.py
View File

@@ -15,6 +15,7 @@ import numpy as np
import unittest
import paddle
import os
import inspect
from distutils.util import strtobool
__all__ = ['get_vocab_list', 'stable_softmax', 'cross_entropy']
@@ -126,3 +127,26 @@ def slow(test):
return unittest.skip("test spends too much time")(test)
else:
return test
def get_tests_dir(append_path=None):
"""
Args:
append_path: optional path to append to the tests dir path
Return:
The full path to the `tests` dir, so that the tests can be invoked from anywhere. Optionally `append_path` is
joined after the `tests` dir the former is provided.
"""
# this function caller's __file__
caller__file__ = inspect.stack()[1][1]
tests_dir = os.path.abspath(os.path.dirname(caller__file__))
while not tests_dir.endswith("tests"):
tests_dir = os.path.dirname(tests_dir)
if append_path:
return os.path.join(tests_dir, append_path)
else:
return tests_dir

578
tests/transformers/bert/test_modeling.py
View File

@@ -1,4 +1,5 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,317 +13,318 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
import os
import unittest
import paddle
import copy
from paddlenlp.transformers import BertModel, BertForPretraining, BertPretrainingCriterion, BertForMaskedLM
from paddlenlp.transformers import BertForQuestionAnswering, BertForSequenceClassification, BertForTokenClassification, BertForMultipleChoice
from common_test import CommonTest
from util import softmax_with_cross_entropy, slow
import unittest
from paddlenlp.transformers import BertModel, BertForQuestionAnswering, BertForSequenceClassification,\
BertForTokenClassification, BertForPretraining, BertForMultipleChoice, BertForMaskedLM, BertPretrainedModel
from tests.transformers.test_modeling_common import ids_tensor, floats_tensor, random_attention_mask, ModelTesterMixin
from tests.testing_utils import slow
def create_input_data(config, seed=None):
'''
the generated input data will be same if a specified seed is set
'''
if seed is not None:
np.random.seed(seed)
class BertModelTester:
input_ids = np.random.randint(low=0,
high=config['vocab_size'],
size=(config["batch_size"],
config["seq_len"]))
num_to_predict = int(config["seq_len"] * 0.15)
masked_lm_positions = np.random.choice(
config["seq_len"], (config["batch_size"], num_to_predict),
replace=False)
masked_lm_positions = np.sort(masked_lm_positions)
pred_padding_len = config["seq_len"] - num_to_predict
temp_masked_lm_positions = np.full(masked_lm_positions.size,
0,
dtype=np.int32)
mask_token_num = 0
for i, x in enumerate(masked_lm_positions):
for j, pos in enumerate(x):
temp_masked_lm_positions[
mask_token_num] = i * config["seq_len"] + pos
mask_token_num += 1
masked_lm_positions = temp_masked_lm_positions
return input_ids, masked_lm_positions
class NpBertPretrainingCriterion(object):
def __init__(self, vocab_size):
def __init__(
self,
parent,
batch_size=13,
seq_length=7,
is_training=True,
use_input_mask=True,
use_token_type_ids=True,
vocab_size=99,
hidden_size=32,
num_hidden_layers=5,
num_attention_heads=4,
intermediate_size=37,
hidden_act="gelu",
hidden_dropout_prob=0.1,
attention_probs_dropout_prob=0.1,
max_position_embeddings=512,
type_vocab_size=16,
initializer_range=0.02,
pad_token_id=0,
pool_act="tanh",
fuse=False,
type_sequence_label_size=2,
num_labels=3,
num_choices=4,
num_classes=3,
scope=None,
):
self.parent = parent
self.batch_size = batch_size
self.seq_length = seq_length
self.is_training = is_training
self.use_input_mask = use_input_mask
self.use_token_type_ids = use_token_type_ids
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.intermediate_size = intermediate_size
self.hidden_act = hidden_act
self.hidden_dropout_prob = hidden_dropout_prob
self.attention_probs_dropout_prob = attention_probs_dropout_prob
self.max_position_embeddings = max_position_embeddings
self.type_vocab_size = type_vocab_size
self.initializer_range = initializer_range
self.pad_token_id = pad_token_id
self.pool_act = pool_act
self.fuse = fuse
self.type_sequence_label_size = type_sequence_label_size
self.num_classes = num_classes
self.num_labels = num_labels
self.num_choices = num_choices
self.scope = scope
def __call__(self, prediction_scores, seq_relationship_score,
masked_lm_labels, next_sentence_labels, masked_lm_scale):
masked_lm_loss = softmax_with_cross_entropy(prediction_scores,
masked_lm_labels,
ignore_index=-1)
masked_lm_loss = masked_lm_loss / masked_lm_scale
next_sentence_loss = softmax_with_cross_entropy(seq_relationship_score,
next_sentence_labels)
return np.sum(masked_lm_loss) + np.mean(next_sentence_loss)
def prepare_config_and_inputs(self):
input_ids = ids_tensor([self.batch_size, self.seq_length],
self.vocab_size)
input_mask = None
if self.use_input_mask:
input_mask = random_attention_mask(
[self.batch_size, self.seq_length])
token_type_ids = None
if self.use_token_type_ids:
token_type_ids = ids_tensor([self.batch_size, self.seq_length],
self.type_vocab_size)
config = self.get_config()
return config, input_ids, token_type_ids, input_mask
def get_config(self):
return {
"vocab_size": self.vocab_size,
"hidden_size": self.hidden_size,
"num_hidden_layers": self.num_hidden_layers,
"num_attention_heads": self.num_attention_heads,
"intermediate_size": self.intermediate_size,
"hidden_act": self.hidden_act,
"hidden_dropout_prob": self.hidden_dropout_prob,
"attention_probs_dropout_prob": self.attention_probs_dropout_prob,
"max_position_embeddings": self.max_position_embeddings,
"type_vocab_size": self.type_vocab_size,
"initializer_range": self.initializer_range,
"pad_token_id": self.pad_token_id,
"pool_act": self.pool_act,
"fuse": self.fuse,
}
def create_and_check_model(
self,
config,
input_ids,
token_type_ids,
input_mask,
):
model = BertModel(**config)
model.eval()
result = model(input_ids,
attention_mask=input_mask,
token_type_ids=token_type_ids)
result = model(input_ids, token_type_ids=token_type_ids)
result = model(input_ids)
self.parent.assertEqual(
result[0].shape,
[self.batch_size, self.seq_length, self.hidden_size])
self.parent.assertEqual(result[1].shape,
[self.batch_size, self.hidden_size])
def create_and_check_for_masked_lm(
self,
config,
input_ids,
token_type_ids,
input_mask,
):
model = BertForMaskedLM(BertModel(**config))
model.eval()
result = model(input_ids,
attention_mask=input_mask,
token_type_ids=token_type_ids)
self.parent.assertEqual(
result.shape, [self.batch_size, self.seq_length, self.vocab_size])
def create_and_check_for_multiple_choice(
self,
config,
input_ids,
token_type_ids,
input_mask,
):
model = BertForMultipleChoice(BertModel(**config),
num_choices=self.num_choices)
model.eval()
multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(
[-1, self.num_choices, -1])
multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(
[-1, self.num_choices, -1])
multiple_choice_input_mask = input_mask.unsqueeze(1).expand(
[-1, self.num_choices, -1])
result = model(
multiple_choice_inputs_ids,
attention_mask=multiple_choice_input_mask,
token_type_ids=multiple_choice_token_type_ids,
)
self.parent.assertEqual(result.shape,
[self.batch_size, self.num_choices])
def create_and_check_for_question_answering(self, config, input_ids,
token_type_ids, input_mask):
model = BertForQuestionAnswering(BertModel(**config))
model.eval()
result = model(
input_ids,
attention_mask=input_mask,
token_type_ids=token_type_ids,
)
self.parent.assertEqual(result[0].shape,
[self.batch_size, self.seq_length])
self.parent.assertEqual(result[1].shape,
[self.batch_size, self.seq_length])
def create_and_check_for_sequence_classification(
self,
config,
input_ids,
token_type_ids,
input_mask,
):
model = BertForSequenceClassification(BertModel(**config),
num_classes=self.num_classes)
model.eval()
result = model(
input_ids,
attention_mask=input_mask,
token_type_ids=token_type_ids,
)
self.parent.assertEqual(result.shape,
[self.batch_size, self.num_classes])
def create_and_check_for_token_classification(
self,
config,
input_ids,
token_type_ids,
input_mask,
):
model = BertForTokenClassification(BertModel(**config),
num_classes=self.num_classes)
model.eval()
result = model(input_ids,
attention_mask=input_mask,
token_type_ids=token_type_ids)
self.parent.assertEqual(
result.shape, [self.batch_size, self.seq_length, self.num_classes])
def prepare_config_and_inputs_for_common(self):
config_and_inputs = self.prepare_config_and_inputs()
(
config,
input_ids,
token_type_ids,
input_mask,
) = config_and_inputs
inputs_dict = {
"input_ids": input_ids,
"token_type_ids": token_type_ids,
"attention_mask": input_mask
}
return config, inputs_dict
class TestBertForSequenceClassification(CommonTest):
class BertModelTest(ModelTesterMixin, unittest.TestCase):
base_model_class = BertModel
def set_input(self):
self.config = copy.deepcopy(
BertModel.pretrained_init_configuration['bert-base-uncased'])
self.config['num_hidden_layers'] = 2
self.config['vocab_size'] = 512
self.config['attention_probs_dropout_prob'] = 0.0
self.config['hidden_dropout_prob'] = 0.0
self.config['intermediate_size'] = 1024
self.config['seq_len'] = 64
self.config['batch_size'] = 3
self.config['max_position_embeddings'] = 512
self.input_ids, self.masked_lm_positions = create_input_data(
self.config)
def set_output(self):
self.expected_shape = (self.config['batch_size'], 2)
def set_model_class(self):
self.TEST_MODEL_CLASS = BertForSequenceClassification
all_model_classes = (
BertModel,
BertForMaskedLM,
BertForMultipleChoice,
BertForPretraining,
BertForQuestionAnswering,
BertForSequenceClassification,
BertForTokenClassification,
)
def setUp(self):
self.set_model_class()
self.set_input()
self.set_output()
self.model_tester = BertModelTester(self)
def check_testcase(self):
self.check_output_equal(self.output.numpy().shape, self.expected_shape)
def test_model(self):
config_and_inputs = self.model_tester.prepare_config_and_inputs()
self.model_tester.create_and_check_model(*config_and_inputs)
def test_forward(self):
config = copy.deepcopy(self.config)
del config['batch_size']
del config['seq_len']
def test_for_masked_lm(self):
config_and_inputs = self.model_tester.prepare_config_and_inputs()
self.model_tester.create_and_check_for_masked_lm(*config_and_inputs)
bert = BertModel(**config)
model = self.TEST_MODEL_CLASS(bert)
input_ids = paddle.to_tensor(self.input_ids, dtype="int64")
self.output = model(input_ids)
self.check_testcase()
def test_for_multiple_choice(self):
config_and_inputs = self.model_tester.prepare_config_and_inputs()
self.model_tester.create_and_check_for_multiple_choice(
*config_and_inputs)
def test_for_question_answering(self):
config_and_inputs = self.model_tester.prepare_config_and_inputs()
self.model_tester.create_and_check_for_question_answering(
*config_and_inputs)
class TestBertForTokenClassification(TestBertForSequenceClassification):
def test_for_sequence_classification(self):
config_and_inputs = self.model_tester.prepare_config_and_inputs()
self.model_tester.create_and_check_for_sequence_classification(
*config_and_inputs)
def set_model_class(self):
self.TEST_MODEL_CLASS = BertForTokenClassification
def set_output(self):
self.expected_shape = (self.config['batch_size'],
self.config['seq_len'], 2)
class TestBertForPretraining(TestBertForSequenceClassification):
def set_model_class(self):
self.TEST_MODEL_CLASS = BertForPretraining
def set_output(self):
self.expected_seq_shape = (self.masked_lm_positions.shape[0],
self.config['vocab_size'])
self.expected_pooled_shape = (self.config['batch_size'], 2)
def test_forward(self):
config = copy.deepcopy(self.config)
del config['batch_size']
del config['seq_len']
bert = BertModel(**config)
model = self.TEST_MODEL_CLASS(bert)
input_ids = paddle.to_tensor(self.input_ids, dtype="int64")
masked_lm_positions = paddle.to_tensor(self.masked_lm_positions,
dtype="int64")
self.output = model(input_ids, masked_positions=masked_lm_positions)
self.check_testcase()
def check_testcase(self):
self.check_output_equal(self.output[0].numpy().shape,
self.expected_seq_shape)
self.check_output_equal(self.output[1].numpy().shape,
self.expected_pooled_shape)
class TestBertForMaskedLM(TestBertForSequenceClassification):
def set_model_class(self):
self.TEST_MODEL_CLASS = BertForMaskedLM
def set_output(self):
self.expected_seq_shape = (self.config['batch_size'],
self.config['seq_len'],
self.config['vocab_size'])
def test_forward(self):
config = copy.deepcopy(self.config)
del config['batch_size']
del config['seq_len']
bert = BertModel(**config)
model = self.TEST_MODEL_CLASS(bert)
input_ids = paddle.to_tensor(self.input_ids, dtype="int64")
self.output = model(input_ids)
self.check_testcase()
def check_testcase(self):
self.check_output_equal(self.output.numpy().shape,
self.expected_seq_shape)
class TestBertForQuestionAnswering(TestBertForSequenceClassification):
def set_model_class(self):
self.TEST_MODEL_CLASS = BertForQuestionAnswering
def set_output(self):
self.expected_start_logit_shape = (self.config['batch_size'],
self.config['seq_len'])
self.expected_end_logit_shape = (self.config['batch_size'],
self.config['seq_len'])
def check_testcase(self):
self.check_output_equal(self.output[0].numpy().shape,
self.expected_start_logit_shape)
self.check_output_equal(self.output[1].numpy().shape,
self.expected_end_logit_shape)
class TestBertForMultipleChoice(TestBertForSequenceClassification):
def set_input(self):
self.config = copy.deepcopy(
BertModel.pretrained_init_configuration['bert-base-uncased'])
self.config['num_hidden_layers'] = 2
self.config['vocab_size'] = 512
self.config['attention_probs_dropout_prob'] = 0.0
self.config['hidden_dropout_prob'] = 0.0
self.config['intermediate_size'] = 1024
self.config['seq_len'] = 64
self.config['batch_size'] = 4
self.config['num_choices'] = 2
self.config['max_position_embeddings'] = 512
self.input_ids, _ = create_input_data(self.config)
# [bs*num_choice,seq_l] -> [bs,num_choice,seq_l]
self.input_ids = np.reshape(self.input_ids, [
self.config['batch_size'] // self.config['num_choices'],
self.config['num_choices'], -1
])
def set_model_class(self):
self.TEST_MODEL_CLASS = BertForMultipleChoice
def set_output(self):
self.expected_logit_shape = (self.config['batch_size'] //
self.config['num_choices'],
self.config['num_choices'])
def check_testcase(self):
self.check_output_equal(self.output.numpy().shape,
self.expected_logit_shape)
def test_forward(self):
config = copy.deepcopy(self.config)
del config["num_choices"]
del config['batch_size']
del config['seq_len']
bert = BertModel(**config)
model = self.TEST_MODEL_CLASS(bert)
input_ids = paddle.to_tensor(self.input_ids, dtype="int64")
self.output = model(input_ids)
self.check_testcase()
class TestBertPretrainingCriterion(CommonTest):
def setUp(self):
self.config['vocab_size'] = 1024
self.criterion = BertPretrainingCriterion(**self.config)
self.np_criterion = NpBertPretrainingCriterion(**self.config)
def _construct_input_data(self, mask_num, vocab_size, batch_size):
prediction_scores = np.random.rand(mask_num, vocab_size).astype(
paddle.get_default_dtype())
seq_relationship_score = np.random.rand(batch_size, 2).astype(
paddle.get_default_dtype())
masked_lm_labels = np.random.randint(0, vocab_size, (mask_num, 1))
next_sentence_labels = np.random.randint(0, 2, (batch_size, 1))
masked_lm_scale = 1.0
masked_lm_weights = np.random.randint(0, 2, (mask_num)).astype(
paddle.get_default_dtype())
return prediction_scores, seq_relationship_score, masked_lm_labels, \
next_sentence_labels, masked_lm_scale, masked_lm_weights
def test_forward(self):
np_prediction_score, np_seq_relationship_score, np_masked_lm_labels, \
np_next_sentence_labels, masked_lm_scale, np_masked_lm_weights \
= self._construct_input_data(20, self.config['vocab_size'], 4)
prediction_score = paddle.to_tensor(np_prediction_score)
seq_relationship_score = paddle.to_tensor(np_seq_relationship_score)
masked_lm_labels = paddle.to_tensor(np_masked_lm_labels, dtype="int64")
next_sentence_labels = paddle.to_tensor(np_next_sentence_labels,
dtype="int64")
masked_lm_weights = paddle.to_tensor(np_masked_lm_weights)
np_loss = self.np_criterion(np_prediction_score,
np_seq_relationship_score,
np_masked_lm_labels,
np_next_sentence_labels, masked_lm_scale)
loss = self.criterion(prediction_score, seq_relationship_score,
masked_lm_labels, next_sentence_labels,
masked_lm_scale)
self.check_output_equal(np_loss, loss.numpy()[0])
class TestBertFromPretrain(CommonTest):
def test_for_token_classification(self):
config_and_inputs = self.model_tester.prepare_config_and_inputs()
self.model_tester.create_and_check_for_token_classification(
*config_and_inputs)
@slow
def test_bert_base_uncased(self):
model = BertModel.from_pretrained('bert-base-uncased',
attention_probs_dropout_prob=0.0,
hidden_dropout_prob=0.0)
self.config = copy.deepcopy(model.config)
self.config['seq_len'] = 32
self.config['batch_size'] = 3
def test_model_from_pretrained(self):
for model_name in list(
BertPretrainedModel.pretrained_init_configuration)[:1]:
model = BertModel.from_pretrained(model_name)
self.assertIsNotNone(model)
input_ids, _ = create_input_data(self.config, 102)
input_ids = paddle.to_tensor(input_ids)
output = model(input_ids)
expected_seq_shape = (self.config['batch_size'], self.config['seq_len'],
self.config['hidden_size'])
expected_pooled_shape = (self.config['batch_size'],
self.config['hidden_size'])
self.check_output_equal(output[0].numpy().shape, expected_seq_shape)
self.check_output_equal(output[1].numpy().shape, expected_pooled_shape)
expected_seq_slice = np.array([[0.17383946, 0.09206937, 0.45788339],
[-0.28287640, 0.06244858, 0.54864359],
[-0.54589444, 0.04811822, 0.50559914]])
# There's output diff about 1e-6 between cpu and gpu
self.check_output_equal(output[0].numpy()[0, 0:3, 0:3],
expected_seq_slice,
atol=1e-6)
class BertModelIntegrationTest(unittest.TestCase):
expected_pooled_slice = np.array(
[[-0.67418981, -0.07148759, 0.85799801],
[-0.62072051, -0.08452632, 0.96691507],
[-0.74019802, -0.10187808, 0.95353240]])
self.check_output_equal(output[1].numpy()[0:3, 0:3],
expected_pooled_slice,
atol=1e-6)
@slow
def test_inference_no_attention(self):
model = BertModel.from_pretrained("bert-base-uncased")
model.eval()
input_ids = paddle.to_tensor(
[[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
attention_mask = paddle.to_tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
with paddle.no_grad():
output = model(input_ids, attention_mask=attention_mask)[0]
expected_shape = [1, 11, 768]
self.assertEqual(output.shape, expected_shape)
expected_slice = paddle.to_tensor([[[0.4249, 0.1008, 0.7531],
[0.3771, 0.1188, 0.7467],
[0.4152, 0.1098, 0.7108]]])
self.assertTrue(
paddle.allclose(output[:, 1:4, 1:4], expected_slice, atol=1e-4))
@slow
def test_inference_with_attention(self):
model = BertModel.from_pretrained("bert-base-uncased")
model.eval()
input_ids = paddle.to_tensor(
[[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
attention_mask = paddle.to_tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
with paddle.no_grad():
output = model(input_ids, attention_mask=attention_mask)[0]
expected_shape = [1, 11, 768]
self.assertEqual(output.shape, expected_shape)
expected_slice = paddle.to_tensor([[[0.4249, 0.1008, 0.7531],
[0.3771, 0.1188, 0.7467],
[0.4152, 0.1098, 0.7108]]])
self.assertTrue(
paddle.allclose(output[:, 1:4, 1:4], expected_slice, atol=1e-4))
if __name__ == "__main__":

576
tests/transformers/bert/test_tokenizer.py
View File

@@ -1,4 +1,5 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,359 +13,286 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
import os
import unittest
from paddlenlp.transformers import BertTokenizer, BasicTokenizer, WordpieceTokenizer
from paddlenlp.data import Vocab
from common_test import CpuCommonTest
from util import slow, assert_raises
import unittest
from paddlenlp.transformers.bert.tokenizer import (
BasicTokenizer,
BertTokenizer,
WordpieceTokenizer,
_is_control,
_is_punctuation,
_is_whitespace,
)
from tests.testing_utils import slow
from tests.transformers.test_tokenizer_common import TokenizerTesterMixin, filter_non_english
class TestBasicTokenizer(CpuCommonTest):
class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
def set_attr(self):
self.do_lower_case = True
def set_test_case(self):
self.text = "This is a SImple text"
self.expected_text_array = ['this', 'is', 'a', 'simple', 'text']
tokenizer_class = BertTokenizer
space_between_special_tokens = True
from_pretrained_filter = filter_non_english
test_seq2seq = True
def setUp(self):
self.set_attr()
self.set_test_case()
self.tokenizer = BasicTokenizer(self.do_lower_case)
super().setUp()
def test_tokenize(self):
text_array = self.tokenizer.tokenize(self.text)
self.check_output_equal(text_array, self.expected_text_array)
class TestBasicTokenizerChinese(TestBasicTokenizer):
def set_test_case(self):
self.text = "这是个Simple的文本。"
self.expected_text_array = ['这', '是', '个', 'simple', '的', '文', '本', '。']
class TestBasicTokenizerCased(TestBasicTokenizer):
def set_attr(self):
self.do_lower_case = False
def set_test_case(self):
self.text = "This is a SImple text"
self.expected_text_array = ['This', 'is', 'a', 'SImple', 'text']
class TestBasicTokenizerChineseCased(TestBasicTokenizerCased):
def set_test_case(self):
self.text = "这是个Simple的文本。"
self.expected_text_array = ['这', '是', '个', 'Simple', '的', '文', '本', '。']
class TestBasicTokenizerControlChar(TestBasicTokenizer):
def set_test_case(self):
self.text = "This\11 is a SImple\10 text"
self.expected_text_array = ['this', 'is', 'a', 'simple', 'text']
class TestBasicTokenizerStripAccents(TestBasicTokenizer):
def set_test_case(self):
self.text = "This is ä SImple text"
self.expected_text_array = ['this', 'is', 'a', 'simple', 'text']
class TestBasicTokenizerPunctuation(TestBasicTokenizer):
def set_test_case(self):
self.text = "This ^ is ä SImple text$\u00A0"
self.expected_text_array = [
'this', '^', 'is', 'a', 'simple', 'text', '$'
vocab_tokens = [
"[UNK]",
"[CLS]",
"[SEP]",
"[PAD]",
"[MASK]",
"want",
"##want",
"##ed",
"wa",
"un",
"runn",
"##ing",
",",
"low",
"lowest",
]
self.vocab_file = os.path.join(
self.tmpdirname, BertTokenizer.resource_files_names["vocab_file"])
with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
class TestBasicTokenizerBytes(TestBasicTokenizer):
def get_input_output_texts(self, tokenizer):
input_text = "UNwant\u00E9d,running"
output_text = "unwanted, running"
return input_text, output_text
def set_test_case(self):
self.text = "This ^ is ä SImple text$\u00A0".encode('utf-8')
self.expected_text_array = [
'this', '^', 'is', 'a', 'simple', 'text', '$'
def test_full_tokenizer(self):
tokenizer = self.tokenizer_class(self.vocab_file)
tokens = tokenizer.tokenize("UNwant\u00E9d,running")
self.assertListEqual(tokens,
["un", "##want", "##ed", ",", "runn", "##ing"])
self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens),
[9, 6, 7, 12, 10, 11])
def test_chinese(self):
tokenizer = BasicTokenizer()
self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"),
["ah", "\u535A", "\u63A8", "zz"])
def test_basic_tokenizer_lower(self):
tokenizer = BasicTokenizer(do_lower_case=True)
self.assertListEqual(tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "),
["hello", "!", "how", "are", "you", "?"])
self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"])
def test_basic_tokenizer_lower_strip_accents_false(self):
tokenizer = BasicTokenizer(do_lower_case=True, strip_accents=False)
self.assertListEqual(tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "),
["hällo", "!", "how", "are", "you", "?"])
self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["h\u00E9llo"])
def test_basic_tokenizer_lower_strip_accents_true(self):
tokenizer = BasicTokenizer(do_lower_case=True)
self.assertListEqual(tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "),
["hallo", "!", "how", "are", "you", "?"])
self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"])
def test_basic_tokenizer_lower_strip_accents_default(self):
tokenizer = BasicTokenizer(do_lower_case=True)
self.assertListEqual(tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "),
["hallo", "!", "how", "are", "you", "?"])
self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"])
def test_basic_tokenizer_no_lower(self):
tokenizer = BasicTokenizer(do_lower_case=False)
self.assertListEqual(tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "),
["HeLLo", "!", "how", "Are", "yoU", "?"])
def test_basic_tokenizer_no_lower_strip_accents_false(self):
tokenizer = BasicTokenizer(do_lower_case=False, strip_accents=False)
self.assertListEqual(tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "),
["HäLLo", "!", "how", "Are", "yoU", "?"])
def test_basic_tokenizer_no_lower_strip_accents_true(self):
tokenizer = BasicTokenizer(do_lower_case=False, strip_accents=True)
self.assertListEqual(tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "),
["HaLLo", "!", "how", "Are", "yoU", "?"])
def test_basic_tokenizer_respects_never_split_tokens(self):
tokenizer = BasicTokenizer(do_lower_case=False, never_split=["[UNK]"])
self.assertListEqual(
tokenizer.tokenize(" \tHeLLo!how \n Are yoU? [UNK]"),
["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])
def test_wordpiece_tokenizer(self):
vocab_tokens = [
"[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un",
"runn", "##ing"
]
vocab = {}
for (i, token) in enumerate(vocab_tokens):
vocab[token] = i
tokenizer = WordpieceTokenizer(vocab=vocab, unk_token="[UNK]")
class TestBasicTokenizerErrorStr(CpuCommonTest):
self.assertListEqual(tokenizer.tokenize(""), [])
@assert_raises(ValueError)
def test_tokenize(self):
self.tokenizer = BasicTokenizer()
self.text = 1
text_array = self.tokenizer.tokenize(self.text)
self.assertListEqual(tokenizer.tokenize("unwanted running"),
["un", "##want", "##ed", "runn", "##ing"])
self.assertListEqual(tokenizer.tokenize("unwantedX running"),
["[UNK]", "runn", "##ing"])
class TestWordpieceTokenizer(CpuCommonTest):
def test_is_whitespace(self):
self.assertTrue(_is_whitespace(" "))
self.assertTrue(_is_whitespace("\t"))
self.assertTrue(_is_whitespace("\r"))
self.assertTrue(_is_whitespace("\n"))
self.assertTrue(_is_whitespace("\u00A0"))
def setUp(self):
vocab = [
"[UNK]", "[CLS]", "[SEP]", "th", "##is", "is", "simple", "text"
]
self.vocab_dict = {}
for i, v in enumerate(vocab):
self.vocab_dict[v] = i
self.assertFalse(_is_whitespace("A"))
self.assertFalse(_is_whitespace("-"))
def test_tokenize(self):
self.tokenizer = WordpieceTokenizer(self.vocab_dict, "[UNK]")
text = "this is a simple text"
expected_text_array = ['th', '##is', 'is', '[UNK]', 'simple', 'text']
text_array = self.tokenizer.tokenize(text)
self.check_output_equal(expected_text_array, text_array)
def test_is_control(self):
self.assertTrue(_is_control("\u0005"))
def test_tokenize_long_token(self):
self.tokenizer = WordpieceTokenizer(self.vocab_dict, "[UNK]", 4)
text = "this is a simple text"
expected_text_array = ['th', '##is', 'is', '[UNK]', '[UNK]', 'text']
text_array = self.tokenizer.tokenize(text)
self.check_output_equal(text_array, expected_text_array)
self.assertFalse(_is_control("A"))
self.assertFalse(_is_control(" "))
self.assertFalse(_is_control("\t"))
self.assertFalse(_is_control("\r"))
def test_is_punctuation(self):
self.assertTrue(_is_punctuation("-"))
self.assertTrue(_is_punctuation("$"))
self.assertTrue(_is_punctuation("`"))
self.assertTrue(_is_punctuation("."))
class TestBertTokenizer(CpuCommonTest):
self.assertFalse(_is_punctuation("A"))
self.assertFalse(_is_punctuation(" "))
def set_attr(self):
self.do_lower_case = True
def test_clean_text(self):
tokenizer = self.get_tokenizer()
def create_input_file(self):
vocab = [
"[UNK]", "[CLS]", "[SEP]", "th", "##is", "is", "simple", "text",
"a", "an", "for", "easy", "which", "children", "[MASK]", "[PAD]"
]
curr_dir = os.path.dirname(os.path.realpath(__file__))
vocab_file = os.path.join(curr_dir, "vocab.txt")
with open(vocab_file, "w") as fw:
for v in vocab:
fw.write(v)
fw.write('\n')
self.vocab_file = vocab_file
self.vocab = vocab
def set_test_case(self):
self.text = "this is a simple text"
self.expected_text_array = ['th', '##is', 'is', 'a', 'simple', 'text']
def setUp(self):
self.set_attr()
self.create_input_file()
self.set_test_case()
self.tokenizer = BertTokenizer(self.vocab_file, self.do_lower_case)
def test_tokenize(self):
text_array = self.tokenizer.tokenize(self.text)
self.check_output_equal(text_array, self.expected_text_array)
self.check_output_equal(
self.tokenizer.convert_tokens_to_string(text_array), self.text)
def test_call(self):
expected_input_ids = [1, 3, 4, 5, 8, 6, 7, 2]
expected_token_type_ids = [0, 0, 0, 0, 0, 0, 0, 0]
expected_attention_mask = [1] * len(expected_input_ids)
expected_tokens_mask = [1, 0, 0, 0, 0, 0, 0, 1]
result = self.tokenizer("This is a simple text",
return_attention_mask=True,
return_length=True,
return_special_tokens_mask=True)
self.check_output_equal(result['input_ids'], expected_input_ids)
self.check_output_equal(result['token_type_ids'],
expected_token_type_ids)
self.check_output_equal(result['seq_len'], len(expected_token_type_ids))
self.check_output_equal(result['attention_mask'],
expected_attention_mask)
self.check_output_equal(result['special_tokens_mask'],
expected_tokens_mask)
def test_call_pair(self):
expected_input_ids = [1, 3, 4, 5, 8, 6, 7, 2, 12, 5, 11, 10, 13, 2]
expected_token_type_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
result = self.tokenizer("This is a simple text",
"which is easy for children")
self.check_output_equal(result['input_ids'], expected_input_ids)
self.check_output_equal(result['token_type_ids'],
expected_token_type_ids)
def test_call_batch(self):
expected_input_ids = [1, 3, 4, 5, 8, 6, 7, 2]
expected_token_type_ids = [0, 0, 0, 0, 0, 0, 0, 0]
results = self.tokenizer(
["This is a simple text", "this Is A simple text"])
for result in results:
self.check_output_equal(result['input_ids'], expected_input_ids)
self.check_output_equal(result['token_type_ids'],
expected_token_type_ids)
def test_call_truncate_seq(self):
expected_input_ids = [1, 3, 2, 3, 2]
expected_token_type_ids = [0, 0, 0, 1, 1]
results = self.tokenizer("This is a simple text",
"this Is A simple text", 5)
self.check_output_equal(results['input_ids'], expected_input_ids)
self.check_output_equal(results['token_type_ids'],
expected_token_type_ids)
# Test PretrainedTokenizer
def test_truncate_only_first(self):
ids = [1, 3, 4, 5, 8, 6, 7, 2]
pair_ids = [12, 5, 11, 10, 13, 2]
truncate_ids, truncate_pair_ids, _ = self.tokenizer.truncate_sequences(
ids, pair_ids, 3, truncation_strategy='only_first')
self.check_output_equal(truncate_ids, ids[:-3])
self.check_output_equal(truncate_pair_ids, pair_ids)
def test_truncate_only_second(self):
ids = [1, 3, 4, 5, 8, 6, 7, 2]
pair_ids = [12, 5, 11, 10, 13, 2]
truncate_ids, truncate_pair_ids, _ = self.tokenizer.truncate_sequences(
ids, pair_ids, 3, truncation_strategy='only_second')
self.check_output_equal(truncate_ids, ids)
self.check_output_equal(truncate_pair_ids, pair_ids[:-3])
@assert_raises(ValueError)
def test_truncate_do_not_truncate(self):
ids = [1, 3, 4, 5, 8, 6, 7, 2]
pair_ids = [12, 5, 11, 10, 13, 2]
truncate_ids, truncate_pair_ids, _ = self.tokenizer.truncate_sequences(
ids, pair_ids, 3, truncation_strategy='do_not_truncate')
@assert_raises(ValueError)
def test_truncate_error_strategy(self):
ids = [1, 3, 4, 5, 8, 6, 7, 2]
pair_ids = [12, 5, 11, 10, 13, 2]
truncate_ids, truncate_pair_ids, _ = self.tokenizer.truncate_sequences(
ids, pair_ids, 1, truncation_strategy='')
def test_save_pretrained(self):
curr_dir = os.path.dirname(os.path.realpath(__file__))
model_path = os.path.join(curr_dir, "pretrained_model")
if not os.path.exists(model_path):
os.mkdir(model_path)
self.tokenizer.save_pretrained(model_path)
vocab_path = os.path.join(
model_path, self.tokenizer.resource_files_names['vocab_file'])
with open(vocab_path, "r") as fr:
vocabs = [vocab.strip() for vocab in fr.readlines()]
self.check_output_equal(vocabs, self.vocab)
@assert_raises(ValueError)
def test_from_pretrained_non_exist(self):
BertTokenizer.from_pretrained("")
def test_vocab_size(self):
self.check_output_equal(self.tokenizer.vocab_size, len(self.vocab))
def test_all_special_tokens(self):
expected_special_tokens_set = set([
self.tokenizer.pad_token, self.tokenizer.mask_token,
self.tokenizer.cls_token, self.tokenizer.unk_token,
self.tokenizer.sep_token
])
self.check_output_equal(set(self.tokenizer.all_special_tokens),
expected_special_tokens_set)
self.check_output_equal(set(self.tokenizer.all_special_ids),
set([0, 1, 2, 14, 15]))
@assert_raises(ValueError)
def test_non_exist_vocab_file(self):
BertTokenizer("non_exist.txt")
class TestBertTokenizerFromPretrained(CpuCommonTest):
# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
self.assertListEqual(
[tokenizer.tokenize(t) for t in ["Test", "\xad", "test"]],
[["[UNK]"], [], ["[UNK]"]])
@slow
def test_from_pretrained(self):
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
text1 = "This is a simple text"
text2 = "which is easy for children"
# test batch_encode
expected_input_ids = [
101, 2023, 2003, 1037, 3722, 3793, 102, 2029, 2003, 3733, 2005,
2336, 102, 0, 0, 0, 0, 0, 0, 0
]
expected_token_type_ids = [
0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0
]
expected_attention_mask = [
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0
]
expected_special_tokens_mask = [
1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1
]
results = tokenizer([text1], [text2],
20,
stride=1,
pad_to_max_seq_len=True,
return_attention_mask=True,
return_special_tokens_mask=True)
def test_sequence_builders(self):
tokenizer = self.tokenizer_class.from_pretrained("bert-base-uncased")
self.check_output_equal(results[0]['input_ids'], expected_input_ids)
self.check_output_equal(results[0]['token_type_ids'],
expected_token_type_ids)
self.check_output_equal(results[0]['attention_mask'],
expected_attention_mask)
self.check_output_equal(results[0]['special_tokens_mask'],
expected_special_tokens_mask)
# test encode
results = tokenizer(text1, text2, 20, stride=1, pad_to_max_seq_len=True)
self.check_output_equal(results['input_ids'], expected_input_ids)
self.check_output_equal(results['token_type_ids'],
expected_token_type_ids)
text = tokenizer.encode("sequence builders",
return_token_type_ids=None,
add_special_tokens=False)["input_ids"]
text_2 = tokenizer.encode("multi-sequence build",
return_token_type_ids=None,
add_special_tokens=False)["input_ids"]
@slow
def test_from_pretrained_pad_left(self):
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
tokenizer.padding_side = "left"
text1 = "This is a simple text"
text2 = "which is easy for children"
# test batch_encode
expected_input_ids = [
0, 0, 0, 0, 0, 0, 0, 101, 2023, 2003, 1037, 3722, 3793, 102, 2029,
2003, 3733, 2005, 2336, 102
]
expected_token_type_ids = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1
]
expected_attention_mask = [
0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
]
expected_special_tokens_mask = [
1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1
]
results = tokenizer([text1], [text2],
20,
stride=1,
pad_to_max_seq_len=True,
return_attention_mask=True,
return_special_tokens_mask=True)
encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)
encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)
self.check_output_equal(results[0]['input_ids'], expected_input_ids)
self.check_output_equal(results[0]['token_type_ids'],
expected_token_type_ids)
self.check_output_equal(results[0]['attention_mask'],
expected_attention_mask)
self.check_output_equal(results[0]['special_tokens_mask'],
expected_special_tokens_mask)
# test encode
results = tokenizer(text1, text2, 20, stride=1, pad_to_max_seq_len=True)
self.check_output_equal(results['input_ids'], expected_input_ids)
self.check_output_equal(results['token_type_ids'],
expected_token_type_ids)
assert encoded_sentence == [101] + text + [102]
assert encoded_pair == [101] + text + [102] + text_2 + [102]
def test_offsets_with_special_characters(self):
for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
with self.subTest(
f"{tokenizer.__class__.__name__} ({pretrained_name})"):
tokenizer = self.tokenizer_class.from_pretrained(
pretrained_name, **kwargs)
if __name__ == "__main__":
unittest.main()
sentence = f"A, naïve {tokenizer.mask_token} AllenNLP sentence."
tokens = tokenizer.encode(
sentence,
return_attention_mask=False,
return_token_type_ids=False,
return_offsets_mapping=True,
add_special_tokens=True,
)
do_lower_case = tokenizer.do_lower_case if hasattr(
tokenizer, "do_lower_case") else False
expected_results = ([
((0, 0), tokenizer.cls_token),
((0, 1), "A"),
((1, 2), ","),
((3, 5), "na"),
((5, 6), "##ï"),
((6, 8), "##ve"),
((9, 15), tokenizer.mask_token),
((16, 21), "Allen"),
((21, 23), "##NL"),
((23, 24), "##P"),
((25, 33), "sentence"),
((33, 34), "."),
((0, 0), tokenizer.sep_token),
] if not do_lower_case else [
((0, 0), tokenizer.cls_token),
((0, 1), "a"),
((1, 2), ","),
((3, 8), "naive"),
((9, 15), tokenizer.mask_token),
((16, 21), "allen"),
((21, 23), "##nl"),
((23, 24), "##p"),
((25, 33), "sentence"),
((33, 34), "."),
((0, 0), tokenizer.sep_token),
])
self.assertEqual([e[1] for e in expected_results],
tokenizer.convert_ids_to_tokens(
tokens["input_ids"]))
self.assertEqual([e[0] for e in expected_results],
tokens["offset_mapping"])
def test_change_tokenize_chinese_chars(self):
list_of_commun_chinese_char = ["的", "人", "有"]
text_with_chinese_char = "".join(list_of_commun_chinese_char)
for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
with self.subTest(
f"{tokenizer.__class__.__name__} ({pretrained_name})"):
kwargs["tokenize_chinese_chars"] = True
tokenizer = self.tokenizer_class.from_pretrained(
pretrained_name, **kwargs)
ids_without_spe_char_p = tokenizer.encode(
text_with_chinese_char,
return_token_type_ids=None,
add_special_tokens=False)["input_ids"]
tokens_without_spe_char_p = tokenizer.convert_ids_to_tokens(
ids_without_spe_char_p)
# it is expected that each Chinese character is not preceded by "##"
self.assertListEqual(tokens_without_spe_char_p,
list_of_commun_chinese_char)
# not yet supported in bert tokenizer
'''
kwargs["tokenize_chinese_chars"] = False
tokenizer = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)
ids_without_spe_char_p = tokenizer.encode(text_with_chinese_char, return_token_type_ids=None,add_special_tokens=False)["input_ids"]
tokens_without_spe_char_p = tokenizer.convert_ids_to_tokens(ids_without_spe_char_p)
# it is expected that only the first Chinese character is not preceded by "##".
expected_tokens = [
f"##{token}" if idx != 0 else token for idx, token in enumerate(list_of_commun_chinese_char)
]
self.assertListEqual(tokens_without_spe_char_p, expected_tokens)
'''

168
tests/transformers/test_modeling_common.py Normal file
View File

@@ -0,0 +1,168 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import random
import copy
import tempfile
import numpy as np
import paddle
global_rng = random.Random()
def ids_tensor(shape, vocab_size):
# Creates a random int32 tensor of the shape within the vocab size
return paddle.randint(low=0, high=vocab_size, dtype="int32", shape=shape)
def random_attention_mask(shape):
attn_mask = ids_tensor(shape, vocab_size=2)
# make sure that at least one token is attended to for each batch
attn_mask[:, -1] = 1
return attn_mask
def floats_tensor(shape, scale=1.0):
"""Creates a random float32 tensor"""
return scale * paddle.randn(shape, dtype="float32")
class ModelTesterMixin:
model_tester = None
base_model_class = None
all_model_classes = ()
all_generative_model_classes = ()
test_resize_embeddings = True
test_resize_position_embeddings = False
test_mismatched_shapes = True
test_missing_keys = True
is_encoder_decoder = False
has_attentions = True
model_split_percents = [0.5, 0.7, 0.9]
def _prepare_for_class(self, inputs_dict, model_class):
inputs_dict = copy.deepcopy(inputs_dict)
if model_class.__name__.endswith("ForMultipleChoice"):
inputs_dict = {
k: v.unsqueeze(1).expand(
shape=[-1, self.model_tester.num_choices, -1])
if isinstance(v, paddle.Tensor) and v.ndim > 1 else v
for k, v in inputs_dict.items()
}
return inputs_dict
def test_save_load(self):
config, input_ids, token_type_ids, input_mask = self.model_tester.prepare_config_and_inputs(
)
inputs_dict = {
"input_ids": input_ids,
"token_type_ids": token_type_ids,
"attention_mask": input_mask,
}
for model_class in self.all_model_classes:
if model_class == self.base_model_class:
model = model_class(**config)
else:
model = model_class(self.base_model_class(**config))
model.eval()
with paddle.no_grad():
outputs = model(
**self._prepare_for_class(inputs_dict, model_class))
out_2 = outputs[0].numpy()
out_2[np.isnan(out_2)] = 0
with tempfile.TemporaryDirectory() as tmpdirname:
model.save_pretrained(tmpdirname)
model = model_class.from_pretrained(tmpdirname)
model.eval()
with paddle.no_grad():
after_outputs = model(
**self._prepare_for_class(inputs_dict, model_class))
# Make sure we don't have nans
out_1 = after_outputs[0].numpy()
out_1[np.isnan(out_1)] = 0
max_diff = np.amax(np.abs(out_1 - out_2))
self.assertLessEqual(max_diff, 1e-5)
def test_resize_tokens_embeddings(self):
(
original_config,
inputs_dict,
) = self.model_tester.prepare_config_and_inputs_for_common()
if not self.test_resize_embeddings:
return
for model_class in self.all_model_classes:
config = copy.deepcopy(original_config)
if model_class == self.base_model_class:
model = model_class(**config)
else:
model = model_class(self.base_model_class(**config))
if self.model_tester.is_training is False:
model.eval()
model_vocab_size = config["vocab_size"]
# Retrieve the embeddings and clone theme
model_embed = model.resize_token_embeddings(model_vocab_size)
cloned_embeddings = model_embed.weight.clone()
# Check that resizing the token embeddings with a larger vocab size increases the model's vocab size
model_embed = model.resize_token_embeddings(model_vocab_size + 10)
self.assertEqual(model.base_model.config["vocab_size"],
model_vocab_size + 10)
# Check that it actually resizes the embeddings matrix
self.assertEqual(model_embed.weight.shape[0],
cloned_embeddings.shape[0] + 10)
# Check that the model can still do a forward pass successfully (every parameter should be resized)
model(**self._prepare_for_class(inputs_dict, model_class))
# Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size
model_embed = model.resize_token_embeddings(model_vocab_size - 15)
self.assertEqual(model.base_model.config["vocab_size"],
model_vocab_size - 15)
# Check that it actually resizes the embeddings matrix
self.assertEqual(model_embed.weight.shape[0],
cloned_embeddings.shape[0] - 15)
# Check that the model can still do a forward pass successfully (every parameter should be resized)
# Input ids should be clamped to the maximum size of the vocabulary
inputs_dict["input_ids"] = paddle.clip(inputs_dict["input_ids"],
max=model_vocab_size - 15 -
1)
# make sure that decoder_input_ids are resized as well
if "decoder_input_ids" in inputs_dict:
inputs_dict["decoder_input_ids"] = paddle.clip(
inputs_dict["decoder_input_ids"],
max=model_vocab_size - 15 - 1)
model(**self._prepare_for_class(inputs_dict, model_class))
# Check that adding and removing tokens has not modified the first part of the embedding matrix.
models_equal = True
for p1, p2 in zip(cloned_embeddings, model_embed.weight):
if not paddle.equal_all(p1, p2).item():
models_equal = False
break
self.assertTrue(models_equal)
def test_model_name_list(self):
config = self.model_tester.get_config()
model = self.base_model_class(**config)
self.assertTrue(len(model.model_name_list) != 0)

147
tests/transformers/test_tokenizer.py
View File

@@ -1,147 +0,0 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
import os
from typing import Type, List, Tuple
import shutil
import unittest
from multiprocessing import Process
from tempfile import TemporaryDirectory
from parameterized import parameterized
from paddle import nn
from paddlenlp.transformers.model_utils import PretrainedModel, MODEL_HOME
from paddlenlp.transformers.tokenizer_utils import PretrainedTokenizer
from paddlenlp.transformers import BertTokenizer, BasicTokenizer, WordpieceTokenizer
from paddlenlp.transformers.bert.modeling import BertForPretraining
from paddlenlp.transformers.gpt.modeling import GPTForPretraining
from paddlenlp.transformers.tinybert.modeling import TinyBertForPretraining
from paddlenlp.transformers.bert.tokenizer import BertTokenizer
from paddlenlp.transformers.gpt.tokenizer import GPTTokenizer, GPTChineseTokenizer
from paddlenlp.transformers.tinybert.tokenizer import TinyBertTokenizer
from tests.common_test import CpuCommonTest, CommonTest
from tests.util import slow, assert_raises
class FakePretrainedModel(PretrainedModel):
def __init__(self):
super().__init__()
self.linear = nn.Linear(2, 2)
def forward(self, *args, **kwargs):
pass
def get_pretrained_models_params() -> List[Tuple[str, Type[PretrainedModel]]]:
"""get all of pretrained model names in some PretrainedModels
Returns:
List[Tuple[str, Type[PretrainedModel]]]: the parameters of unit test method
"""
model_types: List[PretrainedModel] = [
BertForPretraining, GPTForPretraining, TinyBertForPretraining
]
name_class_tuples: List[Tuple[str, Type[PretrainedModel]]] = []
for ModelType in model_types:
for model_name in ModelType.pretrained_resource_files_map.get(
'model_state', {}).keys():
name_class_tuples.append([model_name, ModelType])
return name_class_tuples
def get_pretrained_tokenzier_params(
) -> List[Tuple[str, Type[PretrainedTokenizer]]]:
"""get all of pretrained tokenzier names in some PretrainedTokenzier
Returns:
List[Tuple[str, Type[PretrainedTokenzier]]]: the parameters of unit test method
"""
tokenizer_types: List[PretrainedTokenizer] = [
BertTokenizer, GPTTokenizer, GPTChineseTokenizer, TinyBertTokenizer
]
name_class_params: List[Tuple[str, Type[PretrainedTokenizer]]] = []
for TokenizerType in tokenizer_types:
for model_name in TokenizerType.pretrained_resource_files_map.get(
'vocab_file', {}).keys():
name_class_params.append([model_name, TokenizerType])
return name_class_params
class TestPretrainedFromPretrained(CpuCommonTest):
"""module for test pretrained model"""
def setUp(self):
self.temp_dir = TemporaryDirectory()
model = FakePretrainedModel()
model.save_pretrained(self.temp_dir.name)
def do_pretrained_in_process(self):
FakePretrainedModel.from_pretrained(self.temp_dir.name)
@parameterized.expand([(1, ), (8, ), (20, ), (50, ), (100, ), (1000, )])
def test_model_config_writing(self, process_num: int):
for _ in range(process_num):
process = Process(target=self.do_pretrained_in_process)
process.start()
@parameterized.expand(get_pretrained_models_params())
def test_pretrained_model(self, model_name: str,
PretrainedModelClass: Type[PretrainedModel]):
"""stupid test"""
cache_dir = os.path.join(MODEL_HOME, model_name)
shutil.rmtree(cache_dir, ignore_errors=True)
model: PretrainedModelClass = PretrainedModelClass.from_pretrained(
model_name)
self.assertTrue(
os.path.exists(os.path.join(cache_dir, model.model_config_file)))
# TODO(wj-Mcat): make this test code pass
# from pretrained from the dir
# PretrainedModelClass.from_pretrained(cache_dir)
# remove the cache model file
shutil.rmtree(cache_dir, ignore_errors=True)
@parameterized.expand(get_pretrained_tokenzier_params())
def test_pretrained_tokenizer(
self, tokenizer_name: str,
PretrainedTokenzierClass: Type[PretrainedTokenizer]):
"""stupid test on the pretrained tokenzier"""
cache_dir = os.path.join(MODEL_HOME, tokenizer_name)
shutil.rmtree(cache_dir, ignore_errors=True)
tokenizer: PretrainedTokenzierClass = PretrainedTokenzierClass.from_pretrained(
tokenizer_name)
files = os.listdir(cache_dir)
self.assertTrue(
os.path.exists(
os.path.join(cache_dir, tokenizer.tokenizer_config_file)))
for resource_file_name in tokenizer.resource_files_names.values():
self.assertTrue(
os.path.exists(os.path.join(cache_dir, resource_file_name)))
# TODO(wj-Mcat): make this test code pass
# from_pretrained from the dir
# PretrainedTokenzierClass.from_pretrained(cache_dir)
# remove the cache model file
shutil.rmtree(cache_dir, ignore_errors=True)

2767
tests/transformers/test_tokenizer_common.py Normal file
View File

File diff suppressed because it is too large Load Diff