Kimirock/PaddleNLP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

black format all (#4014)

Browse Source
This commit is contained in:
Sijun He
2022-12-05 19:13:15 +08:00
committed by GitHub
parent 420bfe3fd8
commit 4e56e4890c
1424 changed files with 60685 additions and 82164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
applications/document_intelligence/doc_vqa/Extraction/docvqa.py
View File

@@ -28,14 +28,7 @@ sys.path.insert(0, "../")
class DocVQAExample(object):
def __init__(self,
question,
doc_tokens,
doc_boxes=[],
answer=None,
labels=None,
image=None):
def __init__(self, question, doc_tokens, doc_boxes=[], answer=None, labels=None, image=None):
self.question = question
self.doc_tokens = doc_tokens
self.doc_boxes = doc_boxes
@@ -47,13 +40,7 @@ class DocVQAExample(object):
class DocVQAFeatures(object):
"""A single set of features of data."""
def __init__(self,
example_index,
input_ids,
input_mask,
segment_ids,
boxes=None,
label=None):
def __init__(self, example_index, input_ids, input_mask, segment_ids, boxes=None, label=None):
self.example_index = example_index
self.input_ids = input_ids
self.input_mask = input_mask
@@ -63,15 +50,9 @@ class DocVQAFeatures(object):
class DocVQA(Dataset):
def __init__(self,
args,
tokenizer,
label2id_map,
max_seq_len=512,
max_query_length=20,
max_doc_length=512,
max_span_num=1):
def __init__(
self, args, tokenizer, label2id_map, max_seq_len=512, max_query_length=20, max_doc_length=512, max_span_num=1
):
super(DocVQA, self).__init__()
self.tokenizer = tokenizer
self.label2id_map = label2id_map
@@ -113,17 +94,16 @@ class DocVQA(Dataset):
continue
num_left_context = position - doc_span.start
num_right_context = end - position
score = min(num_left_context,
num_right_context) + 0.01 * doc_span.length
score = min(num_left_context, num_right_context) + 0.01 * doc_span.length
if best_score is None or score > best_score:
best_score = score
best_span_index = span_index
return cur_span_index == best_span_index
def convert_examples_to_features(self, examples, tokenizer, label_map,
max_seq_length, max_span_num,
max_doc_length, max_query_length):
def convert_examples_to_features(
self, examples, tokenizer, label_map, max_seq_length, max_span_num, max_doc_length, max_query_length
):
if "[CLS]" in self.tokenizer.get_vocab():
start_token = "[CLS]"
@@ -188,8 +168,7 @@ class DocVQA(Dataset):
segment_ids.append(0)
for i in range(doc_span.length):
split_token_index = doc_span.start + i
is_max_context = self.check_is_max_context(
doc_spans, doc_span_index, split_token_index)
is_max_context = self.check_is_max_context(doc_spans, doc_span_index, split_token_index)
token_is_max_context[len(tokens)] = is_max_context
tokens.append(all_doc_tokens[split_token_index])
boxes_tokens.append(all_doc_boxes_tokens[split_token_index])
@@ -292,12 +271,10 @@ class DocVQA(Dataset):
question = sample["question"]
doc_tokens = sample["document"]
doc_boxes = sample["document_bbox"]
labels = sample['labels'] if not is_test else []
labels = sample["labels"] if not is_test else []
x_min, y_min = min(doc_boxes, key=lambda x: x[0])[0], min(
doc_boxes, key=lambda x: x[2])[2]
x_max, y_max = max(doc_boxes, key=lambda x: x[1])[1], max(
doc_boxes, key=lambda x: x[3])[3]
x_min, y_min = min(doc_boxes, key=lambda x: x[0])[0], min(doc_boxes, key=lambda x: x[2])[2]
x_max, y_max = max(doc_boxes, key=lambda x: x[1])[1], max(doc_boxes, key=lambda x: x[3])[3]
width = x_max - x_min
height = y_max - y_min
@@ -308,12 +285,15 @@ class DocVQA(Dataset):
scale_x = 1000 / max(width, height)
scale_y = 1000 / max(width, height)
scaled_doc_boxes = [[
round((b[0] - x_min) * scale_x),
round((b[2] - y_min) * scale_y),
round((b[1] - x_min) * scale_x),
round((b[3] - y_min) * scale_y)
] for b in doc_boxes]
scaled_doc_boxes = [
[
round((b[0] - x_min) * scale_x),
round((b[2] - y_min) * scale_y),
round((b[1] - x_min) * scale_x),
round((b[3] - y_min) * scale_y),
]
for b in doc_boxes
]
for box, oribox in zip(scaled_doc_boxes, doc_boxes):
if box[0] < 0:
@@ -326,10 +306,9 @@ class DocVQA(Dataset):
if pos > 1000:
print(width, height, box, oribox)
example = DocVQAExample(question=question,
doc_tokens=doc_tokens,
doc_boxes=scaled_doc_boxes,
labels=labels)
example = DocVQAExample(
question=question, doc_tokens=doc_tokens, doc_boxes=scaled_doc_boxes, labels=labels
)
examples.append(example)
return examples
@@ -339,7 +318,7 @@ class DocVQA(Dataset):
dataset = self.args.train_file
elif self.args.do_test:
dataset = self.args.test_file
with open(dataset, 'r', encoding='utf8') as f:
with open(dataset, "r", encoding="utf8") as f:
for index, line in enumerate(f):
data.append(json.loads(line.strip()))
@@ -353,30 +332,32 @@ class DocVQA(Dataset):
max_seq_length=self.max_seq_len,
max_doc_length=self.max_doc_length,
max_span_num=self.max_span_num,
max_query_length=self.max_query_length)
max_query_length=self.max_query_length,
)
all_input_ids = paddle.to_tensor([f.input_ids for f in features],
dtype="int64")
all_input_mask = paddle.to_tensor([f.input_mask for f in features],
dtype="int64")
all_segment_ids = paddle.to_tensor([f.segment_ids for f in features],
dtype="int64")
all_bboxes = paddle.to_tensor([f.boxes for f in features],
dtype="int64")
all_labels = paddle.to_tensor([f.label for f in features],
dtype="int64")
all_input_ids = paddle.to_tensor([f.input_ids for f in features], dtype="int64")
all_input_mask = paddle.to_tensor([f.input_mask for f in features], dtype="int64")
all_segment_ids = paddle.to_tensor([f.segment_ids for f in features], dtype="int64")
all_bboxes = paddle.to_tensor([f.boxes for f in features], dtype="int64")
all_labels = paddle.to_tensor([f.label for f in features], dtype="int64")
self.sample_list = [
np.array(all_input_ids),
np.array(all_input_mask),
np.array(all_segment_ids),
np.array(all_bboxes),
np.array(all_labels)
np.array(all_labels),
]
def __getitem__(self, idx):
return self.sample_list[0][idx], self.sample_list[1][
idx], self.sample_list[2][idx], self.sample_list[3][
idx], self.sample_list[4][idx]
return (
self.sample_list[0][idx],
self.sample_list[1][idx],
self.sample_list[2][idx],
self.sample_list[3][idx],
self.sample_list[4][idx],
)
def __len__(self, ):
def __len__(
self,
):
return self.sample_list[0].shape[0]

201
applications/document_intelligence/doc_vqa/Extraction/model.py
View File

@@ -21,8 +21,7 @@ from paddlenlp.transformers import LayoutXLMPretrainedModel
class Crf_decoding(paddle.fluid.dygraph.Layer):
def __init__(self, param_attr, size=None, is_test=True, dtype='float32'):
def __init__(self, param_attr, size=None, is_test=True, dtype="float32"):
super(Crf_decoding, self).__init__()
self._dtype = dtype
@@ -30,9 +29,8 @@ class Crf_decoding(paddle.fluid.dygraph.Layer):
self._is_test = is_test
self._param_attr = param_attr
self._transition = self.create_parameter(
attr=self._param_attr,
shape=[self._size + 2, self._size],
dtype=self._dtype)
attr=self._param_attr, shape=[self._size + 2, self._size], dtype=self._dtype
)
@property
def weight(self):
@@ -44,30 +42,23 @@ class Crf_decoding(paddle.fluid.dygraph.Layer):
def forward(self, input, label=None, length=None):
viterbi_path = self._helper.create_variable_for_type_inference(
dtype=self._dtype)
this_inputs = {
"Emission": [input],
"Transition": self._transition,
"Label": label
}
viterbi_path = self._helper.create_variable_for_type_inference(dtype=self._dtype)
this_inputs = {"Emission": [input], "Transition": self._transition, "Label": label}
if length is not None:
this_inputs['Length'] = [length]
self._helper.append_op(type='crf_decoding',
inputs=this_inputs,
outputs={"ViterbiPath": [viterbi_path]},
attrs={
"is_test": self._is_test,
})
this_inputs["Length"] = [length]
self._helper.append_op(
type="crf_decoding",
inputs=this_inputs,
outputs={"ViterbiPath": [viterbi_path]},
attrs={
"is_test": self._is_test,
},
)
return viterbi_path
class Chunk_eval(paddle.fluid.dygraph.Layer):
def __init__(self,
num_chunk_types,
chunk_scheme,
excluded_chunk_types=None):
def __init__(self, num_chunk_types, chunk_scheme, excluded_chunk_types=None):
super(Chunk_eval, self).__init__()
self.num_chunk_types = num_chunk_types
self.chunk_scheme = chunk_scheme
@@ -75,48 +66,39 @@ class Chunk_eval(paddle.fluid.dygraph.Layer):
def forward(self, input, label, seq_length=None):
precision = self._helper.create_variable_for_type_inference(
dtype="float32")
recall = self._helper.create_variable_for_type_inference(
dtype="float32")
f1_score = self._helper.create_variable_for_type_inference(
dtype="float32")
num_infer_chunks = self._helper.create_variable_for_type_inference(
dtype="int64")
num_label_chunks = self._helper.create_variable_for_type_inference(
dtype="int64")
num_correct_chunks = self._helper.create_variable_for_type_inference(
dtype="int64")
precision = self._helper.create_variable_for_type_inference(dtype="float32")
recall = self._helper.create_variable_for_type_inference(dtype="float32")
f1_score = self._helper.create_variable_for_type_inference(dtype="float32")
num_infer_chunks = self._helper.create_variable_for_type_inference(dtype="int64")
num_label_chunks = self._helper.create_variable_for_type_inference(dtype="int64")
num_correct_chunks = self._helper.create_variable_for_type_inference(dtype="int64")
this_input = {"Inference": [input], "Label": [label]}
if seq_length is not None:
this_input["SeqLength"] = [seq_length]
self._helper.append_op(type='chunk_eval',
inputs=this_input,
outputs={
"Precision": [precision],
"Recall": [recall],
"F1-Score": [f1_score],
"NumInferChunks": [num_infer_chunks],
"NumLabelChunks": [num_label_chunks],
"NumCorrectChunks": [num_correct_chunks]
},
attrs={
"num_chunk_types":
self.num_chunk_types,
"chunk_scheme":
self.chunk_scheme,
"excluded_chunk_types":
self.excluded_chunk_types or []
})
return (precision, recall, f1_score, num_infer_chunks, num_label_chunks,
num_correct_chunks)
self._helper.append_op(
type="chunk_eval",
inputs=this_input,
outputs={
"Precision": [precision],
"Recall": [recall],
"F1-Score": [f1_score],
"NumInferChunks": [num_infer_chunks],
"NumLabelChunks": [num_label_chunks],
"NumCorrectChunks": [num_correct_chunks],
},
attrs={
"num_chunk_types": self.num_chunk_types,
"chunk_scheme": self.chunk_scheme,
"excluded_chunk_types": self.excluded_chunk_types or [],
},
)
return (precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks)
class Linear_chain_crf(paddle.fluid.dygraph.Layer):
def __init__(self, param_attr, size=None, is_test=False, dtype='float32'):
def __init__(self, param_attr, size=None, is_test=False, dtype="float32"):
super(Linear_chain_crf, self).__init__()
self._param_attr = param_attr
@@ -124,9 +106,8 @@ class Linear_chain_crf(paddle.fluid.dygraph.Layer):
self._size = size
self._is_test = is_test
self._transition = self.create_parameter(
attr=self._param_attr,
shape=[self._size + 2, self._size],
dtype=self._dtype)
attr=self._param_attr, shape=[self._size + 2, self._size], dtype=self._dtype
)
@property
def weight(self):
@@ -138,37 +119,30 @@ class Linear_chain_crf(paddle.fluid.dygraph.Layer):
def forward(self, input, label, length=None):
alpha = self._helper.create_variable_for_type_inference(
dtype=self._dtype)
emission_exps = self._helper.create_variable_for_type_inference(
dtype=self._dtype)
transition_exps = self._helper.create_variable_for_type_inference(
dtype=self._dtype)
log_likelihood = self._helper.create_variable_for_type_inference(
dtype=self._dtype)
this_inputs = {
"Emission": [input],
"Transition": self._transition,
"Label": [label]
}
alpha = self._helper.create_variable_for_type_inference(dtype=self._dtype)
emission_exps = self._helper.create_variable_for_type_inference(dtype=self._dtype)
transition_exps = self._helper.create_variable_for_type_inference(dtype=self._dtype)
log_likelihood = self._helper.create_variable_for_type_inference(dtype=self._dtype)
this_inputs = {"Emission": [input], "Transition": self._transition, "Label": [label]}
if length is not None:
this_inputs['Length'] = [length]
self._helper.append_op(type='linear_chain_crf',
inputs=this_inputs,
outputs={
"Alpha": [alpha],
"EmissionExps": [emission_exps],
"TransitionExps": transition_exps,
"LogLikelihood": log_likelihood
},
attrs={
"is_test": self._is_test,
})
this_inputs["Length"] = [length]
self._helper.append_op(
type="linear_chain_crf",
inputs=this_inputs,
outputs={
"Alpha": [alpha],
"EmissionExps": [emission_exps],
"TransitionExps": transition_exps,
"LogLikelihood": log_likelihood,
},
attrs={
"is_test": self._is_test,
},
)
return log_likelihood
class LayoutXLMForTokenClassification_with_CRF(LayoutXLMPretrainedModel):
def __init__(self, layoutxlm, num_classes, dropout=None):
super(LayoutXLMForTokenClassification_with_CRF, self).__init__()
self.num_classes = num_classes
@@ -176,40 +150,36 @@ class LayoutXLMForTokenClassification_with_CRF(LayoutXLMPretrainedModel):
self.layoutxlm = LayoutXLMModel(**layoutxlm)
else:
self.layoutxlm = layoutxlm
self.dropout = nn.Dropout(dropout if dropout is not None else self.
layoutxlm.config["hidden_dropout_prob"])
self.emission_classifier = nn.Linear(
self.layoutxlm.config["hidden_size"], self.num_classes)
self.dropout = nn.Dropout(dropout if dropout is not None else self.layoutxlm.config["hidden_dropout_prob"])
self.emission_classifier = nn.Linear(self.layoutxlm.config["hidden_size"], self.num_classes)
self.emission_classifier.apply(self.init_weights)
self.linear_chain_crf = Linear_chain_crf(
size=self.num_classes,
param_attr=paddle.fluid.ParamAttr(name='liner_chain_crfw'))
self.crf_decoding = Crf_decoding(
param_attr=paddle.fluid.ParamAttr(name='crfw_decode'),
size=self.num_classes)
size=self.num_classes, param_attr=paddle.fluid.ParamAttr(name="liner_chain_crfw")
)
self.crf_decoding = Crf_decoding(param_attr=paddle.fluid.ParamAttr(name="crfw_decode"), size=self.num_classes)
self.crf_decoding.weight = self.linear_chain_crf.weight
self.crfw = fluid.layers.create_parameter(
shape=[self.num_classes + 2, self.num_classes],
dtype='float32',
name='crfw')
shape=[self.num_classes + 2, self.num_classes], dtype="float32", name="crfw"
)
self.mask_crfw = fluid.layers.create_parameter(
shape=[self.num_classes + 2, self.num_classes],
dtype='float32',
name='mask_matrix')
shape=[self.num_classes + 2, self.num_classes], dtype="float32", name="mask_matrix"
)
def get_input_embeddings(self):
return self.layoutxlm.embeddings.word_embeddings
def forward(self,
input_ids=None,
bbox=None,
attention_mask=None,
token_type_ids=None,
labels=None,
image=None,
position_ids=None,
head_mask=None,
is_train=False):
def forward(
self,
input_ids=None,
bbox=None,
attention_mask=None,
token_type_ids=None,
labels=None,
image=None,
position_ids=None,
head_mask=None,
is_train=False,
):
input_ids = input_ids.squeeze(axis=1)
bbox = bbox.squeeze(axis=1)
@@ -226,16 +196,13 @@ class LayoutXLMForTokenClassification_with_CRF(LayoutXLMPretrainedModel):
)
seq_length = input_ids.shape[1]
# sequence out and image out
sequence_logits, image_output = outputs[0][:, :seq_length], outputs[
0][:, seq_length:]
sequence_logits, image_output = outputs[0][:, :seq_length], outputs[0][:, seq_length:]
emission = self.emission_classifier(sequence_logits)
length = paddle.sum(attention_mask, axis=1)
labels = labels.reshape([-1, seq_length, 1])
# standard crf loss
crf_cost = self.linear_chain_crf(input=emission,
label=labels,
length=length)
crf_cost = self.linear_chain_crf(input=emission, label=labels, length=length)
crf_decode = self.crf_decoding(input=emission, length=length)
if is_train:
return [crf_cost]

294
applications/document_intelligence/doc_vqa/Extraction/run_docvqa.py
View File

@@ -81,12 +81,10 @@ def get_label_maps():
def main(args):
os.makedirs(args.output_dir, exist_ok=True)
logging.basicConfig(
filename=os.path.join(args.output_dir, "train.log")
if paddle.distributed.get_rank() == 0 else None,
filename=os.path.join(args.output_dir, "train.log") if paddle.distributed.get_rank() == 0 else None,
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if paddle.distributed.get_rank() == 0 else logging.WARN,
level=logging.INFO if paddle.distributed.get_rank() == 0 else logging.WARN,
)
ch = logging.StreamHandler()
@@ -103,74 +101,62 @@ def main(args):
tokenizer = LayoutXLMTokenizer.from_pretrained(args.model_name_or_path)
if args.do_test:
model = LayoutXLMForTokenClassification_with_CRF.from_pretrained(
args.init_checkpoint)
evaluate(args,
model,
tokenizer,
label2id_map,
id2label_map,
pad_token_label_id,
global_step=0)
model = LayoutXLMForTokenClassification_with_CRF.from_pretrained(args.init_checkpoint)
evaluate(args, model, tokenizer, label2id_map, id2label_map, pad_token_label_id, global_step=0)
exit(0)
if args.init_checkpoint:
logger.info('Init checkpoint from {}'.format(args.init_checkpoint))
model = LayoutXLMForTokenClassification_with_CRF.from_pretrained(
args.init_checkpoint)
logger.info("Init checkpoint from {}".format(args.init_checkpoint))
model = LayoutXLMForTokenClassification_with_CRF.from_pretrained(args.init_checkpoint)
else:
base_model = LayoutXLMModel.from_pretrained(args.model_name_or_path)
model = LayoutXLMForTokenClassification_with_CRF(
base_model, num_classes=len(label2id_map), dropout=None)
model = LayoutXLMForTokenClassification_with_CRF(base_model, num_classes=len(label2id_map), dropout=None)
# dist mode
if paddle.distributed.get_world_size() > 1:
model = paddle.DataParallel(model)
train_dataset = DocVQA(args,
tokenizer,
label2id_map,
max_seq_len=args.max_seq_len,
max_query_length=args.max_query_length,
max_doc_length=args.max_doc_length,
max_span_num=args.max_span_num)
train_dataset = DocVQA(
args,
tokenizer,
label2id_map,
max_seq_len=args.max_seq_len,
max_query_length=args.max_query_length,
max_doc_length=args.max_doc_length,
max_span_num=args.max_span_num,
)
train_sampler = paddle.io.DistributedBatchSampler(
train_dataset, batch_size=args.per_gpu_train_batch_size, shuffle=False)
train_dataset, batch_size=args.per_gpu_train_batch_size, shuffle=False
)
args.train_batch_size = args.per_gpu_train_batch_size * max(
1, paddle.distributed.get_world_size())
args.train_batch_size = args.per_gpu_train_batch_size * max(1, paddle.distributed.get_world_size())
train_dataloader = paddle.io.DataLoader(train_dataset,
batch_sampler=train_sampler,
num_workers=0,
use_shared_memory=True,
collate_fn=None)
train_dataloader = paddle.io.DataLoader(
train_dataset, batch_sampler=train_sampler, num_workers=0, use_shared_memory=True, collate_fn=None
)
t_total = len(train_dataloader) * args.num_train_epochs
# build linear decay with warmup lr sch
lr_scheduler = paddle.optimizer.lr.PolynomialDecay(
learning_rate=args.learning_rate,
decay_steps=t_total,
end_lr=0.0,
power=1.0)
learning_rate=args.learning_rate, decay_steps=t_total, end_lr=0.0, power=1.0
)
if args.warmup_steps > 0:
lr_scheduler = paddle.optimizer.lr.LinearWarmup(
lr_scheduler,
args.warmup_steps,
start_lr=0,
end_lr=args.learning_rate)
lr_scheduler, args.warmup_steps, start_lr=0, end_lr=args.learning_rate
)
optimizer = paddle.optimizer.AdamW(learning_rate=lr_scheduler,
parameters=model.parameters(),
epsilon=args.adam_epsilon,
weight_decay=args.weight_decay)
optimizer = paddle.optimizer.AdamW(
learning_rate=lr_scheduler,
parameters=model.parameters(),
epsilon=args.adam_epsilon,
weight_decay=args.weight_decay,
)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_dataset))
logger.info(" Num Epochs = %d", args.num_train_epochs)
logger.info(" Instantaneous batch size per GPU = %d",
args.per_gpu_train_batch_size)
logger.info(" Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
logger.info(
" Total train batch size (w. parallel, distributed) = %d",
args.train_batch_size * paddle.distributed.get_world_size(),
@@ -182,27 +168,34 @@ def main(args):
set_seed(args)
best_metrics = None
for epoch_id in range(args.num_train_epochs):
print('epoch id:{}'.format(epoch_id))
print("epoch id:{}".format(epoch_id))
for step, batch in enumerate(train_dataloader):
model.train()
input_ids, input_mask, segment_ids, bboxes, labels = batch
if input_ids.shape[0] != args.per_gpu_train_batch_size:
continue
outputs = model(input_ids=input_ids,
bbox=bboxes,
attention_mask=input_mask,
token_type_ids=segment_ids,
labels=labels,
is_train=True)
outputs = model(
input_ids=input_ids,
bbox=bboxes,
attention_mask=input_mask,
token_type_ids=segment_ids,
labels=labels,
is_train=True,
)
# model outputs are always tuple in paddlenlp (see doc)
loss = outputs[0]
loss = loss.mean()
if global_step % 50 == 0:
logger.info(
"[epoch {}/{}][iter: {}/{}] lr: {:.5f}, train loss: {:.5f}, "
.format(epoch_id, args.num_train_epochs, step,
len(train_dataloader), lr_scheduler.get_lr(),
loss.numpy()[0]))
"[epoch {}/{}][iter: {}/{}] lr: {:.5f}, train loss: {:.5f}, ".format(
epoch_id,
args.num_train_epochs,
step,
len(train_dataloader),
lr_scheduler.get_lr(),
loss.numpy()[0],
)
)
loss.backward()
tr_loss += loss.item()
@@ -211,17 +204,14 @@ def main(args):
optimizer.clear_grad()
global_step += 1
if paddle.distributed.get_rank(
) == 0 and args.save_steps > 0 and global_step % args.save_steps == 0:
if paddle.distributed.get_rank() == 0 and args.save_steps > 0 and global_step % args.save_steps == 0:
# Save model checkpoint
output_dir = os.path.join(args.output_dir,
"checkpoint-{}".format(global_step))
output_dir = os.path.join(args.output_dir, "checkpoint-{}".format(global_step))
os.makedirs(output_dir, exist_ok=True)
if paddle.distributed.get_rank() == 0:
model.save_pretrained(output_dir)
tokenizer.save_pretrained(output_dir)
paddle.save(args,
os.path.join(output_dir, "training_args.bin"))
paddle.save(args, os.path.join(output_dir, "training_args.bin"))
logger.info("Saving model checkpoint to %s", output_dir)
@@ -242,14 +232,16 @@ def _tokenize_chinese_chars(text):
# as is Japanese Hiragana and Katakana. Those alphabets are used to write
# space-separated words, so they are not treated specially and handled
# like the all of the other languages.
if ((cp >= 0x4E00 and cp <= 0x9FFF) or #
(cp >= 0x3400 and cp <= 0x4DBF) or #
(cp >= 0x20000 and cp <= 0x2A6DF) or #
(cp >= 0x2A700 and cp <= 0x2B73F) or #
(cp >= 0x2B740 and cp <= 0x2B81F) or #
(cp >= 0x2B820 and cp <= 0x2CEAF) or
(cp >= 0xF900 and cp <= 0xFAFF) or #
(cp >= 0x2F800 and cp <= 0x2FA1F)): #
if (
(cp >= 0x4E00 and cp <= 0x9FFF)
or (cp >= 0x3400 and cp <= 0x4DBF) #
or (cp >= 0x20000 and cp <= 0x2A6DF) #
or (cp >= 0x2A700 and cp <= 0x2B73F) #
or (cp >= 0x2B740 and cp <= 0x2B81F) #
or (cp >= 0x2B820 and cp <= 0x2CEAF) #
or (cp >= 0xF900 and cp <= 0xFAFF)
or (cp >= 0x2F800 and cp <= 0x2FA1F) #
): #
return True
return False
@@ -290,9 +282,35 @@ def _normalize(in_str):
"""
in_str = in_str.lower()
sp_char = [
u':', u'_', u'`', u'，', u'。', u'：', u'？', u'！', u'(', u')', u'“', u'”',
u'；', u'’', u'《', u'》', u'……', u'·', u'、', u',', u'「', u'」', u'（', u'）',
u'－', u'～', u'『', u'』', '|'
":",
"_",
"`",
"，",
"。",
"：",
"？",
"！",
"(",
")",
"“",
"”",
"；",
"’",
"《",
"》",
"……",
"·",
"、",
",",
"「",
"」",
"（",
"）",
"－",
"～",
"『",
"』",
"|",
]
out_segs = []
for char in in_str:
@@ -300,7 +318,7 @@ def _normalize(in_str):
continue
else:
out_segs.append(char)
return ''.join(out_segs)
return "".join(out_segs)
def calc_f1_score(answer, prediction):
@@ -318,38 +336,35 @@ def decode(tokenizer, res):
save_f1 = []
for i in range(len(res)):
input_ids, label_ids, predict_ids, bbox = res[i]
remove_pos = len(' '.join(
[str(x) for x in input_ids]).split('2 6 ')[0].strip(' ').split(
' ')) + 2 # remove the question bbox and sep bbox
remove_pos = (
len(" ".join([str(x) for x in input_ids]).split("2 6 ")[0].strip(" ").split(" ")) + 2
) # remove the question bbox and sep bbox
start_pos = input_ids.index(sep_id)
query_text = []
for idx in range(1, start_pos):
input_id = input_ids[idx]
query_text.append(tokenizer._convert_id_to_token(int(input_id)))
#label texts and predict texts
# label texts and predict texts
text_label, text_predict = [], []
label_bbox_index, predict_bbox_index = [], []
for idx in range(start_pos + 1, len(input_ids)):
input_id, label_id, predict_id = input_ids[idx], label_ids[
idx], predict_ids[idx]
input_id, label_id, predict_id = input_ids[idx], label_ids[idx], predict_ids[idx]
if label_id in [1, 2, 3]:
text_label.append(tokenizer._convert_id_to_token(int(input_id)))
label_bbox_index.append(idx - remove_pos + 1)
if predict_id in [1, 2, 3]:
text_predict.append(
tokenizer._convert_id_to_token(int(input_id)))
text_predict.append(tokenizer._convert_id_to_token(int(input_id)))
predict_bbox_index.append(idx - remove_pos + 1)
text_res.append([
''.join(query_text), ''.join(text_label), ''.join(text_predict),
label_bbox_index, predict_bbox_index
])
text_res.append(
["".join(query_text), "".join(text_label), "".join(text_predict), label_bbox_index, predict_bbox_index]
)
f1 = calc_f1_score(''.join(text_label), ''.join(text_predict))
f1 = calc_f1_score("".join(text_label), "".join(text_predict))
save_f1.append(f1)
if len(''.join(text_label)) > 10:
if len("".join(text_label)) > 10:
all_f1.append(f1)
if len(all_f1) > 0:
print("F1: ", sum(all_f1) / len(all_f1))
@@ -358,31 +373,17 @@ def decode(tokenizer, res):
return text_res
def evaluate(args,
model,
tokenizer,
label2id_map,
id2label_map,
pad_token_label_id,
prefix="",
global_step=0):
def evaluate(args, model, tokenizer, label2id_map, id2label_map, pad_token_label_id, prefix="", global_step=0):
eval_dataset = DocVQA(args,
tokenizer,
label2id_map,
max_seq_len=512,
max_query_length=20,
max_doc_length=512,
max_span_num=1)
eval_dataset = DocVQA(
args, tokenizer, label2id_map, max_seq_len=512, max_query_length=20, max_doc_length=512, max_span_num=1
)
args.eval_batch_size = args.per_gpu_eval_batch_size * max(
1, paddle.distributed.get_world_size())
args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, paddle.distributed.get_world_size())
eval_dataloader = paddle.io.DataLoader(eval_dataset,
batch_size=args.eval_batch_size,
num_workers=0,
use_shared_memory=True,
collate_fn=None)
eval_dataloader = paddle.io.DataLoader(
eval_dataset, batch_size=args.eval_batch_size, num_workers=0, use_shared_memory=True, collate_fn=None
)
# Eval!
logger.info("***** Running evaluation %s *****", prefix)
@@ -396,66 +397,65 @@ def evaluate(args,
if input_ids.shape[0] != args.eval_batch_size:
continue
outputs = model(input_ids=input_ids,
bbox=bboxes,
attention_mask=input_mask,
token_type_ids=segment_ids,
labels=labels,
is_train=False)
outputs = model(
input_ids=input_ids,
bbox=bboxes,
attention_mask=input_mask,
token_type_ids=segment_ids,
labels=labels,
is_train=False,
)
labels = labels.numpy()
crf_decode = outputs[1].numpy()
bboxes = bboxes.squeeze().numpy()
input_ids = input_ids.squeeze(axis=1).numpy()
for index in range(input_ids.shape[0]):
res.append([
list(input_ids[index]),
list(labels[index]),
list(crf_decode[index]), bboxes[index]
])
res.append([list(input_ids[index]), list(labels[index]), list(crf_decode[index]), bboxes[index]])
origin_inputs = []
with open(args.test_file, 'r', encoding='utf8') as f:
with open(args.test_file, "r", encoding="utf8") as f:
for line in f:
line = json.loads(line.strip())
origin_inputs.append({
'img_name': line['img_name'],
'question': line['question'],
'bboxes': line['document_bbox'],
'img_id': line['img_id']
})
origin_inputs.append(
{
"img_name": line["img_name"],
"question": line["question"],
"bboxes": line["document_bbox"],
"img_id": line["img_id"],
}
)
text_res = decode(tokenizer, res)
with open(args.save_path, 'w', encoding='utf8') as f:
for line_res, line_text, line_label in zip(res, text_res,
origin_inputs):
with open(args.save_path, "w", encoding="utf8") as f:
for line_res, line_text, line_label in zip(res, text_res, origin_inputs):
line_json = {}
line_json['img_name'] = line_label['img_name']
line_json['img_id'] = line_label['img_id']
line_json['question'] = line_label['question']
line_json['label_answer'] = line_text[1]
line_json['predict_answer'] = line_text[2]
line_json["img_name"] = line_label["img_name"]
line_json["img_id"] = line_label["img_id"]
line_json["question"] = line_label["question"]
line_json["label_answer"] = line_text[1]
line_json["predict_answer"] = line_text[2]
all_boxes = line_res[3]
label_bbox_index, predict_bbox_index = line_text[3], line_text[4]
label_bboxes, predict_bboxes = [], []
for i in range(len(line_label['bboxes'])):
for i in range(len(line_label["bboxes"])):
if i in label_bbox_index:
label_bboxes.append(line_label['bboxes'][i])
label_bboxes.append(line_label["bboxes"][i])
if i in predict_bbox_index:
predict_bboxes.append(line_label['bboxes'][i])
line_json['label_bboxes'] = label_bboxes
line_json['predict_bboxes'] = predict_bboxes
predict_bboxes.append(line_label["bboxes"][i])
line_json["label_bboxes"] = label_bboxes
line_json["predict_bboxes"] = predict_bboxes
json.dump(line_json, f, ensure_ascii=False)
f.write('\n')
f.write("\n")
def print_arguments(args):
"""print arguments"""
print('----------- Configuration Arguments -----------')
print("----------- Configuration Arguments -----------")
for arg, value in sorted(vars(args).items()):
print('%s: %s' % (arg, value))
print('------------------------------------------------')
print("%s: %s" % (arg, value))
print("------------------------------------------------")
if __name__ == "__main__":

10
applications/document_intelligence/doc_vqa/Extraction/view.py
View File

@@ -30,12 +30,12 @@ def _highlight_bbox(img, bbox):
w = bbox[1] - x
y = bbox[2]
h = bbox[3] - y
sub_img = img[y:y + h, x:x + w]
sub_img = img[y : y + h, x : x + w]
colored_rect = np.zeros(sub_img.shape, dtype=np.uint8)
colored_rect[:, :, 2] = 255
colored_rect[:, :, 1] = 255
res = cv2.addWeighted(sub_img, 0.5, colored_rect, 0.5, 1.0)
img[y:y + h, x:x + w] = res
img[y : y + h, x : x + w] = res
def highlight_ans(source_img_path, output_img_path, ans_bbox):
@@ -54,14 +54,14 @@ def highlight_img(source_img_path, output_img_path):
cv2.imwrite(output_img_path, image)
if __name__ == '__main__':
if __name__ == "__main__":
res_path = "./data/decode_res.json"
result = {}
with open(res_path, "r", encoding="utf-8") as f:
line = f.readline()
result = json.loads(line.strip())
img_path = '../OCR_process/demo_pics/demo_{}.png'.format(result["img_id"])
img_path = "../OCR_process/demo_pics/demo_{}.png".format(result["img_id"])
img_save_path = "../answer.png"
highlight_ans(img_path, img_save_path, result['predict_bboxes'])
highlight_ans(img_path, img_save_path, result["predict_bboxes"])
print("extraction result has been saved to answer.png")

69
applications/document_intelligence/doc_vqa/OCR_process/ocr_process.py
View File

@@ -33,7 +33,7 @@ def get_all_chars(tokenizer):
tok_chr = [tc.replace("▁", "") for tc in tok_chr]
while "" in tok_chr:
tok_chr.remove("")
tok_chr = ''.join(tok_chr)
tok_chr = "".join(tok_chr)
if len(tok_chr) != 1:
all_chr.append(i)
return all_chr
@@ -53,8 +53,7 @@ def merge_bbox(tok_bboxes):
height_m += y_max - y_min
width_m += x_max - x_min
height_m = height_m / len(tok_bboxes)
if (height_g -
height_m) < 0.5 * height_m and width_g - width_m < 0.1 * width_m:
if (height_g - height_m) < 0.5 * height_m and width_g - width_m < 0.1 * width_m:
return False, [min_gx, max_gx, min_gy, max_gy]
else:
return True, tok_bboxes[0]
@@ -72,7 +71,7 @@ def xlm_parse(ocr_res, tokenizer):
new_token_boxes.extend(item["token_box"])
# get layoutxlm tokenizer results and get the final results
temp_span_text = ''.join(new_tokens)
temp_span_text = "".join(new_tokens)
temp_span_bbox = new_token_boxes
span_text = ""
span_bbox = []
@@ -138,20 +137,15 @@ def xlm_parse(ocr_res, tokenizer):
for j in range(i, len(span_text)):
if span_text[j].lower() == span_tokens[tid + 1][0]:
break
if span_text[j].lower() == "，" and span_tokens[
tid + 1][0] == ",":
if span_text[j].lower() == "，" and span_tokens[tid + 1][0] == ",":
break
if span_text[j].lower() == "；" and span_tokens[
tid + 1][0] == ";":
if span_text[j].lower() == "；" and span_tokens[tid + 1][0] == ";":
break
if span_text[j].lower() == "）" and span_tokens[
tid + 1][0] == ")":
if span_text[j].lower() == "）" and span_tokens[tid + 1][0] == ")":
break
if span_text[j].lower() == "（" and span_tokens[
tid + 1][0] == "(":
if span_text[j].lower() == "（" and span_tokens[tid + 1][0] == "(":
break
if span_text[j].lower() == "￥" and span_tokens[
tid + 1][0] == "¥":
if span_text[j].lower() == "￥" and span_tokens[tid + 1][0] == "¥":
break
tok_len = j - i
@@ -159,7 +153,7 @@ def xlm_parse(ocr_res, tokenizer):
else:
if "<EFBFBD>" == span_text[i]:
tok_len = len(tok) + 1
elif tok == "......" and "…" in span_text[i:i + 6]:
elif tok == "......" and "…" in span_text[i : i + 6]:
tok_len = len(tok) - 2
elif "ⅱ" in span_text[i + len(tok) - 1]:
if tok == "i":
@@ -172,21 +166,21 @@ def xlm_parse(ocr_res, tokenizer):
tok_len = len(tok)
assert i + tok_len <= len(span_bbox)
tok_bboxes = span_bbox[i:i + tok_len]
tok_bboxes = span_bbox[i : i + tok_len]
_, merged_bbox = merge_bbox(tok_bboxes)
doc_bboxes.append(merged_bbox)
i = i + tok_len
except:
print('Error')
span_tokens = ['▁'] * 512
print("Error")
span_tokens = ["▁"] * 512
doc_bboxes = [[0, 0, 0, 0]] * 512
return span_tokens, doc_bboxes
def tokenize_ocr_res(ocr_reses):
'''
"""
input:
ocr_res: the ocr result of the image
return:
@@ -198,7 +192,7 @@ def tokenize_ocr_res(ocr_reses):
"token_box: bounding box of each chars in ocr_res
}
}
'''
"""
new_reses = []
for img_name, ocr_res in ocr_reses:
new_res = []
@@ -227,16 +221,16 @@ def tokenize_ocr_res(ocr_reses):
tok_y_max = y_max
shift = tok_x_max
new_token_boxes.append(
[round(tok_x_min),
round(tok_x_max), tok_y_min, tok_y_max])
new_token_boxes.append([round(tok_x_min), round(tok_x_max), tok_y_min, tok_y_max])
new_tokens.append(char)
new_res.append({
"text": para["text"],
"bounding_box": para["bbox"],
"tokens": new_tokens,
"token_box": new_token_boxes
})
new_res.append(
{
"text": para["text"],
"bounding_box": para["bbox"],
"tokens": new_tokens,
"token_box": new_token_boxes,
}
)
new_reses.append((img_name, new_res))
return new_reses
@@ -247,19 +241,15 @@ def process_input(ocr_reses, tokenizer, save_ocr_path):
examples = []
for img_name, ocr_res in ocr_reses:
doc_tokens, doc_bboxes = xlm_parse(ocr_res, tokenizer)
doc_tokens.insert(0, '▁')
doc_tokens.insert(0, "▁")
doc_bboxes.insert(0, doc_bboxes[0])
example = {
"img_name": img_name,
"document": doc_tokens,
"document_bbox": doc_bboxes
}
example = {"img_name": img_name, "document": doc_tokens, "document_bbox": doc_bboxes}
examples.append(example)
with open(save_ocr_path, 'w', encoding='utf8') as f:
with open(save_ocr_path, "w", encoding="utf8") as f:
for example in examples:
json.dump(example, f, ensure_ascii=False)
f.write('\n')
f.write("\n")
print(f"ocr parsing results has been save to: {save_ocr_path}")
@@ -267,8 +257,7 @@ def process_input(ocr_reses, tokenizer, save_ocr_path):
def ocr_preprocess(img_dir):
ocr = PaddleOCR(use_angle_cls=True, lang="ch", use_gpu=True)
ocr_reses = []
img_names = sorted(os.listdir(img_dir),
key=lambda x: int(x.split("_")[1].split(".")[0]))
img_names = sorted(os.listdir(img_dir), key=lambda x: int(x.split("_")[1].split(".")[0]))
for img_name in img_names:
img_path = os.path.join(img_dir, img_name)
parsing_res = ocr.ocr(img_path, cls=True)
@@ -280,7 +269,7 @@ def ocr_preprocess(img_dir):
return ocr_reses
if __name__ == '__main__':
if __name__ == "__main__":
img_dir = "./demo_pics"
save_path = "./demo_ocr_res.json"
ocr_results = ocr_preprocess(img_dir)

12
applications/document_intelligence/doc_vqa/Rerank/change_to_rerank.py
View File

@@ -17,17 +17,17 @@ import json
question = sys.argv[1]
with open('../OCR_process/demo_ocr_res.json', 'r', encoding='utf8') as f:
with open("../OCR_process/demo_ocr_res.json", "r", encoding="utf8") as f:
paras = []
for line in f:
line = json.loads(line.strip())
document = line['document']
document = line["document"]
para = []
for token in document:
token = token.replace('▁', '')
token = token.replace("▁", "")
para.append(token)
paras.append(''.join(para))
paras.append("".join(para))
with open('./data/demo.tsv', 'w', encoding='utf8') as f:
with open("./data/demo.tsv", "w", encoding="utf8") as f:
for para in paras:
f.write('{}\t\t{}\t0\n'.format(question, para))
f.write("{}\t\t{}\t0\n".format(question, para))

27
applications/document_intelligence/doc_vqa/Rerank/src/batching.py
View File

@@ -21,13 +21,15 @@ import numpy as np
from six.moves import xrange
def pad_batch_data(insts,
pad_idx=0,
return_pos=False,
return_input_mask=False,
return_max_len=False,
return_num_token=False,
return_seq_lens=False):
def pad_batch_data(
insts,
pad_idx=0,
return_pos=False,
return_input_mask=False,
return_max_len=False,
return_num_token=False,
return_seq_lens=False,
):
"""
Pad the instances to the max sequence length in batch, and generate the
corresponding position data and attention bias.
@@ -37,23 +39,18 @@ def pad_batch_data(insts,
# Any token included in dict can be used to pad, since the paddings' loss
# will be masked out by weights and make no effect on parameter gradients.
inst_data = np.array(
[inst + list([pad_idx] * (max_len - len(inst))) for inst in insts])
inst_data = np.array([inst + list([pad_idx] * (max_len - len(inst))) for inst in insts])
return_list += [inst_data.astype("int64").reshape([-1, max_len, 1])]
# position data
if return_pos:
inst_pos = np.array([
list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst))
for inst in insts
])
inst_pos = np.array([list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) for inst in insts])
return_list += [inst_pos.astype("int64").reshape([-1, max_len, 1])]
if return_input_mask:
# This is used to avoid attention on paddings.
input_mask_data = np.array(
[[1] * len(inst) + [0] * (max_len - len(inst)) for inst in insts])
input_mask_data = np.array([[1] * len(inst) + [0] * (max_len - len(inst)) for inst in insts])
input_mask_data = np.expand_dims(input_mask_data, axis=-1)
return_list += [input_mask_data.astype("float32")]

181
applications/document_intelligence/doc_vqa/Rerank/src/cross_encoder.py
View File

@@ -32,53 +32,48 @@ from model.ernie import ErnieModel
log = logging.getLogger(__name__)
def create_model(args,
pyreader_name,
ernie_config,
is_prediction=False,
task_name=""):
pyreader = fluid.layers.py_reader(capacity=50,
shapes=[[-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1],
[-1, 1], [-1, 1]],
dtypes=[
'int64', 'int64', 'int64', 'int64',
'float32', 'int64', 'int64'
],
lod_levels=[0, 0, 0, 0, 0, 0, 0],
name=task_name + "_" + pyreader_name,
use_double_buffer=True)
def create_model(args, pyreader_name, ernie_config, is_prediction=False, task_name=""):
pyreader = fluid.layers.py_reader(
capacity=50,
shapes=[
[-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1],
[-1, args.max_seq_len, 1],
[-1, 1],
[-1, 1],
],
dtypes=["int64", "int64", "int64", "int64", "float32", "int64", "int64"],
lod_levels=[0, 0, 0, 0, 0, 0, 0],
name=task_name + "_" + pyreader_name,
use_double_buffer=True,
)
(src_ids, sent_ids, pos_ids, task_ids, input_mask, labels,
qids) = fluid.layers.read_file(pyreader)
(src_ids, sent_ids, pos_ids, task_ids, input_mask, labels, qids) = fluid.layers.read_file(pyreader)
def _model(is_noise=False):
ernie = ErnieModel(src_ids=src_ids,
position_ids=pos_ids,
sentence_ids=sent_ids,
task_ids=task_ids,
input_mask=input_mask,
config=ernie_config,
is_noise=is_noise)
ernie = ErnieModel(
src_ids=src_ids,
position_ids=pos_ids,
sentence_ids=sent_ids,
task_ids=task_ids,
input_mask=input_mask,
config=ernie_config,
is_noise=is_noise,
)
cls_feats = ernie.get_pooled_output()
if not is_noise:
cls_feats = fluid.layers.dropout(
x=cls_feats,
dropout_prob=0.1,
dropout_implementation="upscale_in_train")
cls_feats = fluid.layers.dropout(x=cls_feats, dropout_prob=0.1, dropout_implementation="upscale_in_train")
logits = fluid.layers.fc(
input=cls_feats,
size=args.num_labels,
param_attr=fluid.ParamAttr(
name=task_name + "_cls_out_w",
initializer=fluid.initializer.TruncatedNormal(scale=0.02)),
bias_attr=fluid.ParamAttr(
name=task_name + "_cls_out_b",
initializer=fluid.initializer.Constant(0.)))
name=task_name + "_cls_out_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)
),
bias_attr=fluid.ParamAttr(name=task_name + "_cls_out_b", initializer=fluid.initializer.Constant(0.0)),
)
"""
if is_prediction:
probs = fluid.layers.softmax(logits)
@@ -90,37 +85,31 @@ def create_model(args,
return pyreader, probs, feed_targets_name
"""
num_seqs = fluid.layers.create_tensor(dtype='int64')
num_seqs = fluid.layers.create_tensor(dtype="int64")
## add focal loss
ce_loss, probs = fluid.layers.softmax_with_cross_entropy(
logits=logits, label=labels, return_softmax=True)
ce_loss, probs = fluid.layers.softmax_with_cross_entropy(logits=logits, label=labels, return_softmax=True)
loss = fluid.layers.mean(x=ce_loss)
accuracy = fluid.layers.accuracy(input=probs,
label=labels,
total=num_seqs)
accuracy = fluid.layers.accuracy(input=probs, label=labels, total=num_seqs)
graph_vars = {
"loss": loss,
"probs": probs,
"accuracy": accuracy,
"labels": labels,
"num_seqs": num_seqs,
"qids": qids
"qids": qids,
}
return graph_vars
if not is_prediction:
graph_vars = _model(is_noise=True)
old_loss = graph_vars["loss"]
token_emb = fluid.default_main_program().global_block().var(
"word_embedding")
token_emb = fluid.default_main_program().global_block().var("word_embedding")
# print(token_emb)
token_emb.stop_gradient = False
token_gradient = fluid.gradients(old_loss, token_emb)[0]
token_gradient.stop_gradient = False
epsilon = 1e-8
norm = (fluid.layers.sqrt(
fluid.layers.reduce_sum(fluid.layers.square(token_gradient)) +
epsilon))
norm = fluid.layers.sqrt(fluid.layers.reduce_sum(fluid.layers.square(token_gradient)) + epsilon)
gp = (0.01 * token_gradient) / norm
gp.stop_gradient = True
fluid.layers.assign(token_emb + gp, token_emb)
@@ -153,17 +142,10 @@ def evaluate_mrr(preds):
return total_mrr / qnum
def evaluate(exe,
test_program,
test_pyreader,
graph_vars,
eval_phase,
use_multi_gpu_test=False,
metric='simple_accuracy'):
train_fetch_list = [
graph_vars["loss"].name, graph_vars["accuracy"].name,
graph_vars["num_seqs"].name
]
def evaluate(
exe, test_program, test_pyreader, graph_vars, eval_phase, use_multi_gpu_test=False, metric="simple_accuracy"
):
train_fetch_list = [graph_vars["loss"].name, graph_vars["accuracy"].name, graph_vars["num_seqs"].name]
if eval_phase == "train":
if "learning_rate" in graph_vars:
@@ -185,18 +167,21 @@ def evaluate(exe,
time_begin = time.time()
fetch_list = [
graph_vars["loss"].name, graph_vars["accuracy"].name,
graph_vars["probs"].name, graph_vars["labels"].name,
graph_vars["num_seqs"].name, graph_vars["qids"].name
graph_vars["loss"].name,
graph_vars["accuracy"].name,
graph_vars["probs"].name,
graph_vars["labels"].name,
graph_vars["num_seqs"].name,
graph_vars["qids"].name,
]
while True:
try:
if use_multi_gpu_test:
np_loss, np_acc, np_probs, np_labels, np_num_seqs, np_qids = exe.run(
fetch_list=fetch_list)
np_loss, np_acc, np_probs, np_labels, np_num_seqs, np_qids = exe.run(fetch_list=fetch_list)
else:
np_loss, np_acc, np_probs, np_labels, np_num_seqs, np_qids = exe.run(
program=test_program, fetch_list=fetch_list)
program=test_program, fetch_list=fetch_list
)
total_cost += np.sum(np_loss * np_num_seqs)
total_acc += np.sum(np_acc * np_num_seqs)
total_num_seqs += np.sum(np_num_seqs)
@@ -218,31 +203,55 @@ def evaluate(exe,
elapsed_time = time_end - time_begin
evaluate_info = ""
if metric == 'acc_and_f1':
if metric == "acc_and_f1":
ret = acc_and_f1(preds, labels)
evaluate_info = "[%s evaluation] ave loss: %f, ave_acc: %f, f1: %f, data_num: %d, elapsed time: %f s" \
% (eval_phase, cost, ret['acc'], ret['f1'], total_num_seqs, elapsed_time)
elif metric == 'matthews_corrcoef':
evaluate_info = "[%s evaluation] ave loss: %f, ave_acc: %f, f1: %f, data_num: %d, elapsed time: %f s" % (
eval_phase,
cost,
ret["acc"],
ret["f1"],
total_num_seqs,
elapsed_time,
)
elif metric == "matthews_corrcoef":
ret = matthews_corrcoef(preds, labels)
evaluate_info = "[%s evaluation] ave loss: %f, matthews_corrcoef: %f, data_num: %d, elapsed time: %f s" \
% (eval_phase, cost, ret, total_num_seqs, elapsed_time)
elif metric == 'pearson_and_spearman':
evaluate_info = "[%s evaluation] ave loss: %f, matthews_corrcoef: %f, data_num: %d, elapsed time: %f s" % (
eval_phase,
cost,
ret,
total_num_seqs,
elapsed_time,
)
elif metric == "pearson_and_spearman":
ret = pearson_and_spearman(scores, labels)
evaluate_info = "[%s evaluation] ave loss: %f, pearson:%f, spearman:%f, corr:%f, data_num: %d, elapsed time: %f s" \
% (eval_phase, cost, ret['pearson'], ret['spearman'], ret['corr'], total_num_seqs, elapsed_time)
elif metric == 'simple_accuracy':
evaluate_info = (
"[%s evaluation] ave loss: %f, pearson:%f, spearman:%f, corr:%f, data_num: %d, elapsed time: %f s"
% (eval_phase, cost, ret["pearson"], ret["spearman"], ret["corr"], total_num_seqs, elapsed_time)
)
elif metric == "simple_accuracy":
ret = simple_accuracy(preds, labels)
evaluate_info = "[%s evaluation] ave loss: %f, acc:%f, data_num: %d, elapsed time: %f s" \
% (eval_phase, cost, ret, total_num_seqs, elapsed_time)
evaluate_info = "[%s evaluation] ave loss: %f, acc:%f, data_num: %d, elapsed time: %f s" % (
eval_phase,
cost,
ret,
total_num_seqs,
elapsed_time,
)
elif metric == "acc_and_f1_and_mrr":
ret_a = acc_and_f1(preds, labels)
preds = sorted(zip(qids, scores, labels),
key=lambda elem: (elem[0], -elem[1]))
preds = sorted(zip(qids, scores, labels), key=lambda elem: (elem[0], -elem[1]))
ret_b = evaluate_mrr(preds)
evaluate_info = "[%s evaluation] ave loss: %f, acc: %f, f1: %f, mrr: %f, data_num: %d, elapsed time: %f s" \
% (eval_phase, cost, ret_a['acc'], ret_a['f1'], ret_b, total_num_seqs, elapsed_time)
evaluate_info = "[%s evaluation] ave loss: %f, acc: %f, f1: %f, mrr: %f, data_num: %d, elapsed time: %f s" % (
eval_phase,
cost,
ret_a["acc"],
ret_a["f1"],
ret_b,
total_num_seqs,
elapsed_time,
)
else:
raise ValueError('unsupported metric {}'.format(metric))
raise ValueError("unsupported metric {}".format(metric))
return evaluate_info
@@ -254,8 +263,7 @@ def matthews_corrcoef(preds, labels):
fp = np.sum((labels == 0) & (preds == 1))
fn = np.sum((labels == 1) & (preds == 0))
mcc = ((tp * tn) - (fp * fn)) / np.sqrt(
(tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
mcc = ((tp * tn) - (fp * fn)) / np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
return mcc
@@ -315,8 +323,7 @@ def predict(exe, test_program, test_pyreader, graph_vars, dev_count=1):
while True:
try:
if dev_count == 1:
np_probs, np_qids = exe.run(program=test_program,
fetch_list=fetch_list)
np_probs, np_qids = exe.run(program=test_program, fetch_list=fetch_list)
else:
np_probs, np_qids = exe.run(fetch_list=fetch_list)

21
applications/document_intelligence/doc_vqa/Rerank/src/index_search.py
View File

@@ -20,11 +20,11 @@ import numpy as np
def read_embed(file_name, dim=768, bs=3000):
if file_name.endswith('npy'):
if file_name.endswith("npy"):
i = 0
emb_np = np.load(file_name)
while (i < len(emb_np)):
vec_list = emb_np[i:i + bs]
while i < len(emb_np):
vec_list = emb_np[i : i + bs]
i += bs
yield vec_list
else:
@@ -32,7 +32,7 @@ def read_embed(file_name, dim=768, bs=3000):
with open(file_name) as inp:
for line in inp:
data = line.strip()
vector = [float(item) for item in data.split(' ')]
vector = [float(item) for item in data.split(" ")]
assert len(vector) == dim
vec_list.append(vector)
if len(vec_list) == bs:
@@ -47,24 +47,23 @@ def load_qid(file_name):
with open(file_name) as inp:
for line in inp:
line = line.strip()
qid = line.split('\t')[0]
qid = line.split("\t")[0]
qid_list.append(qid)
return qid_list
def search(index, emb_file, qid_list, outfile, top_k):
q_idx = 0
with open(outfile, 'w') as out:
with open(outfile, "w") as out:
for batch_vec in read_embed(emb_file):
q_emb_matrix = np.array(batch_vec)
res_dist, res_p_id = index.search(q_emb_matrix.astype('float32'),
top_k)
res_dist, res_p_id = index.search(q_emb_matrix.astype("float32"), top_k)
for i in range(len(q_emb_matrix)):
qid = qid_list[q_idx]
for j in range(top_k):
pid = res_p_id[i][j]
score = res_dist[i][j]
out.write('%s\t%s\t%s\t%s\n' % (qid, pid, j + 1, score))
out.write("%s\t%s\t%s\t%s\n" % (qid, pid, j + 1, score))
q_idx += 1
@@ -72,12 +71,12 @@ def main():
part = sys.argv[1]
topk = int(sys.argv[2])
q_text_file = sys.argv[3]
outfile = 'output/res.top%s-part%s' % (topk, part)
outfile = "output/res.top%s-part%s" % (topk, part)
qid_list = load_qid(q_text_file)
engine = faiss.read_index("output/para.index.part%s" % part)
emb_file = 'output/query.emb.npy'
emb_file = "output/query.emb.npy"
search(engine, emb_file, qid_list, outfile, topk)

15
applications/document_intelligence/doc_vqa/Rerank/src/merge.py
View File

@@ -20,7 +20,7 @@ total_part = int(sys.argv[3])
f_list = []
for part in range(total_part):
f0 = open('output/res.top%s-part%s' % (top, part))
f0 = open("output/res.top%s-part%s" % (top, part))
f_list.append(f0)
line_list = []
@@ -28,22 +28,21 @@ for part in range(total_part):
line = f_list[part].readline()
line_list.append(line)
out = open('output/dev.res.top%s' % top, 'w')
last_q = ''
out = open("output/dev.res.top%s" % top, "w")
last_q = ""
ans_list = {}
while line_list[-1]:
cur_list = []
for line in line_list:
sub = line.strip().split('\t')
sub = line.strip().split("\t")
cur_list.append(sub)
if last_q == '':
if last_q == "":
last_q = cur_list[0][0]
if cur_list[0][0] != last_q:
rank = sorted(ans_list.items(), key=lambda a: a[1], reverse=True)
for i in range(top):
out.write("%s\t%s\t%s\t%s\n" %
(last_q, rank[i][0], i + 1, rank[i][1]))
out.write("%s\t%s\t%s\t%s\n" % (last_q, rank[i][0], i + 1, rank[i][1]))
ans_list = {}
for i, sub in enumerate(cur_list):
ans_list[int(sub[1]) + shift * i] = float(sub[-1])
@@ -59,4 +58,4 @@ for i in range(top):
out.write("%s\t%s\t%s\t%s\n" % (last_q, rank[i][0], i + 1, rank[i][1]))
out.close()
print('output/dev.res.top%s' % top)
print("output/dev.res.top%s" % top)

193
applications/document_intelligence/doc_vqa/Rerank/src/model/ernie.py
View File

@@ -32,17 +32,15 @@ log = logging.getLogger(__name__)
class ErnieConfig(object):
def __init__(self, config_path):
self._config_dict = self._parse(config_path)
def _parse(self, config_path):
try:
with open(config_path, 'r', encoding='utf8') as json_file:
with open(config_path, "r", encoding="utf8") as json_file:
config_dict = json.load(json_file)
except Exception:
raise IOError("Error in parsing Ernie model config file '%s'" %
config_path)
raise IOError("Error in parsing Ernie model config file '%s'" % config_path)
else:
return config_dict
@@ -51,39 +49,40 @@ class ErnieConfig(object):
def print_config(self):
for arg, value in sorted(six.iteritems(self._config_dict)):
log.info('%s: %s' % (arg, value))
log.info('------------------------------------------------')
log.info("%s: %s" % (arg, value))
log.info("------------------------------------------------")
class ErnieModel(object):
def __init__(
self,
src_ids,
position_ids,
sentence_ids,
task_ids,
input_mask,
config,
weight_sharing=True,
model_name="",
is_noise=False,
):
def __init__(self,
src_ids,
position_ids,
sentence_ids,
task_ids,
input_mask,
config,
weight_sharing=True,
model_name='',
is_noise=False):
self._emb_size = config['hidden_size']
self._n_layer = config['num_hidden_layers']
self._n_head = config['num_attention_heads']
self._voc_size = config['vocab_size']
self._max_position_seq_len = config['max_position_embeddings']
if config['sent_type_vocab_size']:
self._sent_types = config['sent_type_vocab_size']
self._emb_size = config["hidden_size"]
self._n_layer = config["num_hidden_layers"]
self._n_head = config["num_attention_heads"]
self._voc_size = config["vocab_size"]
self._max_position_seq_len = config["max_position_embeddings"]
if config["sent_type_vocab_size"]:
self._sent_types = config["sent_type_vocab_size"]
else:
self._sent_types = config['type_vocab_size']
self._sent_types = config["type_vocab_size"]
self._use_task_id = config['use_task_id']
self._use_task_id = config["use_task_id"]
if self._use_task_id:
self._task_types = config['task_type_vocab_size']
self._hidden_act = config['hidden_act']
self._prepostprocess_dropout = config['hidden_dropout_prob']
self._attention_dropout = config['attention_probs_dropout_prob']
self._task_types = config["task_type_vocab_size"]
self._hidden_act = config["hidden_act"]
self._prepostprocess_dropout = config["hidden_dropout_prob"]
self._attention_dropout = config["attention_probs_dropout_prob"]
if is_noise:
self._prepostprocess_dropout = 0
self._attention_dropout = 0
@@ -98,36 +97,33 @@ class ErnieModel(object):
# Initialize all weigths by truncated normal initializer, and all biases
# will be initialized by constant zero by default.
self._param_initializer = fluid.initializer.TruncatedNormal(
scale=config['initializer_range'])
self._param_initializer = fluid.initializer.TruncatedNormal(scale=config["initializer_range"])
self._build_model(model_name, src_ids, position_ids, sentence_ids,
task_ids, input_mask)
self._build_model(model_name, src_ids, position_ids, sentence_ids, task_ids, input_mask)
def _build_model(self, model_name, src_ids, position_ids, sentence_ids,
task_ids, input_mask):
def _build_model(self, model_name, src_ids, position_ids, sentence_ids, task_ids, input_mask):
# padding id in vocabulary must be set to 0
emb_out = fluid.layers.embedding(
input=src_ids,
size=[self._voc_size, self._emb_size],
dtype=self._emb_dtype,
param_attr=fluid.ParamAttr(name=model_name + self._word_emb_name,
initializer=self._param_initializer),
is_sparse=False)
param_attr=fluid.ParamAttr(name=model_name + self._word_emb_name, initializer=self._param_initializer),
is_sparse=False,
)
position_emb_out = fluid.layers.embedding(
input=position_ids,
size=[self._max_position_seq_len, self._emb_size],
dtype=self._emb_dtype,
param_attr=fluid.ParamAttr(name=model_name + self._pos_emb_name,
initializer=self._param_initializer))
param_attr=fluid.ParamAttr(name=model_name + self._pos_emb_name, initializer=self._param_initializer),
)
sent_emb_out = fluid.layers.embedding(
sentence_ids,
size=[self._sent_types, self._emb_size],
dtype=self._emb_dtype,
param_attr=fluid.ParamAttr(name=model_name + self._sent_emb_name,
initializer=self._param_initializer))
param_attr=fluid.ParamAttr(name=model_name + self._sent_emb_name, initializer=self._param_initializer),
)
emb_out = emb_out + position_emb_out
emb_out = emb_out + sent_emb_out
@@ -137,28 +133,17 @@ class ErnieModel(object):
task_ids,
size=[self._task_types, self._emb_size],
dtype=self._emb_dtype,
param_attr=fluid.ParamAttr(name=model_name +
self._task_emb_name,
initializer=self._param_initializer))
param_attr=fluid.ParamAttr(name=model_name + self._task_emb_name, initializer=self._param_initializer),
)
emb_out = emb_out + task_emb_out
emb_out = pre_process_layer(emb_out,
'nd',
self._prepostprocess_dropout,
name=model_name + 'pre_encoder')
emb_out = pre_process_layer(emb_out, "nd", self._prepostprocess_dropout, name=model_name + "pre_encoder")
self_attn_mask = fluid.layers.matmul(x=input_mask,
y=input_mask,
transpose_y=True)
self_attn_mask = fluid.layers.matmul(x=input_mask, y=input_mask, transpose_y=True)
self_attn_mask = fluid.layers.scale(x=self_attn_mask,
scale=10000.0,
bias=-1.0,
bias_after_scale=False)
n_head_self_attn_mask = fluid.layers.stack(x=[self_attn_mask] *
self._n_head,
axis=1)
self_attn_mask = fluid.layers.scale(x=self_attn_mask, scale=10000.0, bias=-1.0, bias_after_scale=False)
n_head_self_attn_mask = fluid.layers.stack(x=[self_attn_mask] * self._n_head, axis=1)
n_head_self_attn_mask.stop_gradient = True
self._enc_out, self.checkpoints = encoder(
@@ -178,44 +163,38 @@ class ErnieModel(object):
postprocess_cmd="dan",
param_initializer=self._param_initializer,
model_name=model_name,
name=model_name + 'encoder')
name=model_name + "encoder",
)
def get_sequence_output(self):
return self._enc_out
def get_cls_output(self):
"""Get the first feature of each sequence for classification"""
cls_output = fluid.layers.slice(input=self._enc_out,
axes=[1],
starts=[0],
ends=[1])
cls_output = fluid.layers.slice(input=self._enc_out, axes=[1], starts=[0], ends=[1])
cls_output = fluid.layers.squeeze(cls_output, axes=[1])
return cls_output
def get_pooled_output(self):
"""Get the first feature of each sequence for classification"""
next_sent_feat = fluid.layers.slice(input=self._enc_out,
axes=[1],
starts=[0],
ends=[1])
next_sent_feat = fluid.layers.slice(input=self._enc_out, axes=[1], starts=[0], ends=[1])
next_sent_feat = fluid.layers.fc(
input=next_sent_feat,
size=self._emb_size,
act="tanh",
param_attr=fluid.ParamAttr(name="pooled_fc.w_0",
initializer=self._param_initializer),
bias_attr="pooled_fc.b_0")
param_attr=fluid.ParamAttr(name="pooled_fc.w_0", initializer=self._param_initializer),
bias_attr="pooled_fc.b_0",
)
return next_sent_feat
def get_lm_output(self, mask_label, mask_pos):
"""Get the loss & accuracy for pretraining"""
mask_pos = fluid.layers.cast(x=mask_pos, dtype='int32')
mask_pos = fluid.layers.cast(x=mask_pos, dtype="int32")
# extract the first token feature in each sentence
self.next_sent_feat = self.get_pooled_output()
reshaped_emb_out = fluid.layers.reshape(x=self._enc_out,
shape=[-1, self._emb_size])
reshaped_emb_out = fluid.layers.reshape(x=self._enc_out, shape=[-1, self._emb_size])
# extract masked tokens' feature
mask_feat = fluid.layers.gather(input=reshaped_emb_out, index=mask_pos)
@@ -224,61 +203,61 @@ class ErnieModel(object):
input=mask_feat,
size=self._emb_size,
act=self._hidden_act,
param_attr=fluid.ParamAttr(name='mask_lm_trans_fc.w_0',
initializer=self._param_initializer),
bias_attr=fluid.ParamAttr(name='mask_lm_trans_fc.b_0'))
param_attr=fluid.ParamAttr(name="mask_lm_trans_fc.w_0", initializer=self._param_initializer),
bias_attr=fluid.ParamAttr(name="mask_lm_trans_fc.b_0"),
)
# transform: layer norm
mask_trans_feat = fluid.layers.layer_norm(
mask_trans_feat,
begin_norm_axis=len(mask_trans_feat.shape) - 1,
param_attr=fluid.ParamAttr(
name='mask_lm_trans_layer_norm_scale',
initializer=fluid.initializer.Constant(1.)),
name="mask_lm_trans_layer_norm_scale", initializer=fluid.initializer.Constant(1.0)
),
bias_attr=fluid.ParamAttr(
name='mask_lm_trans_layer_norm_bias',
initializer=fluid.initializer.Constant(1.)))
name="mask_lm_trans_layer_norm_bias", initializer=fluid.initializer.Constant(1.0)
),
)
# transform: layer norm
#mask_trans_feat = pre_process_layer(
# mask_trans_feat = pre_process_layer(
# mask_trans_feat, 'n', name='mask_lm_trans')
mask_lm_out_bias_attr = fluid.ParamAttr(
name="mask_lm_out_fc.b_0",
initializer=fluid.initializer.Constant(value=0.0))
name="mask_lm_out_fc.b_0", initializer=fluid.initializer.Constant(value=0.0)
)
if self._weight_sharing:
fc_out = fluid.layers.matmul(
x=mask_trans_feat,
y=fluid.default_main_program().global_block().var(
self._word_emb_name),
transpose_y=True)
fc_out += fluid.layers.create_parameter(shape=[self._voc_size],
dtype=self._emb_dtype,
attr=mask_lm_out_bias_attr,
is_bias=True)
y=fluid.default_main_program().global_block().var(self._word_emb_name),
transpose_y=True,
)
fc_out += fluid.layers.create_parameter(
shape=[self._voc_size], dtype=self._emb_dtype, attr=mask_lm_out_bias_attr, is_bias=True
)
else:
fc_out = fluid.layers.fc(input=mask_trans_feat,
size=self._voc_size,
param_attr=fluid.ParamAttr(
name="mask_lm_out_fc.w_0",
initializer=self._param_initializer),
bias_attr=mask_lm_out_bias_attr)
fc_out = fluid.layers.fc(
input=mask_trans_feat,
size=self._voc_size,
param_attr=fluid.ParamAttr(name="mask_lm_out_fc.w_0", initializer=self._param_initializer),
bias_attr=mask_lm_out_bias_attr,
)
mask_lm_loss = fluid.layers.softmax_with_cross_entropy(logits=fc_out,
label=mask_label)
mask_lm_loss = fluid.layers.softmax_with_cross_entropy(logits=fc_out, label=mask_label)
mean_mask_lm_loss = fluid.layers.mean(mask_lm_loss)
return mean_mask_lm_loss
def get_task_output(self, task, task_labels):
task_fc_out = fluid.layers.fc(input=self.next_sent_feat,
size=task["num_labels"],
param_attr=fluid.ParamAttr(
name=task["task_name"] + "_fc.w_0",
initializer=self._param_initializer),
bias_attr=task["task_name"] + "_fc.b_0")
task_fc_out = fluid.layers.fc(
input=self.next_sent_feat,
size=task["num_labels"],
param_attr=fluid.ParamAttr(name=task["task_name"] + "_fc.w_0", initializer=self._param_initializer),
bias_attr=task["task_name"] + "_fc.b_0",
)
task_loss, task_softmax = fluid.layers.softmax_with_cross_entropy(
logits=task_fc_out, label=task_labels, return_softmax=True)
logits=task_fc_out, label=task_labels, return_softmax=True
)
task_acc = fluid.layers.accuracy(input=task_softmax, label=task_labels)
mean_task_loss = fluid.layers.mean(task_loss)
return mean_task_loss, task_acc

339
applications/document_intelligence/doc_vqa/Rerank/src/model/transformer_encoder.py
View File

@@ -23,18 +23,20 @@ import paddle.fluid as fluid
import paddle.fluid.layers as layers
def multi_head_attention(queries,
keys,
values,
attn_bias,
d_key,
d_value,
d_model,
n_head=1,
dropout_rate=0.,
cache=None,
param_initializer=None,
name='multi_head_att'):
def multi_head_attention(
queries,
keys,
values,
attn_bias,
d_key,
d_value,
d_model,
n_head=1,
dropout_rate=0.0,
cache=None,
param_initializer=None,
name="multi_head_att",
):
"""
Multi-Head Attention. Note that attn_bias is added to the logit before
computing softmax activiation to mask certain selected positions so that
@@ -44,31 +46,33 @@ def multi_head_attention(queries,
values = keys if values is None else values
if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3):
raise ValueError(
"Inputs: quries, keys and values should all be 3-D tensors.")
raise ValueError("Inputs: quries, keys and values should all be 3-D tensors.")
def __compute_qkv(queries, keys, values, n_head, d_key, d_value):
"""
Add linear projection to queries, keys, and values.
"""
q = layers.fc(input=queries,
size=d_key * n_head,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(name=name + '_query_fc.w_0',
initializer=param_initializer),
bias_attr=name + '_query_fc.b_0')
k = layers.fc(input=keys,
size=d_key * n_head,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(name=name + '_key_fc.w_0',
initializer=param_initializer),
bias_attr=name + '_key_fc.b_0')
v = layers.fc(input=values,
size=d_value * n_head,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(name=name + '_value_fc.w_0',
initializer=param_initializer),
bias_attr=name + '_value_fc.b_0')
q = layers.fc(
input=queries,
size=d_key * n_head,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(name=name + "_query_fc.w_0", initializer=param_initializer),
bias_attr=name + "_query_fc.b_0",
)
k = layers.fc(
input=keys,
size=d_key * n_head,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(name=name + "_key_fc.w_0", initializer=param_initializer),
bias_attr=name + "_key_fc.b_0",
)
v = layers.fc(
input=values,
size=d_value * n_head,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(name=name + "_value_fc.w_0", initializer=param_initializer),
bias_attr=name + "_value_fc.b_0",
)
return q, k, v
def __split_heads(x, n_head):
@@ -81,9 +85,7 @@ def multi_head_attention(queries,
hidden_size = x.shape[-1]
# The value 0 in shape attr means copying the corresponding dimension
# size of the input as the output dimension size.
reshaped = layers.reshape(x=x,
shape=[0, 0, n_head, hidden_size // n_head],
inplace=True)
reshaped = layers.reshape(x=x, shape=[0, 0, n_head, hidden_size // n_head], inplace=True)
# permuate the dimensions into:
# [batch_size, n_head, max_sequence_len, hidden_size_per_head]
@@ -94,16 +96,15 @@ def multi_head_attention(queries,
Transpose and then reshape the last two dimensions of inpunt tensor x
so that it becomes one dimension, which is reverse to __split_heads.
"""
if len(x.shape) == 3: return x
if len(x.shape) == 3:
return x
if len(x.shape) != 4:
raise ValueError("Input(x) should be a 4-D Tensor.")
trans_x = layers.transpose(x, perm=[0, 2, 1, 3])
# The value 0 in shape attr means copying the corresponding dimension
# size of the input as the output dimension size.
return layers.reshape(x=trans_x,
shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]],
inplace=True)
return layers.reshape(x=trans_x, shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], inplace=True)
def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate):
"""
@@ -115,10 +116,9 @@ def multi_head_attention(queries,
product += attn_bias
weights = layers.softmax(product)
if dropout_rate:
weights = layers.dropout(weights,
dropout_prob=dropout_rate,
dropout_implementation="upscale_in_train",
is_test=False)
weights = layers.dropout(
weights, dropout_prob=dropout_rate, dropout_implementation="upscale_in_train", is_test=False
)
out = layers.matmul(weights, v)
return out
@@ -128,70 +128,57 @@ def multi_head_attention(queries,
# Since the inplace reshape in __split_heads changes the shape of k and
# v, which is the cache input for next time step, reshape the cache
# input from the previous time step first.
k = cache["k"] = layers.concat(
[layers.reshape(cache["k"], shape=[0, 0, d_model]), k], axis=1)
v = cache["v"] = layers.concat(
[layers.reshape(cache["v"], shape=[0, 0, d_model]), v], axis=1)
k = cache["k"] = layers.concat([layers.reshape(cache["k"], shape=[0, 0, d_model]), k], axis=1)
v = cache["v"] = layers.concat([layers.reshape(cache["v"], shape=[0, 0, d_model]), v], axis=1)
q = __split_heads(q, n_head)
k = __split_heads(k, n_head)
v = __split_heads(v, n_head)
ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_key,
dropout_rate)
ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate)
out = __combine_heads(ctx_multiheads)
# Project back to the model size.
proj_out = layers.fc(input=out,
size=d_model,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(
name=name + '_output_fc.w_0',
initializer=param_initializer),
bias_attr=name + '_output_fc.b_0')
proj_out = layers.fc(
input=out,
size=d_model,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(name=name + "_output_fc.w_0", initializer=param_initializer),
bias_attr=name + "_output_fc.b_0",
)
return proj_out
def positionwise_feed_forward(x,
d_inner_hid,
d_hid,
dropout_rate,
hidden_act,
param_initializer=None,
name='ffn'):
def positionwise_feed_forward(x, d_inner_hid, d_hid, dropout_rate, hidden_act, param_initializer=None, name="ffn"):
"""
Position-wise Feed-Forward Networks.
This module consists of two linear transformations with a ReLU activation
in between, which is applied to each position separately and identically.
"""
hidden = layers.fc(input=x,
size=d_inner_hid,
num_flatten_dims=2,
act=hidden_act,
param_attr=fluid.ParamAttr(
name=name + '_fc_0.w_0',
initializer=param_initializer),
bias_attr=name + '_fc_0.b_0')
hidden = layers.fc(
input=x,
size=d_inner_hid,
num_flatten_dims=2,
act=hidden_act,
param_attr=fluid.ParamAttr(name=name + "_fc_0.w_0", initializer=param_initializer),
bias_attr=name + "_fc_0.b_0",
)
if dropout_rate:
hidden = layers.dropout(hidden,
dropout_prob=dropout_rate,
dropout_implementation="upscale_in_train",
is_test=False)
out = layers.fc(input=hidden,
size=d_hid,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(name=name + '_fc_1.w_0',
initializer=param_initializer),
bias_attr=name + '_fc_1.b_0')
hidden = layers.dropout(
hidden, dropout_prob=dropout_rate, dropout_implementation="upscale_in_train", is_test=False
)
out = layers.fc(
input=hidden,
size=d_hid,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(name=name + "_fc_1.w_0", initializer=param_initializer),
bias_attr=name + "_fc_1.b_0",
)
return out
def pre_post_process_layer(prev_out,
out,
process_cmd,
dropout_rate=0.,
name=''):
def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.0, name=""):
"""
Add residual connection, layer normalization and droput to the out tensor
optionally according to the value of process_cmd.
@@ -209,19 +196,17 @@ def pre_post_process_layer(prev_out,
out,
begin_norm_axis=len(out.shape) - 1,
param_attr=fluid.ParamAttr(
name=name + '_layer_norm_scale',
initializer=fluid.initializer.Constant(1.)),
bias_attr=fluid.ParamAttr(
name=name + '_layer_norm_bias',
initializer=fluid.initializer.Constant(0.)))
name=name + "_layer_norm_scale", initializer=fluid.initializer.Constant(1.0)
),
bias_attr=fluid.ParamAttr(name=name + "_layer_norm_bias", initializer=fluid.initializer.Constant(0.0)),
)
if out_dtype == fluid.core.VarDesc.VarType.FP16:
out = layers.cast(x=out, dtype="float16")
elif cmd == "d": # add dropout
if dropout_rate:
out = layers.dropout(out,
dropout_prob=dropout_rate,
dropout_implementation="upscale_in_train",
is_test=False)
out = layers.dropout(
out, dropout_prob=dropout_rate, dropout_implementation="upscale_in_train", is_test=False
)
return out
@@ -229,68 +214,88 @@ pre_process_layer = partial(pre_post_process_layer, None)
post_process_layer = pre_post_process_layer
def encoder_layer(enc_input,
attn_bias,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd="n",
postprocess_cmd="da",
param_initializer=None,
name=''):
def encoder_layer(
enc_input,
attn_bias,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd="n",
postprocess_cmd="da",
param_initializer=None,
name="",
):
"""The encoder layers that can be stacked to form a deep encoder.
This module consits of a multi-head (self) attention followed by
position-wise feed-forward networks and both the two components companied
with the post_process_layer to add residual connection, layer normalization
and droput.
"""
attn_output = multi_head_attention(pre_process_layer(enc_input,
preprocess_cmd,
prepostprocess_dropout,
name=name +
'_pre_att'),
None,
None,
attn_bias,
d_key,
d_value,
d_model,
n_head,
attention_dropout,
param_initializer=param_initializer,
name=name + '_multi_head_att')
attn_output = post_process_layer(enc_input,
attn_output,
postprocess_cmd,
prepostprocess_dropout,
name=name + '_post_att')
ffd_output = positionwise_feed_forward(pre_process_layer(
attn_output,
preprocess_cmd,
prepostprocess_dropout,
name=name + '_pre_ffn'),
d_inner_hid,
d_model,
relu_dropout,
hidden_act,
param_initializer=param_initializer,
name=name + '_ffn')
return post_process_layer(attn_output,
ffd_output,
postprocess_cmd,
prepostprocess_dropout,
name=name + '_post_ffn'), ffd_output
attn_output = multi_head_attention(
pre_process_layer(enc_input, preprocess_cmd, prepostprocess_dropout, name=name + "_pre_att"),
None,
None,
attn_bias,
d_key,
d_value,
d_model,
n_head,
attention_dropout,
param_initializer=param_initializer,
name=name + "_multi_head_att",
)
attn_output = post_process_layer(
enc_input, attn_output, postprocess_cmd, prepostprocess_dropout, name=name + "_post_att"
)
ffd_output = positionwise_feed_forward(
pre_process_layer(attn_output, preprocess_cmd, prepostprocess_dropout, name=name + "_pre_ffn"),
d_inner_hid,
d_model,
relu_dropout,
hidden_act,
param_initializer=param_initializer,
name=name + "_ffn",
)
return (
post_process_layer(attn_output, ffd_output, postprocess_cmd, prepostprocess_dropout, name=name + "_post_ffn"),
ffd_output,
)
def encoder(enc_input,
def encoder(
enc_input,
attn_bias,
n_layer,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd="n",
postprocess_cmd="da",
param_initializer=None,
model_name="",
name="",
):
"""
The encoder is composed of a stack of identical layers returned by calling
encoder_layer.
"""
checkpoints = []
for i in range(n_layer):
enc_output, cp = encoder_layer(
enc_input,
attn_bias,
n_layer,
n_head,
d_key,
d_value,
@@ -300,37 +305,15 @@ def encoder(enc_input,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd="n",
postprocess_cmd="da",
param_initializer=None,
model_name='',
name=''):
"""
The encoder is composed of a stack of identical layers returned by calling
encoder_layer.
"""
checkpoints = []
for i in range(n_layer):
enc_output, cp = encoder_layer(enc_input,
attn_bias,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd,
postprocess_cmd,
param_initializer=param_initializer,
name=name + '_layer_' + str(i))
preprocess_cmd,
postprocess_cmd,
param_initializer=param_initializer,
name=name + "_layer_" + str(i),
)
checkpoints.append(cp)
enc_input = enc_output
enc_output = pre_process_layer(enc_output,
preprocess_cmd,
prepostprocess_dropout,
name=model_name + "post_encoder")
enc_output = pre_process_layer(
enc_output, preprocess_cmd, prepostprocess_dropout, name=model_name + "post_encoder"
)
return enc_output, checkpoints

86
applications/document_intelligence/doc_vqa/Rerank/src/optimization.py
View File

@@ -26,14 +26,11 @@ from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy
def linear_warmup_decay(learning_rate, warmup_steps, num_train_steps):
""" Applies linear warmup of learning rate from 0 and decay to 0."""
"""Applies linear warmup of learning rate from 0 and decay to 0."""
with fluid.default_main_program()._lr_schedule_guard():
lr = fluid.layers.tensor.create_global_var(
shape=[1],
value=0.0,
dtype='float32',
persistable=True,
name="scheduled_learning_rate")
shape=[1], value=0.0, dtype="float32", persistable=True, name="scheduled_learning_rate"
)
global_step = fluid.layers.learning_rate_scheduler._decay_step_counter()
@@ -47,41 +44,41 @@ def linear_warmup_decay(learning_rate, warmup_steps, num_train_steps):
decay_steps=num_train_steps,
end_learning_rate=0.0,
power=1.0,
cycle=False)
cycle=False,
)
fluid.layers.tensor.assign(decayed_lr, lr)
return lr
def optimization(loss,
warmup_steps,
num_train_steps,
learning_rate,
train_program,
startup_prog,
weight_decay,
scheduler='linear_warmup_decay',
use_dynamic_loss_scaling=False,
incr_every_n_steps=1000,
decr_every_n_nan_or_inf=2,
incr_ratio=2.0,
decr_ratio=0.8,
dist_strategy=None,
use_lamb=False):
def optimization(
loss,
warmup_steps,
num_train_steps,
learning_rate,
train_program,
startup_prog,
weight_decay,
scheduler="linear_warmup_decay",
use_dynamic_loss_scaling=False,
incr_every_n_steps=1000,
decr_every_n_nan_or_inf=2,
incr_ratio=2.0,
decr_ratio=0.8,
dist_strategy=None,
use_lamb=False,
):
if warmup_steps > 0:
if scheduler == 'noam_decay':
scheduled_lr = fluid.layers.learning_rate_scheduler\
.noam_decay(1/(warmup_steps *(learning_rate ** 2)),
warmup_steps)
elif scheduler == 'linear_warmup_decay':
scheduled_lr = linear_warmup_decay(learning_rate, warmup_steps,
num_train_steps)
if scheduler == "noam_decay":
scheduled_lr = fluid.layers.learning_rate_scheduler.noam_decay(
1 / (warmup_steps * (learning_rate**2)), warmup_steps
)
elif scheduler == "linear_warmup_decay":
scheduled_lr = linear_warmup_decay(learning_rate, warmup_steps, num_train_steps)
else:
raise ValueError("Unkown learning rate scheduler, should be "
"'noam_decay' or 'linear_warmup_decay'")
raise ValueError("Unkown learning rate scheduler, should be " "'noam_decay' or 'linear_warmup_decay'")
if use_lamb:
optimizer = fluid.optimizer.LambOptimizer(
learning_rate=scheduled_lr)
optimizer = fluid.optimizer.LambOptimizer(learning_rate=scheduled_lr)
else:
optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr)
else:
@@ -89,18 +86,16 @@ def optimization(loss,
name=fluid.unique_name.generate("learning_rate"),
shape=[1],
value=learning_rate,
dtype='float32',
persistable=True)
dtype="float32",
persistable=True,
)
if use_lamb:
optimizer = fluid.optimizer.LambOptimizer(
learning_rate=scheduled_lr)
optimizer = fluid.optimizer.LambOptimizer(learning_rate=scheduled_lr)
else:
optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr)
optimizer._learning_rate_map[
fluid.default_main_program()] = scheduled_lr
optimizer._learning_rate_map[fluid.default_main_program()] = scheduled_lr
fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByGlobalNorm(
clip_norm=1.0))
fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0))
def exclude_from_weight_decay(name):
if name.find("layer_norm") > -1:
@@ -119,8 +114,7 @@ def optimization(loss,
if dist_strategy is not None:
# use fleet api
optimizer = fleet.distributed_optimizer(optimizer,
strategy=dist_strategy)
optimizer = fleet.distributed_optimizer(optimizer, strategy=dist_strategy)
_, param_grads = optimizer.minimize(loss)
@@ -128,10 +122,8 @@ def optimization(loss,
for param, grad in param_grads:
if exclude_from_weight_decay(param.name):
continue
with param.block.program._optimized_guard(
[param, grad]), fluid.framework.name_scope("weight_decay"):
updated_param = param - param_list[
param.name] * weight_decay * scheduled_lr
with param.block.program._optimized_guard([param, grad]), fluid.framework.name_scope("weight_decay"):
updated_param = param - param_list[param.name] * weight_decay * scheduled_lr
fluid.layers.assign(output=param, input=updated_param)
return scheduled_lr

144
applications/document_intelligence/doc_vqa/Rerank/src/reader_ce.py
View File

@@ -34,16 +34,16 @@ log = logging.getLogger(__name__)
if six.PY3:
import io
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8")
def csv_reader(fd, delimiter='\t', trainer_id=0, trainer_num=1):
def csv_reader(fd, delimiter="\t", trainer_id=0, trainer_num=1):
def gen():
for i, line in enumerate(fd):
if i % trainer_num == trainer_id:
slots = line.rstrip('\n').split(delimiter)
slots = line.rstrip("\n").split(delimiter)
if len(slots) == 1:
yield slots,
else:
@@ -53,22 +53,22 @@ def csv_reader(fd, delimiter='\t', trainer_id=0, trainer_num=1):
class BaseReader(object):
def __init__(self,
vocab_path,
label_map_config=None,
max_seq_len=512,
total_num=0,
do_lower_case=True,
in_tokens=False,
is_inference=False,
random_seed=None,
tokenizer="FullTokenizer",
for_cn=True,
task_id=0):
def __init__(
self,
vocab_path,
label_map_config=None,
max_seq_len=512,
total_num=0,
do_lower_case=True,
in_tokens=False,
is_inference=False,
random_seed=None,
tokenizer="FullTokenizer",
for_cn=True,
task_id=0,
):
self.max_seq_len = max_seq_len
self.tokenizer = tokenization.FullTokenizer(vocab_file=vocab_path,
do_lower_case=do_lower_case)
self.tokenizer = tokenization.FullTokenizer(vocab_file=vocab_path, do_lower_case=do_lower_case)
self.vocab = self.tokenizer.vocab
self.pad_id = self.vocab["[PAD]"]
self.cls_id = self.vocab["[CLS]"]
@@ -86,7 +86,7 @@ class BaseReader(object):
self.total_num = total_num
if label_map_config:
with open(label_map_config, encoding='utf8') as f:
with open(label_map_config, encoding="utf8") as f:
self.label_map = json.load(f)
else:
self.label_map = None
@@ -97,10 +97,10 @@ class BaseReader(object):
def _read_tsv(self, input_file, quotechar=None):
"""Reads a tab separated value file."""
with open(input_file, 'r', encoding='utf8') as f:
with open(input_file, "r", encoding="utf8") as f:
reader = csv_reader(f)
headers = next(reader)
Example = namedtuple('Example', headers)
Example = namedtuple("Example", headers)
examples = []
for line in reader:
@@ -180,30 +180,23 @@ class BaseReader(object):
position_ids = list(range(len(token_ids)))
if self.is_inference:
Record = namedtuple('Record',
['token_ids', 'text_type_ids', 'position_ids'])
record = Record(token_ids=token_ids,
text_type_ids=text_type_ids,
position_ids=position_ids)
Record = namedtuple("Record", ["token_ids", "text_type_ids", "position_ids"])
record = Record(token_ids=token_ids, text_type_ids=text_type_ids, position_ids=position_ids)
else:
if self.label_map:
label_id = self.label_map[example.label]
else:
label_id = example.label
Record = namedtuple('Record', [
'token_ids', 'text_type_ids', 'position_ids', 'label_id', 'qid'
])
Record = namedtuple("Record", ["token_ids", "text_type_ids", "position_ids", "label_id", "qid"])
qid = None
if "qid" in example._fields:
qid = example.qid
record = Record(token_ids=token_ids,
text_type_ids=text_type_ids,
position_ids=position_ids,
label_id=label_id,
qid=qid)
record = Record(
token_ids=token_ids, text_type_ids=text_type_ids, position_ids=position_ids, label_id=label_id, qid=qid
)
return record
def _prepare_batch_data(self, examples, batch_size, phase=None):
@@ -212,8 +205,7 @@ class BaseReader(object):
for index, example in enumerate(examples):
if phase == "train":
self.current_example = index
record = self._convert_example_to_record(example, self.max_seq_len,
self.tokenizer)
record = self._convert_example_to_record(example, self.max_seq_len, self.tokenizer)
max_len = max(max_len, len(record.token_ids))
if self.in_tokens:
to_append = (len(batch_records) + 1) * max_len <= batch_size
@@ -233,25 +225,18 @@ class BaseReader(object):
# return len(examples)
return self.num_examples
def data_generator(self,
input_file,
batch_size,
epoch,
dev_count=1,
trainer_id=0,
trainer_num=1,
shuffle=True,
phase=None):
def data_generator(
self, input_file, batch_size, epoch, dev_count=1, trainer_id=0, trainer_num=1, shuffle=True, phase=None
):
if phase == 'train':
if phase == "train":
# examples = examples[trainer_id: (len(examples) //trainer_num) * trainer_num : trainer_num]
self.num_examples_per_node = self.total_num // trainer_num
self.num_examples = self.num_examples_per_node * trainer_num
examples = self._read_tsv(input_file,
trainer_id=trainer_id,
trainer_num=trainer_num,
num_examples=self.num_examples_per_node)
log.info('apply sharding %d/%d' % (trainer_id, trainer_num))
examples = self._read_tsv(
input_file, trainer_id=trainer_id, trainer_num=trainer_num, num_examples=self.num_examples_per_node
)
log.info("apply sharding %d/%d" % (trainer_id, trainer_num))
else:
examples = self._read_tsv(input_file)
@@ -264,9 +249,7 @@ class BaseReader(object):
if shuffle:
np.random.shuffle(examples)
for batch_data in self._prepare_batch_data(examples,
batch_size,
phase=phase):
for batch_data in self._prepare_batch_data(examples, batch_size, phase=phase):
if len(all_dev_batches) < dev_count:
all_dev_batches.append(batch_data)
if len(all_dev_batches) == dev_count:
@@ -280,30 +263,21 @@ class BaseReader(object):
yield i
except Exception as e:
import traceback
traceback.print_exc()
return f
class ClassifyReader(BaseReader):
def _read_tsv(self,
input_file,
quotechar=None,
trainer_id=0,
trainer_num=1,
num_examples=0):
def _read_tsv(self, input_file, quotechar=None, trainer_id=0, trainer_num=1, num_examples=0):
"""Reads a tab separated value file."""
with open(input_file, 'r', encoding='utf8') as f:
reader = csv_reader(f,
trainer_id=trainer_id,
trainer_num=trainer_num)
with open(input_file, "r", encoding="utf8") as f:
reader = csv_reader(f, trainer_id=trainer_id, trainer_num=trainer_num)
# headers = next(reader)
headers = 'query\ttitle\tpara\tlabel'.split('\t')
text_indices = [
index for index, h in enumerate(headers) if h != "label"
]
Example = namedtuple('Example', headers)
headers = "query\ttitle\tpara\tlabel".split("\t")
text_indices = [index for index, h in enumerate(headers) if h != "label"]
Example = namedtuple("Example", headers)
examples = []
for cnt, line in enumerate(reader):
@@ -312,7 +286,7 @@ class ClassifyReader(BaseReader):
for index, text in enumerate(line):
if index in text_indices:
if self.for_cn:
line[index] = text.replace(' ', '')
line[index] = text.replace(" ", "")
else:
line[index] = text
example = Example(*line)
@@ -326,31 +300,21 @@ class ClassifyReader(BaseReader):
if not self.is_inference:
batch_labels = [record.label_id for record in batch_records]
batch_labels = np.array(batch_labels).astype("int64").reshape(
[-1, 1])
batch_labels = np.array(batch_labels).astype("int64").reshape([-1, 1])
if batch_records[0].qid:
batch_qids = [record.qid for record in batch_records]
batch_qids = np.array(batch_qids).astype("int64").reshape(
[-1, 1])
batch_qids = np.array(batch_qids).astype("int64").reshape([-1, 1])
else:
batch_qids = np.array([]).astype("int64").reshape([-1, 1])
# padding
padded_token_ids, input_mask = pad_batch_data(batch_token_ids,
pad_idx=self.pad_id,
return_input_mask=True)
padded_text_type_ids = pad_batch_data(batch_text_type_ids,
pad_idx=self.pad_id)
padded_position_ids = pad_batch_data(batch_position_ids,
pad_idx=self.pad_id)
padded_task_ids = np.ones_like(padded_token_ids,
dtype="int64") * self.task_id
padded_token_ids, input_mask = pad_batch_data(batch_token_ids, pad_idx=self.pad_id, return_input_mask=True)
padded_text_type_ids = pad_batch_data(batch_text_type_ids, pad_idx=self.pad_id)
padded_position_ids = pad_batch_data(batch_position_ids, pad_idx=self.pad_id)
padded_task_ids = np.ones_like(padded_token_ids, dtype="int64") * self.task_id
return_list = [
padded_token_ids, padded_text_type_ids, padded_position_ids,
padded_task_ids, input_mask
]
return_list = [padded_token_ids, padded_text_type_ids, padded_position_ids, padded_task_ids, input_mask]
if not self.is_inference:
return_list += [batch_labels, batch_qids]

43
applications/document_intelligence/doc_vqa/Rerank/src/tokenization.py
View File

@@ -72,7 +72,7 @@ def printable_text(text):
def load_vocab(vocab_file):
"""Loads a vocabulary file into a dictionary."""
vocab = collections.OrderedDict()
with open(vocab_file, encoding='utf8') as fin:
with open(vocab_file, encoding="utf8") as fin:
for num, line in enumerate(fin):
items = convert_to_unicode(line.strip()).split("\t")
if len(items) > 2:
@@ -245,14 +245,16 @@ class BasicTokenizer(object):
# as is Japanese Hiragana and Katakana. Those alphabets are used to write
# space-separated words, so they are not treated specially and handled
# like the all of the other languages.
if ((cp >= 0x4E00 and cp <= 0x9FFF) or #
(cp >= 0x3400 and cp <= 0x4DBF) or #
(cp >= 0x20000 and cp <= 0x2A6DF) or #
(cp >= 0x2A700 and cp <= 0x2B73F) or #
(cp >= 0x2B740 and cp <= 0x2B81F) or #
(cp >= 0x2B820 and cp <= 0x2CEAF) or
(cp >= 0xF900 and cp <= 0xFAFF) or #
(cp >= 0x2F800 and cp <= 0x2FA1F)): #
if (
(cp >= 0x4E00 and cp <= 0x9FFF)
or (cp >= 0x3400 and cp <= 0x4DBF) #
or (cp >= 0x20000 and cp <= 0x2A6DF) #
or (cp >= 0x2A700 and cp <= 0x2B73F) #
or (cp >= 0x2B740 and cp <= 0x2B81F) #
or (cp >= 0x2B820 and cp <= 0x2CEAF) #
or (cp >= 0xF900 and cp <= 0xFAFF)
or (cp >= 0x2F800 and cp <= 0x2FA1F) #
): #
return True
return False
@@ -262,7 +264,7 @@ class BasicTokenizer(object):
output = []
for char in text:
cp = ord(char)
if cp == 0 or cp == 0xfffd or _is_control(char):
if cp == 0 or cp == 0xFFFD or _is_control(char):
continue
if _is_whitespace(char):
output.append(" ")
@@ -364,8 +366,7 @@ def _is_punctuation(char):
# Characters such as "^", "$", and "`" are not in the Unicode
# Punctuation class but we treat them as punctuation anyways, for
# consistency.
if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64)
or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):
if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):
return True
cat = unicodedata.category(char)
if cat.startswith("P"):
@@ -386,14 +387,16 @@ def tokenize_chinese_chars(text):
# as is Japanese Hiragana and Katakana. Those alphabets are used to write
# space-separated words, so they are not treated specially and handled
# like the all of the other languages.
if ((cp >= 0x4E00 and cp <= 0x9FFF) or #
(cp >= 0x3400 and cp <= 0x4DBF) or #
(cp >= 0x20000 and cp <= 0x2A6DF) or #
(cp >= 0x2A700 and cp <= 0x2B73F) or #
(cp >= 0x2B740 and cp <= 0x2B81F) or #
(cp >= 0x2B820 and cp <= 0x2CEAF) or
(cp >= 0xF900 and cp <= 0xFAFF) or #
(cp >= 0x2F800 and cp <= 0x2FA1F)): #
if (
(cp >= 0x4E00 and cp <= 0x9FFF)
or (cp >= 0x3400 and cp <= 0x4DBF) #
or (cp >= 0x20000 and cp <= 0x2A6DF) #
or (cp >= 0x2A700 and cp <= 0x2B73F) #
or (cp >= 0x2B740 and cp <= 0x2B81F) #
or (cp >= 0x2B820 and cp <= 0x2CEAF) #
or (cp >= 0xF900 and cp <= 0xFAFF)
or (cp >= 0x2F800 and cp <= 0x2FA1F) #
): #
return True
return False

202
applications/document_intelligence/doc_vqa/Rerank/src/train_ce.py
View File

@@ -28,11 +28,12 @@ import multiprocessing
# NOTE(paddle-dev): All of these flags should be
# set before `import paddle`. Otherwise, it would
# not take any effect.
os.environ['FLAGS_eager_delete_tensor_gb'] = '0' # enable gc
os.environ["FLAGS_eager_delete_tensor_gb"] = "0" # enable gc
import paddle
import paddle.fluid as fluid
if hasattr(paddle, 'enable_static'):
if hasattr(paddle, "enable_static"):
paddle.enable_static()
from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy
import paddle.fluid.incubate.fleet.base.role_maker as role_maker
@@ -60,23 +61,24 @@ def main(args):
dev_count = len(dev_list)
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
dev_count = int(os.environ.get("CPU_NUM", multiprocessing.cpu_count()))
exe = fluid.Executor(place)
reader = reader_ce.ClassifyReader(vocab_path=args.vocab_path,
label_map_config=args.label_map_config,
max_seq_len=args.max_seq_len,
total_num=args.train_data_size,
do_lower_case=args.do_lower_case,
in_tokens=args.in_tokens,
random_seed=args.random_seed,
tokenizer=args.tokenizer,
for_cn=args.for_cn,
task_id=args.task_id)
reader = reader_ce.ClassifyReader(
vocab_path=args.vocab_path,
label_map_config=args.label_map_config,
max_seq_len=args.max_seq_len,
total_num=args.train_data_size,
do_lower_case=args.do_lower_case,
in_tokens=args.in_tokens,
random_seed=args.random_seed,
tokenizer=args.tokenizer,
for_cn=args.for_cn,
task_id=args.task_id,
)
if not (args.do_train or args.do_val or args.do_test):
raise ValueError("For args `do_train`, `do_val` and `do_test`, at "
"least one of them must be True.")
raise ValueError("For args `do_train`, `do_val` and `do_test`, at " "least one of them must be True.")
if args.do_test:
assert args.test_save is not None
@@ -100,13 +102,13 @@ def main(args):
trainer_id=fleet.worker_index(),
trainer_num=fleet.worker_num(),
shuffle=True,
phase="train")
phase="train",
)
num_train_examples = reader.get_num_examples(args.train_set)
if args.in_tokens:
max_train_steps = args.epoch * num_train_examples // (
args.batch_size // args.max_seq_len) // dev_count
max_train_steps = args.epoch * num_train_examples // (args.batch_size // args.max_seq_len) // dev_count
else:
max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count
@@ -141,9 +143,8 @@ def main(args):
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
train_pyreader, graph_vars = create_model(
args,
pyreader_name='train_reader',
ernie_config=ernie_config)
args, pyreader_name="train_reader", ernie_config=ernie_config
)
scheduled_lr = optimization(
loss=graph_vars["loss"],
warmup_steps=warmup_steps,
@@ -158,28 +159,27 @@ def main(args):
decr_every_n_nan_or_inf=args.decr_every_n_nan_or_inf,
incr_ratio=args.incr_ratio,
decr_ratio=args.decr_ratio,
dist_strategy=dist_strategy)
dist_strategy=dist_strategy,
)
if args.verbose:
if args.in_tokens:
lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
program=train_program,
batch_size=args.batch_size // args.max_seq_len)
program=train_program, batch_size=args.batch_size // args.max_seq_len
)
else:
lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
program=train_program, batch_size=args.batch_size)
log.info("Theoretical memory usage in training: %.3f - %.3f %s" %
(lower_mem, upper_mem, unit))
program=train_program, batch_size=args.batch_size
)
log.info("Theoretical memory usage in training: %.3f - %.3f %s" % (lower_mem, upper_mem, unit))
if args.do_val or args.do_test:
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_pyreader, graph_vars = create_model(
args,
pyreader_name='test_reader',
ernie_config=ernie_config,
is_prediction=True)
args, pyreader_name="test_reader", ernie_config=ernie_config, is_prediction=True
)
test_prog = test_prog.clone(for_test=True)
@@ -192,19 +192,15 @@ def main(args):
if args.init_checkpoint and args.init_pretraining_params:
log.warning(
"WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
"both are set! Only arg 'init_checkpoint' is made valid.")
"both are set! Only arg 'init_checkpoint' is made valid."
)
if args.init_checkpoint:
init_checkpoint(exe,
args.init_checkpoint,
main_program=startup_prog)
init_checkpoint(exe, args.init_checkpoint, main_program=startup_prog)
elif args.init_pretraining_params:
init_pretraining_params(exe,
args.init_pretraining_params,
main_program=startup_prog)
init_pretraining_params(exe, args.init_pretraining_params, main_program=startup_prog)
elif args.do_val or args.do_test:
if not args.init_checkpoint:
raise ValueError("args 'init_checkpoint' should be set if"
"only doing validation or testing!")
raise ValueError("args 'init_checkpoint' should be set if" "only doing validation or testing!")
init_checkpoint(exe, args.init_checkpoint, main_program=startup_prog)
if args.do_train:
@@ -237,19 +233,15 @@ def main(args):
train_exe.run(fetch_list=[], program=train_program)
else:
outputs = evaluate(train_exe,
train_program,
train_pyreader,
graph_vars,
"train",
metric=args.metric)
outputs = evaluate(
train_exe, train_program, train_pyreader, graph_vars, "train", metric=args.metric
)
if args.verbose:
verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size(
)
verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size()
verbose += "learning rate: %f" % (
outputs["learning_rate"]
if warmup_steps > 0 else args.learning_rate)
outputs["learning_rate"] if warmup_steps > 0 else args.learning_rate
)
log.info(verbose)
current_example, current_epoch = reader.get_train_progress()
@@ -258,47 +250,45 @@ def main(args):
log.info(
"epoch: %d, progress: %d/%d, step: %d, ave loss: %f, "
"ave acc: %f, speed: %f steps/s" %
(current_epoch, current_example * dev_count,
num_train_examples, steps, outputs["loss"],
outputs["accuracy"], args.skip_steps / used_time))
ce_info.append(
[outputs["loss"], outputs["accuracy"], used_time])
"ave acc: %f, speed: %f steps/s"
% (
current_epoch,
current_example * dev_count,
num_train_examples,
steps,
outputs["loss"],
outputs["accuracy"],
args.skip_steps / used_time,
)
)
ce_info.append([outputs["loss"], outputs["accuracy"], used_time])
time_begin = time.time()
if steps % args.save_steps == 0:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps))
fluid.io.save_persistables(exe, save_path,
fleet._origin_program)
save_path = os.path.join(args.checkpoints, "step_" + str(steps))
fluid.io.save_persistables(exe, save_path, fleet._origin_program)
if steps % args.validation_steps == 0:
# evaluate dev set
if args.do_val:
evaluate_wrapper(args, reader, exe, test_prog,
test_pyreader, graph_vars,
current_epoch, steps)
evaluate_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, current_epoch, steps)
if args.do_test:
predict_wrapper(args, reader, exe, test_prog,
test_pyreader, graph_vars,
current_epoch, steps)
predict_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, current_epoch, steps)
if last_epoch != current_epoch:
last_epoch = current_epoch
except fluid.core.EOFException:
save_path = os.path.join(args.checkpoints, "step_" + str(steps))
fluid.io.save_persistables(exe, save_path,
fleet._origin_program)
fluid.io.save_persistables(exe, save_path, fleet._origin_program)
train_pyreader.reset()
break
# final eval on dev set
if args.do_val:
evaluate_wrapper(args, reader, exe, test_prog, test_pyreader,
graph_vars, current_epoch, steps)
evaluate_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, current_epoch, steps)
# final eval on test set
if args.do_test:
@@ -307,68 +297,42 @@ def main(args):
# final eval on dianostic, hack for glue-ax
if args.diagnostic:
test_pyreader.decorate_tensor_provider(
reader.data_generator(args.diagnostic,
batch_size=args.batch_size,
epoch=1,
dev_count=1,
shuffle=False))
reader.data_generator(args.diagnostic, batch_size=args.batch_size, epoch=1, dev_count=1, shuffle=False)
)
log.info("Final diagnostic")
qids, preds, probs = predict(test_exe, test_prog, test_pyreader,
graph_vars)
assert len(qids) == len(preds), '{} v.s. {}'.format(
len(qids), len(preds))
with open(args.diagnostic_save, 'w') as f:
qids, preds, probs = predict(test_exe, test_prog, test_pyreader, graph_vars)
assert len(qids) == len(preds), "{} v.s. {}".format(len(qids), len(preds))
with open(args.diagnostic_save, "w") as f:
for id, s, p in zip(qids, preds, probs):
f.write('{}\t{}\t{}\n'.format(id, s, p))
f.write("{}\t{}\t{}\n".format(id, s, p))
log.info("Done final diagnostic, saving to {}".format(
args.diagnostic_save))
log.info("Done final diagnostic, saving to {}".format(args.diagnostic_save))
def evaluate_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars,
epoch, steps):
def evaluate_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, epoch, steps):
# evaluate dev set
for ds in args.dev_set.split(','):
for ds in args.dev_set.split(","):
test_pyreader.decorate_tensor_provider(
reader.data_generator(ds,
batch_size=args.predict_batch_size,
epoch=1,
dev_count=1,
shuffle=False))
reader.data_generator(ds, batch_size=args.predict_batch_size, epoch=1, dev_count=1, shuffle=False)
)
log.info("validation result of dataset {}:".format(ds))
evaluate_info = evaluate(exe,
test_prog,
test_pyreader,
graph_vars,
"dev",
metric=args.metric)
log.info(evaluate_info +
', file: {}, epoch: {}, steps: {}'.format(ds, epoch, steps))
evaluate_info = evaluate(exe, test_prog, test_pyreader, graph_vars, "dev", metric=args.metric)
log.info(evaluate_info + ", file: {}, epoch: {}, steps: {}".format(ds, epoch, steps))
def predict_wrapper(args,
reader,
exe,
test_prog,
test_pyreader,
graph_vars,
epoch=None,
steps=None):
test_sets = args.test_set.split(',')
save_dirs = args.test_save.split(',')
def predict_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, epoch=None, steps=None):
test_sets = args.test_set.split(",")
save_dirs = args.test_save.split(",")
assert len(test_sets) == len(save_dirs)
for test_f, save_f in zip(test_sets, save_dirs):
test_pyreader.decorate_tensor_provider(
reader.data_generator(test_f,
batch_size=args.predict_batch_size,
epoch=1,
dev_count=1,
shuffle=False))
reader.data_generator(test_f, batch_size=args.predict_batch_size, epoch=1, dev_count=1, shuffle=False)
)
if epoch is not None or steps is not None:
save_path = save_f + '.' + str(epoch) + '.' + str(steps)
save_path = save_f + "." + str(epoch) + "." + str(steps)
else:
save_path = save_f
log.info("testing {}, save to {}".format(test_f, save_path))
@@ -378,14 +342,14 @@ def predict_wrapper(args,
if not os.path.exists(save_dir):
os.makedirs(save_dir)
else:
log.warning('save dir exsits: %s, will skip saving' % save_dir)
log.warning("save dir exsits: %s, will skip saving" % save_dir)
with open(save_path, 'w') as f:
with open(save_path, "w") as f:
for p in probs:
f.write('{}\n'.format(p[1]))
f.write("{}\n".format(p[1]))
if __name__ == '__main__':
if __name__ == "__main__":
prepare_logger(log)
print_arguments(args)
check_cuda(args.use_cuda)

37
applications/document_intelligence/doc_vqa/Rerank/src/utils/args.py
View File

@@ -30,10 +30,7 @@ log = logging.getLogger(__name__)
def prepare_logger(logger, debug=False, save_to_file=None):
formatter = logging.Formatter(
fmt=
'[%(levelname)s] %(asctime)s [%(filename)12s:%(lineno)5d]:\t%(message)s'
)
formatter = logging.Formatter(fmt="[%(levelname)s] %(asctime)s [%(filename)12s:%(lineno)5d]:\t%(message)s")
console_hdl = logging.StreamHandler()
console_hdl.setFormatter(formatter)
logger.addHandler(console_hdl)
@@ -52,37 +49,29 @@ def str2bool(v):
class ArgumentGroup(object):
def __init__(self, parser, title, des):
self._group = parser.add_argument_group(title=title, description=des)
def add_arg(self,
name,
type,
default,
help,
positional_arg=False,
**kwargs):
def add_arg(self, name, type, default, help, positional_arg=False, **kwargs):
prefix = "" if positional_arg else "--"
type = str2bool if type == bool else type
self._group.add_argument(prefix + name,
default=default,
type=type,
help=help + ' Default: %(default)s.',
**kwargs)
self._group.add_argument(
prefix + name, default=default, type=type, help=help + " Default: %(default)s.", **kwargs
)
def print_arguments(args):
log.info('----------- Configuration Arguments -----------')
log.info("----------- Configuration Arguments -----------")
for arg, value in sorted(six.iteritems(vars(args))):
log.info('%s: %s' % (arg, value))
log.info('------------------------------------------------')
log.info("%s: %s" % (arg, value))
log.info("------------------------------------------------")
def check_cuda(use_cuda, err = \
"\nYou can not set use_cuda = True in the model because you are using paddlepaddle-cpu.\n \
Please: 1. Install paddlepaddle-gpu to run your models on GPU or 2. Set use_cuda = False to run models on CPU.\n"
):
def check_cuda(
use_cuda,
err="\nYou can not set use_cuda = True in the model because you are using paddlepaddle-cpu.\n \
Please: 1. Install paddlepaddle-gpu to run your models on GPU or 2. Set use_cuda = False to run models on CPU.\n",
):
try:
if use_cuda == True and fluid.is_compiled_with_cuda() == False:
log.error(err)

29
applications/document_intelligence/doc_vqa/Rerank/src/utils/init.py
View File

@@ -31,43 +31,32 @@ log = logging.getLogger(__name__)
def init_checkpoint(exe, init_checkpoint_path, main_program):
assert os.path.exists(
init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path
assert os.path.exists(init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path
def existed_persitables(var):
if not fluid.io.is_persistable(var):
return False
if not os.path.exists(os.path.join(init_checkpoint_path, var.name)):
print("Var not exists: [%s]\t%s" %
(var.name, os.path.join(init_checkpoint_path, var.name)))
#else:
print("Var not exists: [%s]\t%s" % (var.name, os.path.join(init_checkpoint_path, var.name)))
# else:
# print ("Var exists: [%s]" % (var.name))
return os.path.exists(os.path.join(init_checkpoint_path, var.name))
fluid.io.load_vars(exe,
init_checkpoint_path,
main_program=main_program,
predicate=existed_persitables)
fluid.io.load_vars(exe, init_checkpoint_path, main_program=main_program, predicate=existed_persitables)
log.info("Load model from {}".format(init_checkpoint_path))
def init_pretraining_params(exe, pretraining_params_path, main_program):
assert os.path.exists(pretraining_params_path
), "[%s] cann't be found." % pretraining_params_path
assert os.path.exists(pretraining_params_path), "[%s] cann't be found." % pretraining_params_path
def existed_params(var):
if not isinstance(var, fluid.framework.Parameter):
return False
if not os.path.exists(os.path.join(pretraining_params_path, var.name)):
print("Var not exists: [%s]\t%s" %
(var.name, os.path.join(pretraining_params_path, var.name)))
#else:
print("Var not exists: [%s]\t%s" % (var.name, os.path.join(pretraining_params_path, var.name)))
# else:
# print ("Var exists: [%s]" % (var.name))
return os.path.exists(os.path.join(pretraining_params_path, var.name))
fluid.io.load_vars(exe,
pretraining_params_path,
main_program=main_program,
predicate=existed_params)
log.info(
"Load pretraining parameters from {}.".format(pretraining_params_path))
fluid.io.load_vars(exe, pretraining_params_path, main_program=main_program, predicate=existed_params)
log.info("Load pretraining parameters from {}.".format(pretraining_params_path))

55
applications/neural_search/ranking/cross_encoder/data.py
View File

@@ -15,16 +15,12 @@ import numpy as np
import paddle
def convert_example(example,
tokenizer,
max_seq_length=512,
is_test=False,
is_pair=False):
def convert_example(example, tokenizer, max_seq_length=512, is_test=False, is_pair=False):
"""
Builds model inputs from a sequence or a pair of sequence for sequence classification tasks
by concatenating and adding special tokens. And creates a mask from the two sequences passed
by concatenating and adding special tokens. And creates a mask from the two sequences passed
to be used in a sequence-pair classification task.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
@@ -34,9 +30,9 @@ def convert_example(example,
Args:
example(obj:`list[str]`): List of input data, containing text and label if it have label.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -52,9 +48,7 @@ def convert_example(example,
else:
text = example["text"]
text_pair = None
encoded_inputs = tokenizer(text=text,
text_pair=text_pair,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=text, text_pair=text_pair, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -66,51 +60,40 @@ def convert_example(example,
def read_text_pair(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if len(data) != 3:
continue
yield {'text_a': data[0], 'text_b': data[1]}
yield {"text_a": data[0], "text_b": data[1]}
def read_data(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8', errors='ignore') as f:
with open(data_path, "r", encoding="utf-8", errors="ignore") as f:
for i, line in enumerate(f):
# Skip column name
if (i == 0):
if i == 0:
continue
data = line.rstrip('\n').split("\t")
if (len(data) != 3):
data = line.rstrip("\n").split("\t")
if len(data) != 3:
print(data)
continue
query = data[0]
title = data[1]
label = data[-1]
# breakpoint()
yield {'text_a': query, 'text_b': title, 'label': int(label)}
yield {"text_a": query, "text_b": title, "label": int(label)}
def create_dataloader(dataset,
mode='train',
batch_size=1,
batchify_fn=None,
trans_fn=None):
def create_dataloader(dataset, mode="train", batch_size=1, batchify_fn=None, trans_fn=None):
if trans_fn:
dataset = dataset.map(trans_fn)
shuffle = True if mode == 'train' else False
if mode == 'train':
batch_sampler = paddle.io.DistributedBatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
shuffle = True if mode == "train" else False
if mode == "train":
batch_sampler = paddle.io.DistributedBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
else:
batch_sampler = paddle.io.BatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
batch_sampler = paddle.io.BatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
return paddle.io.DataLoader(dataset=dataset,
batch_sampler=batch_sampler,
collate_fn=batchify_fn,
return_list=True)
return paddle.io.DataLoader(dataset=dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn, return_list=True)

23
applications/neural_search/ranking/cross_encoder/deploy/cpp/http_client.py
View File

@@ -29,9 +29,7 @@ import paddlenlp as ppnlp
def convert_example(example, tokenizer, max_seq_length=512):
query, title = example["query"], example["title"]
encoded_inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -39,34 +37,29 @@ def convert_example(example, tokenizer, max_seq_length=512):
# 启动python客户端
endpoint_list = ['127.0.0.1:8600']
endpoint_list = ["127.0.0.1:8600"]
client = HttpClient()
client.load_client_config('serving_client')
client.load_client_config("serving_client")
client.connect(endpoint_list)
feed_names = client.feed_names_
fetch_names = client.fetch_names_
# 创建tokenizer
tokenizer = AutoTokenizer.from_pretrained('rocketqa-base-cross-encoder')
tokenizer = AutoTokenizer.from_pretrained("rocketqa-base-cross-encoder")
max_seq_len = 64
# 数据预处理
list_data = [{
"query": "加强科研项目管理有效促进医学科研工作",
"title": "科研项目管理策略科研项目,项目管理,实施,必要性,策略"
}]
list_data = [{"query": "加强科研项目管理有效促进医学科研工作", "title": "科研项目管理策略科研项目,项目管理,实施,必要性,策略"}]
input_ids, token_type_ids = [], []
for example in list_data:
input_id, token_type_id = convert_example(example,
tokenizer,
max_seq_length=max_seq_len)
input_id, token_type_id = convert_example(example, tokenizer, max_seq_length=max_seq_len)
input_ids.append(input_id)
token_type_ids.append(token_type_id)
feed_dict = {}
feed_dict['input_ids'] = np.array(input_ids)
feed_dict['token_type_ids'] = np.array(token_type_ids)
feed_dict["input_ids"] = np.array(input_ids)
feed_dict["token_type_ids"] = np.array(token_type_ids)
# batch设置为True表示的是批量预测
b_start = time.time()
result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)

25
applications/neural_search/ranking/cross_encoder/deploy/cpp/rpc_client.py
View File

@@ -27,9 +27,7 @@ import paddlenlp as ppnlp
def convert_example(example, tokenizer, max_seq_length=512):
query, title = example["query"], example["title"]
encoded_inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -38,39 +36,34 @@ def convert_example(example, tokenizer, max_seq_length=512):
# 启动python客户端
endpoint_list = ['127.0.0.1:8600']
endpoint_list = ["127.0.0.1:8600"]
client = Client()
client.load_client_config('serving_client')
client.load_client_config("serving_client")
client.connect(endpoint_list)
feed_names = client.feed_names_
fetch_names = client.fetch_names_
# 创建tokenizer
tokenizer = AutoTokenizer.from_pretrained('rocketqa-base-cross-encoder')
tokenizer = AutoTokenizer.from_pretrained("rocketqa-base-cross-encoder")
max_seq_len = 64
# 数据预处理
list_data = [{
"query": "加强科研项目管理有效促进医学科研工作",
"title": "科研项目管理策略科研项目,项目管理,实施,必要性,策略"
}]
list_data = [{"query": "加强科研项目管理有效促进医学科研工作", "title": "科研项目管理策略科研项目,项目管理,实施,必要性,策略"}]
input_ids, token_type_ids = [], []
for example in list_data:
input_id, token_type_id = convert_example(example,
tokenizer,
max_seq_length=max_seq_len)
input_id, token_type_id = convert_example(example, tokenizer, max_seq_length=max_seq_len)
input_ids.append(input_id)
token_type_ids.append(token_type_id)
feed_dict = {}
feed_dict['input_ids'] = np.array(input_ids)
feed_dict['token_type_ids'] = np.array(token_type_ids)
feed_dict["input_ids"] = np.array(input_ids)
feed_dict["token_type_ids"] = np.array(token_type_ids)
# batch设置为True表示的是批量预测
b_start = time.time()
result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)
# breakpoint()
sim_score = expit(result['predict'])[:, 1]
sim_score = expit(result["predict"])[:, 1]
b_end = time.time()
print("time to cost :{} seconds".format(b_end - b_start))
print(sim_score)

92
applications/neural_search/ranking/cross_encoder/deploy/python/predict.py
View File

@@ -29,7 +29,7 @@ from paddlenlp.datasets import load_dataset
from paddlenlp.utils.log import logger
from paddlenlp.transformers import AutoTokenizer
sys.path.append('.')
sys.path.append(".")
# yapf: disable
parser = argparse.ArgumentParser()
@@ -68,21 +68,19 @@ args = parser.parse_args()
def read_text_pair(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if len(data) != 3:
continue
yield {'query': data[0], 'title': data[1]}
yield {"query": data[0], "title": data[1]}
def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
query, title = example["query"], example["title"]
encoded_inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -95,16 +93,17 @@ def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
class Predictor(object):
def __init__(self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False):
def __init__(
self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False,
):
self.max_seq_length = max_seq_length
self.batch_size = batch_size
@@ -123,14 +122,14 @@ class Predictor(object):
precision_map = {
"fp16": inference.PrecisionType.Half,
"fp32": inference.PrecisionType.Float32,
"int8": inference.PrecisionType.Int8
"int8": inference.PrecisionType.Int8,
}
precision_mode = precision_map[precision]
if args.use_tensorrt:
config.enable_tensorrt_engine(max_batch_size=batch_size,
min_subgraph_size=30,
precision_mode=precision_mode)
config.enable_tensorrt_engine(
max_batch_size=batch_size, min_subgraph_size=30, precision_mode=precision_mode
)
elif device == "cpu":
# set CPU configs accordingly,
# such as enable_mkldnn, set_cpu_math_library_num_threads
@@ -146,15 +145,12 @@ class Predictor(object):
config.switch_use_feed_fetch_ops(False)
self.predictor = paddle.inference.create_predictor(config)
self.input_handles = [
self.predictor.get_input_handle(name)
for name in self.predictor.get_input_names()
]
self.output_handle = self.predictor.get_output_handle(
self.predictor.get_output_names()[0])
self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()]
self.output_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[0])
if args.benchmark:
import auto_log
pid = os.getpid()
self.autolog = auto_log.AutoLogger(
model_name=args.model_name_or_path,
@@ -166,11 +162,10 @@ class Predictor(object):
pids=pid,
process_name=None,
gpu_ids=0,
time_keys=[
'preprocess_time', 'inference_time', 'postprocess_time'
],
time_keys=["preprocess_time", "inference_time", "postprocess_time"],
warmup=0,
logger=logger)
logger=logger,
)
def predict(self, data, tokenizer):
"""
@@ -178,7 +173,7 @@ class Predictor(object):
Args:
data (obj:`List(str)`): The batch data whose each element is a raw text.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
label_map(obj:`dict`): The label id (key) to label str (value) map.
@@ -190,17 +185,12 @@ class Predictor(object):
examples = []
for text in data:
input_ids, segment_ids = convert_example(
text,
tokenizer,
max_seq_length=self.max_seq_length,
is_test=True)
input_ids, segment_ids = convert_example(text, tokenizer, max_seq_length=self.max_seq_length, is_test=True)
examples.append((input_ids, segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # segment
): fn(samples)
if args.benchmark:
@@ -223,27 +213,29 @@ class Predictor(object):
if __name__ == "__main__":
# Define predictor to do prediction.
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
args.batch_size, args.use_tensorrt, args.precision,
args.cpu_threads, args.enable_mkldnn)
predictor = Predictor(
args.model_dir,
args.device,
args.max_seq_length,
args.batch_size,
args.use_tensorrt,
args.precision,
args.cpu_threads,
args.enable_mkldnn,
)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
test_ds = load_dataset(read_text_pair,
data_path=args.input_file,
lazy=False)
test_ds = load_dataset(read_text_pair, data_path=args.input_file, lazy=False)
data = [{'query': d['query'], 'title': d['title']} for d in test_ds]
data = [{"query": d["query"], "title": d["title"]} for d in test_ds]
batches = [
data[idx:idx + args.batch_size]
for idx in range(0, len(data), args.batch_size)
]
batches = [data[idx : idx + args.batch_size] for idx in range(0, len(data), args.batch_size)]
results = []
for batch_data in batches:
results.extend(predictor.predict(batch_data, tokenizer))
for idx, text in enumerate(data):
print('Data: {} \t prob: {}'.format(text, results[idx]))
print("Data: {} \t prob: {}".format(text, results[idx]))
if args.benchmark:
predictor.autolog.report()

7
applications/neural_search/ranking/cross_encoder/deploy/python/rpc_client.py
View File

@@ -17,12 +17,9 @@ import numpy as np
from paddle_serving_server.pipeline import PipelineClient
client = PipelineClient()
client.connect(['127.0.0.1:8089'])
client.connect(["127.0.0.1:8089"])
list_data = [{
"query": "加强科研项目管理有效促进医学科研工作",
"title": "科研项目管理策略科研项目,项目管理,实施,必要性,策略"
}]
list_data = [{"query": "加强科研项目管理有效促进医学科研工作", "title": "科研项目管理策略科研项目,项目管理,实施,必要性,策略"}]
feed = {}
for i, item in enumerate(list_data):
feed[str(i)] = str(item)

24
applications/neural_search/ranking/cross_encoder/deploy/python/web_service.py
View File

@@ -23,9 +23,7 @@ from paddle_serving_server.web_service import WebService, Op
def convert_example(example, tokenizer, max_seq_length=512):
query, title = example["query"], example["title"]
encoded_inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -34,33 +32,30 @@ def convert_example(example, tokenizer, max_seq_length=512):
class ErnieOp(Op):
def init_op(self):
from paddlenlp.transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained(
'rocketqa-base-cross-encoder')
self.tokenizer = AutoTokenizer.from_pretrained("rocketqa-base-cross-encoder")
def preprocess(self, input_dicts, data_id, log_id):
from paddlenlp.data import Stack, Tuple, Pad
(_, input_dict), = input_dicts.items()
((_, input_dict),) = input_dicts.items()
print("input dict", input_dict)
batch_size = len(input_dict.keys())
examples = []
for i in range(batch_size):
example = json.loads(input_dict[str(i)].replace("\'", "\""))
example = json.loads(input_dict[str(i)].replace("'", '"'))
input_ids, segment_ids = convert_example(example, self.tokenizer)
examples.append((input_ids, segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"
), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"), # segment
): fn(samples)
input_ids, segment_ids = batchify_fn(examples)
feed_dict = {}
feed_dict['input_ids'] = input_ids
feed_dict['token_type_ids'] = segment_ids
feed_dict["input_ids"] = input_ids
feed_dict["token_type_ids"] = segment_ids
return feed_dict, False, None, ""
def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
@@ -71,7 +66,6 @@ class ErnieOp(Op):
class ErnieService(WebService):
def get_pipeline_response(self, read_op):
ernie_op = ErnieOp(name="ernie", input_ops=[read_op])
return ernie_op

27
applications/neural_search/ranking/cross_encoder/evaluate.py
View File

@@ -87,35 +87,26 @@ def main():
dev_ds = load_dataset(read_data, data_path=args.test_file, lazy=False)
model = AutoModelForSequenceClassification.from_pretrained(
args.model_name_or_path, num_classes=2)
model = AutoModelForSequenceClassification.from_pretrained(args.model_name_or_path, num_classes=2)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func_eval = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
is_pair=True)
trans_func_eval = partial(convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length, is_pair=True)
batchify_fn_eval = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # pair_segment
Stack(dtype="int64") # label
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # pair_segment
Stack(dtype="int64"), # label
): [data for data in fn(samples)]
dev_data_loader = create_dataloader(dev_ds,
mode='dev',
batch_size=args.batch_size,
batchify_fn=batchify_fn_eval,
trans_fn=trans_func_eval)
dev_data_loader = create_dataloader(
dev_ds, mode="dev", batch_size=args.batch_size, batchify_fn=batchify_fn_eval, trans_fn=trans_func_eval
)
if args.init_from_ckpt and os.path.isfile(args.init_from_ckpt):
state_dict = paddle.load(args.init_from_ckpt)
model.set_dict(state_dict)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
metric = paddle.metric.Auc()
evaluate(model, metric, dev_data_loader, "dev")

15
applications/neural_search/ranking/cross_encoder/export_model.py
View File

@@ -31,27 +31,24 @@ args = parser.parse_args()
if __name__ == "__main__":
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
model = AutoModelForSequenceClassification.from_pretrained(
args.model_name_or_path, num_classes=2)
model = AutoModelForSequenceClassification.from_pretrained(args.model_name_or_path, num_classes=2)
if args.params_path and os.path.isfile(args.params_path):
state_dict = paddle.load(args.params_path)
model.set_dict(state_dict)
print("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
model.eval()
# Convert to static graph with specific input description
model = paddle.jit.to_static(
model,
input_spec=[
paddle.static.InputSpec(shape=[None, None],
dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None],
dtype="int64") # segment_ids
])
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # segment_ids
],
)
# Save in static graph model.
save_path = os.path.join(args.output_path, "inference")
paddle.jit.save(model, save_path)

4
applications/neural_search/ranking/cross_encoder/export_to_serving.py
View File

@@ -14,6 +14,7 @@
import argparse
import paddle_serving_client.io as serving_io
# yapf: disable
parser = argparse.ArgumentParser()
parser.add_argument("--dirname", type=str, required=True,
@@ -44,4 +45,5 @@ if __name__ == "__main__":
params_filename=args.params_filename,
show_proto=args.show_proto,
feed_alias_names=args.feed_alias_names,
fetch_alias_names=args.fetch_alias_names)
fetch_alias_names=args.fetch_alias_names,
)

28
applications/neural_search/ranking/cross_encoder/predict.py
View File

@@ -57,35 +57,28 @@ if __name__ == "__main__":
paddle.set_device(args.device)
test_ds = load_dataset(read_text_pair, data_path=args.test_set, lazy=False)
model = AutoModelForSequenceClassification.from_pretrained(
args.model_name_or_path, num_classes=2)
model = AutoModelForSequenceClassification.from_pretrained(args.model_name_or_path, num_classes=2)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
is_test=True,
is_pair=True)
trans_func = partial(
convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length, is_test=True, is_pair=True
)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # segment
): [data for data in fn(samples)]
test_data_loader = create_dataloader(test_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
test_data_loader = create_dataloader(
test_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
if args.params_path and os.path.isfile(args.params_path):
state_dict = paddle.load(args.params_path)
model.set_dict(state_dict)
print("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
results = predict(model, test_data_loader)
test_ds = load_dataset(read_text_pair, data_path=args.test_set, lazy=False)
text_pairs = []
@@ -93,7 +86,6 @@ if __name__ == "__main__":
text_pair = test_ds[idx]
text_pair["pred_prob"] = prob
text_pairs.append(text_pair)
text_pairs = sorted(text_pairs, key=lambda x: x['pred_prob'],
reverse=True)[:args.topk]
text_pairs = sorted(text_pairs, key=lambda x: x["pred_prob"], reverse=True)[: args.topk]
for item in text_pairs:
print(item)

47
applications/neural_search/ranking/cross_encoder/train_ce.py
View File

@@ -102,31 +102,22 @@ def do_train():
dev_ds = load_dataset(read_data, data_path=args.test_file, lazy=False)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
model = AutoModelForSequenceClassification.from_pretrained(
args.model_name_or_path, num_classes=2)
model = AutoModelForSequenceClassification.from_pretrained(args.model_name_or_path, num_classes=2)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
is_pair=True)
trans_func = partial(convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length, is_pair=True)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # segment
Stack(dtype="int64") # label
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # segment
Stack(dtype="int64"), # label
): [data for data in fn(samples)]
train_data_loader = create_dataloader(train_ds,
mode='train',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
dev_data_loader = create_dataloader(dev_ds,
mode='dev',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
train_data_loader = create_dataloader(
train_ds, mode="train", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
dev_data_loader = create_dataloader(
dev_ds, mode="dev", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
if args.init_from_ckpt and os.path.isfile(args.init_from_ckpt):
state_dict = paddle.load(args.init_from_ckpt)
@@ -147,16 +138,14 @@ def do_train():
# Generate parameter names needed to perform weight decay.
# All bias and LayerNorm parameters are excluded.
decay_params = [
p.name for n, p in model.named_parameters()
if not any(nd in n for nd in ["bias", "norm"])
]
decay_params = [p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])]
optimizer = paddle.optimizer.AdamW(
learning_rate=args.learning_rate,
parameters=model.parameters(),
weight_decay=args.weight_decay,
apply_decay_param_fun=lambda x: x in decay_params,
grad_clip=paddle.nn.ClipGradByGlobalNorm(1.0))
grad_clip=paddle.nn.ClipGradByGlobalNorm(1.0),
)
criterion = paddle.nn.loss.CrossEntropyLoss()
metric = paddle.metric.Auc()
@@ -181,23 +170,21 @@ def do_train():
time_diff = time.time() - tic_train
print(
"global step %d, epoch: %d, batch: %d, loss: %.5f, accuracy: %.5f, speed: %.2f step/s"
% (global_step, epoch, step, loss, acc,
args.logging_steps / time_diff))
% (global_step, epoch, step, loss, acc, args.logging_steps / time_diff)
)
tic_train = time.time()
if global_step % args.eval_step == 0 and rank == 0:
evaluate(model, metric, dev_data_loader, "dev")
if global_step % args.save_steps == 0 and rank == 0:
save_dir = os.path.join(args.save_dir, "model_%d" % global_step)
model_to_save = model._layers if isinstance(
model, paddle.DataParallel) else model
model_to_save = model._layers if isinstance(model, paddle.DataParallel) else model
model_to_save.save_pretrained(save_dir)
tokenizer.save_pretrained(save_dir)
tic_train = time.time()
# save final checkpoint
save_dir = os.path.join(args.save_dir, "model_%d" % global_step)
model_to_save = model._layers if isinstance(model,
paddle.DataParallel) else model
model_to_save = model._layers if isinstance(model, paddle.DataParallel) else model
model_to_save.save_pretrained(save_dir)
tokenizer.save_pretrained(save_dir)

65
applications/neural_search/ranking/ernie_matching/data.py
View File

@@ -18,50 +18,34 @@ import numpy as np
from paddlenlp.datasets import MapDataset
def create_dataloader(dataset,
mode='train',
batch_size=1,
batchify_fn=None,
trans_fn=None):
def create_dataloader(dataset, mode="train", batch_size=1, batchify_fn=None, trans_fn=None):
if trans_fn:
dataset = dataset.map(trans_fn)
shuffle = True if mode == 'train' else False
if mode == 'train':
batch_sampler = paddle.io.DistributedBatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
shuffle = True if mode == "train" else False
if mode == "train":
batch_sampler = paddle.io.DistributedBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
else:
batch_sampler = paddle.io.BatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
batch_sampler = paddle.io.BatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
return paddle.io.DataLoader(dataset=dataset,
batch_sampler=batch_sampler,
collate_fn=batchify_fn,
return_list=True)
return paddle.io.DataLoader(dataset=dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn, return_list=True)
def read_text_pair(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if len(data) != 3:
continue
yield {'query': data[0], 'title': data[1]}
yield {"query": data[0], "title": data[1]}
def convert_pointwise_example(example,
tokenizer,
max_seq_length=512,
is_test=False):
def convert_pointwise_example(example, tokenizer, max_seq_length=512, is_test=False):
query, title = example["query"], example["title"]
encoded_inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -73,36 +57,25 @@ def convert_pointwise_example(example,
return input_ids, token_type_ids
def convert_pairwise_example(example,
tokenizer,
max_seq_length=512,
phase="train"):
def convert_pairwise_example(example, tokenizer, max_seq_length=512, phase="train"):
if phase == "train":
query, pos_title, neg_title = example["query"], example[
"title"], example["neg_title"]
query, pos_title, neg_title = example["query"], example["title"], example["neg_title"]
pos_inputs = tokenizer(text=query,
text_pair=pos_title,
max_seq_len=max_seq_length)
neg_inputs = tokenizer(text=query,
text_pair=neg_title,
max_seq_len=max_seq_length)
pos_inputs = tokenizer(text=query, text_pair=pos_title, max_seq_len=max_seq_length)
neg_inputs = tokenizer(text=query, text_pair=neg_title, max_seq_len=max_seq_length)
pos_input_ids = pos_inputs["input_ids"]
pos_token_type_ids = pos_inputs["token_type_ids"]
neg_input_ids = neg_inputs["input_ids"]
neg_token_type_ids = neg_inputs["token_type_ids"]
return (pos_input_ids, pos_token_type_ids, neg_input_ids,
neg_token_type_ids)
return (pos_input_ids, pos_token_type_ids, neg_input_ids, neg_token_type_ids)
else:
query, title = example["query"], example["title"]
inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = inputs["input_ids"]
token_type_ids = inputs["token_type_ids"]
@@ -115,11 +88,11 @@ def convert_pairwise_example(example,
def gen_pair(dataset, pool_size=100):
"""
"""
Generate triplet randomly based on dataset
Args:
dataset: A `MapDataset` or `IterDataset` or a tuple of those.
dataset: A `MapDataset` or `IterDataset` or a tuple of those.
Each example is composed of 2 texts: exampe["query"], example["title"]
pool_size: the number of example to sample negative example randomly

18
applications/neural_search/ranking/ernie_matching/deploy/cpp/http_client.py
View File

@@ -27,9 +27,7 @@ import paddlenlp as ppnlp
def convert_example(example, tokenizer, max_seq_length=512):
query, title = example["query"], example["title"]
encoded_inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -37,15 +35,15 @@ def convert_example(example, tokenizer, max_seq_length=512):
# 启动python客户端
endpoint_list = ['127.0.0.1:8600']
endpoint_list = ["127.0.0.1:8600"]
client = HttpClient()
client.load_client_config('serving_client')
client.load_client_config("serving_client")
client.connect(endpoint_list)
feed_names = client.feed_names_
fetch_names = client.fetch_names_
# 创建tokenizer
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-gram-zh')
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained("ernie-gram-zh")
max_seq_len = 64
# 数据预处理
@@ -53,15 +51,13 @@ list_data = [{"query": "中西方语言与文化的差异", "title": "第二语
input_ids, token_type_ids = [], []
for example in list_data:
input_id, token_type_id = convert_example(example,
tokenizer,
max_seq_length=max_seq_len)
input_id, token_type_id = convert_example(example, tokenizer, max_seq_length=max_seq_len)
input_ids.append(input_id)
token_type_ids.append(token_type_id)
feed_dict = {}
feed_dict['input_ids'] = np.array(input_ids)
feed_dict['token_type_ids'] = np.array(token_type_ids)
feed_dict["input_ids"] = np.array(input_ids)
feed_dict["token_type_ids"] = np.array(token_type_ids)
# batch设置为True表示的是批量预测
b_start = time.time()
result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)

18
applications/neural_search/ranking/ernie_matching/deploy/cpp/rpc_client.py
View File

@@ -25,9 +25,7 @@ import paddlenlp as ppnlp
def convert_example(example, tokenizer, max_seq_length=512):
query, title = example["query"], example["title"]
encoded_inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -36,15 +34,15 @@ def convert_example(example, tokenizer, max_seq_length=512):
# 启动python客户端
endpoint_list = ['127.0.0.1:8600']
endpoint_list = ["127.0.0.1:8600"]
client = Client()
client.load_client_config('serving_client')
client.load_client_config("serving_client")
client.connect(endpoint_list)
feed_names = client.feed_names_
fetch_names = client.fetch_names_
# 创建tokenizer
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-gram-zh')
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained("ernie-gram-zh")
max_seq_len = 64
# 数据预处理
@@ -52,15 +50,13 @@ list_data = [{"query": "中西方语言与文化的差异", "title": "第二语
input_ids, token_type_ids = [], []
for example in list_data:
input_id, token_type_id = convert_example(example,
tokenizer,
max_seq_length=max_seq_len)
input_id, token_type_id = convert_example(example, tokenizer, max_seq_length=max_seq_len)
input_ids.append(input_id)
token_type_ids.append(token_type_id)
feed_dict = {}
feed_dict['input_ids'] = np.array(input_ids)
feed_dict['token_type_ids'] = np.array(token_type_ids)
feed_dict["input_ids"] = np.array(input_ids)
feed_dict["token_type_ids"] = np.array(token_type_ids)
# batch设置为True表示的是批量预测
b_start = time.time()
result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)

115
applications/neural_search/ranking/ernie_matching/deploy/python/predict.py
View File

@@ -30,7 +30,7 @@ from paddlenlp.datasets import load_dataset
from paddlenlp.utils.log import logger
from paddlenlp.transformers import AutoTokenizer, AutoModel
sys.path.append('.')
sys.path.append(".")
# yapf: disable
parser = argparse.ArgumentParser()
@@ -67,21 +67,19 @@ args = parser.parse_args()
def read_text_pair(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if len(data) != 3:
continue
yield {'query': data[0], 'title': data[1]}
yield {"query": data[0], "title": data[1]}
def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
query, title = example["query"], example["title"]
encoded_inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -94,16 +92,17 @@ def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
class Predictor(object):
def __init__(self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False):
def __init__(
self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False,
):
self.max_seq_length = max_seq_length
self.batch_size = batch_size
@@ -122,14 +121,14 @@ class Predictor(object):
precision_map = {
"fp16": inference.PrecisionType.Half,
"fp32": inference.PrecisionType.Float32,
"int8": inference.PrecisionType.Int8
"int8": inference.PrecisionType.Int8,
}
precision_mode = precision_map[precision]
if args.use_tensorrt:
config.enable_tensorrt_engine(max_batch_size=batch_size,
min_subgraph_size=30,
precision_mode=precision_mode)
config.enable_tensorrt_engine(
max_batch_size=batch_size, min_subgraph_size=30, precision_mode=precision_mode
)
elif device == "cpu":
# set CPU configs accordingly,
# such as enable_mkldnn, set_cpu_math_library_num_threads
@@ -145,32 +144,27 @@ class Predictor(object):
config.switch_use_feed_fetch_ops(False)
self.predictor = paddle.inference.create_predictor(config)
self.input_handles = [
self.predictor.get_input_handle(name)
for name in self.predictor.get_input_names()
]
self.output_handle = self.predictor.get_output_handle(
self.predictor.get_output_names()[0])
self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()]
self.output_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[0])
if args.benchmark:
import auto_log
pid = os.getpid()
self.autolog = auto_log.AutoLogger(model_name="ernie-tiny",
model_precision=precision,
batch_size=self.batch_size,
data_shape="dynamic",
save_path=args.save_log_path,
inference_config=config,
pids=pid,
process_name=None,
gpu_ids=0,
time_keys=[
'preprocess_time',
'inference_time',
'postprocess_time'
],
warmup=0,
logger=logger)
self.autolog = auto_log.AutoLogger(
model_name="ernie-tiny",
model_precision=precision,
batch_size=self.batch_size,
data_shape="dynamic",
save_path=args.save_log_path,
inference_config=config,
pids=pid,
process_name=None,
gpu_ids=0,
time_keys=["preprocess_time", "inference_time", "postprocess_time"],
warmup=0,
logger=logger,
)
def predict(self, data, tokenizer):
"""
@@ -178,7 +172,7 @@ class Predictor(object):
Args:
data (obj:`List(str)`): The batch data whose each element is a raw text.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
label_map(obj:`dict`): The label id (key) to label str (value) map.
@@ -190,17 +184,12 @@ class Predictor(object):
examples = []
for text in data:
input_ids, segment_ids = convert_example(
text,
tokenizer,
max_seq_length=self.max_seq_length,
is_test=True)
input_ids, segment_ids = convert_example(text, tokenizer, max_seq_length=self.max_seq_length, is_test=True)
examples.append((input_ids, segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # segment
): fn(samples)
if args.benchmark:
@@ -223,27 +212,29 @@ class Predictor(object):
if __name__ == "__main__":
# Define predictor to do prediction.
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
args.batch_size, args.use_tensorrt, args.precision,
args.cpu_threads, args.enable_mkldnn)
predictor = Predictor(
args.model_dir,
args.device,
args.max_seq_length,
args.batch_size,
args.use_tensorrt,
args.precision,
args.cpu_threads,
args.enable_mkldnn,
)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
test_ds = load_dataset(read_text_pair,
data_path=args.input_file,
lazy=False)
test_ds = load_dataset(read_text_pair, data_path=args.input_file, lazy=False)
data = [{'query': d['query'], 'title': d['title']} for d in test_ds]
data = [{"query": d["query"], "title": d["title"]} for d in test_ds]
batches = [
data[idx:idx + args.batch_size]
for idx in range(0, len(data), args.batch_size)
]
batches = [data[idx : idx + args.batch_size] for idx in range(0, len(data), args.batch_size)]
results = []
for batch_data in batches:
results.extend(predictor.predict(batch_data, tokenizer))
for idx, text in enumerate(data):
print('Data: {} \t prob: {}'.format(text, results[idx]))
print("Data: {} \t prob: {}".format(text, results[idx]))
if args.benchmark:
predictor.autolog.report()

2
applications/neural_search/ranking/ernie_matching/deploy/python/rpc_client.py
View File

@@ -17,7 +17,7 @@ import numpy as np
from paddle_serving_server.pipeline import PipelineClient
client = PipelineClient()
client.connect(['127.0.0.1:8089'])
client.connect(["127.0.0.1:8089"])
list_data = [{"query": "中西方语言与文化的差异", "title": "第二语言习得的一大障碍就是文化差异。"}]
feed = {}

23
applications/neural_search/ranking/ernie_matching/deploy/python/web_service.py
View File

@@ -22,9 +22,7 @@ from paddle_serving_server.web_service import WebService, Op
def convert_example(example, tokenizer, max_seq_length=512):
query, title = example["query"], example["title"]
encoded_inputs = tokenizer(text=query,
text_pair=title,
max_seq_len=max_seq_length)
encoded_inputs = tokenizer(text=query, text_pair=title, max_seq_len=max_seq_length)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -33,32 +31,30 @@ def convert_example(example, tokenizer, max_seq_length=512):
class ErnieOp(Op):
def init_op(self):
from paddlenlp.transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
self.tokenizer = AutoTokenizer.from_pretrained("ernie-3.0-medium-zh")
def preprocess(self, input_dicts, data_id, log_id):
from paddlenlp.data import Stack, Tuple, Pad
(_, input_dict), = input_dicts.items()
((_, input_dict),) = input_dicts.items()
print("input dict", input_dict)
batch_size = len(input_dict.keys())
examples = []
for i in range(batch_size):
example = json.loads(input_dict[str(i)].replace("\'", "\""))
example = json.loads(input_dict[str(i)].replace("'", '"'))
input_ids, segment_ids = convert_example(example, self.tokenizer)
examples.append((input_ids, segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"
), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"), # segment
): fn(samples)
input_ids, segment_ids = batchify_fn(examples)
feed_dict = {}
feed_dict['input_ids'] = input_ids
feed_dict['token_type_ids'] = segment_ids
feed_dict["input_ids"] = input_ids
feed_dict["token_type_ids"] = segment_ids
return feed_dict, False, None, ""
def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
@@ -68,7 +64,6 @@ class ErnieOp(Op):
class ErnieService(WebService):
def get_pipeline_response(self, read_op):
ernie_op = ErnieOp(name="ernie", input_ops=[read_op])
return ernie_op

44
applications/neural_search/ranking/ernie_matching/evaluate.py
View File

@@ -71,8 +71,7 @@ def evaluate(model, metric, data_loader, phase="dev"):
for idx, batch in enumerate(data_loader):
input_ids, token_type_ids, labels = batch
pos_probs = model.predict(input_ids=input_ids,
token_type_ids=token_type_ids)
pos_probs = model.predict(input_ids=input_ids, token_type_ids=token_type_ids)
neg_probs = 1.0 - pos_probs
@@ -86,21 +85,21 @@ def evaluate(model, metric, data_loader, phase="dev"):
# 构建读取函数，读取原始数据
def read(src_path, is_predict=False):
data = pd.read_csv(src_path, sep='\t')
data = pd.read_csv(src_path, sep="\t")
for index, row in tqdm(data.iterrows()):
query = row['query']
title = row['title']
neg_title = row['neg_title']
yield {'query': query, 'title': title, 'neg_title': neg_title}
query = row["query"]
title = row["title"]
neg_title = row["neg_title"]
yield {"query": query, "title": title, "neg_title": neg_title}
def read_test(src_path, is_predict=False):
data = pd.read_csv(src_path, sep='\t')
data = pd.read_csv(src_path, sep="\t")
for index, row in tqdm(data.iterrows()):
query = row['query']
title = row['title']
label = row['label']
yield {'query': query, 'title': title, 'label': label}
query = row["query"]
title = row["title"]
label = row["label"]
yield {"query": query, "title": title, "label": label}
def main():
@@ -117,24 +116,17 @@ def main():
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func_eval = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
phase="eval")
trans_func_eval = partial(convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length, phase="eval")
batchify_fn_eval = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # pair_segment
Stack(dtype="int64") # label
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # pair_segment
Stack(dtype="int64"), # label
): [data for data in fn(samples)]
dev_data_loader = create_dataloader(dev_ds,
mode='dev',
batch_size=args.batch_size,
batchify_fn=batchify_fn_eval,
trans_fn=trans_func_eval)
dev_data_loader = create_dataloader(
dev_ds, mode="dev", batch_size=args.batch_size, batchify_fn=batchify_fn_eval, trans_fn=trans_func_eval
)
model = PairwiseMatching(pretrained_model, margin=args.margin)

12
applications/neural_search/ranking/ernie_matching/export_model.py
View File

@@ -40,19 +40,17 @@ if __name__ == "__main__":
model.set_dict(state_dict)
print("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
model.eval()
# Convert to static graph with specific input description
model = paddle.jit.to_static(
model,
input_spec=[
paddle.static.InputSpec(shape=[None, None],
dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None],
dtype="int64") # segment_ids
])
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # segment_ids
],
)
# Save in static graph model.
save_path = os.path.join(args.output_path, "inference")
paddle.jit.save(model, save_path)

4
applications/neural_search/ranking/ernie_matching/export_to_serving.py
View File

@@ -14,6 +14,7 @@
import argparse
import paddle_serving_client.io as serving_io
# yapf: disable
parser = argparse.ArgumentParser()
parser.add_argument("--dirname", type=str, required=True,
@@ -44,4 +45,5 @@ if __name__ == "__main__":
params_filename=args.params_filename,
show_proto=args.show_proto,
feed_alias_names=args.feed_alias_names,
fetch_alias_names=args.fetch_alias_names)
fetch_alias_names=args.fetch_alias_names,
)

55
applications/neural_search/ranking/ernie_matching/model.py
View File

@@ -18,7 +18,6 @@ import paddle.nn.functional as F
class PairwiseMatching(nn.Layer):
def __init__(self, pretrained_model, dropout=None, margin=0.1):
super().__init__()
self.ptm = pretrained_model
@@ -28,18 +27,15 @@ class PairwiseMatching(nn.Layer):
# hidden_size -> 1, calculate similarity
self.similarity = nn.Linear(self.ptm.config["hidden_size"], 1)
@paddle.jit.to_static(input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype='int64'),
paddle.static.InputSpec(shape=[None, None], dtype='int64')
])
def predict(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None):
@paddle.jit.to_static(
input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
]
)
def predict(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
attention_mask)
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
cls_embedding = self.dropout(cls_embedding)
sim_score = self.similarity(cls_embedding)
@@ -47,21 +43,21 @@ class PairwiseMatching(nn.Layer):
return sim_score
def forward(self,
pos_input_ids,
neg_input_ids,
pos_token_type_ids=None,
neg_token_type_ids=None,
pos_position_ids=None,
neg_position_ids=None,
pos_attention_mask=None,
neg_attention_mask=None):
def forward(
self,
pos_input_ids,
neg_input_ids,
pos_token_type_ids=None,
neg_token_type_ids=None,
pos_position_ids=None,
neg_position_ids=None,
pos_attention_mask=None,
neg_attention_mask=None,
):
_, pos_cls_embedding = self.ptm(pos_input_ids, pos_token_type_ids,
pos_position_ids, pos_attention_mask)
_, pos_cls_embedding = self.ptm(pos_input_ids, pos_token_type_ids, pos_position_ids, pos_attention_mask)
_, neg_cls_embedding = self.ptm(neg_input_ids, neg_token_type_ids,
neg_position_ids, neg_attention_mask)
_, neg_cls_embedding = self.ptm(neg_input_ids, neg_token_type_ids, neg_position_ids, neg_attention_mask)
pos_embedding = self.dropout(pos_cls_embedding)
neg_embedding = self.dropout(neg_cls_embedding)
@@ -72,13 +68,8 @@ class PairwiseMatching(nn.Layer):
pos_sim = F.sigmoid(pos_sim)
neg_sim = F.sigmoid(neg_sim)
labels = paddle.full(shape=[pos_cls_embedding.shape[0]],
fill_value=1.0,
dtype='float32')
labels = paddle.full(shape=[pos_cls_embedding.shape[0]], fill_value=1.0, dtype="float32")
loss = F.margin_ranking_loss(pos_sim,
neg_sim,
labels,
margin=self.margin)
loss = F.margin_ranking_loss(pos_sim, neg_sim, labels, margin=self.margin)
return loss

32
applications/neural_search/ranking/ernie_matching/predict_pairwise.py
View File

@@ -61,11 +61,10 @@ def predict(model, data_loader):
for batch_data in data_loader:
input_ids, token_type_ids = batch_data
batch_prob = model.predict(input_ids=input_ids,
token_type_ids=token_type_ids).numpy()
batch_prob = model.predict(input_ids=input_ids, token_type_ids=token_type_ids).numpy()
batch_probs.append(batch_prob)
if (len(batch_prob) == 1):
if len(batch_prob) == 1:
batch_probs = np.array(batch_probs)
else:
batch_probs = np.concatenate(batch_probs, axis=0)
@@ -79,26 +78,18 @@ if __name__ == "__main__":
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
phase="predict")
trans_func = partial(convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length, phase="predict")
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input_ids
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # segment_ids
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # segment_ids
): [data for data in fn(samples)]
valid_ds = load_dataset(read_text_pair,
data_path=args.input_file,
lazy=False)
valid_ds = load_dataset(read_text_pair, data_path=args.input_file, lazy=False)
valid_data_loader = create_dataloader(valid_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
valid_data_loader = create_dataloader(
valid_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
model = PairwiseMatching(pretrained_model)
@@ -107,14 +98,11 @@ if __name__ == "__main__":
model.set_dict(state_dict)
print("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
y_probs = predict(model, valid_data_loader)
valid_ds = load_dataset(read_text_pair,
data_path=args.input_file,
lazy=False)
valid_ds = load_dataset(read_text_pair, data_path=args.input_file, lazy=False)
for idx, prob in enumerate(y_probs):
text_pair = valid_ds[idx]

95
applications/neural_search/ranking/ernie_matching/train_pairwise.py
View File

@@ -79,8 +79,7 @@ def evaluate(model, metric, data_loader, phase="dev"):
for idx, batch in enumerate(data_loader):
input_ids, token_type_ids, labels = batch
pos_probs = model.predict(input_ids=input_ids,
token_type_ids=token_type_ids)
pos_probs = model.predict(input_ids=input_ids, token_type_ids=token_type_ids)
neg_probs = 1.0 - pos_probs
@@ -94,21 +93,21 @@ def evaluate(model, metric, data_loader, phase="dev"):
# 构建读取函数，读取原始数据
def read(src_path, is_predict=False):
data = pd.read_csv(src_path, sep='\t')
data = pd.read_csv(src_path, sep="\t")
for index, row in tqdm(data.iterrows()):
query = row['query']
title = row['title']
neg_title = row['neg_title']
yield {'query': query, 'title': title, 'neg_title': neg_title}
query = row["query"]
title = row["title"]
neg_title = row["neg_title"]
yield {"query": query, "title": title, "neg_title": neg_title}
def read_test(src_path, is_predict=False):
data = pd.read_csv(src_path, sep='\t')
data = pd.read_csv(src_path, sep="\t")
for index, row in tqdm(data.iterrows()):
query = row['query']
title = row['title']
label = row['label']
yield {'query': query, 'title': title, 'label': label}
query = row["query"]
title = row["title"]
label = row["label"]
yield {"query": query, "title": title, "label": label}
def do_train():
@@ -125,45 +124,30 @@ def do_train():
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func_train = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length)
trans_func_train = partial(convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length)
trans_func_eval = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
phase="eval")
trans_func_eval = partial(convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length, phase="eval")
batchify_fn_train = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # pos_pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # pos_pair_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # neg_pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
) # neg_pair_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # pos_pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # pos_pair_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # neg_pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # neg_pair_segment
): [data for data in fn(samples)]
batchify_fn_eval = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # pair_segment
Stack(dtype="int64") # label
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # pair_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # pair_segment
Stack(dtype="int64"), # label
): [data for data in fn(samples)]
train_data_loader = create_dataloader(train_ds,
mode='train',
batch_size=args.batch_size,
batchify_fn=batchify_fn_train,
trans_fn=trans_func_train)
train_data_loader = create_dataloader(
train_ds, mode="train", batch_size=args.batch_size, batchify_fn=batchify_fn_train, trans_fn=trans_func_train
)
dev_data_loader = create_dataloader(dev_ds,
mode='dev',
batch_size=args.batch_size,
batchify_fn=batchify_fn_eval,
trans_fn=trans_func_eval)
dev_data_loader = create_dataloader(
dev_ds, mode="dev", batch_size=args.batch_size, batchify_fn=batchify_fn_eval, trans_fn=trans_func_eval
)
model = PairwiseMatching(pretrained_model, margin=args.margin)
@@ -173,20 +157,17 @@ def do_train():
num_training_steps = len(train_data_loader) * args.epochs
lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps,
args.warmup_proportion)
lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps, args.warmup_proportion)
# Generate parameter names needed to perform weight decay.
# All bias and LayerNorm parameters are excluded.
decay_params = [
p.name for n, p in model.named_parameters()
if not any(nd in n for nd in ["bias", "norm"])
]
decay_params = [p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])]
optimizer = paddle.optimizer.AdamW(
learning_rate=lr_scheduler,
parameters=model.parameters(),
weight_decay=args.weight_decay,
apply_decay_param_fun=lambda x: x in decay_params)
apply_decay_param_fun=lambda x: x in decay_params,
)
metric = paddle.metric.Auc()
@@ -196,17 +177,19 @@ def do_train():
for step, batch in enumerate(train_data_loader, start=1):
pos_input_ids, pos_token_type_ids, neg_input_ids, neg_token_type_ids = batch
loss = model(pos_input_ids=pos_input_ids,
neg_input_ids=neg_input_ids,
pos_token_type_ids=pos_token_type_ids,
neg_token_type_ids=neg_token_type_ids)
loss = model(
pos_input_ids=pos_input_ids,
neg_input_ids=neg_input_ids,
pos_token_type_ids=pos_token_type_ids,
neg_token_type_ids=neg_token_type_ids,
)
global_step += 1
if global_step % 10 == 0 and rank == 0:
print(
"global step %d, epoch: %d, batch: %d, loss: %.5f, speed: %.2f step/s"
% (global_step, epoch, step, loss, 10 /
(time.time() - tic_train)))
% (global_step, epoch, step, loss, 10 / (time.time() - tic_train))
)
tic_train = time.time()
loss.backward()
@@ -221,7 +204,7 @@ def do_train():
save_dir = os.path.join(args.save_dir, "model_%d" % global_step)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_param_path = os.path.join(save_dir, 'model_state.pdparams')
save_param_path = os.path.join(save_dir, "model_state.pdparams")
paddle.save(model.state_dict(), save_param_path)
tokenizer.save_pretrained(save_dir)

8
applications/neural_search/recall/in_batch_negative/ann_util.py
View File

@@ -21,9 +21,7 @@ from paddlenlp.utils.log import logger
def build_index(args, data_loader, model):
index = hnswlib.Index(
space='ip',
dim=args.output_emb_size if args.output_emb_size > 0 else 768)
index = hnswlib.Index(space="ip", dim=args.output_emb_size if args.output_emb_size > 0 else 768)
# Initializing index
# max_elements - the maximum number of elements (capacity). Will throw an exception if exceeded
@@ -34,9 +32,7 @@ def build_index(args, data_loader, model):
#
# M - is tightly connected with internal dimensionality of the data. Strongly affects memory consumption (~M)
# Higher M leads to higher accuracy/run_time at fixed ef/efConstruction
index.init_index(max_elements=args.hnsw_max_elements,
ef_construction=args.hnsw_ef,
M=args.hnsw_m)
index.init_index(max_elements=args.hnsw_max_elements, ef_construction=args.hnsw_ef, M=args.hnsw_m)
# Controlling the recall by setting ef:
# higher ef leads to better accuracy, but slower search

133
applications/neural_search/recall/in_batch_negative/base_model.py
View File

@@ -23,7 +23,6 @@ import paddle.nn.functional as F
class SemanticIndexBase(nn.Layer):
def __init__(self, pretrained_model, dropout=None, output_emb_size=None):
super().__init__()
self.ptm = pretrained_model
@@ -35,19 +34,11 @@ class SemanticIndexBase(nn.Layer):
self.output_emb_size = output_emb_size
if output_emb_size > 0:
weight_attr = paddle.ParamAttr(
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768,
output_emb_size,
weight_attr=weight_attr)
weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768, output_emb_size, weight_attr=weight_attr)
def get_pooled_embedding(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
attention_mask)
def get_pooled_embedding(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
if self.output_emb_size > 0:
cls_embedding = self.emb_reduce_linear(cls_embedding)
@@ -62,33 +53,31 @@ class SemanticIndexBase(nn.Layer):
for batch_data in data_loader:
input_ids, token_type_ids = batch_data
text_embeddings = self.get_pooled_embedding(
input_ids, token_type_ids=token_type_ids)
text_embeddings = self.get_pooled_embedding(input_ids, token_type_ids=token_type_ids)
yield text_embeddings
def cosine_sim(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None):
def cosine_sim(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask
)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
axis=-1)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding, axis=-1)
return cosine_sim
@abc.abstractmethod
@@ -97,7 +86,6 @@ class SemanticIndexBase(nn.Layer):
class SemanticIndexBaseStatic(nn.Layer):
def __init__(self, pretrained_model, dropout=None, output_emb_size=None):
super().__init__()
self.ptm = pretrained_model
@@ -109,23 +97,17 @@ class SemanticIndexBaseStatic(nn.Layer):
self.output_emb_size = output_emb_size
if output_emb_size > 0:
weight_attr = paddle.ParamAttr(
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768,
output_emb_size,
weight_attr=weight_attr)
weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768, output_emb_size, weight_attr=weight_attr)
@paddle.jit.to_static(input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype='int64'),
paddle.static.InputSpec(shape=[None, None], dtype='int64')
])
def get_pooled_embedding(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
attention_mask)
@paddle.jit.to_static(
input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
]
)
def get_pooled_embedding(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
if self.output_emb_size > 0:
cls_embedding = self.emb_reduce_linear(cls_embedding)
@@ -140,42 +122,35 @@ class SemanticIndexBaseStatic(nn.Layer):
for batch_data in data_loader:
input_ids, token_type_ids = batch_data
text_embeddings = self.get_pooled_embedding(
input_ids, token_type_ids=token_type_ids)
text_embeddings = self.get_pooled_embedding(input_ids, token_type_ids=token_type_ids)
yield text_embeddings
def cosine_sim(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None):
def cosine_sim(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask
)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
axis=-1)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding, axis=-1)
return cosine_sim
def forward(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
attention_mask)
def forward(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
if self.output_emb_size > 0:
cls_embedding = self.emb_reduce_linear(cls_embedding)

106
applications/neural_search/recall/in_batch_negative/batch_negative/model.py
View File

@@ -23,54 +23,46 @@ from base_model import SemanticIndexBase
class SemanticIndexBatchNeg(SemanticIndexBase):
def __init__(self,
pretrained_model,
dropout=None,
margin=0.3,
scale=30,
output_emb_size=None):
def __init__(self, pretrained_model, dropout=None, margin=0.3, scale=30, output_emb_size=None):
super().__init__(pretrained_model, dropout, output_emb_size)
self.margin = margin
# Used scaling cosine similarity to ease converge
self.sacle = scale
def forward(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None):
def forward(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask
)
cosine_sim = paddle.matmul(query_cls_embedding,
title_cls_embedding,
transpose_y=True)
cosine_sim = paddle.matmul(query_cls_embedding, title_cls_embedding, transpose_y=True)
# Substract margin from all positive samples cosine_sim()
margin_diag = paddle.full(shape=[query_cls_embedding.shape[0]],
fill_value=self.margin,
dtype=paddle.get_default_dtype())
margin_diag = paddle.full(
shape=[query_cls_embedding.shape[0]], fill_value=self.margin, dtype=paddle.get_default_dtype()
)
cosine_sim = cosine_sim - paddle.diag(margin_diag)
# Scale cosine to ease training converge
cosine_sim *= self.sacle
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype='int64')
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype="int64")
labels = paddle.reshape(labels, shape=[-1, 1])
loss = F.cross_entropy(input=cosine_sim, label=labels)
@@ -79,53 +71,43 @@ class SemanticIndexBatchNeg(SemanticIndexBase):
class SemanticIndexCacheNeg(SemanticIndexBase):
def __init__(self,
pretrained_model,
dropout=None,
margin=0.3,
scale=30,
output_emb_size=None):
def __init__(self, pretrained_model, dropout=None, margin=0.3, scale=30, output_emb_size=None):
super().__init__(pretrained_model, dropout, output_emb_size)
self.margin = margin
# Used scaling cosine similarity to ease converge
self.sacle = scale
def forward(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None):
def forward(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask
)
cosine_sim = paddle.matmul(query_cls_embedding,
title_cls_embedding,
transpose_y=True)
cosine_sim = paddle.matmul(query_cls_embedding, title_cls_embedding, transpose_y=True)
# Substract margin from all positive samples cosine_sim()
margin_diag = paddle.full(shape=[query_cls_embedding.shape[0]],
fill_value=self.margin,
dtype=cosine_sim.dtype)
margin_diag = paddle.full(shape=[query_cls_embedding.shape[0]], fill_value=self.margin, dtype=cosine_sim.dtype)
cosine_sim = cosine_sim - paddle.diag(margin_diag)
# Scale cosine to ease training converge
cosine_sim *= self.sacle
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype='int64')
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype="int64")
labels = paddle.reshape(labels, shape=[-1, 1])
return [cosine_sim, labels, query_cls_embedding, title_cls_embedding]

73
applications/neural_search/recall/in_batch_negative/data.py
View File

@@ -17,33 +17,19 @@ import paddle
from paddlenlp.utils.log import logger
def create_dataloader(dataset,
mode='train',
batch_size=1,
batchify_fn=None,
trans_fn=None):
def create_dataloader(dataset, mode="train", batch_size=1, batchify_fn=None, trans_fn=None):
if trans_fn:
dataset = dataset.map(trans_fn)
shuffle = True if mode == 'train' else False
if mode == 'train':
batch_sampler = paddle.io.DistributedBatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
shuffle = True if mode == "train" else False
if mode == "train":
batch_sampler = paddle.io.DistributedBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
else:
batch_sampler = paddle.io.BatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
batch_sampler = paddle.io.BatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
return paddle.io.DataLoader(dataset=dataset,
batch_sampler=batch_sampler,
collate_fn=batchify_fn,
return_list=True)
return paddle.io.DataLoader(dataset=dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn, return_list=True)
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
"""
Builds model inputs from a sequence.
@@ -53,9 +39,9 @@ def convert_example(example,
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -66,9 +52,7 @@ def convert_example(example,
result = []
for key, text in example.items():
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -77,32 +61,28 @@ def convert_example(example,
def read_text_pair(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if len(data) != 2:
continue
yield {'text_a': data[0], 'text_b': data[1]}
yield {"text_a": data[0], "text_b": data[1]}
def read_text_triplet(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if len(data) != 3:
continue
yield {
'text': data[0],
'pos_sample': data[1],
'neg_sample': data[2]
}
yield {"text": data[0], "pos_sample": data[1], "neg_sample": data[2]}
# ANN - active learning ------------------------------------------------------
def get_latest_checkpoint(args):
"""
Return: (latest_checkpint_path, global_step)
Return: (latest_checkpint_path, global_step)
"""
if not os.path.exists(args.save_dir):
return args.init_from_ckpt, 0
@@ -113,14 +93,12 @@ def get_latest_checkpoint(args):
chk_path = os.path.join(args.save_dir, checkpoint)
scheduler_path = os.path.join(chk_path, "model_state.pdparams")
succeed_flag_file = os.path.join(chk_path, "succeed_flag_file")
return os.path.exists(scheduler_path) and os.path.exists(
succeed_flag_file)
return os.path.exists(scheduler_path) and os.path.exists(succeed_flag_file)
trained_steps = [int(s) for s in subdirectories if valid_checkpoint(s)]
if len(trained_steps) > 0:
return os.path.join(args.save_dir, str(max(trained_steps)),
"model_state.pdparams"), max(trained_steps)
return os.path.join(args.save_dir, str(max(trained_steps)), "model_state.pdparams"), max(trained_steps)
return args.init_from_ckpt, 0
@@ -135,19 +113,14 @@ def get_latest_ann_data(ann_data_dir):
def valid_checkpoint(step):
ann_data_file = os.path.join(ann_data_dir, step, "new_ann_data")
# succed_flag_file is an empty file that indicates ann data has been generated
succeed_flag_file = os.path.join(ann_data_dir, step,
"succeed_flag_file")
return os.path.exists(succeed_flag_file) and os.path.exists(
ann_data_file)
succeed_flag_file = os.path.join(ann_data_dir, step, "succeed_flag_file")
return os.path.exists(succeed_flag_file) and os.path.exists(ann_data_file)
ann_data_steps = [int(s) for s in subdirectories if valid_checkpoint(s)]
if len(ann_data_steps) > 0:
latest_ann_data_file = os.path.join(ann_data_dir,
str(max(ann_data_steps)),
"new_ann_data")
logger.info(
"Using lateset ann_data_file:{}".format(latest_ann_data_file))
latest_ann_data_file = os.path.join(ann_data_dir, str(max(ann_data_steps)), "new_ann_data")
logger.info("Using lateset ann_data_file:{}".format(latest_ann_data_file))
return latest_ann_data_file, max(ann_data_steps)
logger.info("no new ann_data, return (None, -1)")
@@ -156,7 +129,7 @@ def get_latest_ann_data(ann_data_dir):
def gen_id2corpus(corpus_file):
id2corpus = {}
with open(corpus_file, 'r', encoding='utf-8') as f:
with open(corpus_file, "r", encoding="utf-8") as f:
for idx, line in enumerate(f):
id2corpus[idx] = line.rstrip()
return id2corpus
@@ -165,7 +138,7 @@ def gen_id2corpus(corpus_file):
def gen_text_file(similar_text_pair_file):
text2similar_text = {}
texts = []
with open(similar_text_pair_file, 'r', encoding='utf-8') as f:
with open(similar_text_pair_file, "r", encoding="utf-8") as f:
for line in f:
splited_line = line.rstrip().split("\t")
if len(splited_line) != 2:

25
applications/neural_search/recall/in_batch_negative/deploy/cpp/http_client.py
View File

@@ -25,16 +25,11 @@ from paddlenlp.transformers import AutoModel, AutoTokenizer
from paddle_serving_client import HttpClient
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=True):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=True):
list_input_ids = []
list_token_type_ids = []
for text in example:
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
@@ -44,9 +39,9 @@ def convert_example(example,
# 启动python客户端
endpoint_list = ['127.0.0.1:9393']
endpoint_list = ["127.0.0.1:9393"]
client = HttpClient()
client.load_client_config('serving_client')
client.load_client_config("serving_client")
client.connect(endpoint_list)
feed_names = client.feed_names_
fetch_names = client.fetch_names_
@@ -54,12 +49,12 @@ print(feed_names)
print(fetch_names)
# 创建tokenizer
tokenizer = AutoTokenizer.from_pretrained('rocketqa-zh-base-query-encoder')
tokenizer = AutoTokenizer.from_pretrained("rocketqa-zh-base-query-encoder")
max_seq_len = 64
# 数据预处理
list_data = ['国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据.', '面向生态系统服务的生态系统分类方案研发与应用']
list_data = ["国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据.", "面向生态系统服务的生态系统分类方案研发与应用"]
# for i in range(5):
# list_data.extend(list_data)
# print(len(list_data))
@@ -67,11 +62,11 @@ examples = convert_example(list_data, tokenizer, max_seq_length=max_seq_len)
print(examples)
feed_dict = {}
feed_dict['input_ids'] = np.array(examples[0])
feed_dict['token_type_ids'] = np.array(examples[1])
feed_dict["input_ids"] = np.array(examples[0])
feed_dict["token_type_ids"] = np.array(examples[1])
print(feed_dict['input_ids'].shape)
print(feed_dict['token_type_ids'].shape)
print(feed_dict["input_ids"].shape)
print(feed_dict["token_type_ids"].shape)
# batch设置为True表示的是批量预测
b_start = time.time()

25
applications/neural_search/recall/in_batch_negative/deploy/cpp/rpc_client.py
View File

@@ -22,16 +22,11 @@ from paddle_serving_client import Client
from paddlenlp.transformers import AutoTokenizer
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=True):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=True):
list_input_ids = []
list_token_type_ids = []
for text in example:
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
list_input_ids.append(input_ids)
@@ -40,9 +35,9 @@ def convert_example(example,
# 启动python客户端
endpoint_list = ['127.0.0.1:9393']
endpoint_list = ["127.0.0.1:9393"]
client = Client()
client.load_client_config('serving_client')
client.load_client_config("serving_client")
client.connect(endpoint_list)
feed_names = client.feed_names_
fetch_names = client.fetch_names_
@@ -50,12 +45,12 @@ print(feed_names)
print(fetch_names)
# 创建tokenizer
tokenizer = AutoTokenizer.from_pretrained('rocketqa-zh-base-query-encoder')
tokenizer = AutoTokenizer.from_pretrained("rocketqa-zh-base-query-encoder")
max_seq_len = 64
# 数据预处理
list_data = ['国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据.', '面向生态系统服务的生态系统分类方案研发与应用']
list_data = ["国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据.", "面向生态系统服务的生态系统分类方案研发与应用"]
# for i in range(5):
# list_data.extend(list_data)
# print(len(list_data))
@@ -63,11 +58,11 @@ examples = convert_example(list_data, tokenizer, max_seq_length=max_seq_len)
print(examples)
feed_dict = {}
feed_dict['input_ids'] = np.array(examples[0])
feed_dict['token_type_ids'] = np.array(examples[1])
feed_dict["input_ids"] = np.array(examples[0])
feed_dict["token_type_ids"] = np.array(examples[1])
print(feed_dict['input_ids'].shape)
print(feed_dict['token_type_ids'].shape)
print(feed_dict["input_ids"].shape)
print(feed_dict["token_type_ids"].shape)
# batch设置为True表示的是批量预测
b_start = time.time()
result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)

98
applications/neural_search/recall/in_batch_negative/deploy/python/predict.py
View File

@@ -26,7 +26,7 @@ from paddlenlp.data import Stack, Tuple, Pad
from paddlenlp.datasets import load_dataset
from paddlenlp.utils.log import logger
sys.path.append('.')
sys.path.append(".")
# yapf: disable
parser = argparse.ArgumentParser()
@@ -59,22 +59,19 @@ args = parser.parse_args()
# yapf: enable
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -85,9 +82,7 @@ def convert_example(example,
result = []
for key, text in example.items():
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -95,16 +90,17 @@ def convert_example(example,
class Predictor(object):
def __init__(self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False):
def __init__(
self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False,
):
self.max_seq_length = max_seq_length
self.batch_size = batch_size
@@ -123,14 +119,14 @@ class Predictor(object):
precision_map = {
"fp16": inference.PrecisionType.Half,
"fp32": inference.PrecisionType.Float32,
"int8": inference.PrecisionType.Int8
"int8": inference.PrecisionType.Int8,
}
precision_mode = precision_map[precision]
if args.use_tensorrt:
config.enable_tensorrt_engine(max_batch_size=batch_size,
min_subgraph_size=30,
precision_mode=precision_mode)
config.enable_tensorrt_engine(
max_batch_size=batch_size, min_subgraph_size=30, precision_mode=precision_mode
)
elif device == "cpu":
# set CPU configs accordingly,
# such as enable_mkldnn, set_cpu_math_library_num_threads
@@ -146,15 +142,12 @@ class Predictor(object):
config.switch_use_feed_fetch_ops(False)
self.predictor = paddle.inference.create_predictor(config)
self.input_handles = [
self.predictor.get_input_handle(name)
for name in self.predictor.get_input_names()
]
self.output_handle = self.predictor.get_output_handle(
self.predictor.get_output_names()[0])
self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()]
self.output_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[0])
if args.benchmark:
import auto_log
pid = os.getpid()
self.autolog = auto_log.AutoLogger(
model_name=args.model_name_or_path,
@@ -166,11 +159,10 @@ class Predictor(object):
pids=pid,
process_name=None,
gpu_ids=0,
time_keys=[
'preprocess_time', 'inference_time', 'postprocess_time'
],
time_keys=["preprocess_time", "inference_time", "postprocess_time"],
warmup=0,
logger=logger)
logger=logger,
)
def extract_embedding(self, data, tokenizer):
"""
@@ -178,7 +170,7 @@ class Predictor(object):
Args:
data (obj:`List(str)`): The batch data whose each element is a raw text.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
Returns:
@@ -219,7 +211,7 @@ class Predictor(object):
Args:
data (obj:`List(str)`): The batch data whose each element is a raw text.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
Returns:
@@ -231,10 +223,8 @@ class Predictor(object):
examples = []
for idx, text in enumerate(data):
input_ids, segment_ids = convert_example({idx: text[0]}, tokenizer)
title_ids, title_segment_ids = convert_example({idx: text[1]},
tokenizer)
examples.append(
(input_ids, segment_ids, title_ids, title_segment_ids))
title_ids, title_segment_ids = convert_example({idx: text[1]}, tokenizer)
examples.append((input_ids, segment_ids, title_ids, title_segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id), # input
@@ -246,8 +236,7 @@ class Predictor(object):
if args.benchmark:
self.autolog.times.stamp()
query_ids, query_segment_ids, title_ids, title_segment_ids = batchify_fn(
examples)
query_ids, query_segment_ids, title_ids, title_segment_ids = batchify_fn(examples)
self.input_handles[0].copy_from_cpu(query_ids)
self.input_handles[1].copy_from_cpu(query_segment_ids)
self.predictor.run()
@@ -263,28 +252,31 @@ class Predictor(object):
if args.benchmark:
self.autolog.times.end(stamp=True)
result = [
float(1 - spatial.distance.cosine(arr1, arr2))
for arr1, arr2 in zip(query_logits, title_logits)
]
result = [float(1 - spatial.distance.cosine(arr1, arr2)) for arr1, arr2 in zip(query_logits, title_logits)]
return result
if __name__ == "__main__":
# Define predictor to do prediction.
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
args.batch_size, args.use_tensorrt, args.precision,
args.cpu_threads, args.enable_mkldnn)
predictor = Predictor(
args.model_dir,
args.device,
args.max_seq_length,
args.batch_size,
args.use_tensorrt,
args.precision,
args.cpu_threads,
args.enable_mkldnn,
)
# ErnieTinyTokenizer is special for ernie-tiny pretained model.
output_emb_size = 256
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
id2corpus = {0: '国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
id2corpus = {0: "国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据"}
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
res = predictor.extract_embedding(corpus_list, tokenizer)
print(res.shape)
print(res)
corpus_list = [['中西方语言与文化的差异', '中西方文化差异以及语言体现中西方文化,差异,语言体现'],
['中西方语言与文化的差异', '飞桨致力于让深度学习技术的创新与应用更简单']]
corpus_list = [["中西方语言与文化的差异", "中西方文化差异以及语言体现中西方文化,差异,语言体现"], ["中西方语言与文化的差异", "飞桨致力于让深度学习技术的创新与应用更简单"]]
res = predictor.predict(corpus_list, tokenizer)
print(res)

7
applications/neural_search/recall/in_batch_negative/deploy/python/rpc_client.py
View File

@@ -17,12 +17,9 @@ import numpy as np
from paddle_serving_server.pipeline import PipelineClient
client = PipelineClient()
client.connect(['127.0.0.1:8080'])
client.connect(["127.0.0.1:8080"])
list_data = [
"国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据",
"试论翻译过程中的文化差异与语言空缺翻译过程,文化差异,语言空缺,文化对比"
]
list_data = ["国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据", "试论翻译过程中的文化差异与语言空缺翻译过程,文化差异,语言空缺,文化对比"]
feed = {}
for i, item in enumerate(list_data):
feed[str(i)] = item

33
applications/neural_search/recall/in_batch_negative/deploy/python/web_service.py
View File

@@ -21,15 +21,10 @@ from paddle_serving_server.web_service import WebService, Op
_LOGGER = logging.getLogger()
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
result = []
for text in example:
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -37,44 +32,38 @@ def convert_example(example,
class ErnieOp(Op):
def init_op(self):
from paddlenlp.transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained(
"rocketqa-zh-base-query-encoder")
self.tokenizer = AutoTokenizer.from_pretrained("rocketqa-zh-base-query-encoder")
def preprocess(self, input_dicts, data_id, log_id):
from paddlenlp.data import Stack, Tuple, Pad
(_, input_dict), = input_dicts.items()
((_, input_dict),) = input_dicts.items()
print("input dict", input_dict)
batch_size = len(input_dict.keys())
examples = []
for i in range(batch_size):
input_ids, segment_ids = convert_example([input_dict[str(i)]],
self.tokenizer)
input_ids, segment_ids = convert_example([input_dict[str(i)]], self.tokenizer)
examples.append((input_ids, segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"
), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"), # segment
): fn(samples)
input_ids, segment_ids = batchify_fn(examples)
feed_dict = {}
feed_dict['input_ids'] = input_ids
feed_dict['token_type_ids'] = segment_ids
feed_dict["input_ids"] = input_ids
feed_dict["token_type_ids"] = segment_ids
return feed_dict, False, None, ""
def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
new_dict = {}
new_dict["output_embedding"] = str(
fetch_dict["output_embedding"].tolist())
new_dict["output_embedding"] = str(fetch_dict["output_embedding"].tolist())
return new_dict, None, ""
class ErnieService(WebService):
def get_pipeline_response(self, read_op):
ernie_op = ErnieOp(name="ernie", input_ops=[read_op])
return ernie_op

12
applications/neural_search/recall/in_batch_negative/evaluate.py
View File

@@ -56,14 +56,14 @@ def recall(rs, N=10):
if __name__ == "__main__":
text2similar = {}
with open(args.similar_text_pair, 'r', encoding='utf-8') as f:
with open(args.similar_text_pair, "r", encoding="utf-8") as f:
for line in f:
text, similar_text = line.rstrip().split("\t")
text2similar[text] = similar_text
rs = []
with open(args.recall_result_file, 'r', encoding='utf-8') as f:
with open(args.recall_result_file, "r", encoding="utf-8") as f:
relevance_labels = []
for index, line in enumerate(f):
@@ -82,11 +82,11 @@ if __name__ == "__main__":
for topN in recall_num:
R = round(100 * recall(rs, N=topN), 3)
recall_N.append(str(R))
result = open('result.tsv', 'a')
result = open("result.tsv", "a")
res = []
timestamp = time.strftime('%Y%m%d-%H%M%S', time.localtime())
timestamp = time.strftime("%Y%m%d-%H%M%S", time.localtime())
res.append(timestamp)
for key, val in zip(recall_num, recall_N):
print('recall@{}={}'.format(key, val))
print("recall@{}={}".format(key, val))
res.append(str(val))
result.write('\t'.join(res) + '\n')
result.write("\t".join(res) + "\n")

15
applications/neural_search/recall/in_batch_negative/export_model.py
View File

@@ -38,26 +38,23 @@ if __name__ == "__main__":
output_emb_size = 256
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
model = SemanticIndexBaseStatic(pretrained_model,
output_emb_size=output_emb_size)
model = SemanticIndexBaseStatic(pretrained_model, output_emb_size=output_emb_size)
if args.params_path and os.path.isfile(args.params_path):
state_dict = paddle.load(args.params_path)
model.set_dict(state_dict)
print("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
model.eval()
# Convert to static graph with specific input description
model = paddle.jit.to_static(
model,
input_spec=[
paddle.static.InputSpec(shape=[None, None],
dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None],
dtype="int64") # segment_ids
])
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # segment_ids
],
)
# Save in static graph model.
save_path = os.path.join(args.output_path, "inference")
paddle.jit.save(model, save_path)

4
applications/neural_search/recall/in_batch_negative/export_to_serving.py
View File

@@ -14,6 +14,7 @@
import argparse
import paddle_serving_client.io as serving_io
# yapf: disable
parser = argparse.ArgumentParser()
parser.add_argument("--dirname", type=str, required=True,
@@ -44,4 +45,5 @@ if __name__ == "__main__":
params_filename=args.params_filename,
show_proto=args.show_proto,
feed_alias_names=args.feed_alias_names,
fetch_alias_names=args.fetch_alias_names)
fetch_alias_names=args.fetch_alias_names,
)

27
applications/neural_search/recall/in_batch_negative/inference.py
View File

@@ -34,19 +34,17 @@ from ann_util import build_index
from tqdm import tqdm
if __name__ == "__main__":
device = 'gpu'
device = "gpu"
max_seq_length = 64
output_emb_size = 256
batch_size = 1
params_path = 'checkpoints/inbatch/model_40/model_state.pdparams'
id2corpus = {0: '国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
params_path = "checkpoints/inbatch/model_40/model_state.pdparams"
id2corpus = {0: "国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据"}
model_name_or_path = "rocketqa-zh-base-query-encoder"
paddle.set_device(device)
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=max_seq_length)
trans_func = partial(convert_example, tokenizer=tokenizer, max_seq_length=max_seq_length)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id), # text_input
@@ -55,8 +53,7 @@ if __name__ == "__main__":
pretrained_model = AutoModel.from_pretrained(model_name_or_path)
model = SemanticIndexBaseStatic(pretrained_model,
output_emb_size=output_emb_size)
model = SemanticIndexBaseStatic(pretrained_model, output_emb_size=output_emb_size)
# Load pretrained semantic model
if params_path and os.path.isfile(params_path):
@@ -64,18 +61,15 @@ if __name__ == "__main__":
model.set_dict(state_dict)
print("Loaded parameters from %s" % params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
# conver_example function's input must be dict
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
corpus_ds = MapDataset(corpus_list)
corpus_data_loader = create_dataloader(corpus_ds,
mode='predict',
batch_size=batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
corpus_data_loader = create_dataloader(
corpus_ds, mode="predict", batch_size=batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
all_embeddings = []
model.eval()
@@ -83,8 +77,7 @@ if __name__ == "__main__":
for batch_data in corpus_data_loader:
input_ids, token_type_ids = batch_data
text_embeddings = model.get_pooled_embedding(
input_ids, token_type_ids)
text_embeddings = model.get_pooled_embedding(input_ids, token_type_ids)
all_embeddings.append(text_embeddings)
text_embedding = all_embeddings[0]

45
applications/neural_search/recall/in_batch_negative/predict.py
View File

@@ -72,7 +72,8 @@ def predict(model, data_loader):
query_input_ids=query_input_ids,
title_input_ids=title_input_ids,
query_token_type_ids=query_token_type_ids,
title_token_type_ids=title_token_type_ids).numpy()
title_token_type_ids=title_token_type_ids,
).numpy()
cosine_sims.append(batch_cosine_sim)
@@ -86,46 +87,38 @@ if __name__ == "__main__":
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
pad_to_max_seq_len=args.pad_to_max_seq_len)
trans_func = partial(
convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
pad_to_max_seq_len=args.pad_to_max_seq_len,
)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'
), # query_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int64'
), # query_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'
), # title_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int64'
), # tilte_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # query_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # query_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # title_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # tilte_segment
): [data for data in fn(samples)]
valid_ds = load_dataset(read_text_pair,
data_path=args.text_pair_file,
lazy=False)
valid_ds = load_dataset(read_text_pair, data_path=args.text_pair_file, lazy=False)
valid_data_loader = create_dataloader(valid_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
valid_data_loader = create_dataloader(
valid_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
model = SemanticIndexBase(pretrained_model,
output_emb_size=args.output_emb_size)
model = SemanticIndexBase(pretrained_model, output_emb_size=args.output_emb_size)
if args.params_path and os.path.isfile(args.params_path):
state_dict = paddle.load(args.params_path)
model.set_dict(state_dict)
print("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
cosin_sim = predict(model, valid_data_loader)
for idx, cosine in enumerate(cosin_sim):
print('{}'.format(cosine))
print("{}".format(cosine))

48
applications/neural_search/recall/in_batch_negative/recall.py
View File

@@ -76,21 +76,16 @@ if __name__ == "__main__":
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length)
trans_func = partial(convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # text_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # text_segment
): [data for data in fn(samples)]
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
model = SemanticIndexBase(pretrained_model,
output_emb_size=args.output_emb_size)
model = SemanticIndexBase(pretrained_model, output_emb_size=args.output_emb_size)
model = paddle.DataParallel(model)
# Load pretrained semantic model
@@ -99,8 +94,7 @@ if __name__ == "__main__":
model.set_dict(state_dict)
logger.info("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
id2corpus = gen_id2corpus(args.corpus_file)
@@ -108,11 +102,9 @@ if __name__ == "__main__":
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
corpus_ds = MapDataset(corpus_list)
corpus_data_loader = create_dataloader(corpus_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
corpus_data_loader = create_dataloader(
corpus_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
# Need better way to get inner model of DataParallel
inner_model = model._layers
@@ -123,29 +115,27 @@ if __name__ == "__main__":
query_ds = MapDataset(text_list)
query_data_loader = create_dataloader(query_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
query_data_loader = create_dataloader(
query_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
query_embedding = inner_model.get_semantic_embedding(query_data_loader)
if not os.path.exists(args.recall_result_dir):
os.mkdir(args.recall_result_dir)
recall_result_file = os.path.join(args.recall_result_dir,
args.recall_result_file)
with open(recall_result_file, 'w', encoding='utf-8') as f:
recall_result_file = os.path.join(args.recall_result_dir, args.recall_result_file)
with open(recall_result_file, "w", encoding="utf-8") as f:
for batch_index, batch_query_embedding in enumerate(query_embedding):
recalled_idx, cosine_sims = final_index.knn_query(
batch_query_embedding.numpy(), args.recall_num)
recalled_idx, cosine_sims = final_index.knn_query(batch_query_embedding.numpy(), args.recall_num)
batch_size = len(cosine_sims)
for row_index in range(batch_size):
text_index = args.batch_size * batch_index + row_index
for idx, doc_idx in enumerate(recalled_idx[row_index]):
f.write("{}\t{}\t{}\n".format(
text_list[text_index]["text"], id2corpus[doc_idx],
1.0 - cosine_sims[row_index][idx]))
f.write(
"{}\t{}\t{}\n".format(
text_list[text_index]["text"], id2corpus[doc_idx], 1.0 - cosine_sims[row_index][idx]
)
)

223
applications/neural_search/recall/in_batch_negative/train_batch_neg.py
View File

@@ -118,30 +118,30 @@ def recall(rs, N=10):
@paddle.no_grad()
def evaluate(model, corpus_data_loader, query_data_loader, recall_result_file,
text_list, id2corpus):
def evaluate(model, corpus_data_loader, query_data_loader, recall_result_file, text_list, id2corpus):
# Load pretrained semantic model
inner_model = model._layers
final_index = build_index(args, corpus_data_loader, inner_model)
query_embedding = inner_model.get_semantic_embedding(query_data_loader)
with open(recall_result_file, 'w', encoding='utf-8') as f:
with open(recall_result_file, "w", encoding="utf-8") as f:
for batch_index, batch_query_embedding in enumerate(query_embedding):
recalled_idx, cosine_sims = final_index.knn_query(
batch_query_embedding.numpy(), args.recall_num)
recalled_idx, cosine_sims = final_index.knn_query(batch_query_embedding.numpy(), args.recall_num)
batch_size = len(cosine_sims)
for row_index in range(batch_size):
text_index = args.batch_size * batch_index + row_index
for idx, doc_idx in enumerate(recalled_idx[row_index]):
f.write("{}\t{}\t{}\n".format(
text_list[text_index]["text"], id2corpus[doc_idx],
1.0 - cosine_sims[row_index][idx]))
f.write(
"{}\t{}\t{}\n".format(
text_list[text_index]["text"], id2corpus[doc_idx], 1.0 - cosine_sims[row_index][idx]
)
)
text2similar = {}
with open(args.similar_text_pair_file, 'r', encoding='utf-8') as f:
with open(args.similar_text_pair_file, "r", encoding="utf-8") as f:
for line in f:
text, similar_text = line.rstrip().split("\t")
text2similar[text] = similar_text
rs = []
with open(recall_result_file, 'r', encoding='utf-8') as f:
with open(recall_result_file, "r", encoding="utf-8") as f:
relevance_labels = []
for index, line in enumerate(f):
if index % args.recall_num == 0 and index != 0:
@@ -160,22 +160,31 @@ def evaluate(model, corpus_data_loader, query_data_loader, recall_result_file,
for topN in recall_num:
R = round(100 * recall(rs, N=topN), 3)
recall_N.append(str(R))
evaluate_result_file = os.path.join(args.recall_result_dir,
args.evaluate_result)
result = open(evaluate_result_file, 'a')
evaluate_result_file = os.path.join(args.recall_result_dir, args.evaluate_result)
result = open(evaluate_result_file, "a")
res = []
timestamp = time.strftime('%Y%m%d-%H%M%S', time.localtime())
timestamp = time.strftime("%Y%m%d-%H%M%S", time.localtime())
res.append(timestamp)
for key, val in zip(recall_num, recall_N):
print('recall@{}={}'.format(key, val))
print("recall@{}={}".format(key, val))
res.append(str(val))
result.write('\t'.join(res) + '\n')
result.write("\t".join(res) + "\n")
return float(recall_N[1])
def train(train_data_loader, model, optimizer, lr_scheduler, rank,
corpus_data_loader, query_data_loader, recall_result_file, text_list,
id2corpus, tokenizer):
def train(
train_data_loader,
model,
optimizer,
lr_scheduler,
rank,
corpus_data_loader,
query_data_loader,
recall_result_file,
text_list,
id2corpus,
tokenizer,
):
global_step = 0
best_recall = 0.0
tic_train = time.time()
@@ -183,17 +192,19 @@ def train(train_data_loader, model, optimizer, lr_scheduler, rank,
for step, batch in enumerate(train_data_loader, start=1):
query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids = batch
loss = model(query_input_ids=query_input_ids,
title_input_ids=title_input_ids,
query_token_type_ids=query_token_type_ids,
title_token_type_ids=title_token_type_ids)
loss = model(
query_input_ids=query_input_ids,
title_input_ids=title_input_ids,
query_token_type_ids=query_token_type_ids,
title_token_type_ids=title_token_type_ids,
)
global_step += 1
if global_step % args.log_steps == 0 and rank == 0:
print(
"global step %d, epoch: %d, batch: %d, loss: %.5f, speed: %.2f step/s"
% (global_step, epoch, step, loss, args.log_steps /
(time.time() - tic_train)))
% (global_step, epoch, step, loss, args.log_steps / (time.time() - tic_train))
)
tic_train = time.time()
loss.backward()
optimizer.step()
@@ -201,36 +212,29 @@ def train(train_data_loader, model, optimizer, lr_scheduler, rank,
optimizer.clear_grad()
if not args.evaluate:
if global_step % args.save_steps == 0 and rank == 0:
save_dir = os.path.join(args.save_dir,
"model_%d" % global_step)
save_dir = os.path.join(args.save_dir, "model_%d" % global_step)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_param_path = os.path.join(save_dir,
'model_state.pdparams')
save_param_path = os.path.join(save_dir, "model_state.pdparams")
paddle.save(model.state_dict(), save_param_path)
tokenizer.save_pretrained(save_dir)
if args.evaluate and rank == 0:
print("evaluating")
recall_5 = evaluate(model, corpus_data_loader, query_data_loader,
recall_result_file, text_list, id2corpus)
recall_5 = evaluate(model, corpus_data_loader, query_data_loader, recall_result_file, text_list, id2corpus)
if recall_5 > best_recall:
best_recall = recall_5
save_dir = os.path.join(args.save_dir, "model_best")
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_param_path = os.path.join(save_dir, 'model_state.pdparams')
save_param_path = os.path.join(save_dir, "model_state.pdparams")
paddle.save(model.state_dict(), save_param_path)
tokenizer.save_pretrained(save_dir)
with open(os.path.join(save_dir, "train_result.txt"),
'a',
encoding='utf-8') as fp:
fp.write('epoch=%d, global_step: %d, recall: %s\n' %
(epoch, global_step, recall_5))
with open(os.path.join(save_dir, "train_result.txt"), "a", encoding="utf-8") as fp:
fp.write("epoch=%d, global_step: %d, recall: %s\n" % (epoch, global_step, recall_5))
def gradient_cache_train(train_data_loader, model, optimizer, lr_scheduler,
rank, tokenizer):
def gradient_cache_train(train_data_loader, model, optimizer, lr_scheduler, rank, tokenizer):
if args.use_amp:
scaler = paddle.amp.GradScaler(init_loss_scaling=args.amp_loss_scale)
@@ -264,20 +268,23 @@ def gradient_cache_train(train_data_loader, model, optimizer, lr_scheduler,
for sub_batch in sub_batchs:
all_reps = []
all_labels = []
sub_query_input_ids, sub_query_token_type_ids, sub_title_input_ids, sub_title_token_type_ids = sub_batch
with paddle.amp.auto_cast(
args.use_amp,
custom_white_list=["layer_norm", "softmax", "gelu"]):
(
sub_query_input_ids,
sub_query_token_type_ids,
sub_title_input_ids,
sub_title_token_type_ids,
) = sub_batch
with paddle.amp.auto_cast(args.use_amp, custom_white_list=["layer_norm", "softmax", "gelu"]):
with paddle.no_grad():
sub_CUDA_rnd_state = paddle.framework.random.get_cuda_rng_state(
)
sub_CUDA_rnd_state = paddle.framework.random.get_cuda_rng_state()
all_CUDA_rnd_state.append(sub_CUDA_rnd_state)
sub_cosine_sim, sub_label, query_embedding, title_embedding = model(
query_input_ids=sub_query_input_ids,
title_input_ids=sub_title_input_ids,
query_token_type_ids=sub_query_token_type_ids,
title_token_type_ids=sub_title_token_type_ids)
title_token_type_ids=sub_title_token_type_ids,
)
all_reps.append(sub_cosine_sim)
all_labels.append(sub_label)
all_title.append(title_embedding)
@@ -300,18 +307,22 @@ def gradient_cache_train(train_data_loader, model, optimizer, lr_scheduler,
# Store gradients
all_grads.append(model_reps.grad)
for sub_batch, CUDA_state, grad in zip(sub_batchs,
all_CUDA_rnd_state,
all_grads):
for sub_batch, CUDA_state, grad in zip(sub_batchs, all_CUDA_rnd_state, all_grads):
sub_query_input_ids, sub_query_token_type_ids, sub_title_input_ids, sub_title_token_type_ids = sub_batch
(
sub_query_input_ids,
sub_query_token_type_ids,
sub_title_input_ids,
sub_title_token_type_ids,
) = sub_batch
paddle.framework.random.set_cuda_rng_state(CUDA_state)
# Recompute the forward propogation
sub_cosine_sim, sub_label, query_embedding, title_embedding = model(
query_input_ids=sub_query_input_ids,
title_input_ids=sub_title_input_ids,
query_token_type_ids=sub_query_token_type_ids,
title_token_type_ids=sub_title_token_type_ids)
title_token_type_ids=sub_title_token_type_ids,
)
# Chain rule
surrogate = paddle.dot(sub_cosine_sim, grad)
# Backward propogation
@@ -330,8 +341,8 @@ def gradient_cache_train(train_data_loader, model, optimizer, lr_scheduler,
if global_step % args.log_steps == 0 and rank == 0:
print(
"global step %d, epoch: %d, batch: %d, loss: %.5f, speed: %.2f step/s"
% (global_step, epoch, step, loss, args.log_steps /
(time.time() - tic_train)))
% (global_step, epoch, step, loss, args.log_steps / (time.time() - tic_train))
)
tic_train = time.time()
lr_scheduler.step()
@@ -341,7 +352,7 @@ def gradient_cache_train(train_data_loader, model, optimizer, lr_scheduler,
save_dir = os.path.join(args.save_dir, "model_%d" % global_step)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_param_path = os.path.join(save_dir, 'model_state.pdparams')
save_param_path = os.path.join(save_dir, "model_state.pdparams")
paddle.save(model.state_dict(), save_param_path)
tokenizer.save_pretrained(save_dir)
@@ -354,45 +365,32 @@ def do_train():
set_seed(args.seed)
train_ds = load_dataset(read_text_pair,
data_path=args.train_set_file,
lazy=False)
train_ds = load_dataset(read_text_pair, data_path=args.train_set_file, lazy=False)
pretrained_model = AutoModel.from_pretrained(
args.model_name_or_path, enable_recompute=args.use_recompute)
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path, enable_recompute=args.use_recompute)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length)
trans_func = partial(convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'
), # query_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int64'
), # query_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'
), # title_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int64'
), # tilte_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # query_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # query_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # title_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # tilte_segment
): [data for data in fn(samples)]
train_data_loader = create_dataloader(train_ds,
mode='train',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
if (args.use_gradient_cache):
model = SemanticIndexCacheNeg(pretrained_model,
margin=args.margin,
scale=args.scale,
output_emb_size=args.output_emb_size)
train_data_loader = create_dataloader(
train_ds, mode="train", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
if args.use_gradient_cache:
model = SemanticIndexCacheNeg(
pretrained_model, margin=args.margin, scale=args.scale, output_emb_size=args.output_emb_size
)
else:
model = SemanticIndexBatchNeg(pretrained_model,
margin=args.margin,
scale=args.scale,
output_emb_size=args.output_emb_size)
model = SemanticIndexBatchNeg(
pretrained_model, margin=args.margin, scale=args.scale, output_emb_size=args.output_emb_size
)
if args.init_from_ckpt and os.path.isfile(args.init_from_ckpt):
state_dict = paddle.load(args.init_from_ckpt)
@@ -402,10 +400,8 @@ def do_train():
model = paddle.DataParallel(model)
batchify_fn_dev = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'
), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int64'
), # text_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # text_segment
): [data for data in fn(samples)]
id2corpus = gen_id2corpus(args.corpus_file)
@@ -414,52 +410,53 @@ def do_train():
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
corpus_ds = MapDataset(corpus_list)
corpus_data_loader = create_dataloader(corpus_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn_dev,
trans_fn=trans_func)
corpus_data_loader = create_dataloader(
corpus_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn_dev, trans_fn=trans_func
)
text_list, text2similar_text = gen_text_file(args.similar_text_pair_file)
query_ds = MapDataset(text_list)
query_data_loader = create_dataloader(query_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn_dev,
trans_fn=trans_func)
query_data_loader = create_dataloader(
query_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn_dev, trans_fn=trans_func
)
if not os.path.exists(args.recall_result_dir):
os.mkdir(args.recall_result_dir)
recall_result_file = os.path.join(args.recall_result_dir,
args.recall_result_file)
recall_result_file = os.path.join(args.recall_result_dir, args.recall_result_file)
num_training_steps = len(train_data_loader) * args.epochs
lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps,
args.warmup_proportion)
lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps, args.warmup_proportion)
# Generate parameter names needed to perform weight decay.
# All bias and LayerNorm parameters are excluded.
decay_params = [
p.name for n, p in model.named_parameters()
if not any(nd in n for nd in ["bias", "norm"])
]
decay_params = [p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])]
optimizer = paddle.optimizer.AdamW(
learning_rate=lr_scheduler,
parameters=model.parameters(),
weight_decay=args.weight_decay,
apply_decay_param_fun=lambda x: x in decay_params)
apply_decay_param_fun=lambda x: x in decay_params,
)
if (args.use_gradient_cache):
gradient_cache_train(train_data_loader, model, optimizer, lr_scheduler,
rank, tokenizer)
if args.use_gradient_cache:
gradient_cache_train(train_data_loader, model, optimizer, lr_scheduler, rank, tokenizer)
else:
train(train_data_loader, model, optimizer, lr_scheduler, rank,
corpus_data_loader, query_data_loader, recall_result_file,
text_list, id2corpus, tokenizer)
train(
train_data_loader,
model,
optimizer,
lr_scheduler,
rank,
corpus_data_loader,
query_data_loader,
recall_result_file,
text_list,
id2corpus,
tokenizer,
)
if __name__ == "__main__":

143
applications/neural_search/recall/milvus/base_model.py
View File

@@ -23,7 +23,6 @@ import paddle.nn.functional as F
class SemanticIndexBase(nn.Layer):
def __init__(self, pretrained_model, dropout=None, output_emb_size=None):
super().__init__()
self.ptm = pretrained_model
@@ -35,23 +34,17 @@ class SemanticIndexBase(nn.Layer):
self.output_emb_size = output_emb_size
if output_emb_size > 0:
weight_attr = paddle.ParamAttr(
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768,
output_emb_size,
weight_attr=weight_attr)
weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768, output_emb_size, weight_attr=weight_attr)
@paddle.jit.to_static(input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype='int64'),
paddle.static.InputSpec(shape=[None, None], dtype='int64')
])
def get_pooled_embedding(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
attention_mask)
@paddle.jit.to_static(
input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
]
)
def get_pooled_embedding(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
if self.output_emb_size > 0:
cls_embedding = self.emb_reduce_linear(cls_embedding)
@@ -68,33 +61,31 @@ class SemanticIndexBase(nn.Layer):
input_ids = paddle.to_tensor(input_ids)
token_type_ids = paddle.to_tensor(token_type_ids)
text_embeddings = self.get_pooled_embedding(
input_ids, token_type_ids=token_type_ids)
text_embeddings = self.get_pooled_embedding(input_ids, token_type_ids=token_type_ids)
yield text_embeddings
def cosine_sim(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None):
def cosine_sim(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask
)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
axis=-1)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding, axis=-1)
return cosine_sim
@abc.abstractmethod
@@ -103,7 +94,6 @@ class SemanticIndexBase(nn.Layer):
class SemanticIndexBaseStatic(nn.Layer):
def __init__(self, pretrained_model, dropout=None, output_emb_size=None):
super().__init__()
self.ptm = pretrained_model
@@ -115,23 +105,17 @@ class SemanticIndexBaseStatic(nn.Layer):
self.output_emb_size = output_emb_size
if output_emb_size > 0:
weight_attr = paddle.ParamAttr(
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768,
output_emb_size,
weight_attr=weight_attr)
weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768, output_emb_size, weight_attr=weight_attr)
@paddle.jit.to_static(input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype='int64'),
paddle.static.InputSpec(shape=[None, None], dtype='int64')
])
def get_pooled_embedding(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
attention_mask)
@paddle.jit.to_static(
input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
]
)
def get_pooled_embedding(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
if self.output_emb_size > 0:
cls_embedding = self.emb_reduce_linear(cls_embedding)
@@ -148,42 +132,35 @@ class SemanticIndexBaseStatic(nn.Layer):
input_ids = paddle.to_tensor(input_ids)
token_type_ids = paddle.to_tensor(token_type_ids)
text_embeddings = self.get_pooled_embedding(
input_ids, token_type_ids=token_type_ids)
text_embeddings = self.get_pooled_embedding(input_ids, token_type_ids=token_type_ids)
yield text_embeddings
def cosine_sim(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None):
def cosine_sim(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask
)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
axis=-1)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding, axis=-1)
return cosine_sim
def forward(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
attention_mask)
def forward(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
if self.output_emb_size > 0:
cls_embedding = self.emb_reduce_linear(cls_embedding)
cls_embedding = self.dropout(cls_embedding)

18
applications/neural_search/recall/milvus/config.py
View File

@@ -12,25 +12,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
MILVUS_HOST = '10.21.226.175'
MILVUS_HOST = "10.21.226.175"
MILVUS_PORT = 8530
data_dim = 256
top_k = 100
collection_name = 'literature_search'
partition_tag = 'partition_2'
embedding_name = 'embeddings'
collection_name = "literature_search"
partition_tag = "partition_2"
embedding_name = "embeddings"
index_config = {
"index_type": "IVF_FLAT",
"metric_type": "L2",
"params": {
"nlist": 1000
},
"params": {"nlist": 1000},
}
search_params = {
"metric_type": "L2",
"params": {
"nprobe": top_k
},
}
"params": {"nprobe": top_k},
}

75
applications/neural_search/recall/milvus/data.py
View File

@@ -19,45 +19,31 @@ import paddle
from paddlenlp.utils.log import logger
def create_dataloader(dataset,
mode='train',
batch_size=1,
batchify_fn=None,
trans_fn=None):
def create_dataloader(dataset, mode="train", batch_size=1, batchify_fn=None, trans_fn=None):
if trans_fn:
dataset = dataset.map(trans_fn)
shuffle = True if mode == 'train' else False
if mode == 'train':
batch_sampler = paddle.io.DistributedBatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
shuffle = True if mode == "train" else False
if mode == "train":
batch_sampler = paddle.io.DistributedBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
else:
batch_sampler = paddle.io.BatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
batch_sampler = paddle.io.BatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
return paddle.io.DataLoader(dataset=dataset,
batch_sampler=batch_sampler,
collate_fn=batchify_fn,
return_list=True)
return paddle.io.DataLoader(dataset=dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn, return_list=True)
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -68,9 +54,7 @@ def convert_example(example,
result = []
for key, text in example.items():
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -79,32 +63,28 @@ def convert_example(example,
def read_text_pair(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if len(data) != 2:
continue
yield {'text_a': data[0], 'text_b': data[1]}
yield {"text_a": data[0], "text_b": data[1]}
def read_text_triplet(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if len(data) != 3:
continue
yield {
'text': data[0],
'pos_sample': data[1],
'neg_sample': data[2]
}
yield {"text": data[0], "pos_sample": data[1], "neg_sample": data[2]}
# ANN - active learning ------------------------------------------------------
def get_latest_checkpoint(args):
"""
Return: (latest_checkpint_path, global_step)
Return: (latest_checkpint_path, global_step)
"""
if not os.path.exists(args.save_dir):
return args.init_from_ckpt, 0
@@ -115,14 +95,12 @@ def get_latest_checkpoint(args):
chk_path = os.path.join(args.save_dir, checkpoint)
scheduler_path = os.path.join(chk_path, "model_state.pdparams")
succeed_flag_file = os.path.join(chk_path, "succeed_flag_file")
return os.path.exists(scheduler_path) and os.path.exists(
succeed_flag_file)
return os.path.exists(scheduler_path) and os.path.exists(succeed_flag_file)
trained_steps = [int(s) for s in subdirectories if valid_checkpoint(s)]
if len(trained_steps) > 0:
return os.path.join(args.save_dir, str(max(trained_steps)),
"model_state.pdparams"), max(trained_steps)
return os.path.join(args.save_dir, str(max(trained_steps)), "model_state.pdparams"), max(trained_steps)
return args.init_from_ckpt, 0
@@ -137,19 +115,14 @@ def get_latest_ann_data(ann_data_dir):
def valid_checkpoint(step):
ann_data_file = os.path.join(ann_data_dir, step, "new_ann_data")
# succed_flag_file is an empty file that indicates ann data has been generated
succeed_flag_file = os.path.join(ann_data_dir, step,
"succeed_flag_file")
return os.path.exists(succeed_flag_file) and os.path.exists(
ann_data_file)
succeed_flag_file = os.path.join(ann_data_dir, step, "succeed_flag_file")
return os.path.exists(succeed_flag_file) and os.path.exists(ann_data_file)
ann_data_steps = [int(s) for s in subdirectories if valid_checkpoint(s)]
if len(ann_data_steps) > 0:
latest_ann_data_file = os.path.join(ann_data_dir,
str(max(ann_data_steps)),
"new_ann_data")
logger.info(
"Using lateset ann_data_file:{}".format(latest_ann_data_file))
latest_ann_data_file = os.path.join(ann_data_dir, str(max(ann_data_steps)), "new_ann_data")
logger.info("Using lateset ann_data_file:{}".format(latest_ann_data_file))
return latest_ann_data_file, max(ann_data_steps)
logger.info("no new ann_data, return (None, -1)")
@@ -158,7 +131,7 @@ def get_latest_ann_data(ann_data_dir):
def gen_id2corpus(corpus_file):
id2corpus = {}
with open(corpus_file, 'r', encoding='utf-8') as f:
with open(corpus_file, "r", encoding="utf-8") as f:
for idx, line in enumerate(f):
id2corpus[idx] = line.rstrip()
return id2corpus
@@ -167,7 +140,7 @@ def gen_id2corpus(corpus_file):
def gen_text_file(similar_text_pair_file):
text2similar_text = {}
texts = []
with open(similar_text_pair_file, 'r', encoding='utf-8') as f:
with open(similar_text_pair_file, "r", encoding="utf-8") as f:
for line in f:
splited_line = line.rstrip().split("\t")
if len(splited_line) != 2:

69
applications/neural_search/recall/milvus/feature_extract.py
View File

@@ -25,7 +25,7 @@ from paddlenlp.data import Stack, Tuple, Pad
from paddlenlp.datasets import load_dataset
from paddlenlp.utils.log import logger
sys.path.append('.')
sys.path.append(".")
from data import convert_example
@@ -57,16 +57,17 @@ args = parser.parse_args()
class Predictor(object):
def __init__(self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False):
def __init__(
self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False,
):
self.max_seq_length = max_seq_length
self.batch_size = batch_size
@@ -85,14 +86,14 @@ class Predictor(object):
precision_map = {
"fp16": inference.PrecisionType.Half,
"fp32": inference.PrecisionType.Float32,
"int8": inference.PrecisionType.Int8
"int8": inference.PrecisionType.Int8,
}
precision_mode = precision_map[precision]
if args.use_tensorrt:
config.enable_tensorrt_engine(max_batch_size=batch_size,
min_subgraph_size=30,
precision_mode=precision_mode)
config.enable_tensorrt_engine(
max_batch_size=batch_size, min_subgraph_size=30, precision_mode=precision_mode
)
elif device == "cpu":
# set CPU configs accordingly,
# such as enable_mkldnn, set_cpu_math_library_num_threads
@@ -108,12 +109,8 @@ class Predictor(object):
config.switch_use_feed_fetch_ops(False)
self.predictor = paddle.inference.create_predictor(config)
self.input_handles = [
self.predictor.get_input_handle(name)
for name in self.predictor.get_input_names()
]
self.output_handle = self.predictor.get_output_handle(
self.predictor.get_output_names()[0])
self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()]
self.output_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[0])
def predict(self, data, tokenizer):
"""
@@ -121,7 +118,7 @@ class Predictor(object):
Args:
data (obj:`List(str)`): The batch data whose each element is a raw text.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
Returns:
@@ -129,20 +126,17 @@ class Predictor(object):
"""
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # segment
): fn(samples)
all_embeddings = []
examples = []
for idx, text in enumerate(tqdm(data)):
input_ids, segment_ids = convert_example(
text,
tokenizer,
max_seq_length=self.max_seq_length,
pad_to_max_seq_len=True)
text, tokenizer, max_seq_length=self.max_seq_length, pad_to_max_seq_len=True
)
examples.append((input_ids, segment_ids))
if (len(examples) > self.batch_size):
if len(examples) > self.batch_size:
input_ids, segment_ids = batchify_fn(examples)
self.input_handles[0].copy_from_cpu(input_ids)
self.input_handles[1].copy_from_cpu(segment_ids)
@@ -150,7 +144,7 @@ class Predictor(object):
logits = self.output_handle.copy_to_cpu()
all_embeddings.append(logits)
examples = []
if (len(examples) > 0):
if len(examples) > 0:
input_ids, segment_ids = batchify_fn(examples)
self.input_handles[0].copy_from_cpu(input_ids)
self.input_handles[1].copy_from_cpu(segment_ids)
@@ -158,7 +152,7 @@ class Predictor(object):
logits = self.output_handle.copy_to_cpu()
all_embeddings.append(logits)
all_embeddings = np.concatenate(all_embeddings, axis=0)
np.save('corpus_embedding', all_embeddings)
np.save("corpus_embedding", all_embeddings)
def read_text(file_path):
@@ -170,9 +164,16 @@ def read_text(file_path):
if __name__ == "__main__":
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
args.batch_size, args.use_tensorrt, args.precision,
args.cpu_threads, args.enable_mkldnn)
predictor = Predictor(
args.model_dir,
args.device,
args.max_seq_length,
args.batch_size,
args.use_tensorrt,
args.precision,
args.cpu_threads,
args.enable_mkldnn,
)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
id2corpus = read_text(args.corpus_file)

45
applications/neural_search/recall/milvus/inference.py
View File

@@ -37,62 +37,55 @@ from config import embedding_name, collection_name, partition_tag
def search_in_milvus(text_embedding):
recall_client = RecallByMilvus()
result = recall_client.search(text_embedding.numpy(),
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=['pk', 'text'])
result = recall_client.search(
text_embedding.numpy(),
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=["pk", "text"],
)
for hits in result:
for hit in hits:
print(f"hit: {hit}, text field: {hit.entity.get('text')}")
if __name__ == "__main__":
device = 'gpu'
device = "gpu"
max_seq_length = 64
output_emb_size = 256
batch_size = 1
params_path = 'checkpoints/model_40/model_state.pdparams'
id2corpus = {0: '国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
params_path = "checkpoints/model_40/model_state.pdparams"
id2corpus = {0: "国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据"}
model_name_or_path = "rocketqa-zh-base-query-encoder"
paddle.set_device(device)
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=max_seq_length)
trans_func = partial(convert_example, tokenizer=tokenizer, max_seq_length=max_seq_length)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # text_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # text_segment
): [data for data in fn(samples)]
pretrained_model = AutoModel.from_pretrained(model_name_or_path)
model = SemanticIndexBaseStatic(pretrained_model,
output_emb_size=output_emb_size)
model = SemanticIndexBaseStatic(pretrained_model, output_emb_size=output_emb_size)
# Load pretrained semantic model
if params_path and os.path.isfile(params_path):
state_dict = paddle.load(params_path)
model.set_dict(state_dict)
print("Loaded parameters from %s" % params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
# conver_example function's input must be dict
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
corpus_ds = MapDataset(corpus_list)
corpus_data_loader = create_dataloader(corpus_ds,
mode='predict',
batch_size=batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
corpus_data_loader = create_dataloader(
corpus_ds, mode="predict", batch_size=batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
# Need better way to get inner model of DataParallel
all_embeddings = []
model.eval()
with paddle.no_grad():
for batch_data in corpus_data_loader:
input_ids, token_type_ids = batch_data
text_embeddings = model.get_pooled_embedding(
input_ids, token_type_ids)
text_embeddings = model.get_pooled_embedding(input_ids, token_type_ids)
all_embeddings.append(text_embeddings)
text_embedding = all_embeddings[0]
print(text_embedding.shape)

65
applications/neural_search/recall/milvus/milvus_ann_search.py
View File

@@ -22,30 +22,16 @@ from milvus_util import VecToMilvus, RecallByMilvus, text_max_len
from config import collection_name, partition_tag, embedding_name
parser = argparse.ArgumentParser()
parser.add_argument("--data_path",
default='milvus/milvus_data.csv',
type=str,
required=True,
help="The data for vector extraction.")
parser.add_argument("--embedding_path",
default='corpus_embedding.npy',
type=str,
required=True,
help="The vector path for data.")
parser.add_argument('--index',
default=0,
type=int,
help='index of the vector for search')
parser.add_argument('--insert',
action='store_true',
help='whether to insert data')
parser.add_argument('--search',
action='store_true',
help='whether to search data')
parser.add_argument('--batch_size',
default=100000,
type=int,
help='number of examples to insert each time')
parser.add_argument(
"--data_path", default="milvus/milvus_data.csv", type=str, required=True, help="The data for vector extraction."
)
parser.add_argument(
"--embedding_path", default="corpus_embedding.npy", type=str, required=True, help="The vector path for data."
)
parser.add_argument("--index", default=0, type=int, help="index of the vector for search")
parser.add_argument("--insert", action="store_true", help="whether to insert data")
parser.add_argument("--search", action="store_true", help="whether to search data")
parser.add_argument("--batch_size", default=100000, type=int, help="number of examples to insert each time")
args = parser.parse_args()
@@ -66,44 +52,41 @@ def milvus_data_insert(data_path, embedding_path, batch_size):
data_size = len(embedding_ids)
for i in tqdm(range(0, data_size, batch_size)):
cur_end = i + batch_size
if (cur_end > data_size):
if cur_end > data_size:
cur_end = data_size
batch_emb = embeddings[np.arange(i, cur_end)]
entities = [
[j for j in range(i, cur_end, 1)],
[corpus_list[j][:text_max_len - 1] for j in range(i, cur_end, 1)],
batch_emb # field embeddings, supports numpy.ndarray and list
[corpus_list[j][: text_max_len - 1] for j in range(i, cur_end, 1)],
batch_emb, # field embeddings, supports numpy.ndarray and list
]
client.insert(collection_name=collection_name,
entities=entities,
index_name=embedding_name,
partition_tag=partition_tag)
client.insert(
collection_name=collection_name, entities=entities, index_name=embedding_name, partition_tag=partition_tag
)
def milvus_data_recall(embedding_path, index):
embeddings = np.load(embedding_path)
embedding_ids = [i for i in range(embeddings.shape[0])]
recall_client = RecallByMilvus()
if (index > len(embedding_ids)):
print('Index should not be larger than embedding szie')
if index > len(embedding_ids):
print("Index should not be larger than embedding szie")
return
embeddings = embeddings[np.arange(index, index + 1)]
time_start = time.time()
result = recall_client.search(embeddings,
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=['pk', 'text'])
result = recall_client.search(
embeddings, embedding_name, collection_name, partition_names=[partition_tag], output_fields=["pk", "text"]
)
time_end = time.time()
sum_t = time_end - time_start
print('time cost', sum_t, 's')
print("time cost", sum_t, "s")
for hits in result:
for hit in hits:
print(f"hit: {hit}, text field: {hit.entity.get('text')}")
if __name__ == "__main__":
if (args.insert):
if args.insert:
milvus_data_insert(args.data_path, args.embedding_path, args.batch_size)
if (args.search):
if args.search:
milvus_data_recall(args.embedding_path, args.index)

86
applications/neural_search/recall/milvus/milvus_util.py
View File

@@ -29,19 +29,14 @@ from config import MILVUS_HOST, MILVUS_PORT, data_dim, index_config, top_k, sear
fmt = "\n=== {:30} ===\n"
text_max_len = 1000
fields = [
FieldSchema(name="pk",
dtype=DataType.INT64,
is_primary=True,
auto_id=False,
max_length=100),
FieldSchema(name="pk", dtype=DataType.INT64, is_primary=True, auto_id=False, max_length=100),
FieldSchema(name="text", dtype=DataType.VARCHAR, max_length=text_max_len),
FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=data_dim)
FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=data_dim),
]
schema = CollectionSchema(fields, "Neural Search Index")
class VecToMilvus():
class VecToMilvus:
def __init__(self):
print(fmt.format("start connecting to Milvus"))
connections.connect("default", host=MILVUS_HOST, port=MILVUS_PORT)
@@ -58,9 +53,7 @@ class VecToMilvus():
def creat_collection(self, collection_name):
try:
print(fmt.format("Create collection {}".format(collection_name)))
self.collection = Collection(collection_name,
schema,
consistency_level="Strong")
self.collection = Collection(collection_name, schema, consistency_level="Strong")
except Exception as e:
print("Milvus create collection error:", e)
@@ -89,9 +82,9 @@ class VecToMilvus():
def create_partition(self, partition_tag):
try:
self.collection.create_partition(partition_tag)
print('create partition {} successfully'.format(partition_tag))
print("create partition {} successfully".format(partition_tag))
except Exception as e:
print('Milvus create partition error: ', e)
print("Milvus create partition error: ", e)
def insert(self, entities, collection_name, index_name, partition_tag=None):
try:
@@ -100,20 +93,16 @@ class VecToMilvus():
self.create_index(index_name)
else:
self.collection = Collection(collection_name)
if (partition_tag
is not None) and (not self.has_partition(partition_tag)):
if (partition_tag is not None) and (not self.has_partition(partition_tag)):
self.create_partition(partition_tag)
self.collection.insert(entities, partition_name=partition_tag)
print(
f"Number of entities in Milvus: {self.collection.num_entities}"
) # check the num_entites
print(f"Number of entities in Milvus: {self.collection.num_entities}") # check the num_entites
except Exception as e:
print("Milvus insert error:", e)
class RecallByMilvus():
class RecallByMilvus:
def __init__(self):
print(fmt.format("start connecting to Milvus"))
connections.connect("default", host=MILVUS_HOST, port=MILVUS_PORT)
@@ -126,55 +115,50 @@ class RecallByMilvus():
except Exception as e:
print("Milvus create collection error:", e)
def search(self,
vectors,
embedding_name,
collection_name,
partition_names=[],
output_fields=[]):
def search(self, vectors, embedding_name, collection_name, partition_names=[], output_fields=[]):
try:
self.get_collection(collection_name)
result = self.collection.search(vectors,
embedding_name,
search_params,
limit=top_k,
partition_names=partition_names,
output_fields=output_fields)
result = self.collection.search(
vectors,
embedding_name,
search_params,
limit=top_k,
partition_names=partition_names,
output_fields=output_fields,
)
return result
except Exception as e:
print('Milvus recall error: ', e)
print("Milvus recall error: ", e)
if __name__ == '__main__':
if __name__ == "__main__":
print(fmt.format("Start inserting entities"))
rng = np.random.default_rng(seed=19530)
num_entities = 3000
entities = [
# provide the pk field because `auto_id` is set to False
[i for i in range(num_entities)],
['第{}个样本'.format(i)
for i in range(num_entities)], # field text, only supports list
rng.random(
(num_entities,
data_dim)), # field embeddings, supports numpy.ndarray and list
["第{}个样本".format(i) for i in range(num_entities)], # field text, only supports list
rng.random((num_entities, data_dim)), # field embeddings, supports numpy.ndarray and list
]
print(entities[-1].shape)
collection_name = 'test1'
partition_tag = 'partition_1'
embedding_name = 'embeddings'
collection_name = "test1"
partition_tag = "partition_1"
embedding_name = "embeddings"
client = VecToMilvus()
client.insert(collection_name=collection_name,
entities=entities,
index_name=embedding_name,
partition_tag=partition_tag)
client.insert(
collection_name=collection_name, entities=entities, index_name=embedding_name, partition_tag=partition_tag
)
print(fmt.format("Start searching entities"))
vectors_to_search = entities[-1][-2:]
recall_client = RecallByMilvus()
result = recall_client.search(vectors_to_search,
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=['pk', 'text'])
result = recall_client.search(
vectors_to_search,
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=["pk", "text"],
)
for hits in result:
for hit in hits:
print(f"hit: {hit}, random field: {hit.entity.get('text')}")

6
applications/neural_search/recall/simcse/ann_util.py
View File

@@ -21,7 +21,7 @@ from paddlenlp.utils.log import logger
def build_index(args, data_loader, model):
index = hnswlib.Index(space='ip', dim=args.output_emb_size)
index = hnswlib.Index(space="ip", dim=args.output_emb_size)
# Initializing index
# max_elements - the maximum number of elements (capacity). Will throw an exception if exceeded
@@ -32,9 +32,7 @@ def build_index(args, data_loader, model):
#
# M - is tightly connected with internal dimensionality of the data. Strongly affects memory consumption (~M)
# Higher M leads to higher accuracy/run_time at fixed ef/efConstruction
index.init_index(max_elements=args.hnsw_max_elements,
ef_construction=args.hnsw_ef,
M=args.hnsw_m)
index.init_index(max_elements=args.hnsw_max_elements, ef_construction=args.hnsw_ef, M=args.hnsw_m)
# Controlling the recall by setting ef:
# higher ef leads to better accuracy, but slower search

62
applications/neural_search/recall/simcse/data.py
View File

@@ -19,46 +19,32 @@ import paddle
from paddlenlp.utils.log import logger
def create_dataloader(dataset,
mode='train',
batch_size=1,
batchify_fn=None,
trans_fn=None):
def create_dataloader(dataset, mode="train", batch_size=1, batchify_fn=None, trans_fn=None):
if trans_fn:
dataset = dataset.map(trans_fn)
shuffle = True if mode == 'train' else False
if mode == 'train':
batch_sampler = paddle.io.DistributedBatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
shuffle = True if mode == "train" else False
if mode == "train":
batch_sampler = paddle.io.DistributedBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
else:
batch_sampler = paddle.io.BatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
batch_sampler = paddle.io.BatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
return paddle.io.DataLoader(dataset=dataset,
batch_sampler=batch_sampler,
collate_fn=batchify_fn,
return_list=True)
return paddle.io.DataLoader(dataset=dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn, return_list=True)
def convert_example_test(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example_test(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -69,9 +55,7 @@ def convert_example_test(example,
result = []
for key, text in example.items():
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -81,16 +65,16 @@ def convert_example_test(example,
def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -102,9 +86,9 @@ def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
result = []
for key, text in example.items():
if 'label' in key:
if "label" in key:
# do_evaluate
result += [example['label']]
result += [example["label"]]
else:
# do_train
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length)
@@ -117,7 +101,7 @@ def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
def gen_id2corpus(corpus_file):
id2corpus = {}
with open(corpus_file, 'r', encoding='utf-8') as f:
with open(corpus_file, "r", encoding="utf-8") as f:
for idx, line in enumerate(f):
id2corpus[idx] = line.rstrip()
return id2corpus
@@ -126,7 +110,7 @@ def gen_id2corpus(corpus_file):
def gen_text_file(similar_text_pair_file):
text2similar_text = {}
texts = []
with open(similar_text_pair_file, 'r', encoding='utf-8') as f:
with open(similar_text_pair_file, "r", encoding="utf-8") as f:
for line in f:
splited_line = line.rstrip().split("\t")
if len(splited_line) != 2:
@@ -144,22 +128,22 @@ def gen_text_file(similar_text_pair_file):
def read_simcse_text(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip()
yield {'text_a': data, 'text_b': data}
yield {"text_a": data, "text_b": data}
def read_text_pair(data_path, is_test=False):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if is_test == False:
if len(data) != 3:
continue
yield {'text_a': data[0], 'text_b': data[1], 'label': data[2]}
yield {"text_a": data[0], "text_b": data[1], "label": data[2]}
else:
if len(data) != 2:
continue
yield {'text_a': data[0], 'text_b': data[1]}
yield {"text_a": data[0], "text_b": data[1]}

89
applications/neural_search/recall/simcse/deploy/python/predict.py
View File

@@ -26,7 +26,7 @@ from paddlenlp.data import Stack, Tuple, Pad
from paddlenlp.datasets import load_dataset
from paddlenlp.utils.log import logger
sys.path.append('.')
sys.path.append(".")
# yapf: disable
parser = argparse.ArgumentParser()
@@ -62,16 +62,16 @@ args = parser.parse_args()
def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -92,16 +92,17 @@ def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
class Predictor(object):
def __init__(self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False):
def __init__(
self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False,
):
self.max_seq_length = max_seq_length
self.batch_size = batch_size
@@ -120,14 +121,14 @@ class Predictor(object):
precision_map = {
"fp16": inference.PrecisionType.Half,
"fp32": inference.PrecisionType.Float32,
"int8": inference.PrecisionType.Int8
"int8": inference.PrecisionType.Int8,
}
precision_mode = precision_map[precision]
if args.use_tensorrt:
config.enable_tensorrt_engine(max_batch_size=batch_size,
min_subgraph_size=30,
precision_mode=precision_mode)
config.enable_tensorrt_engine(
max_batch_size=batch_size, min_subgraph_size=30, precision_mode=precision_mode
)
elif device == "cpu":
# set CPU configs accordingly,
# such as enable_mkldnn, set_cpu_math_library_num_threads
@@ -143,15 +144,12 @@ class Predictor(object):
config.switch_use_feed_fetch_ops(False)
self.predictor = paddle.inference.create_predictor(config)
self.input_handles = [
self.predictor.get_input_handle(name)
for name in self.predictor.get_input_names()
]
self.output_handle = self.predictor.get_output_handle(
self.predictor.get_output_names()[0])
self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()]
self.output_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[0])
if args.benchmark:
import auto_log
pid = os.getpid()
self.autolog = auto_log.AutoLogger(
model_name=args.model_name_or_path,
@@ -163,11 +161,10 @@ class Predictor(object):
pids=pid,
process_name=None,
gpu_ids=0,
time_keys=[
'preprocess_time', 'inference_time', 'postprocess_time'
],
time_keys=["preprocess_time", "inference_time", "postprocess_time"],
warmup=0,
logger=logger)
logger=logger,
)
def extract_embedding(self, data, tokenizer):
"""
@@ -175,7 +172,7 @@ class Predictor(object):
Args:
data (obj:`List(str)`): The batch data whose each element is a raw text.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
Returns:
@@ -216,7 +213,7 @@ class Predictor(object):
Args:
data (obj:`List(str)`): The batch data whose each element is a raw text.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
Returns:
@@ -228,10 +225,8 @@ class Predictor(object):
examples = []
for idx, text in enumerate(data):
input_ids, segment_ids = convert_example({idx: text[0]}, tokenizer)
title_ids, title_segment_ids = convert_example({idx: text[1]},
tokenizer)
examples.append(
(input_ids, segment_ids, title_ids, title_segment_ids))
title_ids, title_segment_ids = convert_example({idx: text[1]}, tokenizer)
examples.append((input_ids, segment_ids, title_ids, title_segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id), # input
@@ -243,8 +238,7 @@ class Predictor(object):
if args.benchmark:
self.autolog.times.stamp()
query_ids, query_segment_ids, title_ids, title_segment_ids = batchify_fn(
examples)
query_ids, query_segment_ids, title_ids, title_segment_ids = batchify_fn(examples)
self.input_handles[0].copy_from_cpu(query_ids)
self.input_handles[1].copy_from_cpu(query_segment_ids)
self.predictor.run()
@@ -260,28 +254,31 @@ class Predictor(object):
if args.benchmark:
self.autolog.times.end(stamp=True)
result = [
float(1 - spatial.distance.cosine(arr1, arr2))
for arr1, arr2 in zip(query_logits, title_logits)
]
result = [float(1 - spatial.distance.cosine(arr1, arr2)) for arr1, arr2 in zip(query_logits, title_logits)]
return result
if __name__ == "__main__":
# Define predictor to do prediction.
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
args.batch_size, args.use_tensorrt, args.precision,
args.cpu_threads, args.enable_mkldnn)
predictor = Predictor(
args.model_dir,
args.device,
args.max_seq_length,
args.batch_size,
args.use_tensorrt,
args.precision,
args.cpu_threads,
args.enable_mkldnn,
)
# ErnieTinyTokenizer is special for ernie-tiny pretained model.
output_emb_size = 256
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
id2corpus = {0: '国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
id2corpus = {0: "国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据"}
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
res = predictor.extract_embedding(corpus_list, tokenizer)
print(res.shape)
print(res)
corpus_list = [['中西方语言与文化的差异', '中西方文化差异以及语言体现中西方文化,差异,语言体现'],
['中西方语言与文化的差异', '飞桨致力于让深度学习技术的创新与应用更简单']]
corpus_list = [["中西方语言与文化的差异", "中西方文化差异以及语言体现中西方文化,差异,语言体现"], ["中西方语言与文化的差异", "飞桨致力于让深度学习技术的创新与应用更简单"]]
res = predictor.predict(corpus_list, tokenizer)
print(res)

10
applications/neural_search/recall/simcse/evaluate.py
View File

@@ -52,14 +52,14 @@ def recall(rs, N=10):
if __name__ == "__main__":
text2similar = {}
with open(args.similar_text_pair, 'r', encoding='utf-8') as f:
with open(args.similar_text_pair, "r", encoding="utf-8") as f:
for line in f:
text, similar_text = line.rstrip().split("\t")
text2similar[text] = similar_text
rs = []
with open(args.recall_result_file, 'r', encoding='utf-8') as f:
with open(args.recall_result_file, "r", encoding="utf-8") as f:
relevance_labels = []
for index, line in enumerate(f):
@@ -75,12 +75,12 @@ if __name__ == "__main__":
recall_N = []
recall_num = [1, 5, 10, 20, 50]
result = open('result.tsv', 'a')
result = open("result.tsv", "a")
res = []
for topN in recall_num:
R = round(100 * recall(rs, N=topN), 3)
recall_N.append(str(R))
for key, val in zip(recall_num, recall_N):
print('recall@{}={}'.format(key, val))
print("recall@{}={}".format(key, val))
res.append(str(val))
result.write('\t'.join(res) + '\n')
result.write("\t".join(res) + "\n")

9
applications/neural_search/recall/simcse/export_model.py
View File

@@ -51,11 +51,10 @@ if __name__ == "__main__":
model = paddle.jit.to_static(
model,
input_spec=[
paddle.static.InputSpec(shape=[None, None],
dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None],
dtype="int64") # segment_ids
])
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # segment_ids
],
)
# Save in static graph model.
save_path = os.path.join(args.output_path, "inference")
paddle.jit.save(model, save_path)

36
applications/neural_search/recall/simcse/inference.py
View File

@@ -35,16 +35,16 @@ from tqdm import tqdm
def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -65,25 +65,21 @@ def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
if __name__ == "__main__":
device = 'gpu'
device = "gpu"
max_seq_length = 64
output_emb_size = 256
batch_size = 1
params_path = 'checkpoints/model_20000/model_state.pdparams'
id2corpus = {0: '国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
params_path = "checkpoints/model_20000/model_state.pdparams"
id2corpus = {0: "国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据"}
model_name_or_path = "rocketqa-zh-base-query-encoder"
paddle.set_device(device)
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=max_seq_length)
trans_func = partial(convert_example, tokenizer=tokenizer, max_seq_length=max_seq_length)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # text_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # text_segment
): [data for data in fn(samples)]
pretrained_model = AutoModel.from_pretrained(model_name_or_path)
@@ -96,18 +92,15 @@ if __name__ == "__main__":
model.set_dict(state_dict)
print("Loaded parameters from %s" % params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
# conver_example function's input must be dict
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
corpus_ds = MapDataset(corpus_list)
corpus_data_loader = create_dataloader(corpus_ds,
mode='predict',
batch_size=batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
corpus_data_loader = create_dataloader(
corpus_ds, mode="predict", batch_size=batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
all_embeddings = []
model.eval()
@@ -115,8 +108,7 @@ if __name__ == "__main__":
for batch_data in corpus_data_loader:
input_ids, token_type_ids = batch_data
text_embeddings = model.get_pooled_embedding(
input_ids, token_type_ids)
text_embeddings = model.get_pooled_embedding(input_ids, token_type_ids)
all_embeddings.append(text_embeddings)
text_embedding = all_embeddings[0]

127
applications/neural_search/recall/simcse/model.py
View File

@@ -23,13 +23,7 @@ import paddle.nn.functional as F
class SimCSE(nn.Layer):
def __init__(self,
pretrained_model,
dropout=None,
margin=0.0,
scale=20,
output_emb_size=None):
def __init__(self, pretrained_model, dropout=None, margin=0.0, scale=20, output_emb_size=None):
super().__init__()
@@ -41,30 +35,25 @@ class SimCSE(nn.Layer):
# recall performance and efficiency
self.output_emb_size = output_emb_size
if output_emb_size > 0:
weight_attr = paddle.ParamAttr(
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768,
output_emb_size,
weight_attr=weight_attr)
weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768, output_emb_size, weight_attr=weight_attr)
self.margin = margin
# Used scaling cosine similarity to ease converge
self.sacle = scale
@paddle.jit.to_static(input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype='int64'),
paddle.static.InputSpec(shape=[None, None], dtype='int64')
])
def get_pooled_embedding(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None,
with_pooler=True):
@paddle.jit.to_static(
input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
]
)
def get_pooled_embedding(
self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None, with_pooler=True
):
# Note: cls_embedding is poolerd embedding with act tanh
sequence_output, cls_embedding = self.ptm(input_ids, token_type_ids,
position_ids, attention_mask)
sequence_output, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
if with_pooler == False:
cls_embedding = sequence_output[:, 0, :]
@@ -85,73 +74,67 @@ class SimCSE(nn.Layer):
input_ids = paddle.to_tensor(input_ids)
token_type_ids = paddle.to_tensor(token_type_ids)
text_embeddings = self.get_pooled_embedding(
input_ids, token_type_ids=token_type_ids)
text_embeddings = self.get_pooled_embedding(input_ids, token_type_ids=token_type_ids)
yield text_embeddings
def cosine_sim(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
with_pooler=True):
def cosine_sim(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
with_pooler=True,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask,
with_pooler=with_pooler)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask, with_pooler=with_pooler
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask,
with_pooler=with_pooler)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask, with_pooler=with_pooler
)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
axis=-1)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding, axis=-1)
return cosine_sim
def forward(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None):
def forward(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask
)
cosine_sim = paddle.matmul(query_cls_embedding,
title_cls_embedding,
transpose_y=True)
cosine_sim = paddle.matmul(query_cls_embedding, title_cls_embedding, transpose_y=True)
# substract margin from all positive samples cosine_sim()
margin_diag = paddle.full(shape=[query_cls_embedding.shape[0]],
fill_value=self.margin,
dtype=paddle.get_default_dtype())
margin_diag = paddle.full(
shape=[query_cls_embedding.shape[0]], fill_value=self.margin, dtype=paddle.get_default_dtype()
)
cosine_sim = cosine_sim - paddle.diag(margin_diag)
# scale cosine to ease training converge
cosine_sim *= self.sacle
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype='int64')
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype="int64")
labels = paddle.reshape(labels, shape=[-1, 1])
loss = F.cross_entropy(input=cosine_sim, label=labels)

30
applications/neural_search/recall/simcse/predict.py
View File

@@ -70,7 +70,8 @@ def predict(model, data_loader):
query_input_ids=query_input_ids,
title_input_ids=title_input_ids,
query_token_type_ids=query_token_type_ids,
title_token_type_ids=title_token_type_ids).numpy()
title_token_type_ids=title_token_type_ids,
).numpy()
cosine_sims.append(batch_cosine_sim)
@@ -84,9 +85,7 @@ if __name__ == "__main__":
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func = partial(convert_example,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length)
trans_func = partial(convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_length)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id), # query_input
@@ -95,33 +94,24 @@ if __name__ == "__main__":
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # tilte_segment
): [data for data in fn(samples)]
valid_ds = load_dataset(read_text_pair,
data_path=args.text_pair_file,
lazy=False,
is_test=True)
valid_ds = load_dataset(read_text_pair, data_path=args.text_pair_file, lazy=False, is_test=True)
valid_data_loader = create_dataloader(valid_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
valid_data_loader = create_dataloader(
valid_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
model = SimCSE(pretrained_model,
margin=args.margin,
scale=args.scale,
output_emb_size=args.output_emb_size)
model = SimCSE(pretrained_model, margin=args.margin, scale=args.scale, output_emb_size=args.output_emb_size)
if args.params_path and os.path.isfile(args.params_path):
state_dict = paddle.load(args.params_path)
model.set_dict(state_dict)
print("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
cosin_sim = predict(model, valid_data_loader)
for idx, cosine in enumerate(cosin_sim):
print('{}'.format(cosine))
print("{}".format(cosine))

45
applications/neural_search/recall/simcse/recall.py
View File

@@ -63,15 +63,11 @@ if __name__ == "__main__":
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func = partial(convert_example_test,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length)
trans_func = partial(convert_example_test, tokenizer=tokenizer, max_seq_length=args.max_seq_length)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
), # text_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # text_segment
): [data for data in fn(samples)]
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
@@ -85,8 +81,7 @@ if __name__ == "__main__":
model.set_dict(state_dict)
logger.info("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
id2corpus = gen_id2corpus(args.corpus_file)
@@ -94,11 +89,9 @@ if __name__ == "__main__":
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
corpus_ds = MapDataset(corpus_list)
corpus_data_loader = create_dataloader(corpus_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
corpus_data_loader = create_dataloader(
corpus_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
# Need better way to get inner model of DataParallel
inner_model = model._layers
@@ -109,29 +102,27 @@ if __name__ == "__main__":
query_ds = MapDataset(text_list)
query_data_loader = create_dataloader(query_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
query_data_loader = create_dataloader(
query_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
query_embedding = inner_model.get_semantic_embedding(query_data_loader)
if not os.path.exists(args.recall_result_dir):
os.mkdir(args.recall_result_dir)
recall_result_file = os.path.join(args.recall_result_dir,
args.recall_result_file)
with open(recall_result_file, 'w', encoding='utf-8') as f:
recall_result_file = os.path.join(args.recall_result_dir, args.recall_result_file)
with open(recall_result_file, "w", encoding="utf-8") as f:
for batch_index, batch_query_embedding in enumerate(query_embedding):
recalled_idx, cosine_sims = final_index.knn_query(
batch_query_embedding.numpy(), args.recall_num)
recalled_idx, cosine_sims = final_index.knn_query(batch_query_embedding.numpy(), args.recall_num)
batch_size = len(cosine_sims)
for row_index in range(batch_size):
text_index = args.batch_size * batch_index + row_index
for idx, doc_idx in enumerate(recalled_idx[row_index]):
f.write("{}\t{}\t{}\n".format(
text_list[text_index]["text"], id2corpus[doc_idx],
1.0 - cosine_sims[row_index][idx]))
f.write(
"{}\t{}\t{}\n".format(
text_list[text_index]["text"], id2corpus[doc_idx], 1.0 - cosine_sims[row_index][idx]
)
)

32
applications/neural_search/run_system.py
View File

@@ -19,7 +19,7 @@ import sys
import random
import time
sys.path.append('./recall/milvus')
sys.path.append("./recall/milvus")
import numpy as np
import pandas as pd
from tqdm import tqdm
@@ -30,15 +30,14 @@ from config import collection_name, partition_tag, embedding_name
def recall_result(list_data):
client = PipelineClient()
client.connect(['127.0.0.1:8080'])
client.connect(["127.0.0.1:8080"])
feed = {}
for i, item in enumerate(list_data):
feed[str(i)] = item
start_time = time.time()
ret = client.predict(feed_dict=feed)
end_time = time.time()
print("Extract feature time to cost :{} seconds".format(end_time -
start_time))
print("Extract feature time to cost :{} seconds".format(end_time - start_time))
result = np.array(eval(ret.value[0]))
return result
@@ -46,32 +45,29 @@ def recall_result(list_data):
def search_in_milvus(embeddings, query_text):
recall_client = RecallByMilvus()
start_time = time.time()
results = recall_client.search(embeddings,
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=['pk', 'text'])
results = recall_client.search(
embeddings, embedding_name, collection_name, partition_names=[partition_tag], output_fields=["pk", "text"]
)
end_time = time.time()
print('Search milvus time cost is {} seconds '.format(end_time -
start_time))
print("Search milvus time cost is {} seconds ".format(end_time - start_time))
list_data = []
for line in results:
for item in line:
idx = item.id
distance = item.distance
text = item.entity.get('text')
text = item.entity.get("text")
list_data.append([query_text, text, distance])
df = pd.DataFrame(list_data, columns=['query_text', 'text', 'distance'])
df.to_csv('recall_result.csv', index=False)
df = pd.DataFrame(list_data, columns=["query_text", "text", "distance"])
df.to_csv("recall_result.csv", index=False)
return df
def rerank(df):
client = PipelineClient()
client.connect(['127.0.0.1:8089'])
client.connect(["127.0.0.1:8089"])
list_data = []
for index, row in df.iterrows():
example = {"query": row['query_text'], "title": row['text']}
example = {"query": row["query_text"], "title": row["text"]}
list_data.append(example)
feed = {}
for i, item in enumerate(list_data):
@@ -82,9 +78,9 @@ def rerank(df):
end_time = time.time()
print("time to cost :{} seconds".format(end_time - start_time))
result = np.array(eval(ret.value[0]))
df['distance'] = result
df["distance"] = result
df = df.sort_values(by=["distance"], ascending=False)
df.to_csv('rank_result.csv', index=False)
df.to_csv("rank_result.csv", index=False)
if __name__ == "__main__":

8
applications/question_answering/faq_finance/ann_util.py
View File

@@ -19,9 +19,7 @@ from paddlenlp.utils.log import logger
def build_index(args, data_loader, model):
index = hnswlib.Index(
space='ip',
dim=args.output_emb_size if args.output_emb_size > 0 else 768)
index = hnswlib.Index(space="ip", dim=args.output_emb_size if args.output_emb_size > 0 else 768)
# Initializing index
# max_elements - the maximum number of elements (capacity). Will throw an exception if exceeded
@@ -32,9 +30,7 @@ def build_index(args, data_loader, model):
#
# M - is tightly connected with internal dimensionality of the data. Strongly affects memory consumption (~M)
# Higher M leads to higher accuracy/run_time at fixed ef/efConstruction
index.init_index(max_elements=args.hnsw_max_elements,
ef_construction=args.hnsw_ef,
M=args.hnsw_m)
index.init_index(max_elements=args.hnsw_max_elements, ef_construction=args.hnsw_ef, M=args.hnsw_m)
# Controlling the recall by setting ef:
# higher ef leads to better accuracy, but slower search

18
applications/question_answering/faq_finance/config.py
View File

@@ -12,27 +12,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
search_param = {'nprobe': 20}
collection_name = 'faq_finance'
partition_tag = 'partition_1'
search_param = {"nprobe": 20}
collection_name = "faq_finance"
partition_tag = "partition_1"
MILVUS_HOST = '10.21.226.175'
MILVUS_HOST = "10.21.226.175"
MILVUS_PORT = 8530
data_dim = 256
top_k = 10
embedding_name = 'embeddings'
embedding_name = "embeddings"
index_config = {
"index_type": "IVF_FLAT",
"metric_type": "L2",
"params": {
"nlist": 1000
},
"params": {"nlist": 1000},
}
search_params = {
"metric_type": "L2",
"params": {
"nprobe": top_k
},
"params": {"nprobe": top_k},
}

73
applications/question_answering/faq_finance/data.py
View File

@@ -21,49 +21,38 @@ import paddle
def gen_id2corpus(corpus_file):
id2corpus = {}
with open(corpus_file, 'r', encoding='utf-8') as f:
with open(corpus_file, "r", encoding="utf-8") as f:
for idx, line in enumerate(f):
id2corpus[idx] = line.rstrip()
return id2corpus
def create_dataloader(dataset,
mode='train',
batch_size=1,
batchify_fn=None,
trans_fn=None):
def create_dataloader(dataset, mode="train", batch_size=1, batchify_fn=None, trans_fn=None):
if trans_fn:
dataset = dataset.map(trans_fn)
shuffle = True if mode == 'train' else False
if mode == 'train':
batch_sampler = paddle.io.DistributedBatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
shuffle = True if mode == "train" else False
if mode == "train":
batch_sampler = paddle.io.DistributedBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
else:
batch_sampler = paddle.io.BatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
batch_sampler = paddle.io.BatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
return paddle.io.DataLoader(dataset=dataset,
batch_sampler=batch_sampler,
collate_fn=batchify_fn,
return_list=True)
return paddle.io.DataLoader(dataset=dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn, return_list=True)
def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -75,9 +64,9 @@ def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
result = []
for key, text in example.items():
if 'label' in key:
if "label" in key:
# do_evaluate
result += [example['label']]
result += [example["label"]]
else:
# do_train
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length)
@@ -88,22 +77,19 @@ def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
return result
def convert_example_test(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example_test(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -114,9 +100,7 @@ def convert_example_test(example,
result = []
for key, text in example.items():
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -125,32 +109,32 @@ def convert_example_test(example,
def read_simcse_text(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip()
yield {'text_a': data, 'text_b': data}
yield {"text_a": data, "text_b": data}
def read_text_pair(data_path, is_test=False):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if is_test == True:
if len(data) != 3:
continue
yield {'text_a': data[0], 'text_b': data[1], 'label': data[2]}
yield {"text_a": data[0], "text_b": data[1], "label": data[2]}
else:
if len(data) != 2:
continue
yield {'text_a': data[0], 'text_b': data[1]}
yield {"text_a": data[0], "text_b": data[1]}
def gen_text_file(similar_text_pair_file):
text2similar_text = {}
texts = []
with open(similar_text_pair_file, 'r', encoding='utf-8') as f:
with open(similar_text_pair_file, "r", encoding="utf-8") as f:
for line in f:
splited_line = line.rstrip().split("\t")
if len(splited_line) != 2:
@@ -180,11 +164,10 @@ def word_repetition(input_ids, token_type_ids, dup_rate=0.32):
actual_len = np.count_nonzero(cur_input_id)
dup_word_index = []
# If sequence length is less than 5, skip it
if (actual_len > 5):
if actual_len > 5:
dup_len = random.randint(a=0, b=max(2, int(dup_rate * actual_len)))
# Skip cls and sep position
dup_word_index = random.sample(list(range(1, actual_len - 1)),
k=dup_len)
dup_word_index = random.sample(list(range(1, actual_len - 1)), k=dup_len)
r_input_id = []
r_token_type_id = []
@@ -208,6 +191,6 @@ def word_repetition(input_ids, token_type_ids, dup_rate=0.32):
repetitied_input_ids[batch_id] += [0] * pad_len
repetitied_token_type_ids[batch_id] += [0] * pad_len
return paddle.to_tensor(repetitied_input_ids,
dtype='int64'), paddle.to_tensor(
repetitied_token_type_ids, dtype='int64')
return paddle.to_tensor(repetitied_input_ids, dtype="int64"), paddle.to_tensor(
repetitied_token_type_ids, dtype="int64"
)

4
applications/question_answering/faq_finance/deploy/python/http_client.py
View File

@@ -27,5 +27,5 @@ print(data)
r = requests.post(url=url, headers=headers, data=data)
print(r.json())
json_data = r.json()
data = np.array(json_data['result']['output_embedding'])
print(data.shape)
data = np.array(json_data["result"]["output_embedding"])
print(data.shape)

2
applications/question_answering/faq_finance/deploy/python/rpc_client.py
View File

@@ -17,7 +17,7 @@ import numpy as np
from paddle_serving_server.pipeline import PipelineClient
client = PipelineClient()
client.connect(['127.0.0.1:8080'])
client.connect(["127.0.0.1:8080"])
list_data = ["买了社保，是不是就不用买商业保险了？"]
feed = {}

30
applications/question_answering/faq_finance/deploy/python/web_service.py
View File

@@ -25,15 +25,10 @@ args = parser.parse_args()
# yapf: enable
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
result = []
for text in example:
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -41,43 +36,38 @@ def convert_example(example,
class ErnieOp(Op):
def init_op(self):
from paddlenlp.transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
def preprocess(self, input_dicts, data_id, log_id):
from paddlenlp.data import Stack, Tuple, Pad
(_, input_dict), = input_dicts.items()
((_, input_dict),) = input_dicts.items()
print("input dict", input_dict)
batch_size = len(input_dict.keys())
examples = []
for i in range(batch_size):
input_ids, segment_ids = convert_example([input_dict[str(i)]],
self.tokenizer)
input_ids, segment_ids = convert_example([input_dict[str(i)]], self.tokenizer)
examples.append((input_ids, segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"
), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"), # segment
): fn(samples)
input_ids, segment_ids = batchify_fn(examples)
feed_dict = {}
feed_dict['input_ids'] = input_ids
feed_dict['token_type_ids'] = segment_ids
feed_dict["input_ids"] = input_ids
feed_dict["token_type_ids"] = segment_ids
return feed_dict, False, None, ""
def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
new_dict = {}
new_dict["output_embedding"] = str(
fetch_dict["output_embedding"].tolist())
new_dict["output_embedding"] = str(fetch_dict["output_embedding"].tolist())
return new_dict, None, ""
class ErnieService(WebService):
def get_pipeline_response(self, read_op):
ernie_op = ErnieOp(name="ernie", input_ops=[read_op])
return ernie_op

22
applications/question_answering/faq_finance/deploy/python/web_service_windows.py
View File

@@ -25,15 +25,10 @@ args = parser.parse_args()
# yapf: enable
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
result = []
for text in example:
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -41,13 +36,14 @@ def convert_example(example,
class ErnieService(WebService):
def init_service(self):
from paddlenlp.transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
def preprocess(self, feed=[], fetch=[]):
from paddlenlp.data import Stack, Tuple, Pad
print("input dict", feed)
batch_size = len(feed)
is_batch = True
@@ -56,15 +52,13 @@ class ErnieService(WebService):
input_ids, segment_ids = convert_example([feed[i]], self.tokenizer)
examples.append((input_ids, segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"
), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"), # segment
): fn(samples)
input_ids, segment_ids = batchify_fn(examples)
feed_dict = {}
feed_dict['input_ids'] = input_ids
feed_dict['token_type_ids'] = segment_ids
feed_dict["input_ids"] = input_ids
feed_dict["token_type_ids"] = segment_ids
return feed_dict, fetch, is_batch
def postprocess(self, feed=[], fetch=[], fetch_map=None):

10
applications/question_answering/faq_finance/evaluate.py
View File

@@ -50,14 +50,14 @@ def recall(rs, N=10):
if __name__ == "__main__":
text2similar = {}
with open(args.similar_text_pair, 'r', encoding='utf-8') as f:
with open(args.similar_text_pair, "r", encoding="utf-8") as f:
for line in f:
text, similar_text = line.rstrip().split("\t")
text2similar[text] = similar_text
rs = []
with open(args.recall_result_file, 'r', encoding='utf-8') as f:
with open(args.recall_result_file, "r", encoding="utf-8") as f:
relevance_labels = []
for index, line in enumerate(f):
@@ -72,13 +72,13 @@ if __name__ == "__main__":
relevance_labels.append(0)
recall_N = []
recall_num = [1, 5, 10]
result = open('result.tsv', 'a')
result = open("result.tsv", "a")
res = []
for topN in recall_num:
R = round(100 * recall(rs, N=topN), 3)
recall_N.append(str(R))
for key, val in zip(recall_num, recall_N):
print('recall@{}={}'.format(key, val))
print("recall@{}={}".format(key, val))
res.append(str(val))
result.write('\t'.join(res) + '\n')
result.write("\t".join(res) + "\n")
# print("\t".join(recall_N))

12
applications/question_answering/faq_finance/export_model.py
View File

@@ -47,18 +47,16 @@ if __name__ == "__main__":
model.set_dict(state_dict)
print("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
model.eval()
# Convert to static graph with specific input description
model = paddle.jit.to_static(
model,
input_spec=[
paddle.static.InputSpec(shape=[None, None],
dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None],
dtype="int64") # segment_ids
])
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # segment_ids
],
)
# Save in static graph model.
save_path = os.path.join(args.output_path, "inference")
paddle.jit.save(model, save_path)

4
applications/question_answering/faq_finance/export_to_serving.py
View File

@@ -14,6 +14,7 @@
import argparse
import paddle_serving_client.io as serving_io
# yapf: disable
parser = argparse.ArgumentParser()
parser.add_argument("--dirname", type=str, required=True,
@@ -44,4 +45,5 @@ if __name__ == "__main__":
params_filename=args.params_filename,
show_proto=args.show_proto,
feed_alias_names=args.feed_alias_names,
fetch_alias_names=args.fetch_alias_names)
fetch_alias_names=args.fetch_alias_names,
)

84
applications/question_answering/faq_finance/feature_extract.py
View File

@@ -58,22 +58,19 @@ args = parser.parse_args()
# yapf: enable
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -84,9 +81,7 @@ def convert_example(example,
result = []
for key, text in example.items():
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -94,16 +89,17 @@ def convert_example(example,
class Predictor(object):
def __init__(self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False):
def __init__(
self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False,
):
self.max_seq_length = max_seq_length
self.batch_size = batch_size
@@ -122,14 +118,14 @@ class Predictor(object):
precision_map = {
"fp16": inference.PrecisionType.Half,
"fp32": inference.PrecisionType.Float32,
"int8": inference.PrecisionType.Int8
"int8": inference.PrecisionType.Int8,
}
precision_mode = precision_map[precision]
if args.use_tensorrt:
config.enable_tensorrt_engine(max_batch_size=batch_size,
min_subgraph_size=30,
precision_mode=precision_mode)
config.enable_tensorrt_engine(
max_batch_size=batch_size, min_subgraph_size=30, precision_mode=precision_mode
)
elif device == "cpu":
# set CPU configs accordingly,
# such as enable_mkldnn, set_cpu_math_library_num_threads
@@ -145,12 +141,8 @@ class Predictor(object):
config.switch_use_feed_fetch_ops(False)
self.predictor = paddle.inference.create_predictor(config)
self.input_handles = [
self.predictor.get_input_handle(name)
for name in self.predictor.get_input_names()
]
self.output_handle = self.predictor.get_output_handle(
self.predictor.get_output_names()[0])
self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()]
self.output_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[0])
def predict(self, data, tokenizer):
"""
@@ -158,7 +150,7 @@ class Predictor(object):
Args:
data (obj:`List(str)`): The batch data whose each element is a raw text.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
Returns:
@@ -166,21 +158,18 @@ class Predictor(object):
"""
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'), # input
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'
), # segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # segment
): fn(samples)
all_embeddings = []
examples = []
for idx, text in enumerate(tqdm(data)):
input_ids, segment_ids = convert_example(
text,
tokenizer,
max_seq_length=self.max_seq_length,
pad_to_max_seq_len=True)
text, tokenizer, max_seq_length=self.max_seq_length, pad_to_max_seq_len=True
)
examples.append((input_ids, segment_ids))
if (len(examples) >= self.batch_size):
if len(examples) >= self.batch_size:
input_ids, segment_ids = batchify_fn(examples)
self.input_handles[0].copy_from_cpu(input_ids)
self.input_handles[1].copy_from_cpu(segment_ids)
@@ -189,7 +178,7 @@ class Predictor(object):
all_embeddings.append(logits)
examples = []
if (len(examples) > 0):
if len(examples) > 0:
input_ids, segment_ids = batchify_fn(examples)
self.input_handles[0].copy_from_cpu(input_ids)
self.input_handles[1].copy_from_cpu(segment_ids)
@@ -198,7 +187,7 @@ class Predictor(object):
all_embeddings.append(logits)
all_embeddings = np.concatenate(all_embeddings, axis=0)
np.save('corpus_embedding', all_embeddings)
np.save("corpus_embedding", all_embeddings)
def read_text(file_path):
@@ -210,9 +199,16 @@ def read_text(file_path):
if __name__ == "__main__":
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
args.batch_size, args.use_tensorrt, args.precision,
args.cpu_threads, args.enable_mkldnn)
predictor = Predictor(
args.model_dir,
args.device,
args.max_seq_length,
args.batch_size,
args.use_tensorrt,
args.precision,
args.cpu_threads,
args.enable_mkldnn,
)
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
id2corpus = read_text(args.corpus_file)

77
applications/question_answering/faq_finance/milvus_ann_search.py
View File

@@ -22,30 +22,16 @@ from milvus_util import VecToMilvus, RecallByMilvus, text_max_len
from config import collection_name, partition_tag, embedding_name
parser = argparse.ArgumentParser()
parser.add_argument("--data_path",
default='data/corpus.csv',
type=str,
required=True,
help="The data for vector extraction.")
parser.add_argument("--embedding_path",
default='corpus_embedding.npy',
type=str,
required=True,
help="The vector path for data.")
parser.add_argument('--index',
default=0,
type=int,
help='index of the vector for search')
parser.add_argument('--insert',
action='store_true',
help='whether to insert data')
parser.add_argument('--search',
action='store_true',
help='whether to search data')
parser.add_argument('--batch_size',
default=100000,
type=int,
help='number of examples to insert each time')
parser.add_argument(
"--data_path", default="data/corpus.csv", type=str, required=True, help="The data for vector extraction."
)
parser.add_argument(
"--embedding_path", default="corpus_embedding.npy", type=str, required=True, help="The vector path for data."
)
parser.add_argument("--index", default=0, type=int, help="index of the vector for search")
parser.add_argument("--insert", action="store_true", help="whether to insert data")
parser.add_argument("--search", action="store_true", help="whether to search data")
parser.add_argument("--batch_size", default=100000, type=int, help="number of examples to insert each time")
args = parser.parse_args()
@@ -53,8 +39,8 @@ def read_text(file_path):
file = open(file_path)
id2corpus = []
for idx, data in enumerate(file.readlines()):
question, answer = data.strip().split('\t')
id2corpus.append({'question': question, 'answer': answer})
question, answer = data.strip().split("\t")
id2corpus.append({"question": question, "answer": answer})
return id2corpus
@@ -67,51 +53,42 @@ def milvus_data_insert(data_path, embedding_path, batch_size):
data_size = len(embedding_ids)
for i in tqdm(range(0, data_size, batch_size)):
cur_end = i + batch_size
if (cur_end > data_size):
if cur_end > data_size:
cur_end = data_size
batch_emb = embeddings[np.arange(i, cur_end)]
entities = [
[j for j in range(i, cur_end, 1)],
[
corpus_list[j]['question'][:text_max_len - 1]
for j in range(i, cur_end, 1)
],
[
corpus_list[j]['answer'][:text_max_len - 1]
for j in range(i, cur_end, 1)
],
batch_emb # field embeddings, supports numpy.ndarray and list
[corpus_list[j]["question"][: text_max_len - 1] for j in range(i, cur_end, 1)],
[corpus_list[j]["answer"][: text_max_len - 1] for j in range(i, cur_end, 1)],
batch_emb, # field embeddings, supports numpy.ndarray and list
]
client.insert(collection_name=collection_name,
entities=entities,
index_name=embedding_name,
partition_tag=partition_tag)
client.insert(
collection_name=collection_name, entities=entities, index_name=embedding_name, partition_tag=partition_tag
)
def milvus_data_recall(embedding_path, index):
embeddings = np.load(embedding_path)
embedding_ids = [i for i in range(embeddings.shape[0])]
recall_client = RecallByMilvus()
if (index > len(embedding_ids)):
print('Index should not be larger than embedding szie')
if index > len(embedding_ids):
print("Index should not be larger than embedding szie")
return
embeddings = embeddings[np.arange(index, index + 1)]
time_start = time.time()
result = recall_client.search(embeddings,
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=['pk', 'text'])
result = recall_client.search(
embeddings, embedding_name, collection_name, partition_names=[partition_tag], output_fields=["pk", "text"]
)
time_end = time.time()
sum_t = time_end - time_start
print('time cost', sum_t, 's')
print("time cost", sum_t, "s")
for hits in result:
for hit in hits:
print(f"hit: {hit}, text field: {hit.entity.get('text')}")
if __name__ == "__main__":
if (args.insert):
if args.insert:
milvus_data_insert(args.data_path, args.embedding_path, args.batch_size)
if (args.search):
if args.search:
milvus_data_recall(args.embedding_path, args.index)

90
applications/question_answering/faq_finance/milvus_util.py
View File

@@ -29,22 +29,15 @@ from config import MILVUS_HOST, MILVUS_PORT, data_dim, index_config, top_k, sear
fmt = "\n=== {:30} ===\n"
text_max_len = 1000
fields = [
FieldSchema(name="pk",
dtype=DataType.INT64,
is_primary=True,
auto_id=False,
max_length=100),
FieldSchema(name="question",
dtype=DataType.VARCHAR,
max_length=text_max_len),
FieldSchema(name="pk", dtype=DataType.INT64, is_primary=True, auto_id=False, max_length=100),
FieldSchema(name="question", dtype=DataType.VARCHAR, max_length=text_max_len),
FieldSchema(name="answer", dtype=DataType.VARCHAR, max_length=text_max_len),
FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=data_dim)
FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=data_dim),
]
schema = CollectionSchema(fields, "Neural Search Index")
class VecToMilvus():
class VecToMilvus:
def __init__(self):
print(fmt.format("start connecting to Milvus"))
connections.connect("default", host=MILVUS_HOST, port=MILVUS_PORT)
@@ -61,9 +54,7 @@ class VecToMilvus():
def creat_collection(self, collection_name):
try:
print(fmt.format("Create collection {}".format(collection_name)))
self.collection = Collection(collection_name,
schema,
consistency_level="Strong")
self.collection = Collection(collection_name, schema, consistency_level="Strong")
except Exception as e:
print("Milvus create collection error:", e)
@@ -92,9 +83,9 @@ class VecToMilvus():
def create_partition(self, partition_tag):
try:
self.collection.create_partition(partition_tag)
print('create partition {} successfully'.format(partition_tag))
print("create partition {} successfully".format(partition_tag))
except Exception as e:
print('Milvus create partition error: ', e)
print("Milvus create partition error: ", e)
def insert(self, entities, collection_name, index_name, partition_tag=None):
try:
@@ -103,20 +94,16 @@ class VecToMilvus():
self.create_index(index_name)
else:
self.collection = Collection(collection_name)
if (partition_tag
is not None) and (not self.has_partition(partition_tag)):
if (partition_tag is not None) and (not self.has_partition(partition_tag)):
self.create_partition(partition_tag)
self.collection.insert(entities, partition_name=partition_tag)
print(
f"Number of entities in Milvus: {self.collection.num_entities}"
) # check the num_entites
print(f"Number of entities in Milvus: {self.collection.num_entities}") # check the num_entites
except Exception as e:
print("Milvus insert error:", e)
class RecallByMilvus():
class RecallByMilvus:
def __init__(self):
print(fmt.format("start connecting to Milvus"))
connections.connect("default", host=MILVUS_HOST, port=MILVUS_PORT)
@@ -129,55 +116,50 @@ class RecallByMilvus():
except Exception as e:
print("Milvus create collection error:", e)
def search(self,
vectors,
embedding_name,
collection_name,
partition_names=[],
output_fields=[]):
def search(self, vectors, embedding_name, collection_name, partition_names=[], output_fields=[]):
try:
self.get_collection(collection_name)
result = self.collection.search(vectors,
embedding_name,
search_params,
limit=top_k,
partition_names=partition_names,
output_fields=output_fields)
result = self.collection.search(
vectors,
embedding_name,
search_params,
limit=top_k,
partition_names=partition_names,
output_fields=output_fields,
)
return result
except Exception as e:
print('Milvus recall error: ', e)
print("Milvus recall error: ", e)
if __name__ == '__main__':
if __name__ == "__main__":
print(fmt.format("Start inserting entities"))
rng = np.random.default_rng(seed=19530)
num_entities = 3000
entities = [
# provide the pk field because `auto_id` is set to False
[i for i in range(num_entities)],
['第{}个样本'.format(i)
for i in range(num_entities)], # field text, only supports list
rng.random(
(num_entities,
data_dim)), # field embeddings, supports numpy.ndarray and list
["第{}个样本".format(i) for i in range(num_entities)], # field text, only supports list
rng.random((num_entities, data_dim)), # field embeddings, supports numpy.ndarray and list
]
print(entities[-1].shape)
collection_name = 'test1'
partition_tag = 'partition_1'
embedding_name = 'embeddings'
collection_name = "test1"
partition_tag = "partition_1"
embedding_name = "embeddings"
client = VecToMilvus()
client.insert(collection_name=collection_name,
entities=entities,
index_name=embedding_name,
partition_tag=partition_tag)
client.insert(
collection_name=collection_name, entities=entities, index_name=embedding_name, partition_tag=partition_tag
)
print(fmt.format("Start searching entities"))
vectors_to_search = entities[-1][-2:]
recall_client = RecallByMilvus()
result = recall_client.search(vectors_to_search,
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=['pk', 'text'])
result = recall_client.search(
vectors_to_search,
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=["pk", "text"],
)
for hits in result:
for hit in hits:
print(f"hit: {hit}, random field: {hit.entity.get('text')}")

127
applications/question_answering/faq_finance/model.py
View File

@@ -25,13 +25,7 @@ import paddlenlp
class SimCSE(nn.Layer):
def __init__(self,
pretrained_model,
dropout=None,
margin=0.0,
scale=20,
output_emb_size=None):
def __init__(self, pretrained_model, dropout=None, margin=0.0, scale=20, output_emb_size=None):
super().__init__()
@@ -43,11 +37,8 @@ class SimCSE(nn.Layer):
# recall performance and efficiency
self.output_emb_size = output_emb_size
if output_emb_size > 0:
weight_attr = paddle.ParamAttr(
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768,
output_emb_size,
weight_attr=weight_attr)
weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
self.emb_reduce_linear = paddle.nn.Linear(768, output_emb_size, weight_attr=weight_attr)
self.margin = margin
# Used scaling cosine similarity to ease converge
@@ -55,20 +46,18 @@ class SimCSE(nn.Layer):
self.classifier = nn.Linear(output_emb_size, 2)
self.rdrop_loss = paddlenlp.losses.RDropLoss()
@paddle.jit.to_static(input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype='int64'),
paddle.static.InputSpec(shape=[None, None], dtype='int64')
])
def get_pooled_embedding(self,
input_ids,
token_type_ids=None,
position_ids=None,
attention_mask=None,
with_pooler=True):
@paddle.jit.to_static(
input_spec=[
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
paddle.static.InputSpec(shape=[None, None], dtype="int64"),
]
)
def get_pooled_embedding(
self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None, with_pooler=True
):
# Note: cls_embedding is poolerd embedding with act tanh
sequence_output, cls_embedding = self.ptm(input_ids, token_type_ids,
position_ids, attention_mask)
sequence_output, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids, attention_mask)
if with_pooler == False:
cls_embedding = sequence_output[:, 0, :]
@@ -86,77 +75,71 @@ class SimCSE(nn.Layer):
with paddle.no_grad():
for batch_data in data_loader:
input_ids, token_type_ids = batch_data
text_embeddings = self.get_pooled_embedding(
input_ids, token_type_ids=token_type_ids)
text_embeddings = self.get_pooled_embedding(input_ids, token_type_ids=token_type_ids)
yield text_embeddings
def cosine_sim(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
with_pooler=True):
def cosine_sim(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
with_pooler=True,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask,
with_pooler=with_pooler)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask, with_pooler=with_pooler
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask,
with_pooler=with_pooler)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask, with_pooler=with_pooler
)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
axis=-1)
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding, axis=-1)
return cosine_sim
def forward(self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None):
def forward(
self,
query_input_ids,
title_input_ids,
query_token_type_ids=None,
query_position_ids=None,
query_attention_mask=None,
title_token_type_ids=None,
title_position_ids=None,
title_attention_mask=None,
):
query_cls_embedding = self.get_pooled_embedding(query_input_ids,
query_token_type_ids,
query_position_ids,
query_attention_mask)
query_cls_embedding = self.get_pooled_embedding(
query_input_ids, query_token_type_ids, query_position_ids, query_attention_mask
)
title_cls_embedding = self.get_pooled_embedding(title_input_ids,
title_token_type_ids,
title_position_ids,
title_attention_mask)
title_cls_embedding = self.get_pooled_embedding(
title_input_ids, title_token_type_ids, title_position_ids, title_attention_mask
)
logits1 = self.classifier(query_cls_embedding)
logits2 = self.classifier(title_cls_embedding)
kl_loss = self.rdrop_loss(logits1, logits2)
cosine_sim = paddle.matmul(query_cls_embedding,
title_cls_embedding,
transpose_y=True)
cosine_sim = paddle.matmul(query_cls_embedding, title_cls_embedding, transpose_y=True)
# substract margin from all positive samples cosine_sim()
margin_diag = paddle.full(shape=[query_cls_embedding.shape[0]],
fill_value=self.margin,
dtype=paddle.get_default_dtype())
margin_diag = paddle.full(
shape=[query_cls_embedding.shape[0]], fill_value=self.margin, dtype=paddle.get_default_dtype()
)
cosine_sim = cosine_sim - paddle.diag(margin_diag)
# scale cosine to ease training converge
cosine_sim *= self.sacle
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype='int64')
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype="int64")
labels = paddle.reshape(labels, shape=[-1, 1])
loss = F.cross_entropy(input=cosine_sim, label=labels)

45
applications/question_answering/faq_finance/recall.py
View File

@@ -61,15 +61,11 @@ if __name__ == "__main__":
paddle.distributed.init_parallel_env()
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
trans_func = partial(convert_example_test,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length)
trans_func = partial(convert_example_test, tokenizer=tokenizer, max_seq_length=args.max_seq_length)
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'
), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int64'
), # text_segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # text_input
Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # text_segment
): [data for data in fn(samples)]
pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
@@ -83,8 +79,7 @@ if __name__ == "__main__":
model.set_dict(state_dict)
logger.info("Loaded parameters from %s" % args.params_path)
else:
raise ValueError(
"Please set --params_path with correct pretrained model file")
raise ValueError("Please set --params_path with correct pretrained model file")
id2corpus = gen_id2corpus(args.corpus_file)
@@ -92,11 +87,9 @@ if __name__ == "__main__":
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
corpus_ds = MapDataset(corpus_list)
corpus_data_loader = create_dataloader(corpus_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
corpus_data_loader = create_dataloader(
corpus_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
# Need better way to get inner model of DataParallel
inner_model = model._layers
@@ -107,29 +100,27 @@ if __name__ == "__main__":
query_ds = MapDataset(text_list)
query_data_loader = create_dataloader(query_ds,
mode='predict',
batch_size=args.batch_size,
batchify_fn=batchify_fn,
trans_fn=trans_func)
query_data_loader = create_dataloader(
query_ds, mode="predict", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func
)
query_embedding = inner_model.get_semantic_embedding(query_data_loader)
if not os.path.exists(args.recall_result_dir):
os.mkdir(args.recall_result_dir)
recall_result_file = os.path.join(args.recall_result_dir,
args.recall_result_file)
with open(recall_result_file, 'w', encoding='utf-8') as f:
recall_result_file = os.path.join(args.recall_result_dir, args.recall_result_file)
with open(recall_result_file, "w", encoding="utf-8") as f:
for batch_index, batch_query_embedding in enumerate(query_embedding):
recalled_idx, cosine_sims = final_index.knn_query(
batch_query_embedding.numpy(), args.recall_num)
recalled_idx, cosine_sims = final_index.knn_query(batch_query_embedding.numpy(), args.recall_num)
batch_size = len(cosine_sims)
for row_index in range(batch_size):
text_index = args.batch_size * batch_index + row_index
for idx, doc_idx in enumerate(recalled_idx[row_index]):
f.write("{}\t{}\t{}\n".format(
text_list[text_index]["text"], id2corpus[doc_idx],
1.0 - cosine_sims[row_index][idx]))
f.write(
"{}\t{}\t{}\n".format(
text_list[text_index]["text"], id2corpus[doc_idx], 1.0 - cosine_sims[row_index][idx]
)
)

29
applications/question_answering/faq_finance/run_system.py
View File

@@ -29,15 +29,14 @@ from config import collection_name, partition_tag, embedding_name
def recall_result(list_data):
client = PipelineClient()
client.connect(['127.0.0.1:8080'])
client.connect(["127.0.0.1:8080"])
feed = {}
for i, item in enumerate(list_data):
feed[str(i)] = item
start_time = time.time()
ret = client.predict(feed_dict=feed)
end_time = time.time()
print("Extract feature time to cost :{} seconds".format(end_time -
start_time))
print("Extract feature time to cost :{} seconds".format(end_time - start_time))
result = np.array(eval(ret.value[0]))
return result
@@ -45,26 +44,26 @@ def recall_result(list_data):
def search_in_milvus(embeddings, query_text):
recall_client = RecallByMilvus()
start_time = time.time()
results = recall_client.search(embeddings,
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=['pk', 'question', 'answer'])
results = recall_client.search(
embeddings,
embedding_name,
collection_name,
partition_names=[partition_tag],
output_fields=["pk", "question", "answer"],
)
end_time = time.time()
print('Search milvus time cost is {} seconds '.format(end_time -
start_time))
print("Search milvus time cost is {} seconds ".format(end_time - start_time))
list_data = []
for line in results:
for item in line:
idx = item.id
distance = item.distance
question = item.entity.get('question')
answer = item.entity.get('answer')
question = item.entity.get("question")
answer = item.entity.get("answer")
print(question, answer, distance)
list_data.append([query_text, question, answer, distance])
df = pd.DataFrame(list_data,
columns=['query_text', 'question', 'answer', 'distance'])
df.to_csv('faq_result.csv', index=False)
df = pd.DataFrame(list_data, columns=["query_text", "question", "answer", "distance"])
df.to_csv("faq_result.csv", index=False)
if __name__ == "__main__":

8
applications/question_answering/faq_system/ann_util.py
View File

@@ -19,9 +19,7 @@ from paddlenlp.utils.log import logger
def build_index(args, data_loader, model):
index = hnswlib.Index(
space='ip',
dim=args.output_emb_size if args.output_emb_size > 0 else 768)
index = hnswlib.Index(space="ip", dim=args.output_emb_size if args.output_emb_size > 0 else 768)
# Initializing index
# max_elements - the maximum number of elements (capacity). Will throw an exception if exceeded
@@ -32,9 +30,7 @@ def build_index(args, data_loader, model):
#
# M - is tightly connected with internal dimensionality of the data. Strongly affects memory consumption (~M)
# Higher M leads to higher accuracy/run_time at fixed ef/efConstruction
index.init_index(max_elements=args.hnsw_max_elements,
ef_construction=args.hnsw_ef,
M=args.hnsw_m)
index.init_index(max_elements=args.hnsw_max_elements, ef_construction=args.hnsw_ef, M=args.hnsw_m)
# Controlling the recall by setting ef:
# higher ef leads to better accuracy, but slower search

12
applications/question_answering/faq_system/config.py
View File

@@ -15,17 +15,13 @@
import os
from milvus import MetricType, IndexType
MILVUS_HOST = '10.21.226.173'
MILVUS_HOST = "10.21.226.173"
MILVUS_PORT = 8530
collection_param = {
'dimension': 256,
'index_file_size': 256,
'metric_type': MetricType.L2
}
collection_param = {"dimension": 256, "index_file_size": 256, "metric_type": MetricType.L2}
index_type = IndexType.IVF_FLAT
index_param = {'nlist': 1000}
index_param = {"nlist": 1000}
top_k = 100
search_param = {'nprobe': 20}
search_param = {"nprobe": 20}

73
applications/question_answering/faq_system/data.py
View File

@@ -21,49 +21,38 @@ import paddle
def gen_id2corpus(corpus_file):
id2corpus = {}
with open(corpus_file, 'r', encoding='utf-8') as f:
with open(corpus_file, "r", encoding="utf-8") as f:
for idx, line in enumerate(f):
id2corpus[idx] = line.rstrip()
return id2corpus
def create_dataloader(dataset,
mode='train',
batch_size=1,
batchify_fn=None,
trans_fn=None):
def create_dataloader(dataset, mode="train", batch_size=1, batchify_fn=None, trans_fn=None):
if trans_fn:
dataset = dataset.map(trans_fn)
shuffle = True if mode == 'train' else False
if mode == 'train':
batch_sampler = paddle.io.DistributedBatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
shuffle = True if mode == "train" else False
if mode == "train":
batch_sampler = paddle.io.DistributedBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
else:
batch_sampler = paddle.io.BatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
batch_sampler = paddle.io.BatchSampler(dataset, batch_size=batch_size, shuffle=shuffle)
return paddle.io.DataLoader(dataset=dataset,
batch_sampler=batch_sampler,
collate_fn=batchify_fn,
return_list=True)
return paddle.io.DataLoader(dataset=dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn, return_list=True)
def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -75,9 +64,9 @@ def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
result = []
for key, text in example.items():
if 'label' in key:
if "label" in key:
# do_evaluate
result += [example['label']]
result += [example["label"]]
else:
# do_train
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length)
@@ -88,22 +77,19 @@ def convert_example(example, tokenizer, max_seq_length=512, do_evalute=False):
return result
def convert_example_test(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example_test(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -114,9 +100,7 @@ def convert_example_test(example,
result = []
for key, text in example.items():
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -125,31 +109,31 @@ def convert_example_test(example,
def read_simcse_text(data_path):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip()
yield {'text_a': data, 'text_b': data}
yield {"text_a": data, "text_b": data}
def read_text_pair(data_path, is_test=False):
"""Reads data."""
with open(data_path, 'r', encoding='utf-8') as f:
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip().split("\t")
if is_test == False:
if len(data) != 3:
continue
yield {'text_a': data[0], 'text_b': data[1], 'label': data[2]}
yield {"text_a": data[0], "text_b": data[1], "label": data[2]}
else:
if len(data) != 2:
continue
yield {'text_a': data[0], 'text_b': data[1]}
yield {"text_a": data[0], "text_b": data[1]}
def gen_text_file(similar_text_pair_file):
text2similar_text = {}
texts = []
with open(similar_text_pair_file, 'r', encoding='utf-8') as f:
with open(similar_text_pair_file, "r", encoding="utf-8") as f:
for line in f:
splited_line = line.rstrip().split("\t")
if len(splited_line) != 2:
@@ -179,11 +163,10 @@ def word_repetition(input_ids, token_type_ids, dup_rate=0.32):
actual_len = np.count_nonzero(cur_input_id)
dup_word_index = []
# If sequence length is less than 5, skip it
if (actual_len > 5):
if actual_len > 5:
dup_len = random.randint(a=0, b=max(2, int(dup_rate * actual_len)))
# Skip cls and sep position
dup_word_index = random.sample(list(range(1, actual_len - 1)),
k=dup_len)
dup_word_index = random.sample(list(range(1, actual_len - 1)), k=dup_len)
r_input_id = []
r_token_type_id = []
@@ -207,6 +190,6 @@ def word_repetition(input_ids, token_type_ids, dup_rate=0.32):
repetitied_input_ids[batch_id] += [0] * pad_len
repetitied_token_type_ids[batch_id] += [0] * pad_len
return paddle.to_tensor(repetitied_input_ids,
dtype='int64'), paddle.to_tensor(
repetitied_token_type_ids, dtype='int64')
return paddle.to_tensor(repetitied_input_ids, dtype="int64"), paddle.to_tensor(
repetitied_token_type_ids, dtype="int64"
)

2
applications/question_answering/faq_system/deploy/python/rpc_client.py
View File

@@ -17,7 +17,7 @@ import numpy as np
from paddle_serving_server.pipeline import PipelineClient
client = PipelineClient()
client.connect(['127.0.0.1:8080'])
client.connect(["127.0.0.1:8080"])
list_data = ["湖北省为什么鼓励缴费人通过线上缴费渠道缴费？", "佛山市救助站有多少个救助床位"]
feed = {}

32
applications/question_answering/faq_system/deploy/python/web_service.py
View File

@@ -18,15 +18,10 @@ import sys
from paddle_serving_server.web_service import WebService, Op
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
result = []
for text in example:
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -34,43 +29,38 @@ def convert_example(example,
class ErnieOp(Op):
def init_op(self):
from paddlenlp.transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
self.tokenizer = AutoTokenizer.from_pretrained("ernie-3.0-medium-zh")
def preprocess(self, input_dicts, data_id, log_id):
from paddlenlp.data import Stack, Tuple, Pad
(_, input_dict), = input_dicts.items()
((_, input_dict),) = input_dicts.items()
print("input dict", input_dict)
batch_size = len(input_dict.keys())
examples = []
for i in range(batch_size):
input_ids, segment_ids = convert_example([input_dict[str(i)]],
self.tokenizer)
input_ids, segment_ids = convert_example([input_dict[str(i)]], self.tokenizer)
examples.append((input_ids, segment_ids))
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"
), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"
), # segment
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"), # segment
): fn(samples)
input_ids, segment_ids = batchify_fn(examples)
feed_dict = {}
feed_dict['input_ids'] = input_ids
feed_dict['token_type_ids'] = segment_ids
feed_dict["input_ids"] = input_ids
feed_dict["token_type_ids"] = segment_ids
return feed_dict, False, None, ""
def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
new_dict = {}
new_dict["output_embedding"] = str(
fetch_dict["output_embedding"].tolist())
new_dict["output_embedding"] = str(fetch_dict["output_embedding"].tolist())
return new_dict, None, ""
class ErnieService(WebService):
def get_pipeline_response(self, read_op):
ernie_op = ErnieOp(name="ernie", input_ops=[read_op])
return ernie_op

10
applications/question_answering/faq_system/evaluate.py
View File

@@ -50,14 +50,14 @@ def recall(rs, N=10):
if __name__ == "__main__":
text2similar = {}
with open(args.similar_text_pair, 'r', encoding='utf-8') as f:
with open(args.similar_text_pair, "r", encoding="utf-8") as f:
for line in f:
text, similar_text = line.rstrip().split("\t")
text2similar[text] = similar_text
rs = []
with open(args.recall_result_file, 'r', encoding='utf-8') as f:
with open(args.recall_result_file, "r", encoding="utf-8") as f:
relevance_labels = []
for index, line in enumerate(f):
@@ -72,13 +72,13 @@ if __name__ == "__main__":
relevance_labels.append(0)
recall_N = []
recall_num = [1, 5, 10]
result = open('result.tsv', 'a')
result = open("result.tsv", "a")
res = []
for topN in recall_num:
R = round(100 * recall(rs, N=topN), 3)
recall_N.append(str(R))
for key, val in zip(recall_num, recall_N):
print('recall@{}={}'.format(key, val))
print("recall@{}={}".format(key, val))
res.append(str(val))
result.write('\t'.join(res) + '\n')
result.write("\t".join(res) + "\n")
# print("\t".join(recall_N))

11
applications/question_answering/faq_system/export_model.py
View File

@@ -39,7 +39,7 @@ if __name__ == "__main__":
pretrained_model = AutoModel.from_pretrained("ernie-3.0-medium-zh")
tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
tokenizer = AutoTokenizer.from_pretrained("ernie-3.0-medium-zh")
model = SimCSE(pretrained_model, output_emb_size=output_emb_size)
if args.params_path and os.path.isfile(args.params_path):
@@ -52,11 +52,10 @@ if __name__ == "__main__":
model = paddle.jit.to_static(
model,
input_spec=[
paddle.static.InputSpec(shape=[None, None],
dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None],
dtype="int64") # segment_ids
])
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # input_ids
paddle.static.InputSpec(shape=[None, None], dtype="int64"), # segment_ids
],
)
# Save in static graph model.
save_path = os.path.join(args.output_path, "inference")
paddle.jit.save(model, save_path)

4
applications/question_answering/faq_system/export_to_serving.py
View File

@@ -14,6 +14,7 @@
import argparse
import paddle_serving_client.io as serving_io
# yapf: disable
parser = argparse.ArgumentParser()
parser.add_argument("--dirname", type=str, required=True,
@@ -44,4 +45,5 @@ if __name__ == "__main__":
params_filename=args.params_filename,
show_proto=args.show_proto,
feed_alias_names=args.feed_alias_names,
fetch_alias_names=args.fetch_alias_names)
fetch_alias_names=args.fetch_alias_names,
)

86
applications/question_answering/faq_system/feature_extract.py
View File

@@ -58,22 +58,19 @@ args = parser.parse_args()
# yapf: enable
def convert_example(example,
tokenizer,
max_seq_length=512,
pad_to_max_seq_len=False):
def convert_example(example, tokenizer, max_seq_length=512, pad_to_max_seq_len=False):
"""
Builds model inputs from a sequence.
A BERT sequence has the following format:
- single sequence: ``[CLS] X [SEP]``
Args:
example(obj:`list(str)`): The list of text to be converted to ids.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
@@ -84,9 +81,7 @@ def convert_example(example,
result = []
for key, text in example.items():
encoded_inputs = tokenizer(text=text,
max_seq_len=max_seq_length,
pad_to_max_seq_len=pad_to_max_seq_len)
encoded_inputs = tokenizer(text=text, max_seq_len=max_seq_length, pad_to_max_seq_len=pad_to_max_seq_len)
input_ids = encoded_inputs["input_ids"]
token_type_ids = encoded_inputs["token_type_ids"]
result += [input_ids, token_type_ids]
@@ -94,16 +89,17 @@ def convert_example(example,
class Predictor(object):
def __init__(self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False):
def __init__(
self,
model_dir,
device="gpu",
max_seq_length=128,
batch_size=32,
use_tensorrt=False,
precision="fp32",
cpu_threads=10,
enable_mkldnn=False,
):
self.max_seq_length = max_seq_length
self.batch_size = batch_size
@@ -122,14 +118,14 @@ class Predictor(object):
precision_map = {
"fp16": inference.PrecisionType.Half,
"fp32": inference.PrecisionType.Float32,
"int8": inference.PrecisionType.Int8
"int8": inference.PrecisionType.Int8,
}
precision_mode = precision_map[precision]
if args.use_tensorrt:
config.enable_tensorrt_engine(max_batch_size=batch_size,
min_subgraph_size=30,
precision_mode=precision_mode)
config.enable_tensorrt_engine(
max_batch_size=batch_size, min_subgraph_size=30, precision_mode=precision_mode
)
elif device == "cpu":
# set CPU configs accordingly,
# such as enable_mkldnn, set_cpu_math_library_num_threads
@@ -145,12 +141,8 @@ class Predictor(object):
config.switch_use_feed_fetch_ops(False)
self.predictor = paddle.inference.create_predictor(config)
self.input_handles = [
self.predictor.get_input_handle(name)
for name in self.predictor.get_input_names()
]
self.output_handle = self.predictor.get_output_handle(
self.predictor.get_output_names()[0])
self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()]
self.output_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[0])
def predict(self, data, tokenizer):
"""
@@ -158,7 +150,7 @@ class Predictor(object):
Args:
data (obj:`List(str)`): The batch data whose each element is a raw text.
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
Returns:
@@ -166,21 +158,18 @@ class Predictor(object):
"""
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'), # input
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int64'
), # segment
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input
Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # segment
): fn(samples)
all_embeddings = []
examples = []
for idx, text in enumerate(tqdm(data)):
input_ids, segment_ids = convert_example(
text,
tokenizer,
max_seq_length=self.max_seq_length,
pad_to_max_seq_len=True)
text, tokenizer, max_seq_length=self.max_seq_length, pad_to_max_seq_len=True
)
examples.append((input_ids, segment_ids))
if (len(examples) >= self.batch_size):
if len(examples) >= self.batch_size:
input_ids, segment_ids = batchify_fn(examples)
self.input_handles[0].copy_from_cpu(input_ids)
self.input_handles[1].copy_from_cpu(segment_ids)
@@ -189,7 +178,7 @@ class Predictor(object):
all_embeddings.append(logits)
examples = []
if (len(examples) > 0):
if len(examples) > 0:
input_ids, segment_ids = batchify_fn(examples)
self.input_handles[0].copy_from_cpu(input_ids)
self.input_handles[1].copy_from_cpu(segment_ids)
@@ -198,7 +187,7 @@ class Predictor(object):
all_embeddings.append(logits)
all_embeddings = np.concatenate(all_embeddings, axis=0)
np.save('corpus_embedding', all_embeddings)
np.save("corpus_embedding", all_embeddings)
def read_text(file_path):
@@ -210,11 +199,18 @@ def read_text(file_path):
if __name__ == "__main__":
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
args.batch_size, args.use_tensorrt, args.precision,
args.cpu_threads, args.enable_mkldnn)
predictor = Predictor(
args.model_dir,
args.device,
args.max_seq_length,
args.batch_size,
args.use_tensorrt,
args.precision,
args.cpu_threads,
args.enable_mkldnn,
)
tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
tokenizer = AutoTokenizer.from_pretrained("ernie-3.0-medium-zh")
id2corpus = read_text(args.corpus_file)
corpus_list = [{idx: text} for idx, text in id2corpus.items()]

46
applications/question_answering/faq_system/milvus_util.py
View File

@@ -17,8 +17,7 @@ from config import MILVUS_HOST, MILVUS_PORT, collection_param, index_type, index
from config import top_k, search_param
class VecToMilvus():
class VecToMilvus:
def __init__(self):
self.client = Milvus(host=MILVUS_HOST, port=MILVUS_PORT)
@@ -31,7 +30,7 @@ class VecToMilvus():
def creat_collection(self, collection_name):
try:
collection_param['collection_name'] = collection_name
collection_param["collection_name"] = collection_name
status = self.client.create_collection(collection_param)
print(status)
return status
@@ -40,8 +39,7 @@ class VecToMilvus():
def create_index(self, collection_name):
try:
status = self.client.create_index(collection_name, index_type,
index_param)
status = self.client.create_index(collection_name, index_type, index_param)
print(status)
return status
except Exception as e:
@@ -49,47 +47,42 @@ class VecToMilvus():
def has_partition(self, collection_name, partition_tag):
try:
status, ok = self.client.has_partition(collection_name,
partition_tag)
status, ok = self.client.has_partition(collection_name, partition_tag)
return ok
except Exception as e:
print("Milvus has partition error: ", e)
def create_partition(self, collection_name, partition_tag):
try:
status = self.client.create_partition(collection_name,
partition_tag)
print('create partition {} successfully'.format(partition_tag))
status = self.client.create_partition(collection_name, partition_tag)
print("create partition {} successfully".format(partition_tag))
return status
except Exception as e:
print('Milvus create partition error: ', e)
print("Milvus create partition error: ", e)
def insert(self, vectors, collection_name, ids=None, partition_tag=None):
try:
if not self.has_collection(collection_name):
self.creat_collection(collection_name)
self.create_index(collection_name)
print('collection info: {}'.format(
self.client.get_collection_info(collection_name)[1]))
if (partition_tag is not None) and (not self.has_partition(
collection_name, partition_tag)):
print("collection info: {}".format(self.client.get_collection_info(collection_name)[1]))
if (partition_tag is not None) and (not self.has_partition(collection_name, partition_tag)):
self.create_partition(collection_name, partition_tag)
status, ids = self.client.insert(collection_name=collection_name,
records=vectors,
ids=ids,
partition_tag=partition_tag)
status, ids = self.client.insert(
collection_name=collection_name, records=vectors, ids=ids, partition_tag=partition_tag
)
self.client.flush([collection_name])
print(
'Insert {} entities, there are {} entities after insert data.'.
format(len(ids),
self.client.count_entities(collection_name)[1]))
"Insert {} entities, there are {} entities after insert data.".format(
len(ids), self.client.count_entities(collection_name)[1]
)
)
return status, ids
except Exception as e:
print("Milvus insert error:", e)
class RecallByMilvus():
class RecallByMilvus:
def __init__(self):
self.client = Milvus(host=MILVUS_HOST, port=MILVUS_PORT)
@@ -100,7 +93,8 @@ class RecallByMilvus():
query_records=vectors,
top_k=top_k,
params=search_param,
partition_tag=partition_tag)
partition_tag=partition_tag,
)
return status, results
except Exception as e:
print('Milvus recall error: ', e)
print("Milvus recall error: ", e)

Some files were not shown because too many files have changed in this diff Show More