Kimirock/PaddleNLP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[TaskFlow] Fix pir for taskflow (#9822)

Browse Source 
* fix pir for taskflow

* update suffix for PIR(ON/OFF)

* fix and revert skip

* update dependency_parsing

* fix pir bug

* fix pir bug

* fix lint

* fix ci

* disable test case

---------

Co-authored-by: Zhong Hui <zhonghui.net@gmail.com>
Co-authored-by: Fantasy-02 <1026931135@qq.com>
This commit is contained in:
Weiguo Zhu
2025-02-20 10:21:53 +08:00
committed by GitHub
parent 11684064d6
commit 777fc2793f
18 changed files with 454 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
llm/predict/predictor.py
View File

@@ -24,7 +24,7 @@ from threading import Thread
import numpy as np
import paddle
import paddle.incubate.multiprocessing as mp
from paddle.base.framework import in_cinn_mode, in_pir_executor_mode, use_pir_api
from paddle.base.framework import in_cinn_mode, in_pir_executor_mode
from paddle.distributed import fleet
try:
@@ -51,7 +51,12 @@ from paddlenlp.transformers import (
PretrainedTokenizer,
)
from paddlenlp.trl import llm_utils
from paddlenlp.utils.env import MAX_BSZ, MAX_DRAFT_TOKENS
from paddlenlp.utils.env import (
MAX_BSZ,
MAX_DRAFT_TOKENS,
PADDLE_INFERENCE_MODEL_SUFFIX,
PADDLE_INFERENCE_WEIGHTS_SUFFIX,
)
from paddlenlp.utils.import_utils import is_paddlenlp_ops_available
from paddlenlp.utils.log import logger
@@ -670,10 +675,11 @@ class StaticGraphInferencePredictor(InferencePredictorMixin):
infer_model_path = llm_utils.get_infer_model_path(
predictor_args.model_name_or_path, predictor_args.model_prefix
)
if use_pir_api():
config = paddle.inference.Config(infer_model_path + ".json", infer_model_path + ".pdiparams")
else:
config = paddle.inference.Config(infer_model_path + ".pdmodel", infer_model_path + ".pdiparams")
config = paddle.inference.Config(
infer_model_path + PADDLE_INFERENCE_MODEL_SUFFIX,
infer_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX,
)
config.switch_ir_optim(True)
# remove `gpu_cpu_map_matmul_v2_to_matmul_pass` to avoid mapping matmul_v2 -> matmul op
@@ -1105,7 +1111,7 @@ class DygraphBlockInferencePredictor(BlockInferencePredictorMixin):
self.full_hidden_states = self._infer(self.model_inputs)
else:
self._infer(self.model_inputs)
logger.info(f"running spend {time.time() - s_time}")
logger.info(f"running spend {time.time() - s_time}")
if self.proposer is not None:
self.proposer.postprocess(base_model_inputs=self.model_inputs)
@@ -1192,10 +1198,10 @@ class StaticGraphBlockInferencePredictor(BlockInferencePredictorMixin):
predictor_args.model_name_or_path, predictor_args.model_prefix
)
if use_pir_api():
config = paddle.inference.Config(infer_model_path + ".json", infer_model_path + ".pdiparams")
else:
config = paddle.inference.Config(infer_model_path + ".pdmodel", infer_model_path + ".pdiparams")
config = paddle.inference.Config(
infer_model_path + PADDLE_INFERENCE_MODEL_SUFFIX,
infer_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX,
)
config.switch_ir_optim(False)
if predictor_args.device in paddle.device.get_all_custom_device_type():
@@ -1232,7 +1238,7 @@ class StaticGraphBlockInferencePredictor(BlockInferencePredictorMixin):
self.proposer.insert_query(
base_model_inputs=self.model_inputs, real_bs=len(input_texts), seq_lens=self.seq_lens
)
logger.info(f"preprocess spend {time.time() - s_time}")
logger.info(f"preprocess spend {time.time() - s_time}")
result_queue = mp.Queue()
tensor_queue = mp.Queue()
@@ -1271,7 +1277,7 @@ class StaticGraphBlockInferencePredictor(BlockInferencePredictorMixin):
self.full_hidden_states = self.predictor.run(list(self.model_inputs.values()))[0]
else:
self.predictor.run(list(self.model_inputs.values()))
logger.info(f"running spend {time.time() - s_time}")
logger.info(f"running spend {time.time() - s_time}")
if self.proposer is not None:
self.proposer.postprocess(base_model_inputs=self.model_inputs)
@@ -1305,7 +1311,7 @@ class AutoPredictor:
config: PretrainedConfig,
model_args: ModelArgument,
tokenizer: PretrainedTokenizer = None,
**kwargs
**kwargs,
):
"""
Create a predictor

2
llm/server/docs/deploy_usage_tutorial.md
View File

@@ -99,7 +99,7 @@ cd /home/workspace/models_dir
# ├── rank_mapping.csv # 多卡模型会有此文件，如为单卡模型，则无此文件（可选，仅在多卡部署模式下需要）
# └── rank_0 # 保存模型结构和权重文件的目录
# ├── model.pdiparams
# └── model.pdmodel
# └── model.pdmodel 或者 model.json # Paddle 3.0 版本模型为model.json，Paddle 2.x 版本模型为model.pdmodel
```
### 创建容器

454
llm/server/server/server/engine/infer.py
View File

@@ -26,14 +26,19 @@ import paddle
import paddle.distributed as dist
import paddle.distributed.fleet as fleet
from paddle.base.framework import use_pir_api
from paddlenlp.trl.llm_utils import get_rotary_position_embedding
from paddlenlp_ops import step_paddle
from server.data.processor import DataProcessor
from server.engine.config import Config
from paddlenlp.experimental.transformers import InferenceWithReferenceProposer
from server.utils import get_logger
from task_queue_manager import TaskQueueManager
from paddlenlp.experimental.transformers import InferenceWithReferenceProposer
from paddlenlp.trl.llm_utils import get_rotary_position_embedding
from paddlenlp.utils.env import (
PADDLE_INFERENCE_MODEL_SUFFIX,
PADDLE_INFERENCE_WEIGHTS_SUFFIX,
)
File_Path = os.path.realpath(sys.argv[0])
Dir_Path = os.path.dirname(File_Path)
logger = get_logger("infer_server", "infer.log")
@@ -75,13 +80,14 @@ class ModelRunner:
self.init_inputs()
if self.is_speculate_decoding:
logger.info(f'Using speculate decoding, method: {self.speculate_config.speculate_method}.')
logger.info(f"Using speculate decoding, method: {self.speculate_config.speculate_method}.")
if self.speculate_config.speculate_method == "inference_with_reference":
self.proposer = InferenceWithReferenceProposer(
self.speculate_config.speculate_max_draft_token_num,
self.speculate_config.speculate_max_ngram_size,
self.args.max_batch_size,
self.args.max_seq_len)
self.args.max_seq_len,
)
else:
self.proposer = None
@@ -91,12 +97,13 @@ class ModelRunner:
if not os.path.exists(model_rank_path):
model_rank_path = self.args.model_dir
self.infer_engine = InferenceEngine(model_dir=model_rank_path,
share_inputs=self.share_inputs,
cache_kvs=self.cache_kvs,
config=self.config,
mp_degree=self.nranks
)
self.infer_engine = InferenceEngine(
model_dir=model_rank_path,
share_inputs=self.share_inputs,
cache_kvs=self.cache_kvs,
config=self.config,
mp_degree=self.nranks,
)
def read_model_config(self):
"""
@@ -105,7 +112,7 @@ class ModelRunner:
Returns:
model_config_json: dict, model config file
"""
model_config_json = json.load(open(self.config_file, 'r', encoding='utf-8'))
model_config_json = json.load(open(self.config_file, "r", encoding="utf-8"))
return model_config_json
def get_value(self, cfg, names):
@@ -118,9 +125,7 @@ class ModelRunner:
if name in cfg:
return cfg[name]
break
raise Exception(
"Cannot find any one of key in {} in configuration file.".format(
names))
raise Exception("Cannot find any one of key in {} in configuration file.".format(names))
def format_print_configuration(self):
"""
@@ -140,13 +145,13 @@ class ModelRunner:
"""
self.top_p = self.model_cfg.get("top_p", 0.0)
self.temperature = self.model_cfg.get("temperature", 1.0)
self.rope_theta = self.model_cfg.get('rope_theta', 10000.0)
self.rope_scaling = self.model_cfg.get('rope_scaling', None)
self.penalty_score = self.model_cfg.get('penalty_score', 1.0)
self.frequency_score = self.model_cfg.get('frequency_score', 0.0)
self.presence_score = self.model_cfg.get('presence_score', 0.0)
self.min_length = self.model_cfg.get('min_length', 1)
self.max_length = self.model_cfg.get('max_length', 1024)
self.rope_theta = self.model_cfg.get("rope_theta", 10000.0)
self.rope_scaling = self.model_cfg.get("rope_scaling", None)
self.penalty_score = self.model_cfg.get("penalty_score", 1.0)
self.frequency_score = self.model_cfg.get("frequency_score", 0.0)
self.presence_score = self.model_cfg.get("presence_score", 0.0)
self.min_length = self.model_cfg.get("min_length", 1)
self.max_length = self.model_cfg.get("max_length", 1024)
data_processor = DataProcessor()
# reserve an eos token for request
@@ -172,9 +177,11 @@ class ModelRunner:
def init_inputs(self):
# init all inputs
if "num_key_value_heads" in self.model_cfg and \
self.model_cfg["num_key_value_heads"] is not None and \
int(self.model_cfg["num_key_value_heads"]) > 0:
if (
"num_key_value_heads" in self.model_cfg
and self.model_cfg["num_key_value_heads"] is not None
and int(self.model_cfg["num_key_value_heads"]) > 0
):
kv_num_head = int(self.model_cfg["num_key_value_heads"]) // self.nranks
else:
kv_num_head = self.args.num_attention_heads // self.nranks
@@ -205,87 +212,108 @@ class ModelRunner:
self.args.block_size, self.args.hidden_size // self.args.num_attention_heads
], fill_value=0, dtype=cache_type)
pre_max_block_num = (self.args.max_seq_len + self.args.block_size - 1) // self.args.block_size + self.args.enc_dec_block_num
pre_max_block_num = (
self.args.max_seq_len + self.args.block_size - 1
) // self.args.block_size + self.args.enc_dec_block_num
self.share_inputs["block_tables"] = paddle.full(
shape=[self.args.max_batch_size, pre_max_block_num], fill_value=-1, dtype="int32")
shape=[self.args.max_batch_size, pre_max_block_num], fill_value=-1, dtype="int32"
)
self.share_inputs['pre_ids'] = paddle.to_tensor(
np.full((self.args.max_batch_size, self.args.max_dec_len), -1, dtype='int64'))
self.share_inputs["pre_ids"] = paddle.to_tensor(
np.full((self.args.max_batch_size, self.args.max_dec_len), -1, dtype="int64")
)
tmp_position_ids = paddle.arange(self.args.max_seq_len).reshape((1, -1))
self.share_inputs['rope_emb'] = get_rotary_position_embedding(tmp_position_ids,
self.args.hidden_size // self.args.num_attention_heads,
self.rope_theta, self.rope_scaling)
self.share_inputs['input_ids'] = paddle.full(
shape=[self.args.max_batch_size, self.args.max_seq_len],
fill_value=self.pad_token_id, dtype='int64')
self.share_inputs['top_p'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.top_p, dtype="float32")
self.share_inputs['temperature'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.temperature, dtype="float32")
self.share_inputs['eos_token_id'] = paddle.to_tensor(
np.zeros((self.eos_tokens_lens, 1)).reshape(-1, 1).astype("int64"))
self.share_inputs['penalty_score'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.penalty_score, dtype="float32")
self.share_inputs['frequency_score'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.frequency_score, dtype="float32")
self.share_inputs['presence_score'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.presence_score, dtype="float32")
self.share_inputs['seq_lens_this_time'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32")
self.share_inputs['seq_lens_encoder'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32")
self.share_inputs['step_seq_lens_encoder'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32")
self.share_inputs['seq_lens_decoder'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32")
self.share_inputs['step_idx'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int64")
self.share_inputs['min_length'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.min_length, dtype="int64")
self.share_inputs['max_length'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.max_length, dtype="int64")
self.share_inputs['not_need_stop'] = paddle.full(
shape=[1], fill_value=False, dtype="bool")
self.share_inputs['stop_flags'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=True, dtype="bool")
self.share_inputs['stop_nums'] = paddle.full(
shape=[1], fill_value=self.args.max_batch_size, dtype="int64")
self.share_inputs['bad_tokens'] = paddle.full(
shape=[1], fill_value=-1, dtype="int64")
self.share_inputs['next_tokens'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=-1, dtype="int64")
self.share_inputs['is_block_step'] = paddle.full(
shape=[self.args.max_batch_size], fill_value=False, dtype="bool")
self.share_inputs['encoder_block_lens'] = paddle.full(
shape=[self.args.max_batch_size], fill_value=0, dtype="int32")
self.share_inputs['step_block_list'] = paddle.full(
shape=[self.args.max_batch_size], fill_value=-1, dtype="int32")
self.share_inputs['step_lens'] = paddle.full(shape=[1], fill_value=0, dtype="int32")
self.share_inputs['recover_block_list'] = paddle.full(
shape=[self.args.max_batch_size], fill_value=-1, dtype="int32")
self.share_inputs['recover_lens'] = paddle.full(
shape=[1], fill_value=0, dtype="int32")
self.share_inputs['need_block_list'] = paddle.full(
shape=[self.args.max_batch_size], fill_value=-1, dtype="int32")
self.share_inputs['need_block_len'] = paddle.full(
shape=[1], fill_value=0, dtype="int32")
self.share_inputs['used_list_len'] = paddle.full(
shape=[self.args.max_batch_size], fill_value=0, dtype="int32")
self.share_inputs['infer_seed'] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int64")
self.share_inputs["rope_emb"] = get_rotary_position_embedding(
tmp_position_ids,
self.args.hidden_size // self.args.num_attention_heads,
self.rope_theta,
self.rope_scaling,
)
self.share_inputs["input_ids"] = paddle.full(
shape=[self.args.max_batch_size, self.args.max_seq_len], fill_value=self.pad_token_id, dtype="int64"
)
self.share_inputs["top_p"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.top_p, dtype="float32"
)
self.share_inputs["temperature"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.temperature, dtype="float32"
)
self.share_inputs["eos_token_id"] = paddle.to_tensor(
np.zeros((self.eos_tokens_lens, 1)).reshape(-1, 1).astype("int64")
)
self.share_inputs["penalty_score"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.penalty_score, dtype="float32"
)
self.share_inputs["frequency_score"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.frequency_score, dtype="float32"
)
self.share_inputs["presence_score"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.presence_score, dtype="float32"
)
self.share_inputs["seq_lens_this_time"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32"
)
self.share_inputs["seq_lens_encoder"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32"
)
self.share_inputs["step_seq_lens_encoder"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32"
)
self.share_inputs["seq_lens_decoder"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32"
)
self.share_inputs["step_idx"] = paddle.full(shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int64")
self.share_inputs["min_length"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.min_length, dtype="int64"
)
self.share_inputs["max_length"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=self.max_length, dtype="int64"
)
self.share_inputs["not_need_stop"] = paddle.full(shape=[1], fill_value=False, dtype="bool")
self.share_inputs["stop_flags"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=True, dtype="bool"
)
self.share_inputs["stop_nums"] = paddle.full(shape=[1], fill_value=self.args.max_batch_size, dtype="int64")
self.share_inputs["bad_tokens"] = paddle.full(shape=[1], fill_value=-1, dtype="int64")
self.share_inputs["next_tokens"] = paddle.full(
shape=[self.args.max_batch_size, 1], fill_value=-1, dtype="int64"
)
self.share_inputs["is_block_step"] = paddle.full(
shape=[self.args.max_batch_size], fill_value=False, dtype="bool"
)
self.share_inputs["encoder_block_lens"] = paddle.full(
shape=[self.args.max_batch_size], fill_value=0, dtype="int32"
)
self.share_inputs["step_block_list"] = paddle.full(
shape=[self.args.max_batch_size], fill_value=-1, dtype="int32"
)
self.share_inputs["step_lens"] = paddle.full(shape=[1], fill_value=0, dtype="int32")
self.share_inputs["recover_block_list"] = paddle.full(
shape=[self.args.max_batch_size], fill_value=-1, dtype="int32"
)
self.share_inputs["recover_lens"] = paddle.full(shape=[1], fill_value=0, dtype="int32")
self.share_inputs["need_block_list"] = paddle.full(
shape=[self.args.max_batch_size], fill_value=-1, dtype="int32"
)
self.share_inputs["need_block_len"] = paddle.full(shape=[1], fill_value=0, dtype="int32")
self.share_inputs["used_list_len"] = paddle.full(shape=[self.args.max_batch_size], fill_value=0, dtype="int32")
self.share_inputs["infer_seed"] = paddle.full(shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int64")
free_list = list(range(int(self.args.max_block_num * self.args.block_ratio)))
self.free_list_len = len(free_list)
self.share_inputs['free_list'] = paddle.to_tensor(free_list, dtype="int32")
self.share_inputs['free_list_len'] = paddle.full(
shape=[1], fill_value=self.free_list_len, dtype="int32")
self.share_inputs["free_list"] = paddle.to_tensor(free_list, dtype="int32")
self.share_inputs["free_list_len"] = paddle.full(shape=[1], fill_value=self.free_list_len, dtype="int32")
self.share_inputs['stop_seqs_len'] = paddle.full(shape=[self.max_stop_seqs_num,],
fill_value=0,
dtype="int32")
self.share_inputs['stop_seqs'] = paddle.full(shape=[self.max_stop_seqs_num, self.stop_seqs_max_len],
fill_value=-1,
dtype="int64")
self.share_inputs["stop_seqs_len"] = paddle.full(
shape=[
self.max_stop_seqs_num,
],
fill_value=0,
dtype="int32",
)
self.share_inputs["stop_seqs"] = paddle.full(
shape=[self.max_stop_seqs_num, self.stop_seqs_max_len], fill_value=-1, dtype="int64"
)
self.share_inputs["first_token_ids"] = paddle.full(
@@ -295,14 +323,22 @@ class ModelRunner:
# speculate decoding input
if self.is_speculate_decoding:
self.share_inputs["accept_tokens"] = paddle.full(
shape=[self.args.max_batch_size, self.speculate_config.speculate_max_draft_token_num + 1], fill_value=0, dtype="int64"
shape=[self.args.max_batch_size, self.speculate_config.speculate_max_draft_token_num + 1],
fill_value=0,
dtype="int64",
)
self.share_inputs["accept_num"] = paddle.full(
shape=[self.args.max_batch_size], fill_value=0, dtype="int32"
)
self.share_inputs["accept_num"] = paddle.full(shape=[self.args.max_batch_size], fill_value=0, dtype="int32")
self.share_inputs["draft_tokens"] = paddle.full(
shape=[self.args.max_batch_size, self.speculate_config.speculate_max_draft_token_num + 1], fill_value=0, dtype="int64"
shape=[self.args.max_batch_size, self.speculate_config.speculate_max_draft_token_num + 1],
fill_value=0,
dtype="int64",
)
self.share_inputs["actual_draft_token_num"] = paddle.full(
shape=[self.args.max_batch_size], fill_value=self.speculate_config.speculate_max_draft_token_num, dtype="int32"
shape=[self.args.max_batch_size],
fill_value=self.speculate_config.speculate_max_draft_token_num,
dtype="int32",
)
def dy_input_preprocess(self, tasks):
@@ -311,58 +347,63 @@ class ModelRunner:
"""
for i in range(len(tasks)):
task = tasks[i]
idx = task['idx']
length = len(task['input_ids'])
self.share_inputs['input_ids'][idx:idx + 1, :length] = np.array(task['input_ids'])
if len(task['eos_token_ids']) < self.eos_tokens_lens:
task['eos_token_ids'].append(task['eos_token_ids'][0])
self.share_inputs['eos_token_id'][:] = np.array(task['eos_token_ids'], dtype="int64").reshape(-1, 1)
self.share_inputs['pre_ids'][idx:idx + 1] = -1
self.share_inputs['top_p'][idx:idx + 1] = task.get('topp', 0.7)
self.share_inputs['temperature'][idx:idx + 1] = task.get('temperature', 0.95)
self.share_inputs['penalty_score'][idx:idx + 1] = task.get('penalty_score', 1.0)
self.share_inputs['frequency_score'][idx:idx + 1] = task.get('frequency_score', 0.0)
self.share_inputs['presence_score'][idx:idx + 1] = task.get('presence_score', 0.0)
self.share_inputs['seq_lens_this_time'][idx:idx + 1] = length
self.share_inputs['step_seq_lens_encoder'][idx:idx + 1] = length
self.share_inputs['seq_lens_encoder'][idx:idx + 1] = length
self.share_inputs['seq_lens_decoder'][idx:idx + 1] = 0
self.share_inputs['step_idx'][idx:idx + 1] = 0
self.share_inputs['min_length'][idx:idx + 1] = task.get('min_dec_len', 1)
idx = task["idx"]
length = len(task["input_ids"])
self.share_inputs["input_ids"][idx : idx + 1, :length] = np.array(task["input_ids"])
if len(task["eos_token_ids"]) < self.eos_tokens_lens:
task["eos_token_ids"].append(task["eos_token_ids"][0])
self.share_inputs["eos_token_id"][:] = np.array(task["eos_token_ids"], dtype="int64").reshape(-1, 1)
self.share_inputs["pre_ids"][idx : idx + 1] = -1
self.share_inputs["top_p"][idx : idx + 1] = task.get("topp", 0.7)
self.share_inputs["temperature"][idx : idx + 1] = task.get("temperature", 0.95)
self.share_inputs["penalty_score"][idx : idx + 1] = task.get("penalty_score", 1.0)
self.share_inputs["frequency_score"][idx : idx + 1] = task.get("frequency_score", 0.0)
self.share_inputs["presence_score"][idx : idx + 1] = task.get("presence_score", 0.0)
self.share_inputs["seq_lens_this_time"][idx : idx + 1] = length
self.share_inputs["step_seq_lens_encoder"][idx : idx + 1] = length
self.share_inputs["seq_lens_encoder"][idx : idx + 1] = length
self.share_inputs["seq_lens_decoder"][idx : idx + 1] = 0
self.share_inputs["step_idx"][idx : idx + 1] = 0
self.share_inputs["min_length"][idx : idx + 1] = task.get("min_dec_len", 1)
if "max_dec_len" in task:
max_dec_len = task['max_dec_len']
max_dec_len = task["max_dec_len"]
elif "seq_len" in task:
max_dec_len = task['seq_len']
max_dec_len = task["seq_len"]
else:
max_dec_len = self.args.max_dec_len
self.share_inputs['max_length'][idx:idx + 1] = max_dec_len
self.share_inputs['stop_flags'][idx:idx + 1] = False
self.share_inputs["max_length"][idx : idx + 1] = max_dec_len
self.share_inputs["stop_flags"][idx : idx + 1] = False
self.share_inputs['first_token_ids'][idx:idx + 1] = self.share_inputs['input_ids'][idx:idx + 1, :1]
self.share_inputs["ori_seq_lens_encoder"][idx:idx + 1] = length
if "infer_seed" in task:
self.share_inputs['infer_seed'][idx:idx + 1] = task['infer_seed']
self.share_inputs["infer_seed"][idx : idx + 1] = task["infer_seed"]
encoder_block_num = len(task['block_tables'])
self.share_inputs['encoder_block_lens'][idx:idx + 1] = encoder_block_num
self.share_inputs["block_tables"][idx:idx + 1, :] = -1
self.share_inputs["block_tables"][idx:idx + 1, :encoder_block_num] = np.array(
task['block_tables'], dtype="int32")
encoder_block_num = len(task["block_tables"])
self.share_inputs["encoder_block_lens"][idx : idx + 1] = encoder_block_num
self.share_inputs["block_tables"][idx : idx + 1, :] = -1
self.share_inputs["block_tables"][idx : idx + 1, :encoder_block_num] = np.array(
task["block_tables"], dtype="int32"
)
if "stop_seqs_len" in task:
stop_seqs_num = len(task["stop_seqs_len"])
for i in range(stop_seqs_num, self.max_stop_seqs_num):
task["stop_seqs_len"].append(0)
self.share_inputs['stop_seqs_len'][:] = np.array(
task["stop_seqs_len"], dtype="int32")
self.share_inputs['stop_seqs'][:stop_seqs_num, :len(task['stop_seqs'][0])] = np.array(
task["stop_seqs"], dtype="int64")
self.share_inputs["stop_seqs_len"][:] = np.array(task["stop_seqs_len"], dtype="int32")
self.share_inputs["stop_seqs"][:stop_seqs_num, : len(task["stop_seqs"][0])] = np.array(
task["stop_seqs"], dtype="int64"
)
if self.is_speculate_decoding:
self.share_inputs["draft_tokens"][idx:idx + 1] = np.zeros([self.speculate_config.speculate_max_draft_token_num + 1])
self.share_inputs["actual_draft_token_num"][idx:idx + 1] = np.array([self.speculate_config.speculate_max_draft_token_num])
self.share_inputs["draft_tokens"][idx : idx + 1] = np.zeros(
[self.speculate_config.speculate_max_draft_token_num + 1]
)
self.share_inputs["actual_draft_token_num"][idx : idx + 1] = np.array(
[self.speculate_config.speculate_max_draft_token_num]
)
def step_cuda(self, seq_lens_this_time):
"""
@@ -398,10 +439,11 @@ class ModelRunner:
"""
engine_ready_check_flag = np.zeros([1], dtype=np.int32)
shm_engine_ready_check_flag = shared_memory.SharedMemory(
name=self.config.get_unique_name("engine_ready_check_flag"))
engine_ready_check_flag_array = np.ndarray(engine_ready_check_flag.shape,
dtype=engine_ready_check_flag.dtype,
buffer=shm_engine_ready_check_flag.buf)
name=self.config.get_unique_name("engine_ready_check_flag")
)
engine_ready_check_flag_array = np.ndarray(
engine_ready_check_flag.shape, dtype=engine_ready_check_flag.dtype, buffer=shm_engine_ready_check_flag.buf
)
return shm_engine_ready_check_flag, engine_ready_check_flag_array
def initialize_engine_live_flag(self):
@@ -411,9 +453,9 @@ class ModelRunner:
Returns:
infer_live_flag_shm: infer live flag
"""
infer_live_flag_shm = shared_memory.SharedMemory(create=True,
size=1,
name=self.config.get_unique_name("shm_flag_infer_{}_live".format(self.rank)))
infer_live_flag_shm = shared_memory.SharedMemory(
create=True, size=1, name=self.config.get_unique_name("shm_flag_infer_{}_live".format(self.rank))
)
return infer_live_flag_shm
def initialize_engine_healthy_recorded_time_flag(self):
@@ -425,10 +467,13 @@ class ModelRunner:
"""
engine_healthy_recorded_time = np.zeros([1], dtype=float)
shm_engine_healthy_recorded_time = shared_memory.SharedMemory(
name=self.config.get_unique_name("engine_healthy_recorded_time"))
engine_healthy_recorded_time_array = np.ndarray(engine_healthy_recorded_time.shape,
dtype=engine_healthy_recorded_time.dtype,
buffer=shm_engine_healthy_recorded_time.buf)
name=self.config.get_unique_name("engine_healthy_recorded_time")
)
engine_healthy_recorded_time_array = np.ndarray(
engine_healthy_recorded_time.shape,
dtype=engine_healthy_recorded_time.dtype,
buffer=shm_engine_healthy_recorded_time.buf,
)
return shm_engine_healthy_recorded_time, engine_healthy_recorded_time_array
def run(self):
@@ -437,35 +482,38 @@ class ModelRunner:
"""
flag_array = np.zeros([1], dtype=np.int32)
shm_flag_broadcast = shared_memory.SharedMemory(
name=self.config.get_unique_name("shm_pd_infer_flag_broadcast"))
flag_broadcast_array = np.ndarray(flag_array.shape,
dtype=flag_array.dtype,
buffer=shm_flag_broadcast.buf)
name=self.config.get_unique_name("shm_pd_infer_flag_broadcast")
)
flag_broadcast_array = np.ndarray(flag_array.shape, dtype=flag_array.dtype, buffer=shm_flag_broadcast.buf)
flag_array = np.zeros([self.nranks], dtype=np.int32)
shm_flag_ready = shared_memory.SharedMemory(name=self.config.get_unique_name("shm_flag_infer_ready"))
flag_ready_array = np.ndarray(flag_array.shape,
dtype=flag_array.dtype,
buffer=shm_flag_ready.buf)
flag_ready_array = np.ndarray(flag_array.shape, dtype=flag_array.dtype, buffer=shm_flag_ready.buf)
flag_ready_array[self.rank] = 1
flag_array = np.zeros([1], dtype=np.int32)
shm_flag_has_block_step = shared_memory.SharedMemory(name=self.config.get_unique_name("shm_flag_has_block_step"))
flag_has_block_step_array = np.ndarray(flag_array.shape,
dtype=flag_array.dtype,
buffer=shm_flag_has_block_step.buf)
shm_flag_has_block_step = shared_memory.SharedMemory(
name=self.config.get_unique_name("shm_flag_has_block_step")
)
flag_has_block_step_array = np.ndarray( # noqa: F841
flag_array.shape, dtype=flag_array.dtype, buffer=shm_flag_has_block_step.buf
)
use_custom_health_checker = self.config.use_custom_health_checker
if use_custom_health_checker:
shm_engine_ready_check_flag_array, engine_ready_check_flag_array = self.initialize_engine_ready_check_flag()
(
shm_engine_ready_check_flag_array,
engine_ready_check_flag_array,
) = self.initialize_engine_ready_check_flag()
engine_ready_check_flag_array[0] = 1
shm_engine_healthy_recorded_time_array, engine_healthy_recorded_time_array = self.initialize_engine_healthy_recorded_time_flag()
(
shm_engine_healthy_recorded_time_array,
engine_healthy_recorded_time_array,
) = self.initialize_engine_healthy_recorded_time_flag()
engine_healthy_recorded_time_array[0] = time.time()
infer_live_flag_shm = self.initialize_engine_live_flag()
infer_seed_increment = paddle.full(shape=[self.args.max_batch_size, 1],
fill_value=4,
dtype="int64")
thread_executor = ThreadPoolExecutor(max_workers=1)
infer_live_flag_shm = self.initialize_engine_live_flag() # noqa: F841
infer_seed_increment = paddle.full(shape=[self.args.max_batch_size, 1], fill_value=4, dtype="int64")
thread_executor = ThreadPoolExecutor(max_workers=1) # noqa: F841
seq_lens_this_time = None
real_bsz = None
@@ -497,18 +545,14 @@ class ModelRunner:
for req_dict, bsz in tasks:
real_bsz = int(bsz)
req_dicts.extend(req_dict)
logger.info(
f'rank: {self.rank}, real_bsz: {real_bsz}, query_num: {len(req_dicts)}'
)
logger.info(f"rank: {self.rank}, real_bsz: {real_bsz}, query_num: {len(req_dicts)}")
self.dy_input_preprocess(req_dicts)
seq_lens_this_time = copy.deepcopy(
self.share_inputs['seq_lens_this_time'][:real_bsz])
self.infer_engine.seq_lens_handle.share_external_data(
seq_lens_this_time)
self.share_inputs['not_need_stop'][0] = True
seq_lens_this_time = copy.deepcopy(self.share_inputs["seq_lens_this_time"][:real_bsz])
self.infer_engine.seq_lens_handle.share_external_data(seq_lens_this_time)
self.share_inputs["not_need_stop"][0] = True
if not self.share_inputs['not_need_stop']:
if not self.share_inputs["not_need_stop"]:
if self.nranks > 1:
paddle.distributed.barrier()
@@ -523,8 +567,8 @@ class ModelRunner:
)
self.infer_engine.predictor.run()
self.share_inputs['infer_seed'].add_(infer_seed_increment)
self.share_inputs['infer_seed'][:] %= self.MAX_INFER_SEED
self.share_inputs["infer_seed"].add_(infer_seed_increment)
self.share_inputs["infer_seed"][:] %= self.MAX_INFER_SEED
if self.free_list_len > 0:
self.step_cuda(seq_lens_this_time)
@@ -537,6 +581,7 @@ class InferenceEngine(object):
model_dir (string): root directory of inference model
mp_degree (int): model parallel size
"""
def __init__(self, model_dir, share_inputs, cache_kvs, config, mp_degree=1):
self.config = config
self.model_dir = model_dir
@@ -576,7 +621,7 @@ class InferenceEngine(object):
self.predictor = paddle.inference.create_predictor(config)
self.input_names = self.predictor.get_input_names()
self.seq_lens_handle = self.predictor.get_input_handle('seq_lens_this_time')
self.seq_lens_handle = self.predictor.get_input_handle("seq_lens_this_time")
def share_data(self):
"""
@@ -599,51 +644,18 @@ def parse_args():
parse args from command line
"""
parser = argparse.ArgumentParser("FastDeploy LLM Inference")
parser.add_argument('-m',
'--model_dir',
type=str,
default='./output',
help='model dir')
parser.add_argument('-mp',
'--mp_degree',
type=int,
default=1,
help='mp degree')
parser.add_argument('-mbs',
'--max_batch_size',
type=int,
default=34,
help='max batch size')
parser.add_argument('--max_block_num', type=int, default=2000)
parser.add_argument("-m", "--model_dir", type=str, default="./output", help="model dir")
parser.add_argument("-mp", "--mp_degree", type=int, default=1, help="mp degree")
parser.add_argument("-mbs", "--max_batch_size", type=int, default=34, help="max batch size")
parser.add_argument("--max_block_num", type=int, default=2000)
parser.add_argument("--block_size", type=int, default=128)
parser.add_argument('--max_seq_len',
type=int,
default=3072,
help='max_seq_len')
parser.add_argument('--max_dec_len',
type=int,
default=1024,
help='max_dec_len')
parser.add_argument('--use_cache_kv_int8',
type=int,
default=0,
help='use cache kv int8')
parser.add_argument('--dtype',
type=str,
default="bfloat16",
help='input dtype')
parser.add_argument('--enc_dec_block_num',
type=int,
default=1,
help="encoder's decoder num")
parser.add_argument('--block_ratio',
type=float,
default=0.7,
help="block ratio")
parser.add_argument('--first_token_id',
type=int,
default=1,
help="first token id")
parser.add_argument("--max_seq_len", type=int, default=3072, help="max_seq_len")
parser.add_argument("--max_dec_len", type=int, default=1024, help="max_dec_len")
parser.add_argument("--use_cache_kv_int8", type=int, default=0, help="use cache kv int8")
parser.add_argument("--dtype", type=str, default="bfloat16", help="input dtype")
parser.add_argument("--enc_dec_block_num", type=int, default=1, help="encoder's decoder num")
parser.add_argument("--block_ratio", type=float, default=0.7, help="block ratio")
parser.add_argument("--first_token_id", type=int, default=1, help="first token id")
args = parser.parse_args()
return args

17
paddlenlp/experimental/autonlp/text_classification.py
View File

@@ -47,6 +47,7 @@ from ...transformers import (
from ...utils.log import logger
from .auto_trainer_base import AutoTrainerBase
from .utils import UTCLoss
from .utils.env import PADDLE_INFERENCE_MODEL_SUFFIX, PADDLE_INFERENCE_WEIGHTS_SUFFIX
class AutoTrainerForTextClassification(AutoTrainerBase):
@@ -560,16 +561,16 @@ class AutoTrainerForTextClassification(AutoTrainerBase):
if os.path.exists(default_export_path):
if "utc" in model_config["model_name_or_path"]:
files = [
"model.pdiparams",
"model.pdmodel",
f"model{PADDLE_INFERENCE_WEIGHTS_SUFFIX}",
f"model{PADDLE_INFERENCE_MODEL_SUFFIX}",
"tokenizer_config.json",
"vocab.txt",
"taskflow_config.json",
]
else:
files = [
"model.pdiparams",
"model.pdmodel",
f"model{PADDLE_INFERENCE_WEIGHTS_SUFFIX}",
f"model{PADDLE_INFERENCE_MODEL_SUFFIX}",
"tokenizer_config.json",
"vocab.txt",
"taskflow_config.json",
@@ -735,8 +736,8 @@ class AutoTrainerForTextClassification(AutoTrainerBase):
executor=exe,
batch_generator=_batch_generator_func,
model_dir=export_path,
model_filename="model.pdmodel",
params_filename="model.pdiparams",
model_filename=f"model{PADDLE_INFERENCE_MODEL_SUFFIX}",
params_filename=f"model{PADDLE_INFERENCE_WEIGHTS_SUFFIX}",
batch_size=batch_size,
batch_nums=batch_nums,
scope=None,
@@ -757,8 +758,8 @@ class AutoTrainerForTextClassification(AutoTrainerBase):
post_training_quantization.quantize()
post_training_quantization.save_quantized_model(
save_model_path=compress_path,
model_filename="model.pdmodel",
params_filename="model.pdiparams",
model_filename=f"model{PADDLE_INFERENCE_MODEL_SUFFIX}",
params_filename=f"model{PADDLE_INFERENCE_WEIGHTS_SUFFIX}",
)
paddle.disable_static()

57
paddlenlp/generation/utils.py
View File

@@ -1437,16 +1437,8 @@ class GenerationMixin(object):
outputs, input_ids, cur_len_gpu, origin_len_gpu, scores, unfinished_flag, model_kwargs, pad_token_id
)
if hasattr(paddle.framework, "_no_check_dy2st_diff"):
# TODO(daisiming): _no_check_dy2st_diff is used to turn off the checking of behavior
# inconsistency between dynamic graph and static graph. _no_check_dy2st_diff should be
# removed after static graphs support inplace and stride.
with paddle.framework._no_check_dy2st_diff():
paddle.increment(cur_len)
paddle.increment(cur_len_gpu)
else:
paddle.increment(cur_len)
paddle.increment(cur_len_gpu)
cur_len += 1
cur_len_gpu += 1
attn_mask = model_kwargs["attention_mask"]
# make the shape of attention_mask = (-1, -1, -1, -1) in dy2static.
@@ -1454,38 +1446,19 @@ class GenerationMixin(object):
model_kwargs["cache"] = outputs[1] if isinstance(outputs, tuple) else None
max_new_tokens = paddle.full([1], max_new_tokens + cur_len - 1, dtype="int64")
if hasattr(paddle.framework, "_no_check_dy2st_diff"):
# TODO(daisiming): _no_check_dy2st_diff is used to turn off the checking of behavior
# inconsistency between dynamic graph and static graph. _no_check_dy2st_diff should be
# removed after static graphs support inplace and stride.
with paddle.framework._no_check_dy2st_diff():
while cur_len < max_new_tokens and paddle.any(unfinished_flag):
input_ids, scores, unfinished_flag, model_kwargs = _post_process_(
_forward_(**model_kwargs),
input_ids,
cur_len_gpu,
origin_len_gpu,
scores,
unfinished_flag,
model_kwargs,
pad_token_id,
)
paddle.increment(cur_len)
paddle.increment(cur_len_gpu)
else:
while cur_len < max_new_tokens and paddle.any(unfinished_flag):
input_ids, scores, unfinished_flag, model_kwargs = _post_process_(
_forward_(**model_kwargs),
input_ids,
cur_len_gpu,
origin_len_gpu,
scores,
unfinished_flag,
model_kwargs,
pad_token_id,
)
paddle.increment(cur_len)
paddle.increment(cur_len_gpu)
while cur_len < max_new_tokens and paddle.any(unfinished_flag):
input_ids, scores, unfinished_flag, model_kwargs = _post_process_(
_forward_(**model_kwargs),
input_ids,
cur_len_gpu,
origin_len_gpu,
scores,
unfinished_flag,
model_kwargs,
pad_token_id,
)
cur_len += 1
cur_len_gpu += 1
return input_ids[:, origin_len:], scores

16
paddlenlp/server/predictor.py
View File

@@ -21,6 +21,7 @@ from multiprocessing import cpu_count
import paddle
from ..utils.env import PADDLE_INFERENCE_MODEL_SUFFIX, PADDLE_INFERENCE_WEIGHTS_SUFFIX
from ..utils.log import logger
@@ -40,13 +41,15 @@ class Predictor:
def _get_default_static_model_path(self):
# The model path had the static_model_path
static_model_path = os.path.join(self._model_path, self._default_static_model_path, "inference.pdmodel")
static_model_path = os.path.join(
self._model_path, self._default_static_model_path, f"inference{PADDLE_INFERENCE_MODEL_SUFFIX}"
)
if os.path.exists(static_model_path):
return os.path.join(self._model_path, self._default_static_model_path, "inference")
for file_name in os.listdir(self._model_path):
# FIXME(wawltor) The path maybe not correct
if file_name.count(".pdmodel"):
return os.path.join(self._model_path, file_name[:-8])
if file_name.count(PADDLE_INFERENCE_MODEL_SUFFIX):
return os.path.join(self._model_path, file_name[: -len(PADDLE_INFERENCE_MODEL_SUFFIX)])
return None
def _is_int8_model(self, model_path):
@@ -110,7 +113,10 @@ class Predictor:
"""
Construct the input data and predictor in the PaddlePaddele static mode.
"""
self._config = paddle.inference.Config(static_model_path + ".pdmodel", static_model_path + ".pdiparams")
self._config = paddle.inference.Config(
static_model_path + PADDLE_INFERENCE_MODEL_SUFFIX,
static_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX,
)
self._config.disable_glog_info()
if paddle.get_device() == "cpu":
self._config.disable_gpu()
@@ -146,7 +152,7 @@ class Predictor:
os.mkdir(onnx_dir)
float_onnx_file = os.path.join(onnx_dir, "model.onnx")
if not os.path.exists(float_onnx_file):
model_path = static_model_path + ".pdmodel"
model_path = static_model_path + PADDLE_INFERENCE_MODEL_SUFFIX
params_file = static_model_path + ".pdiparams"
onnx_model = paddle2onnx.command.c_paddle_to_onnx(
model_file=model_path, params_file=params_file, opset_version=13, enable_onnx_checker=True

21
paddlenlp/taskflow/multimodal_feature_extraction.py
View File

@@ -19,6 +19,7 @@ import paddle
from PIL import Image
from ..transformers import AutoModel, AutoProcessor
from ..utils.env import PADDLE_INFERENCE_MODEL_SUFFIX, PADDLE_INFERENCE_WEIGHTS_SUFFIX
from ..utils.log import logger
from .task import Task
from .utils import dygraph_mode_guard, static_mode_guard
@@ -411,9 +412,9 @@ class MultimodalFeatureExtractionTask(Task):
self.inference_image_model_path = os.path.join(_base_path, "static", "get_image_features")
self.inference_text_model_path = os.path.join(_base_path, "static", "get_text_features")
if (
not os.path.exists(self.inference_image_model_path + ".pdiparams")
not os.path.exists(self.inference_image_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX)
or self._param_updated
or not os.path.exists(self.inference_text_model_path + ".pdiparams")
or not os.path.exists(self.inference_text_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX)
):
with dygraph_mode_guard():
self._construct_model(self.model)
@@ -422,8 +423,8 @@ class MultimodalFeatureExtractionTask(Task):
if self._predictor_type == "paddle-inference":
# Get text inference model
self.inference_model_path = self.inference_text_model_path
self._static_model_file = self.inference_model_path + ".pdmodel"
self._static_params_file = self.inference_model_path + ".pdiparams"
self._static_model_file = self.inference_model_path + PADDLE_INFERENCE_MODEL_SUFFIX
self._static_params_file = self.inference_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX
self._config = paddle.inference.Config(self._static_model_file, self._static_params_file)
self._prepare_static_mode()
@@ -435,8 +436,8 @@ class MultimodalFeatureExtractionTask(Task):
# Get image inference model
self.inference_model_path = self.inference_image_model_path
self._static_model_file = self.inference_model_path + ".pdmodel"
self._static_params_file = self.inference_model_path + ".pdiparams"
self._static_model_file = self.inference_model_path + PADDLE_INFERENCE_MODEL_SUFFIX
self._static_params_file = self.inference_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX
self._config = paddle.inference.Config(self._static_model_file, self._static_params_file)
self._prepare_static_mode()
@@ -449,15 +450,15 @@ class MultimodalFeatureExtractionTask(Task):
# Get text onnx model
self.export_type = "text"
self.inference_model_path = self.inference_text_model_path
self._static_model_file = self.inference_model_path + ".pdmodel"
self._static_params_file = self.inference_model_path + ".pdiparams"
self._static_model_file = self.inference_model_path + PADDLE_INFERENCE_MODEL_SUFFIX
self._static_params_file = self.inference_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX
self._prepare_onnx_mode()
self.predictor_map["text"] = self.predictor
# Get image onnx model
self.export_type = "image"
self.inference_model_path = self.inference_image_model_path
self._static_model_file = self.inference_model_path + ".pdmodel"
self._static_params_file = self.inference_model_path + ".pdiparams"
self._static_model_file = self.inference_model_path + PADDLE_INFERENCE_MODEL_SUFFIX
self._static_params_file = self.inference_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX
self._prepare_onnx_mode()
self.predictor_map["image"] = self.predictor

41
paddlenlp/taskflow/task.py
View File

@@ -22,7 +22,11 @@ from multiprocessing import cpu_count
import paddle
from paddle.dataset.common import md5file
from ..utils.env import PPNLP_HOME
from ..utils.env import (
PADDLE_INFERENCE_MODEL_SUFFIX,
PADDLE_INFERENCE_WEIGHTS_SUFFIX,
PPNLP_HOME,
)
from ..utils.log import logger
from .utils import cut_chinese_sent, download_check, download_file, dygraph_mode_guard
@@ -118,12 +122,12 @@ class Task(metaclass=abc.ABCMeta):
def _get_static_model_name(self):
names = []
for file_name in os.listdir(self._task_path):
if ".pdmodel" in file_name:
names.append(file_name[:-8])
if PADDLE_INFERENCE_MODEL_SUFFIX in file_name:
names.append(file_name[: -len(PADDLE_INFERENCE_MODEL_SUFFIX)])
if len(names) == 0:
raise IOError(f"{self._task_path} should include '.pdmodel' file.")
raise IOError(f"{self._task_path} should include '{PADDLE_INFERENCE_MODEL_SUFFIX}' file.")
if len(names) > 1:
logger.warning(f"{self._task_path} includes more than one '.pdmodel' file.")
logger.warning(f"{self._task_path} includes more than one '{PADDLE_INFERENCE_MODEL_SUFFIX}' file.")
return names[0]
def _check_task_files(self):
@@ -223,7 +227,6 @@ class Task(metaclass=abc.ABCMeta):
self._config.switch_ir_optim(False)
if self.model == "uie-data-distill-gp":
self._config.enable_memory_optim(False)
self.predictor = paddle.inference.create_predictor(self._config)
self.input_names = [name for name in self.predictor.get_input_names()]
self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()]
@@ -286,7 +289,10 @@ class Task(metaclass=abc.ABCMeta):
cache_info_path = os.path.join(self._task_path, ".cache_info")
md5 = md5file(param_path)
self._param_updated = True
if os.path.exists(cache_info_path) and open(cache_info_path).read()[:-8] == md5:
if (
os.path.exists(cache_info_path)
and open(cache_info_path).read()[: -len(PADDLE_INFERENCE_MODEL_SUFFIX)] == md5
):
self._param_updated = False
elif self.task == "information_extraction" and self.model != "uie-data-distill-gp":
# UIE related models are moved to paddlenlp.transformers after v2.4.5
@@ -318,11 +324,11 @@ class Task(metaclass=abc.ABCMeta):
# When the user-provided model path is already a static model, skip to_static conversion
if self.is_static_model:
self.inference_model_path = os.path.join(self._task_path, self._static_model_name)
if not os.path.exists(self.inference_model_path + ".pdmodel") or not os.path.exists(
self.inference_model_path + ".pdiparams"
if not os.path.exists(self.inference_model_path + PADDLE_INFERENCE_MODEL_SUFFIX) or not os.path.exists(
self.inference_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX
):
raise IOError(
f"{self._task_path} should include {self._static_model_name + '.pdmodel'} and {self._static_model_name + '.pdiparams'} while is_static_model is True"
f"{self._task_path} should include {self._static_model_name + PADDLE_INFERENCE_MODEL_SUFFIX} and {self._static_model_name + PADDLE_INFERENCE_WEIGHTS_SUFFIX} while is_static_model is True"
)
if self.paddle_quantize_model(self.inference_model_path):
self._infer_precision = "int8"
@@ -336,19 +342,19 @@ class Task(metaclass=abc.ABCMeta):
else os.path.join(self._home_path, "taskflow", self.task, self._task_path)
)
self.inference_model_path = os.path.join(_base_path, "static", "inference")
if not os.path.exists(self.inference_model_path + ".pdiparams") or self._param_updated:
if not os.path.exists(self.inference_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX) or self._param_updated:
with dygraph_mode_guard():
self._construct_model(self.model)
self._construct_input_spec()
self._convert_dygraph_to_static()
self._static_model_file = self.inference_model_path + ".pdmodel"
self._static_params_file = self.inference_model_path + ".pdiparams"
self._static_model_file = self.inference_model_path + PADDLE_INFERENCE_MODEL_SUFFIX
self._static_params_file = self.inference_model_path + PADDLE_INFERENCE_WEIGHTS_SUFFIX
if paddle.get_device().split(":", 1)[0] == "npu" and self._infer_precision == "fp16":
# transform fp32 model tp fp16 model
self._static_fp16_model_file = self.inference_model_path + "-fp16.pdmodel"
self._static_fp16_params_file = self.inference_model_path + "-fp16.pdiparams"
self._static_fp16_model_file = self.inference_model_path + f"-fp16{PADDLE_INFERENCE_MODEL_SUFFIX}"
self._static_fp16_params_file = self.inference_model_path + f"-fp16{PADDLE_INFERENCE_WEIGHTS_SUFFIX}"
if not os.path.exists(self._static_fp16_model_file) and not os.path.exists(self._static_fp16_params_file):
logger.info("Converting to the inference model from fp32 to fp16.")
paddle.inference.convert_to_mixed_precision(
@@ -384,8 +390,7 @@ class Task(metaclass=abc.ABCMeta):
self._input_spec is not None
), "The input spec must be created before converting the dygraph model to static model."
logger.info("Converting to the inference model cost a little time.")
static_model = paddle.jit.to_static(self._model, input_spec=self._input_spec)
static_model = paddle.jit.to_static(self._model, input_spec=self._input_spec, full_graph=True)
paddle.jit.save(static_model, self.inference_model_path)
logger.info("The inference model save in the path:{}".format(self.inference_model_path))
@@ -512,7 +517,7 @@ class Task(metaclass=abc.ABCMeta):
program = model.program()
for block in program.blocks:
for op in block.ops:
if op.type.count("quantize"):
if "quantize" in op.name():
return True
return False

3
paddlenlp/taskflow/text_similarity.py
View File

@@ -196,7 +196,7 @@ class TextSimilarityTask(Task):
"""
if "rocketqav2-en" in model or "ernie-search" in model:
self._model = ErnieCrossEncoder(self._task_path, num_classes=1, reinitialize=True)
self._model = ErnieCrossEncoder(self._task_path, num_classes=2, reinitialize=True)
elif "rocketqa" in model:
self._model = ErnieCrossEncoder(self._task_path, num_classes=2)
else:
@@ -274,7 +274,6 @@ class TextSimilarityTask(Task):
if "rocketqa" in self.model_name or "ernie-search" in self.model_name:
with static_mode_guard():
for batch in inputs["data_loader"]:
if self._predictor_type == "paddle-inference":
input_ids, segment_ids = self._batchify_fn(batch)
self.input_handles[0].copy_from_cpu(input_ids)

44
paddlenlp/taskflow/zero_shot_text_classification.py
View File

@@ -259,6 +259,7 @@ class ZeroShotTextClassificationTask(Task):
def __init__(self, task: str, model: str, schema: list = None, **kwargs):
super().__init__(task=task, model=model, **kwargs)
self._static_mode = False
self._set_utc_schema(schema)
self._max_seq_len = kwargs.get("max_seq_len", 512)
self._batch_size = kwargs.get("batch_size", 1)
@@ -269,7 +270,10 @@ class ZeroShotTextClassificationTask(Task):
self._check_task_files()
self._construct_tokenizer()
self._check_predictor_type()
self._get_inference_model()
if self._static_mode:
self._get_inference_model()
else:
self._construct_model(model)
def _set_utc_schema(self, schema):
if schema is None:
@@ -293,7 +297,7 @@ class ZeroShotTextClassificationTask(Task):
InputSpec(shape=[None, None], dtype="int64", name="input_ids"),
InputSpec(shape=[None, None], dtype="int64", name="token_type_ids"),
InputSpec(shape=[None, None], dtype="int64", name="position_ids"),
InputSpec(shape=[None, None, None, None], dtype="float32", name="attention_mask"),
InputSpec(shape=[None, None], dtype="float32", name="attention_mask"),
InputSpec(shape=[None, None], dtype="int64", name="omask_positions"),
InputSpec(shape=[None], dtype="int64", name="cls_positions"),
]
@@ -311,7 +315,10 @@ class ZeroShotTextClassificationTask(Task):
Construct the tokenizer for the predictor.
"""
self._tokenizer = AutoTokenizer.from_pretrained(self._task_path, from_hf_hub=self.from_hf_hub)
self._collator = PromptDataCollatorWithPadding(self._tokenizer, return_tensors="np")
if self._static_mode:
self._collator = PromptDataCollatorWithPadding(self._tokenizer, return_tensors="np")
else:
self._collator = PromptDataCollatorWithPadding(self._tokenizer, return_tensors="pd")
self._template = UTCTemplate(self._tokenizer, self._max_seq_len)
def _check_input_text(self, inputs):
@@ -381,19 +388,26 @@ class ZeroShotTextClassificationTask(Task):
"omask_positions": "int64",
"cls_positions": "int64",
}
with static_mode_guard():
if self._static_mode:
with static_mode_guard():
for batch in inputs["batches"]:
if self._predictor_type == "paddle-inference":
for i, input_name in enumerate(self.input_names):
self.input_handles[i].copy_from_cpu(batch[input_name].astype(dtype_dict[input_name]))
self.predictor.run()
logits = self.output_handle[0].copy_to_cpu().tolist()
else:
input_dict = {}
for input_name in dtype_dict:
input_dict[input_name] = batch[input_name].astype(dtype_dict[input_name])
logits = self.predictor.run(None, input_dict)[0].tolist()
outputs["batch_logits"].append(logits)
else:
for batch in inputs["batches"]:
if self._predictor_type == "paddle-inference":
for i, input_name in enumerate(self.input_names):
self.input_handles[i].copy_from_cpu(batch[input_name].astype(dtype_dict[input_name]))
self.predictor.run()
logits = self.output_handle[0].copy_to_cpu().tolist()
else:
input_dict = {}
for input_name in dtype_dict:
input_dict[input_name] = batch[input_name].astype(dtype_dict[input_name])
logits = self.predictor.run(None, input_dict)[0].tolist()
outputs["batch_logits"].append(logits)
if batch["soft_token_ids"] is not None:
del batch["soft_token_ids"]
logits = self._model(**batch)
outputs["batch_logits"].append(np.array(logits))
return outputs

9
paddlenlp/trainer/trainer_compress.py
View File

@@ -38,6 +38,7 @@ from ..transformers.ofa_utils import (
prepare_qkv_ofa,
reorder_neuron_head,
)
from ..utils.env import PADDLE_INFERENCE_MODEL_SUFFIX, PADDLE_INFERENCE_WEIGHTS_SUFFIX
from ..utils.log import logger
from .trainer import Trainer
@@ -651,8 +652,8 @@ def _post_training_quantization_grid_search(self, model_dir):
executor=exe,
batch_generator=_batch_generator_func,
model_dir=model_dir,
model_filename=args.input_filename_prefix + ".pdmodel",
params_filename=args.input_filename_prefix + ".pdiparams",
model_filename=args.input_filename_prefix + PADDLE_INFERENCE_MODEL_SUFFIX,
params_filename=args.input_filename_prefix + PADDLE_INFERENCE_WEIGHTS_SUFFIX,
batch_size=batch_size,
batch_nums=batch_nums,
scope=None,
@@ -675,8 +676,8 @@ def _post_training_quantization_grid_search(self, model_dir):
save_model_path = os.path.join(model_dir, algo + "_".join([str(batch_size), str(batch_nums)]))
post_training_quantization.save_quantized_model(
save_model_path=save_model_path,
model_filename=args.output_filename_prefix + ".pdmodel",
params_filename=args.output_filename_prefix + ".pdiparams",
model_filename=args.output_filename_prefix + PADDLE_INFERENCE_MODEL_SUFFIX,
params_filename=args.output_filename_prefix + PADDLE_INFERENCE_WEIGHTS_SUFFIX,
)
output_dir_list.append(save_model_path)

9
paddlenlp/transformers/chatglm/modeling.py
View File

@@ -132,14 +132,7 @@ class RotaryEmbeddings(nn.Layer):
cos_cached = emb.cos().unsqueeze(1).cast(self.default_dtype)
sin_cached = emb.sin().unsqueeze(1).cast(self.default_dtype)
if hasattr(paddle.framework, "_no_check_dy2st_diff"):
# TODO(daisiming): _no_check_dy2st_diff is used to turn off the checking of behavior
# inconsistency between dynamic graph and static graph. _no_check_dy2st_diff should be
# removed after static graphs support inplace and stride.
with paddle.framework._no_check_dy2st_diff():
self.cos_cached, self.sin_cached = cos_cached, sin_cached
else:
self.cos_cached, self.sin_cached = cos_cached, sin_cached
self.cos_cached, self.sin_cached = cos_cached, sin_cached
cos, sin = self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]
if self.position_encoding_2d:

5
paddlenlp/transformers/semantic_search/modeling.py
View File

@@ -282,8 +282,9 @@ class ErnieCrossEncoder(nn.Layer):
input_ids, token_type_ids=token_type_ids, position_ids=position_ids, attention_mask=attention_mask
)
pooled_output = self.ernie.dropout(sequence_output[:, 0])
probs = self.ernie.classifier(pooled_output)
return probs
cls_embedding = self.ernie.classifier(pooled_output)
probs = F.softmax(cls_embedding, axis=1)
return probs[:, 1]
def matching_v3(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
"""Use the pooled_output as the feature for listwise prediction, eg. ERNIE-Search"""

15
paddlenlp/utils/env.py
View File

@@ -20,6 +20,13 @@ PPNLP_HOME --> the root directory for storing PaddleNLP related da
"""
import os
try:
from paddle.base.framework import use_pir_api
pir_enabled = use_pir_api()
except ImportError:
pir_enabled = False
def _get_user_home():
return os.path.expanduser("~")
@@ -132,4 +139,12 @@ MAX_QUANTIZATION_TIMES = 1
MAX_BSZ = 512
SPECULATE_MAX_BSZ = 256
MAX_DRAFT_TOKENS = 6
if pir_enabled:
PADDLE_INFERENCE_MODEL_SUFFIX = ".json"
PADDLE_INFERENCE_WEIGHTS_SUFFIX = ".pdiparams"
else:
PADDLE_INFERENCE_MODEL_SUFFIX = ".pdmodel"
PADDLE_INFERENCE_WEIGHTS_SUFFIX = ".pdiparams"
USE_FAST_TOKENIZER: bool = _get_bool_env("USE_FAST_TOKENIZER", "false")

15
scripts/regression/ci_case.sh
View File

@@ -22,6 +22,17 @@ export CXX_COMPILER_PATH=$(which g++)
export CC=$(which gcc)
export CXX=$(which g++)
export PADDLE_INFERENCE_MODEL_SUFFIX=$(python -c "
import paddle
try:
from paddle.base.framework import use_pir_api
pir_enabled = use_pir_api()
except ImportError:
pir_enabled = False
model_suffix = '.json' if pir_enabled else '.pdmodel'
print(model_suffix)
")
if [ ! -d "model_logs" ]; then
mkdir model_logs
fi
@@ -363,7 +374,7 @@ lexical_analysis(){
print_info $? lexical_analysis_predict
# deploy
time (python deploy/predict.py \
--model_file=infer_model/static_graph_params.pdmodel \
--model_file=infer_model/static_graph_params${PADDLE_INFERENCE_MODEL_SUFFIX} \
--params_file=infer_model/static_graph_params.pdiparams \
--data_dir lexical_analysis_dataset_tiny >${log_path}/lexical_analysis_deploy) >>${log_path}/lexical_analysis_deploy 2>&1
print_info $? lexical_analysis_deploy
@@ -467,7 +478,7 @@ ernie-csc() {
python export_model.py --params_path ./checkpoints/best_model.pdparams --output_path ./infer_model/static_graph_params >${log_path}/ernie-csc_export >>${log_path}/ernie-csc_export 2>&1
print_info $? ernie-csc_export
#python deploy
python predict.py --model_file infer_model/static_graph_params.pdmodel --params_file infer_model/static_graph_params.pdiparams >${log_path}/ernie-csc_deploy >>${log_path}/ernie-csc_deploy 2>&1
python predict.py --model_file infer_model/static_graph_params${PADDLE_INFERENCE_MODEL_SUFFIX} --params_file infer_model/static_graph_params.pdiparams >${log_path}/ernie-csc_deploy >>${log_path}/ernie-csc_deploy 2>&1
print_info $? ernie-csc_deploy
}

2
scripts/regression/test_taskflow.py
View File

@@ -13,6 +13,7 @@
# limitations under the License.
"""Test taskflow."""
import os
import unittest
from paddlenlp import Taskflow
@@ -68,6 +69,7 @@ def test_corrector():
corrector("遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇，这样我们才能朝著成功之路前进。")
@unittest.skip("dependency_parsing is not support for Paddle >= 2.6.1")
def test_dependency_parsing():
"""
test_dependency_parsing

103
tests/taskflow/test_text_classification.py
View File

@@ -24,7 +24,6 @@ from paddlenlp.prompt import (
PromptModelForSequenceClassification,
SoftVerbalizer,
)
from paddlenlp.taskflow import Taskflow
from paddlenlp.taskflow.text_classification import TextClassificationTask
from paddlenlp.transformers import (
AutoModelForMaskedLM,
@@ -145,60 +144,60 @@ class TestTextClassificationTask(unittest.TestCase):
if model == "multi_label":
self.assertGreater(dygraph_pred["score"], dygraph_taskflow.multilabel_threshold)
@unittest.skip("numerical error")
@parameterized.expand(
[
(1, "multi_class", "finetune"),
(1, "multi_class", "prompt"),
(1, "multi_label", "finetune"),
(1, "multi_label", "prompt"),
]
)
def test_taskflow_task(self, batch_size, problem_type, mode):
input_text = ["百度", "深度学习框架", "飞桨", "PaddleNLP"]
id2label = {
0: "negative",
1: "positive",
}
if mode == "finetune":
dygraph_model_path = self.finetune_dygraph_model_path
static_model_path = self.finetune_static_model_path
else:
dygraph_model_path = self.prompt_dygraph_model_path
static_model_path = self.prompt_static_model_path
# @unittest.skip("numerical error")
# @parameterized.expand(
# [
# (1, "multi_class", "finetune"),
# (1, "multi_class", "prompt"),
# (1, "multi_label", "finetune"),
# (1, "multi_label", "prompt"),
# ]
# )
# def test_taskflow_task(self, batch_size, problem_type, mode):
# input_text = ["百度", "深度学习框架", "飞桨", "PaddleNLP"]
# id2label = {
# 0: "negative",
# 1: "positive",
# }
# if mode == "finetune":
# dygraph_model_path = self.finetune_dygraph_model_path
# static_model_path = self.finetune_static_model_path
# else:
# dygraph_model_path = self.prompt_dygraph_model_path
# static_model_path = self.prompt_static_model_path
dygraph_taskflow = Taskflow(
mode=mode,
task="text_classification",
task_path=dygraph_model_path,
id2label=id2label,
batch_size=batch_size,
device_id=0,
problem_type=problem_type,
)
# dygraph_taskflow = Taskflow(
# mode=mode,
# task="text_classification",
# task_path=dygraph_model_path,
# id2label=id2label,
# batch_size=batch_size,
# device_id=0,
# problem_type=problem_type,
# )
dygraph_results = dygraph_taskflow(input_text)
# dygraph_results = dygraph_taskflow(input_text)
self.assertEqual(len(dygraph_results), len(input_text))
# self.assertEqual(len(dygraph_results), len(input_text))
static_taskflow = Taskflow(
mode=mode,
task="text_classification",
is_static_model=True,
task_path=static_model_path,
id2label=id2label,
batch_size=batch_size,
device_id=0,
problem_type=problem_type,
)
# static_taskflow = Taskflow(
# mode=mode,
# task="text_classification",
# is_static_model=True,
# task_path=static_model_path,
# id2label=id2label,
# batch_size=batch_size,
# device_id=0,
# problem_type=problem_type,
# )
static_results = static_taskflow(input_text)
self.assertEqual(len(static_results), len(input_text))
# static_results = static_taskflow(input_text)
# self.assertEqual(len(static_results), len(input_text))
for dygraph_result, static_result in zip(dygraph_results, static_results):
for dygraph_pred, static_pred in zip(dygraph_result["predictions"], static_result["predictions"]):
self.assertEqual(dygraph_pred["label"], static_pred["label"])
self.assertAlmostEqual(dygraph_pred["score"], static_pred["score"], delta=1e-6)
# if multi_label, all predictions should be greater than the threshold
if mode == "multi_label":
self.assertGreater(dygraph_pred["score"], dygraph_taskflow.task_instance.multilabel_threshold)
# for dygraph_result, static_result in zip(dygraph_results, static_results):
# for dygraph_pred, static_pred in zip(dygraph_result["predictions"], static_result["predictions"]):
# self.assertEqual(dygraph_pred["label"], static_pred["label"])
# self.assertAlmostEqual(dygraph_pred["score"], static_pred["score"], delta=1e-6)
# # if multi_label, all predictions should be greater than the threshold
# if mode == "multi_label":
# self.assertGreater(dygraph_pred["score"], dygraph_taskflow.task_instance.multilabel_threshold)

22
tests/transformers/test_modeling_common.py
View File

@@ -38,7 +38,13 @@ from paddlenlp.taskflow.utils import static_mode_guard
from paddlenlp.transformers import AutoModelForCausalLM, AutoTokenizer
from paddlenlp.transformers.configuration_utils import PretrainedConfig
from paddlenlp.transformers.model_utils import PretrainedModel
from paddlenlp.utils.env import CONFIG_NAME, LEGACY_CONFIG_NAME, MODEL_HOME
from paddlenlp.utils.env import (
CONFIG_NAME,
LEGACY_CONFIG_NAME,
MODEL_HOME,
PADDLE_INFERENCE_MODEL_SUFFIX,
PADDLE_INFERENCE_WEIGHTS_SUFFIX,
)
from ..testing_utils import slow
@@ -968,11 +974,8 @@ class GenerationD2STestMixin:
use_top_p=False,
),
)
if paddle.framework.use_pir_api():
model_path = os.path.join(tempdir, "model.json")
else:
model_path = os.path.join(tempdir, "model.pdmodel")
params_path = os.path.join(tempdir, "model.pdiparams")
model_path = os.path.join(tempdir, f"model{PADDLE_INFERENCE_MODEL_SUFFIX}")
params_path = os.path.join(tempdir, f"model{PADDLE_INFERENCE_WEIGHTS_SUFFIX}")
config = paddle.inference.Config(model_path, params_path)
config.disable_gpu()
@@ -1040,11 +1043,8 @@ class GenerationD2STestMixin:
),
)
if paddle.framework.use_pir_api():
model_path = os.path.join(tempdir, "model.json")
else:
model_path = os.path.join(tempdir, "model.pdmodel")
params_path = os.path.join(tempdir, "model.pdiparams")
model_path = os.path.join(tempdir, f"model{PADDLE_INFERENCE_MODEL_SUFFIX}")
params_path = os.path.join(tempdir, f"model{PADDLE_INFERENCE_WEIGHTS_SUFFIX}")
config = paddle.inference.Config(model_path, params_path)
config.disable_gpu()