Kimirock/PaddleNLP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[BugFix] Fix gpt (#7716)

Browse Source
This commit is contained in:
Zhong Hui
2023-12-26 10:51:07 +08:00
committed by GitHub
parent 79d1d5b42d
commit d97016ec84
22 changed files with 13 additions and 962 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
paddlenlp/transformers/gpt/configuration.py
View File

@@ -18,7 +18,7 @@ from typing import Dict
from paddlenlp.transformers.configuration_utils import PretrainedConfig
__all__ = ["GPT_PRETRAINED_INIT_CONFIGURATION", "GPTConfig", "GPT_PRETRAINED_RESOURCE_FILES_MAP"]
__all__ = ["GPT_PRETRAINED_INIT_CONFIGURATION", "GPTConfig"]
GPT_PRETRAINED_INIT_CONFIGURATION = {
"gpt-cpm-large-cn": { # 2.6B
@@ -130,66 +130,6 @@ GPT_PRETRAINED_INIT_CONFIGURATION = {
"eos_token_id": 50256,
"eol_token_id": 198,
},
"gpt2-xl-en": { # 1558M
"vocab_size": 50257,
"hidden_size": 1600,
"num_hidden_layers": 48,
"num_attention_heads": 25,
"intermediate_size": 6400,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.1,
"attention_probs_dropout_prob": 0.1,
"max_position_embeddings": 1024,
"type_vocab_size": 1, # no use
"initializer_range": 0.02,
"eos_token_id": 50256,
"eol_token_id": 198,
},
"gpt2-large-en": { # 774M
"vocab_size": 50257,
"hidden_size": 1280,
"num_hidden_layers": 36,
"num_attention_heads": 20,
"intermediate_size": 5120,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.1,
"attention_probs_dropout_prob": 0.1,
"max_position_embeddings": 1024,
"type_vocab_size": 1, # no use
"initializer_range": 0.02,
"eos_token_id": 50256,
"eol_token_id": 198,
},
"gpt2-medium-en": { # 345M
"vocab_size": 50304,
"hidden_size": 1024,
"num_hidden_layers": 24,
"num_attention_heads": 16,
"intermediate_size": 4096,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.1,
"attention_probs_dropout_prob": 0.1,
"max_position_embeddings": 1024,
"type_vocab_size": 1, # no use
"initializer_range": 0.02,
"eos_token_id": 50256,
"eol_token_id": 198,
},
"gpt2-en": { # 117M
"vocab_size": 50257,
"hidden_size": 768,
"num_hidden_layers": 12,
"num_attention_heads": 12,
"intermediate_size": 3072,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.1,
"attention_probs_dropout_prob": 0.1,
"max_position_embeddings": 1024,
"type_vocab_size": 1, # no use
"initializer_range": 0.02,
"eos_token_id": 50256,
"eol_token_id": 198,
},
"gpt2-small-en": { # config for CE
"vocab_size": 50304,
"hidden_size": 1024,
@@ -207,17 +147,6 @@ GPT_PRETRAINED_INIT_CONFIGURATION = {
},
}
GPT_PRETRAINED_RESOURCE_FILES_MAP = {
"model_state": {
"gpt-cpm-large-cn": "https://bj.bcebos.com/paddlenlp/models/transformers/gpt/gpt-cpm-large-cn.pdparams",
"gpt-cpm-small-cn-distill": "https://bj.bcebos.com/paddlenlp/models/transformers/gpt/gpt-cpm-small-cn-distill.pdparams",
"gpt2-en": "https://bj.bcebos.com/paddlenlp/models/transformers/gpt/gpt2-en.pdparams",
"gpt2-medium-en": "https://bj.bcebos.com/paddlenlp/models/transformers/gpt/gpt2-medium-en.pdparams",
"gpt2-large-en": "https://bj.bcebos.com/paddlenlp/models/transformers/gpt/gpt2-large-en.pdparams",
"gpt2-xl-en": "https://bj.bcebos.com/paddlenlp/models/transformers/gpt/gpt2-xl-en.pdparams",
}
}
class GPTConfig(PretrainedConfig):
r"""
@@ -331,7 +260,6 @@ class GPTConfig(PretrainedConfig):
fused_linear: bool = False,
fuse_attention_qkv: bool = False,
fuse_attention_ffn: bool = False,
enable_fuse_transformer: bool = False,
fused_softmax_with_triangular: bool = False,
virtual_pp_degree: int = 1,
sequence_parallel=False,
@@ -371,7 +299,6 @@ class GPTConfig(PretrainedConfig):
self.output_attentions = output_attentions
self.ignore_index = ignore_index
self.fused_linear = fused_linear
self.enable_fuse_transformer = enable_fuse_transformer
self.use_fused_dropout_add = use_fused_dropout_add
self.fused_softmax_with_triangular = fused_softmax_with_triangular
self.virtual_pp_degree = virtual_pp_degree

7
paddlenlp/transformers/gpt/modeling.py
View File

@@ -48,11 +48,7 @@ from ..sequence_parallel_utils import (
ScatterOp,
mark_as_sequence_parallel_parameter,
)
from .configuration import (
GPT_PRETRAINED_INIT_CONFIGURATION,
GPT_PRETRAINED_RESOURCE_FILES_MAP,
GPTConfig,
)
from .configuration import GPT_PRETRAINED_INIT_CONFIGURATION, GPTConfig
try:
from paddle.nn.functional.flash_attention import flash_attention
@@ -759,7 +755,6 @@ class GPTPretrainedModel(PretrainedModel):
base_model_prefix = "gpt"
config_class = GPTConfig
pretrained_init_configuration = GPT_PRETRAINED_INIT_CONFIGURATION
pretrained_resource_files_map = GPT_PRETRAINED_RESOURCE_FILES_MAP
@classmethod
def _get_tensor_parallel_mappings(cls, config, is_split=True):

3
scripts/distribute/ci_case_dy.sh
View File

@@ -413,7 +413,6 @@ function llm_gpt_recompute_bs32_bf16_MP2-SD4-stage1() {
log_dir=mylog
rm -rf $log_dir
python -m paddle.distributed.launch --log_dir=./mylog --devices=0,1,2,3,4,5,6,7 run_pretrain.py \
--model_type gpt \
--model_name_or_path gpt2-medium-en \
--tokenizer_name_or_path gpt2-medium-en \
--input_dir ./data \
@@ -649,4 +648,4 @@ else
echo -e "\033[31m ---- Invalid exec_case $exec_case \033[0m"
fi
$1
$1

109
scripts/regression/ci_case.sh
View File

@@ -234,116 +234,7 @@ time (python -u ./run_pretrain.py \
--device gpu >${log_path}/electra_pretrain) >>${log_path}/electra_pretrain 2>&1
print_info $? electra_pretrain
}
fast_gpt(){
# FT
cd ${nlp_dir}/
export PYTHONPATH=$PWD/PaddleNLP/:$PYTHONPATH
wget -q https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-TagBuild-Infer-Linux-Gpu-Cuda120-Cudnn89-Trt86-Mkl-Avx-Gcc122/latest/paddle_inference.tgz
tar -zxf paddle_inference.tgz
cd ${nlp_dir}/paddlenlp/ops
#python
mkdir build_gpt_so
cd build_gpt_so/
cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=${C_COMPILER_PATH} -DCMAKE_CXX_COMPILER=${CXX_COMPILER_PATH} \
-DPADDLE_LIB=${nlp_dir}/paddle_inference/ -DDEMO=${nlp_dir}/paddlenlp/ops/fast_transformer/src/demo/gpt.cc \
-DPY_CMD=python -DWITH_GPT=ON -DON_INFER=ON -DWITH_MKL=ON -DWITH_ONNXRUNTIME=ON
make -j >${log_path}/GPT_python_FT >>${log_path}/gpt_python_FT 2>&1
print_info $? gpt_python_FT
cd ../
#c++
mkdir build_gpt_cc
cd build_gpt_cc/
cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=${C_COMPILER_PATH} -DCMAKE_CXX_COMPILER=${CXX_COMPILER_PATH} \
-DPADDLE_LIB=${nlp_dir}/paddle_inference/ -DDEMO=${nlp_dir}/paddlenlp/ops/fast_transformer/src/demo/gpt.cc \
-DWITH_GPT=ON -DON_INFER=ON -DWITH_MKL=ON -DWITH_ONNXRUNTIME=ON
make -j >${log_path}/GPT_C_FT >>${log_path}/gpt_C_FT 2>&1
print_info $? gpt_C_FT
#depoly python
cd ${nlp_dir}/model_zoo/gpt/fast_gpt/
python infer.py \
--model_name_or_path gpt2-medium-en \
--batch_size 1 \
--topk 4 \
--topp 0.0 \
--max_length 32 \
--start_token "<|endoftext|>" \
--end_token "<|endoftext|>" \
--temperature 1.0 >${log_path}/gpt_deploy_P_FT >>${log_path}/gpt_deploy_P_FT 2>&1
print_info $? gpt_deploy_P_FT
#depoly C++
python export_model.py \
--model_name_or_path gpt2-medium-en \
--decoding_lib ${nlp_dir}/paddlenlp/ops/build_gpt_so/lib/libdecoding_op.so \
--topk 4 \
--topp 0.0 \
--max_out_len 32 \
--temperature 1.0 \
--inference_model_dir ./infer_model/
mv infer_model/ ${nlp_dir}/paddlenlp/ops/build_gpt_cc/bin/
cd ${nlp_dir}/paddlenlp/ops/build_gpt_cc/bin/
./gpt -batch_size 1 -gpu_id 0 -model_dir ./infer_model -vocab_file ./infer_model/vocab.txt -start_token "<|endoftext|>" -end_token "<|endoftext|>" >${log_path}/gpt_deploy_C_FT >>${log_path}/gpt_deploy_C_FT 2>&1
print_info $? gpt_deploy_C_FT
}
# 8 gpt
gpt(){
cd ${nlp_dir}/model_zoo/ernie-1.0/data_tools
sed -i "s/python3/python/g" Makefile
sed -i "s/python-config/python3.7m-config/g" Makefile
#pretrain
cd ${nlp_dir}/model_zoo/gpt/
mkdir data
cd ./data
wget -q https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_ids.npy
wget -q https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_idx.npz
cd ../
time (python run_pretrain_trainer.py \
--model_type "gpt" \
--model_name_or_path "gpt2-en" \
--tokenizer_name_or_path "gpt2-en" \
--input_dir "${nlp_dir}/model_zoo/gpt/data" \
--output_dir "output" \
--split 949,50,1 \
--max_seq_length 1024 \
--per_device_train_batch_size 8 \
--per_device_eval_batch_size 8 \
--fp16 \
--fp16_opt_level "O2" \
--learning_rate 0.0001 \
--min_learning_rate 0.00001 \
--max_steps 10 \
--save_steps 5 \
--weight_decay 0.01 \
--warmup_ratio 0.01 \
--max_grad_norm 1.0 \
--logging_steps 1 \
--dataloader_num_workers 1 \
--eval_steps 5 \
--report_to "visualdl" \
--disable_tqdm true \
--do_train \
--do_eval \
--device "gpu" >${log_path}/gpt_pretrain) >>${log_path}/gpt_pretrain 2>&1
print_info $? gpt_pretrain
time (python export_model.py --model_type=gpt \
--model_path "gpt2-medium-en" \
--output_path ${nlp_dir}/model_zoo/gpt/infer_model/model >${log_path}/gpt_export) >>${log_path}/gpt_export 2>&1
print_info $? gpt_export
time (python deploy/python/inference.py \
--model_type "gpt" \
--model_path ${nlp_dir}/model_zoo/gpt/infer_model/model >${log_path}/gpt_p_depoly) >>${log_path}/gpt_p_depoly 2>&1
print_info $? gpt_p_depoly
# echo 'run gpt test with pytest'
# cd ${nlp_dir}
# python -m pytest ./tests/model_zoo/test_gpt.py >${log_path}/gpt >>${log_path}/gpt 2>&1
# print_info $? gpt
fast_gpt
cd ${nlp_dir}/fast_generation/samples
python gpt_sample.py >${log_path}/fast_generation_gpt >>${log_path}/fast_generation_gpt 2>&1
print_info $? fast_generation_gpt
}
# 9 ernie
ernie(){
#data process

28
tests/examples/gpt/acc_dp_dygraph.sh
View File

@@ -1,28 +0,0 @@
set -x
task_name="gpt-acc-dp-dygraph"
base_out="tests/output"
rm -rf $base_out/$task_name/log
unset CUDA_VISIBLE_DEVICES
PYTHONPATH=../../../ python -m paddle.distributed.launch \
--gpus "0,1" \
--log_dir "$base_out/$task_name/log" run_pretrain.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data"\
--output_dir "$base_out/$task_name"\
--max_seq_len 1024 \
--micro_batch_size 4\
--max_lr 0.00015\
--min_lr 0.00001\
--max_steps 20\
--save_steps 100000\
--decay_steps 320000\
--weight_decay 0.01\
--warmup_rate 0.01\
--grad_clip 1.0\
--logging_freq 1\
--eval_freq 500\
--check_accuracy true\
--device "gpu"

45
tests/examples/gpt/acc_dp_static.sh
View File

@@ -1,45 +0,0 @@
set -x
export PADDLE_WITH_GLOO=0
export FLAGS_call_stack_level=2
export GLOG_v=-1
unset CUDA_VISIBLE_DEVICES
rm -rf *.prototxt
rm -rf core.*
rm -rf start_sharding*
rm -rf main_sharding*
task_name="gpt-acc-dp-static"
base_out="tests/output"
rm -rf $base_out/$task_name/log
PYTHONPATH=../../../ python -u -m paddle.distributed.launch \
--gpus "0,1" \
--log_dir "$base_out/$task_name/log" run_pretrain_static.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data" \
--output_dir "$base_out/$task_name" \
--max_seq_len 1024 \
--micro_batch_size 4 \
--global_batch_size 8 \
--sharding_degree 1\
--mp_degree 1 \
--dp_degree 2 \
--pp_degree 1 \
--use_sharding false \
--use_amp false \
--use_recompute false \
--max_lr 0.00015 \
--min_lr 0.00001 \
--max_steps 20 \
--save_steps 10000 \
--decay_steps 320000 \
--weight_decay 0.01\
--warmup_rate 0.01 \
--grad_clip 1.0 \
--logging_freq 1\
--eval_freq 500 \
--check_accuracy true \
--device "gpu"

47
tests/examples/gpt/acc_mp_static.sh
View File

@@ -1,47 +0,0 @@
set -x
export PADDLE_WITH_GLOO=0
export FLAGS_call_stack_level=2
export FLAGS_allocator_strategy=naive_best_fit
unset CUDA_VISIBLE_DEVICES
rm -rf *.prototxt
rm -rf core.*
rm -rf start_sharding*
rm -rf main_sharding*
task_name="gpt-acc-mp-static"
base_out="tests/output"
rm -rf $base_out/$task_name/log
PYTHONPATH=../../../ python -u -m paddle.distributed.launch \
--gpus "0,1" \
--log_dir "$base_out/$task_name/log" run_pretrain_static.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data" \
--output_dir "$base_out/$task_name" \
--max_seq_len 1024 \
--micro_batch_size 8 \
--global_batch_size 8\
--sharding_degree 1\
--mp_degree 2 \
--dp_degree 1 \
--pp_degree 1 \
--use_sharding true \
--use_amp false \
--use_recompute false \
--max_lr 0.00015 \
--min_lr 0.00001 \
--max_steps 20 \
--save_steps 10000 \
--decay_steps 320000 \
--weight_decay 0.01\
--warmup_rate 0.01 \
--grad_clip 1.0 \
--logging_freq 1\
--eval_freq 500 \
--check_accuracy true\
--device "gpu"
# Not support pipeline for this version, don't change pp_degree.

45
tests/examples/gpt/acc_sharding_static.sh
View File

@@ -1,45 +0,0 @@
set -x
export PADDLE_WITH_GLOO=0
export FLAGS_call_stack_level=2
export GLOG_v=-1
unset CUDA_VISIBLE_DEVICES
rm -rf *.prototxt
rm -rf core.*
rm -rf start_sharding*
rm -rf main_sharding*
task_name="gpt-acc-sharding-static"
base_out="tests/output"
rm -rf $base_out/$task_name/log
PYTHONPATH=../../../ python -u -m paddle.distributed.launch \
--gpus "0,1" \
--log_dir "$base_out/$task_name/log" run_pretrain_static.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data" \
--output_dir "$base_out/$task_name" \
--max_seq_len 1024 \
--micro_batch_size 4 \
--global_batch_size 8 \
--sharding_degree 2\
--mp_degree 1 \
--dp_degree 1 \
--pp_degree 1 \
--use_sharding false \
--use_amp false \
--use_recompute false \
--max_lr 0.00015 \
--min_lr 0.00001 \
--max_steps 20 \
--save_steps 10000 \
--decay_steps 320000 \
--weight_decay 0.01\
--warmup_rate 0.01 \
--grad_clip 1.0 \
--logging_freq 1\
--eval_freq 500 \
--check_accuracy true \
--device "gpu"

28
tests/examples/gpt/acc_single_dygraph.sh
View File

@@ -1,28 +0,0 @@
set -x
task_name="gpt-acc-single-dygraph"
base_out="tests/output"
rm -rf $base_out/$task_name/log
unset CUDA_VISIBLE_DEVICES
PYTHONPATH=../../../ python -m paddle.distributed.launch \
--gpus "0" \
--log_dir "$base_out/$task_name/log" run_pretrain.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data"\
--output_dir "$base_out/$task_name"\
--max_seq_len 1024 \
--micro_batch_size 8\
--max_lr 0.00015\
--min_lr 0.00001\
--max_steps 20\
--save_steps 100000\
--decay_steps 320000\
--weight_decay 0.01\
--warmup_rate 0.01\
--grad_clip 1.0\
--logging_freq 1\
--eval_freq 500\
--check_accuracy true\
--device "gpu"

46
tests/examples/gpt/acc_single_static.sh
View File

@@ -1,46 +0,0 @@
set -x
export PADDLE_WITH_GLOO=0
export FLAGS_call_stack_level=2
export FLAGS_allocator_strategy=naive_best_fit
unset CUDA_VISIBLE_DEVICES
rm -rf *.prototxt
rm -rf core.*
rm -rf start_sharding*
rm -rf main_sharding*
task_name="gpt-acc-single-static"
base_out="tests/output"
rm -rf $base_out/$task_name/log
PYTHONPATH=../../../ python -u -m paddle.distributed.launch \
--gpus "0" \
--log_dir "$base_out/$task_name/log" run_pretrain_static.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data" \
--output_dir "$base_out/$task_name" \
--max_seq_len 1024 \
--micro_batch_size 8 \
--global_batch_size 8\
--sharding_degree 1\
--mp_degree 1 \
--dp_degree 1 \
--pp_degree 1 \
--use_sharding false \
--use_amp false \
--use_recompute false \
--max_lr 0.00015 \
--min_lr 0.00001 \
--max_steps 20 \
--save_steps 10000 \
--decay_steps 320000 \
--weight_decay 0.01\
--warmup_rate 0.01 \
--grad_clip 1.0 \
--logging_freq 1\
--eval_freq 500 \
--check_accuracy true\
--device "gpu"
# Not support pipeline for this version, don't change pp_degree.

44
tests/examples/gpt/benchmark_mp_sharding_static.sh
View File

@@ -1,44 +0,0 @@
set -x
export PADDLE_WITH_GLOO=0
export FLAGS_call_stack_level=2
#export FLAGS_allocator_strategy=naive_best_fit
export GLOG_v=-1
unset CUDA_VISIBLE_DEVICES
rm -rf *.prototxt
rm -rf core.*
rm -rf start_sharding*
rm -rf main_sharding*
task_name="gpt-benchmark-mp-sharding-static"
rm -rf output/$task_name/log
PYTHONPATH=../../../ python -u -m paddle.distributed.fleet.launch \
--gpus "0,1,2,3" \
--log_dir "output/$task_name/log" run_pretrain_static.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data" \
--output_dir "output/$task_name" \
--max_seq_len 1024 \
--micro_batch_size 16 \
--global_batch_size 32 \
--sharding_degree 2\
--mp_degree 2 \
--dp_degree 1 \
--pp_degree 1 \
--use_sharding true \
--use_amp true \
--use_recompute true \
--max_lr 0.00015 \
--min_lr 0.00001 \
--max_steps 70000 \
--save_steps 10000 \
--decay_steps 320000 \
--weight_decay 0.01\
--warmup_rate 0.01 \
--grad_clip 1.0 \
--logging_freq 1\
--eval_freq 500 \
--device "gpu"

44
tests/examples/gpt/benchmark_mp_static.sh
View File

@@ -1,44 +0,0 @@
set -x
export PADDLE_WITH_GLOO=0
export FLAGS_call_stack_level=2
#export FLAGS_allocator_strategy=naive_best_fit
export GLOG_v=-1
unset CUDA_VISIBLE_DEVICES
rm -rf *.prototxt
rm -rf core.*
rm -rf start_sharding*
rm -rf main_sharding*
task_name="gpt-benchmark-mp-static"
rm -rf output/$task_name/log
PYTHONPATH=../../../ python -u -m paddle.distributed.fleet.launch \
--gpus "0,1" \
--log_dir "output/$task_name/log" run_pretrain_static.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data" \
--output_dir "output/$task_name" \
--max_seq_len 1024 \
--micro_batch_size 32 \
--global_batch_size 32 \
--sharding_degree 1\
--mp_degree 2 \
--dp_degree 1 \
--pp_degree 1 \
--use_sharding true \
--use_amp true \
--use_recompute true \
--max_lr 0.00015 \
--min_lr 0.00001 \
--max_steps 70000 \
--save_steps 10000 \
--decay_steps 320000 \
--weight_decay 0.01\
--warmup_rate 0.01 \
--grad_clip 1.0 \
--logging_freq 1\
--eval_freq 500 \
--device "gpu"

44
tests/examples/gpt/benchmark_sharding_static.sh
View File

@@ -1,44 +0,0 @@
set -x
export PADDLE_WITH_GLOO=0
export FLAGS_call_stack_level=2
#export FLAGS_allocator_strategy=naive_best_fit
export GLOG_v=-1
unset CUDA_VISIBLE_DEVICES
rm -rf *.prototxt
rm -rf core.*
rm -rf start_sharding*
rm -rf main_sharding*
task_name="gpt-benchmark-sharding-static"
rm -rf output/$task_name/log
PYTHONPATH=../../../ python -u -m paddle.distributed.fleet.launch \
--gpus "0,1" \
--log_dir "output/$task_name/log" run_pretrain_static.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data" \
--output_dir "output/$task_name" \
--max_seq_len 1024 \
--micro_batch_size 16 \
--global_batch_size 32 \
--sharding_degree 2\
--mp_degree 1 \
--dp_degree 1 \
--pp_degree 1 \
--use_sharding true \
--use_amp true \
--use_recompute true \
--max_lr 0.00015 \
--min_lr 0.00001 \
--max_steps 70000 \
--save_steps 10000 \
--decay_steps 320000 \
--weight_decay 0.01\
--warmup_rate 0.01 \
--grad_clip 1.0 \
--logging_freq 1\
--eval_freq 500 \
--device "gpu"

44
tests/examples/gpt/benchmark_single_static.sh
View File

@@ -1,44 +0,0 @@
set -x
export PADDLE_WITH_GLOO=0
export FLAGS_call_stack_level=2
#export FLAGS_allocator_strategy=naive_best_fit
export GLOG_v=-1
unset CUDA_VISIBLE_DEVICES
rm -rf *.prototxt
rm -rf core.*
rm -rf start_sharding*
rm -rf main_sharding*
task_name="gpt-benchmark-single-static"
rm -rf output/$task_name/log
PYTHONPATH=../../../ python -u -m paddle.distributed.fleet.launch \
--gpus "0" \
--log_dir "output/$task_name/log" run_pretrain_static.py \
--model_type "gpt" \
--model_name_or_path "./ckpt/gpt2-small-en-init-checkpoint"\
--input_dir "./data" \
--output_dir "output/$task_name" \
--max_seq_len 1024 \
--micro_batch_size 32 \
--global_batch_size 32 \
--sharding_degree 1\
--mp_degree 1 \
--dp_degree 1 \
--pp_degree 1 \
--use_sharding false \
--use_amp true \
--use_recompute true \
--max_lr 0.00015 \
--min_lr 0.00001 \
--max_steps 70000 \
--save_steps 10000 \
--decay_steps 320000 \
--weight_decay 0.01\
--warmup_rate 0.01 \
--grad_clip 1.0 \
--logging_freq 1\
--eval_freq 500 \
--device "gpu"

160
tests/examples/gpt/test_accuracy.py
View File

@@ -1,160 +0,0 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import unittest
def check_dataset():
return True
def check_init_checkpoint():
return True
def get_groundtruth():
res = {
1: {"loss": 11.008564949},
20: {"loss": 10.876321793},
}
return res
def get_gpt_path():
return "../../../examples/language_model/gpt"
def get_scripts_path():
return "../../../tests/examples/gpt"
def parse_log(path=None):
path = os.path.join(get_gpt_path(), "tests", path)
if not os.path.exists(path):
raise ValueError("File not found in %s ." % path)
os.system('cat %s | grep "global step" > tmp.log' % path)
res = {}
with open(os.path.join(".", "tmp.log"), "r") as f:
line = f.readline()
while line:
line = line.split(" - ")[-1]
kw = line.split(",")
global_step = int(kw[0].split(" ")[-1])
loss = float(kw[3].split(" ")[-1])
lr = float(kw[6].split(" ")[-1].split("\x1b")[0])
res[global_step] = {"loss": loss, "lr": lr}
line = f.readline()
os.system("rm tmp.log")
return res
def print_test_results(name):
print("\n" * 5)
print("---- This is test reports for %s task: ----" % name)
class GPTAccuarcy(unittest.TestCase):
"""
Train accuarcy test for GPT
"""
def test_acc_single_card(self):
check_dataset()
check_init_checkpoint()
for task_name in ["acc_single_dygraph", "acc_single_static"]:
ret = os.system("cd %s && sh %s/%s.sh" % (get_gpt_path(), get_scripts_path(), task_name))
if ret != 0:
print(ret)
raise ValueError("Train script failed")
gt = get_groundtruth()
res = parse_log("./output/gpt-%s/log/workerlog.0" % task_name.replace("_", "-"))
print_test_results(task_name)
for k in gt.keys():
print("%s step: %d, gt:%.9f res:%.9f " % (task_name, k, gt[k]["loss"], res[k]["loss"]))
self.assertAlmostEqual(gt[k]["loss"], res[k]["loss"], delta=1e-6)
print("\n" * 5)
def test_acc_dp(self):
check_dataset()
check_init_checkpoint()
for task_name in ["acc_dp_dygraph", "acc_dp_static"]:
ret = os.system("cd %s && sh %s/%s.sh" % (get_gpt_path(), get_scripts_path(), task_name))
if ret != 0:
print(ret)
raise ValueError("Train script failed")
gt = get_groundtruth()
res1 = parse_log("./output/gpt-%s/log/workerlog.0" % task_name.replace("_", "-"))
res2 = parse_log("./output/gpt-%s/log/workerlog.1" % task_name.replace("_", "-"))
print_test_results(task_name)
for k in gt.keys():
mean = (res1[k]["loss"] + res2[k]["loss"]) / 2
print("%s step: %d, gt:%.9f res:%.9f " % (task_name, k, gt[k]["loss"], mean))
self.assertAlmostEqual(gt[k]["loss"], mean, delta=5e-6)
print("\n" * 5)
def test_acc_sharding_static(self):
check_dataset()
check_init_checkpoint()
for task_name in ["acc_sharding_static"]:
ret = os.system("cd %s && sh %s/%s.sh" % (get_gpt_path(), get_scripts_path(), task_name))
if ret != 0:
print(ret)
raise ValueError("Train script failed")
gt = get_groundtruth()
res1 = parse_log("./output/gpt-%s/log/workerlog.0" % task_name.replace("_", "-"))
res2 = parse_log("./output/gpt-%s/log/workerlog.1" % task_name.replace("_", "-"))
print_test_results(task_name)
for k in gt.keys():
mean = (res1[k]["loss"] + res2[k]["loss"]) / 2
print("%s step: %d, gt:%.9f res:%.9f " % (task_name, k, gt[k]["loss"], mean))
self.assertAlmostEqual(gt[k]["loss"], mean, delta=5e-6)
print("\n" * 5)
@unittest.skipIf(True, "This folder not support MP. Please use MP in GPT-3.")
def test_acc_mp_static(self):
check_dataset()
check_init_checkpoint()
for task_name in ["acc_mp_static"]:
ret = os.system("cd %s && sh %s/%s.sh" % (get_gpt_path(), get_gpt_path(), task_name))
if ret != 0:
print(ret)
raise ValueError("Train script failed")
gt = get_groundtruth()
res1 = parse_log("./output/gpt-%s/log/workerlog.0" % task_name.replace("_", "-"))
res2 = parse_log("./output/gpt-%s/log/workerlog.1" % task_name.replace("_", "-"))
print_test_results(task_name)
for k in gt.keys():
self.assertAlmostEqual(res1[k]["loss"], res2[k]["loss"], delta=1e-7)
mean = (res1[k]["loss"] + res2[k]["loss"]) / 2
print("%s step: %d, gt:%.9f res:%.9f " % (task_name, k, gt[k]["loss"], mean))
self.assertAlmostEqual(gt[k]["loss"], res1[k]["loss"], delta=1e-7)
print("\n" * 5)
if __name__ == "__main__":
unittest.main()

2
tests/fixtures/llm/pretrain.yaml vendored
View File

@@ -23,6 +23,8 @@ pretrain:
model_name_or_path: __internal_testing__/tiny-random-llama
qwen:
model_name_or_path: __internal_testing__/tiny-fused-qwen
gpt:
model_name_or_path: __internal_testing__/gpt
inference-predict:
default:

94
tests/fixtures/model_zoo/gpt.yaml vendored
View File

@@ -1,94 +0,0 @@
pretrain:
default:
model_type: gpt
model_name_or_path: __internal_testing__/gpt
input_dir: ./data
output_dir: ./output_dir/pretrain
weight_decay: 0.01
max_steps: 2
save_steps: 10
warmup_steps: 10
warmup_ratio: 0.01
per_device_train_batch_size: 4
device: cpu
eval_steps: 10
do_train: true
do_predict: true
slow:
model_type: gpt
model_name_or_path: gpt2-en
warmup_steps: 320000
warmup_ratio: 0.01
per_device_train_batch_size: 4
device: cpu
eval_steps: 100
do_train: true
do_predict: true
msra_ner:
default:
model_name_or_path: __internal_testing__/gpt_cn
max_seq_length: 32
per_device_eval_batch_size: 4
learning_rate: 2e-5
num_train_epochs: 3
logging_steps: 25
save_steps: 250
output_dir: ./output_dir/msra_ner/
device: cpu
do_train: true
max_steps: 10
slow:
model_name_or_path: gpt-cpm-small-cn-distill
max_seq_length: 128
per_device_eval_batch_size: 32
learning_rate: 2e-5
num_train_epochs: 3
logging_steps: 25
save_steps: 250
output_dir: ./tmp/msra_ner/
device: gpu
do_train: true
glue:
default:
model_name_or_path: __internal_testing__/gpt
task_name: SST-2
max_seq_length: 128
per_device_train_batch_size: 32
learning_rate: 2e-5
num_train_epochs: 3
logging_steps: 1
save_steps: 500
output_dir: ./output_dir/glue
max_steps: 10
eval_steps: 1
device: cpu
do_train: true
slow:
model_name_or_path: gpt2-medium-en
task_name: SST-2
max_seq_length: 128
per_device_train_batch_size: 32
learning_rate: 2e-5
num_train_epochs: 3
logging_steps: 1
save_steps: 500
output_dir: ./output_dir/glue
eval_steps: 1
device: gpu
do_train: true
generation:
default:
model_type: gpt2
model_name_or_path: hf-internal-testing/tiny-random-gpt2
from_hf_hub: true
slow:
model_type: gpt2-cn
model_name_or_path: gpt-cpm-small-cn-distill

16
tests/llm/test_pretrain.py
View File

@@ -13,7 +13,6 @@
# limitations under the License.
from __future__ import annotations
import os
import shutil
import sys
import tempfile
@@ -32,6 +31,7 @@ from .testing_utils import LLMTest
[
["llama"],
["qwen"],
["gpt"],
],
)
class PretrainTest(LLMTest, unittest.TestCase):
@@ -42,18 +42,10 @@ class PretrainTest(LLMTest, unittest.TestCase):
LLMTest.setUp(self)
self.dataset_dir = tempfile.mkdtemp()
if self.model_dir != "qwen":
self.model_codes_dir = os.path.join(self.root_path, self.model_dir)
sys.path.insert(0, self.model_codes_dir)
else:
self.model_codes_dir = self.root_path
self.model_codes_dir = self.root_path
def tearDown(self) -> None:
LLMTest.tearDown(self)
if self.model_dir != "qwen":
sys.path.remove(self.model_codes_dir)
shutil.rmtree(self.dataset_dir)
def test_pretrain(self):
@@ -82,6 +74,8 @@ class PretrainTest(LLMTest, unittest.TestCase):
main()
if self.model_dir != "qwen":
# Now, only work for llama, not gpt or qwen
if self.model_dir == "llama":
self.run_predictor({"inference_model": True})
self.run_predictor({"inference_model": False})

85
tests/model_zoo/test_gpt.py
View File

@@ -1,85 +0,0 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import os
import sys
import unittest
from unittest import TestCase
from tests.testing_utils import argv_context_guard, load_test_config
class GPTTest(TestCase):
def setUp(self) -> None:
self.path = "./model_zoo/gpt"
self.config_path = "./tests/fixtures/model_zoo/gpt.yaml"
sys.path.insert(0, self.path)
def tearDown(self) -> None:
sys.path.remove(self.path)
# TODO(wj-Mcat): disable old gpt `run_pretrain.py` and will be uncomment in: https://github.com/PaddlePaddle/PaddleNLP/pull/4614
@unittest.skip("disable for old gpt")
def test_pretrain(self):
# 1. run pretrain
pretrain_config = load_test_config(self.config_path, "pretrain")
with argv_context_guard(pretrain_config):
from run_pretrain import do_train
do_train()
# 2. export model
export_config = {
"model_type": pretrain_config["model_type"],
"model_path": pretrain_config["output_dir"],
"output_path": os.path.join(pretrain_config["output_dir"], "export_model"),
}
with argv_context_guard(export_config):
from export_model import main
main()
# 3. infer model
infer_config = {
"model_type": export_config["model_type"],
"model_path": export_config["output_path"],
"select_device": pretrain_config["device"],
}
with argv_context_guard(infer_config):
from deploy.python.inference import main
main()
def test_msra_ner(self):
config = load_test_config(self.config_path, "msra_ner")
with argv_context_guard(config):
from run_msra_ner import do_train
do_train()
def test_run_glue(self):
config = load_test_config(self.config_path, "glue")
with argv_context_guard(config):
from run_glue import do_train
do_train()
def test_generation(self):
config = load_test_config(self.config_path, "generation")
with argv_context_guard(config):
from run_generation import run
run()

3
tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh
View File

@@ -99,8 +99,7 @@ function _train(){
fi
model_config="gpt2-medium-en"
train_cmd="--model_type gpt \
--model_name_or_path ${model_config} \
train_cmd="--model_name_or_path ${model_config} \
--tokenizer_name_or_path ${model_config} \
--input_dir ./data\
--output_dir output\

3
tests/test_tipc/dygraph/hybrid_parallelism/gpt/benchmark_common/run_benchmark.sh
View File

@@ -92,8 +92,7 @@ function _train(){
model_config="gpt2-medium-en"
[ ${mp_degree} -lt 8 ] && model_config="gpt2-small-en"
if [ "fp16" = ${fp_item} ]; then use_pure_fp16=True; fi
train_cmd="--model_type gpt \
--model_name_or_path ${model_config} \
train_cmd="--model_name_or_path ${model_config} \
--input_dir ./data\
--output_dir output\
--weight_decay 0.01\

3
tests/test_tipc/dygraph/hybrid_parallelism/llama/benchmark_common/run_benchmark.sh
View File

@@ -104,8 +104,7 @@ function _train(){
fi
use_pure_fp16=False
train_cmd="--model_type llama \
--model_name_or_path ${model_name_or_path} \
train_cmd="--model_name_or_path ${model_name_or_path} \
--tokenizer_name_or_path ${model_name_or_path} \
--input_dir ./data \
--output_dir ./output \