Kimirock/PaddleNLP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[BugFix] Add CE for mp2-sp2-pp2-dp2 (#7678)

Browse Source
This commit is contained in:
yinwei
2023-12-19 11:33:25 +08:00
committed by GitHub
parent 6299680263
commit 1757c5d52d
3 changed files with 24 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/CE_gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh
View File

@@ -20,7 +20,12 @@ bs_item=32
fp_item=bf16
run_mode=MP2-SP2-PP2-DP2-mbs8-acc2
device_num=N1C8
max_iter=100
max_iter=50000
virtual_pp_degree=1
use_recompute=True
eval_freq=25
use_pipeline_parallel=True
sequence_parallel=True
model=gpt
micro_bs=8
@@ -28,4 +33,4 @@ micro_bs=8
bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh
# run
bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \
${max_iter} ${sharding} ${sharding_degree} 2>&1;
${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} 2>&1;

7
tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/N1C8/gpt-345m_seqlen1024_pretrain_bs32_bf16_MP2-SP2-PP2-DP2-mbs8-acc2.sh
View File

@@ -21,6 +21,11 @@ fp_item=bf16
run_mode=MP2-SP2-PP2-DP2-mbs8-acc2
device_num=N1C8
max_iter=100
virtual_pp_degree=1
use_recompute=True
eval_freq=25
use_pipeline_parallel=True
sequence_parallel=True
model=gpt
micro_bs=8
@@ -28,4 +33,4 @@ micro_bs=8
bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/prepare.sh
# run
bash ./test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \
${max_iter} ${sharding} ${sharding_degree} 2>&1;
${max_iter} ${sharding} ${sharding_degree} ${virtual_pp_degree} ${use_recompute} ${eval_freq} ${use_pipeline_parallel} ${sequence_parallel} 2>&1;

14
tests/test_tipc/dygraph/hybrid_parallelism/ce_gpt/benchmark_common/run_benchmark.sh
View File

@@ -40,6 +40,8 @@ function _set_params(){
virtual_pp_degree=${13:-"2"} # (可选) virtualpp数据并行度
use_recompute=${14:-"True"} # (可选)是否打开recompute
eval_freq=${15:-"25"} # (可选)模型评估间隔
use_pipeline_parallel=${16:-"False"} # (可选)是否开启pipeline_parallel_config
sequence_parallel=${17:-"False"} # (可选)是否开启sequence_parallel
# 以下为通用执行命令，无特殊可不用修改
model_name=${model_item}_bs${global_batch_size}_${fp_item}_${run_mode} # (必填) 且格式不要改动,与竞品名称对齐
device=${CUDA_VISIBLE_DEVICES//,/ }
@@ -66,7 +68,7 @@ function _train(){
echo "current CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}, model_name=${model_name}, device_num=${device_num}, is profiling=${profiling}"
if [ ${profiling} = "true" ];then
if [ ${profiling} = "True" ];then
add_options="--profiler_options=\"batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile\""
log_file=${profiling_log_file}
else
@@ -90,6 +92,12 @@ function _train(){
sharding_parallel_degree="--sharding_parallel_degree ${sharding_degree}"
fi
if [ "False" = ${use_pipeline_parallel} ]; then
pp_config_disable_partial_send_recv = ""
else
pp_config_disable_partial_send_recv = "--pipeline_parallel_config \"disable_partial_send_recv\""
fi
model_config="gpt2-medium-en"
train_cmd="--model_type gpt \
--model_name_or_path ${model_config} \
@@ -100,8 +108,8 @@ function _train(){
${sharding_parallel_degree} \
--tensor_parallel_degree ${mp_degree} \
--pipeline_parallel_degree ${pp_degree} \
--pipeline_parallel_config "disable_partial_send_recv" \
--sequence_parallel true \
${pp_config_disable_partial_send_recv} \
--sequence_parallel ${sequence_parallel} \
--split 949,50,1 \
--max_seq_length 1024 \
--seed 1234 \