58 lines
1.7 KiB
JSON
58 lines
1.7 KiB
JSON
{
|
|
"train_datasets": "PKU-SafeRLHF/train",
|
|
"eval_datasets": "PKU-SafeRLHF/test",
|
|
"ptx_datasets": "alpaca",
|
|
"actor_model_name_or_path": "PKU-Alignment/alpaca-7b-reproduced",
|
|
"reward_model_name_or_path": "PKU-Alignment/beaver-7b-v1.0-reward",
|
|
"output_dir": "/root/paddlejob/workspace/guosheng/ckpts/ppo-reshard-sd38",
|
|
"max_length": 512,
|
|
"top_p": 0.8,
|
|
"temperature": 1.0,
|
|
"num_return_sequences":1,
|
|
"repetition_penalty": 1.0,
|
|
"num_train_epochs": 1,
|
|
"update_iters": 1,
|
|
"per_device_prompt_batch_size": 16,
|
|
"per_device_train_batch_size": 16,
|
|
"gradient_accumulation_steps": 1,
|
|
"learning_rate": 1e-5,
|
|
"weight_decay": 0.01,
|
|
"lr_scheduler_type": "cosine",
|
|
"warmup_ratio": 0.03,
|
|
"recompute": true,
|
|
"critic_learning_rate": 5e-6,
|
|
"critic_weight_decay": 0.0,
|
|
"critic_lr_scheduler_type": "constant",
|
|
"critic_warmup_ratio": 0.03,
|
|
"critic_recompute": true,
|
|
"normalize_reward": false,
|
|
"kl_coeff": 0.02,
|
|
"clip_range_ratio": 0.2,
|
|
"clip_range_score": 50.0,
|
|
"clip_range_value": 5.0,
|
|
"ptx_coeff": 16.0,
|
|
"per_device_eval_batch_size": 16,
|
|
"logging_steps": 1,
|
|
"evaluation_strategy": "steps",
|
|
"eval_steps": 100,
|
|
"save_strategy": "epoch",
|
|
"save_steps": 100000,
|
|
"bf16": true,
|
|
"fp16_opt_level": "O2",
|
|
"do_train": true,
|
|
"do_eval": true,
|
|
"disable_tqdm": true,
|
|
"save_total_limit": 1,
|
|
"sharding_parallel_degree": 4,
|
|
"sharding": "stage1",
|
|
"tensor_parallel_degree": 2,
|
|
"pipeline_parallel_degree": 1,
|
|
"pipeline_parallel_config": "disable_p2p_cache_shape",
|
|
"max_grad_norm": 1.0,
|
|
"adam_beta1": 0.9,
|
|
"adam_beta2": 0.95,
|
|
"dataloader_drop_last": false,
|
|
"eval_mode": "",
|
|
"offload_level": "freeze_model"
|
|
}
|