From 3c51eda5d6deb9455cfa0eff371138250d5a514f Mon Sep 17 00:00:00 2001 From: zhongtianyun <79952610+tianyumyum@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:53:53 +0800 Subject: [PATCH] [Embedding] Fix readme and env for contrastive_training (#11108) modified: csrc/gpu/unittest/test_get_padding_offset_v2.py modified: paddlenlp/experimental/transformers/llama/modeling.py modified: slm/pipelines/examples/contrastive_training/README.md modified: slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.py modified: slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.sh modified: slm/pipelines/examples/contrastive_training/evaluation/modelling_quant.py modified: slm/pipelines/examples/contrastive_training/shortgpt_prune.py --- .../unittest/test_get_padding_offset_v2.py | 1 - .../transformers/llama/modeling.py | 183 +++++++++++++----- .../examples/contrastive_training/README.md | 63 ++++-- .../evaluation/eval_mteb.py | 2 +- .../evaluation/eval_mteb.sh | 85 ++++++-- .../evaluation/modelling_quant.py | 4 +- .../contrastive_training/shortgpt_prune.py | 2 +- 7 files changed, 256 insertions(+), 84 deletions(-) diff --git a/csrc/gpu/unittest/test_get_padding_offset_v2.py b/csrc/gpu/unittest/test_get_padding_offset_v2.py index aabaa7e971..3e9f3d4934 100644 --- a/csrc/gpu/unittest/test_get_padding_offset_v2.py +++ b/csrc/gpu/unittest/test_get_padding_offset_v2.py @@ -64,6 +64,5 @@ class GetPaddingOffsetV2Test(unittest.TestCase): assert sum(ref_cu_seqlens_q - cu_seqlens_q) == 0, "Check cu_seqlens_q failed." assert sum(ref_cu_seqlens_k - cu_seqlens_k) == 0, "Check cu_seqlens_k failed." - if __name__ == "__main__": unittest.main() diff --git a/paddlenlp/experimental/transformers/llama/modeling.py b/paddlenlp/experimental/transformers/llama/modeling.py index 3a32e3c7eb..031c58a571 100644 --- a/paddlenlp/experimental/transformers/llama/modeling.py +++ b/paddlenlp/experimental/transformers/llama/modeling.py @@ -150,13 +150,15 @@ class LlamaAvxInferenceModel(LlamaPretrainedModel): ln_scale_attrs = [paddle.ParamAttr(name="fusellama.{}.ln_scale".format(i)) for i in range(self.num_layers)] qkv_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.qkv_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.qkv_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] out_proj_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.out_proj_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.out_proj_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] @@ -165,19 +167,22 @@ class LlamaAvxInferenceModel(LlamaPretrainedModel): ] gate_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.gate_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.gate_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] up_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.up_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.up_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] ffn2_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.ffn2_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.ffn2_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] @@ -328,7 +333,8 @@ class LlamaAvxInferenceModel(LlamaPretrainedModel): ) # reshape(3, self.num_attention_heself.hidden_sizeads // self.config.tensor_parallel_degree, head_size, ) if "llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx) in state_dict.keys(): concated_ffn1_weight = np.concatenate( - split_fn(state_dict["llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx)]), axis=-1 + split_fn(state_dict["llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx)]), + axis=-1, ) else: unfused_state_dict["mlp.gate_proj.weight"] = state_dict[ @@ -336,7 +342,11 @@ class LlamaAvxInferenceModel(LlamaPretrainedModel): ] unfused_state_dict["mlp.up_proj.weight"] = state_dict["llama.layers.{}.mlp.up_proj.weight".format(idx)] concated_ffn1_weight = np.concatenate( - [unfused_state_dict["mlp.gate_proj.weight"], unfused_state_dict["mlp.up_proj.weight"]], axis=-1 + [ + unfused_state_dict["mlp.gate_proj.weight"], + unfused_state_dict["mlp.up_proj.weight"], + ], + axis=-1, ) gate_up_list = split_fn(concated_ffn1_weight) gate_weight_tensor = paddle.to_tensor(gate_up_list[0]) @@ -475,13 +485,15 @@ class LlamaInferenceModel(LlamaPretrainedModel): ln_scale_attrs = [paddle.ParamAttr(name="fusellama.{}.ln_scale".format(i)) for i in range(self.num_layers)] qkv_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.qkv_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.qkv_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] out_proj_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.out_proj_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.out_proj_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] @@ -491,26 +503,30 @@ class LlamaInferenceModel(LlamaPretrainedModel): if "fp8" in self.quant_type: ffn1_0_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.ffn1_0_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.ffn1_0_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] ffn1_1_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.ffn1_1_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.ffn1_1_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] else: ffn1_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.ffn1_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.ffn1_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] ffn2_weight_attrs = [ paddle.ParamAttr( - name="fusellama.{}.ffn2_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0) + name="fusellama.{}.ffn2_weight".format(i), + initializer=paddle.nn.initializer.Constant(value=0), ) for i in range(self.num_layers) ] @@ -784,7 +800,12 @@ class LlamaInferenceModel(LlamaPretrainedModel): from paddlenlp_ops import fused_get_rotary_embedding new_rope = fused_get_rotary_embedding( - input_ids, position_ids, self.head_dim_shape_tensor, position_offset, self.rope_theta, self.use_neox + input_ids, + position_ids, + self.head_dim_shape_tensor, + position_offset, + self.rope_theta, + self.use_neox, ) with dy2st_nocheck_guard_context(): @@ -800,7 +821,7 @@ class LlamaInferenceModel(LlamaPretrainedModel): seq_lens=seq_lens, rotary_embs=new_rope, rotary_emb_dims=1, - time_step=paddle.increment(paddle.shape(attention_mask)[-1], -1) if is_decoder else None, + time_step=(paddle.increment(paddle.shape(attention_mask)[-1], -1) if is_decoder else None), ) hidden_states = self.norm(hidden_states) @@ -831,14 +852,18 @@ class LlamaInferenceModel(LlamaPretrainedModel): weight_scale_json_path = resolve_file_path(self.quant_model_path, "weight_scales.json") if self.config.tensor_parallel_degree > 1 and not self.config.single_card_ptq: act_scale_json_path = resolve_file_path( - self.quant_model_path, f"act_scales_{self.config.tensor_parallel_rank}.json" + self.quant_model_path, + f"act_scales_{self.config.tensor_parallel_rank}.json", ) weight_scale_json_path = resolve_file_path( - self.quant_model_path, f"weight_scales_{self.config.tensor_parallel_rank}.json" + self.quant_model_path, + f"weight_scales_{self.config.tensor_parallel_rank}.json", ) act_scales = ActScalesLoader( - act_scale_json_path, act_scale_map_dict, num_of_layers=self.config.num_hidden_layers + act_scale_json_path, + act_scale_map_dict, + num_of_layers=self.config.num_hidden_layers, ) weight_scales = PerTensorWeightScalesLoader( @@ -869,13 +894,17 @@ class LlamaInferenceModel(LlamaPretrainedModel): weight_scale_json_path = resolve_file_path(self.quant_model_path, "weight_scales.json") if self.config.tensor_parallel_degree > 1 and not self.config.single_card_ptq: act_scale_json_path = resolve_file_path( - self.quant_model_path, f"act_scales_{self.config.tensor_parallel_rank}.json" + self.quant_model_path, + f"act_scales_{self.config.tensor_parallel_rank}.json", ) weight_scale_json_path = resolve_file_path( - self.quant_model_path, f"weight_scales_{self.config.tensor_parallel_rank}.json" + self.quant_model_path, + f"weight_scales_{self.config.tensor_parallel_rank}.json", ) act_scale_loader = ActScalesLoader( - act_scale_json_path, act_scale_map_dict, num_of_layers=self.config.num_hidden_layers + act_scale_json_path, + act_scale_map_dict, + num_of_layers=self.config.num_hidden_layers, ) weight_scales_loader = WeightScalesLoader( weight_scale_json_path, @@ -956,7 +985,8 @@ class LlamaInferenceModel(LlamaPretrainedModel): cache_scale_json_path = resolve_file_path(self.quant_model_path, "cachekv_scales.json") if self.config.tensor_parallel_degree > 1 and not self.config.single_card_ptq: cache_scale_json_path = resolve_file_path( - self.quant_model_path, f"cachekv_scales_{self.config.tensor_parallel_rank}.json" + self.quant_model_path, + f"cachekv_scales_{self.config.tensor_parallel_rank}.json", ) cache_scales_loader = CacheScaleLoader( cache_scale_json_path, @@ -1095,7 +1125,9 @@ class LlamaInferenceModel(LlamaPretrainedModel): if self.use_weight_only: qkv_weight_tensor = paddle.transpose(qkv_weight_tensor, perm=[1, 0]) qkv_quanted_weight_tensor, qkv_weight_scale_tensor = weight_quantize( - qkv_weight_tensor.cpu(), algo=self.quant_algo, group_size=self.weightonly_group_size + qkv_weight_tensor.cpu(), + algo=self.quant_algo, + group_size=self.weightonly_group_size, ) self.transformer_block.qkv_weights[idx].set_value(qkv_quanted_weight_tensor.cuda()) self.transformer_block.qkv_weights_scale[idx].set_value(qkv_weight_scale_tensor.cuda()) @@ -1113,7 +1145,9 @@ class LlamaInferenceModel(LlamaPretrainedModel): ).cast(paddle.get_default_dtype()) if self.use_weight_only: linear_quanted_weight_tensor, linear_weight_scale_tensor = weight_quantize( - linear_weight_tensor.cpu(), algo=self.quant_algo, group_size=self.weightonly_group_size + linear_weight_tensor.cpu(), + algo=self.quant_algo, + group_size=self.weightonly_group_size, ) self.transformer_block.linear_weights[idx].set_value(linear_quanted_weight_tensor.cuda()) self.transformer_block.linear_weights_scale[idx].set_value(linear_weight_scale_tensor.cuda()) @@ -1160,7 +1194,8 @@ class LlamaInferenceModel(LlamaPretrainedModel): if "llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx) in state_dict.keys(): concated_ffn1_weight = np.concatenate( - split_fn(state_dict["llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx)]), axis=-1 + split_fn(state_dict["llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx)]), + axis=-1, ) else: unfused_state_dict["mlp.gate_proj.weight"] = state_dict[ @@ -1168,13 +1203,19 @@ class LlamaInferenceModel(LlamaPretrainedModel): ] unfused_state_dict["mlp.up_proj.weight"] = state_dict["llama.layers.{}.mlp.up_proj.weight".format(idx)] concated_ffn1_weight = np.concatenate( - [unfused_state_dict["mlp.gate_proj.weight"], unfused_state_dict["mlp.up_proj.weight"]], axis=-1 + [ + unfused_state_dict["mlp.gate_proj.weight"], + unfused_state_dict["mlp.up_proj.weight"], + ], + axis=-1, ) ffn1_weight_tensor = paddle.to_tensor(concated_ffn1_weight).cast(paddle.get_default_dtype()) if self.use_weight_only: ffn1_quanted_weight_tensor, ffn1_weight_scale_tensor = weight_quantize( - ffn1_weight_tensor.cpu(), algo=self.quant_algo, group_size=self.weightonly_group_size + ffn1_weight_tensor.cpu(), + algo=self.quant_algo, + group_size=self.weightonly_group_size, ) self.transformer_block.ffn1_weights[idx].set_value(ffn1_quanted_weight_tensor.cuda()) self.transformer_block.ffn1_weights_scale[idx].set_value(ffn1_weight_scale_tensor.cuda()) @@ -1201,7 +1242,10 @@ class LlamaInferenceModel(LlamaPretrainedModel): ) else: self.transformer_block.ffn1_weights[idx].set_value( - paddle.cast(paddle.to_tensor(concated_ffn1_weight).transpose((1, 0)), w_dtype) + paddle.cast( + paddle.to_tensor(concated_ffn1_weight).transpose((1, 0)), + w_dtype, + ) ) else: self.transformer_block.ffn1_weights[idx].set_value(ffn1_weight_tensor) @@ -1211,7 +1255,9 @@ class LlamaInferenceModel(LlamaPretrainedModel): ) if self.use_weight_only: ffn2_quanted_weight_tensor, ffn2_weight_scale_tensor = weight_quantize( - ffn2_weight_tensor.cpu(), algo=self.quant_algo, group_size=self.weightonly_group_size + ffn2_weight_tensor.cpu(), + algo=self.quant_algo, + group_size=self.weightonly_group_size, ) self.transformer_block.ffn2_weights[idx].set_value(ffn2_quanted_weight_tensor.cuda()) self.transformer_block.ffn2_weights_scale[idx].set_value(ffn2_weight_scale_tensor.cuda()) @@ -1231,7 +1277,8 @@ class LlamaInferenceModel(LlamaPretrainedModel): if paddle.is_compiled_with_rocm(): self.transformer_block.ffn2_weights[idx].set_value( paddle.cast( - paddle.to_tensor(state_dict["llama.layers.{}.mlp.down_proj.weight".format(idx)]), w_dtype + paddle.to_tensor(state_dict["llama.layers.{}.mlp.down_proj.weight".format(idx)]), + w_dtype, ) ) else: @@ -1297,7 +1344,8 @@ class LlamaInferenceModel(LlamaPretrainedModel): if self.use_fake_parameter: if "llama.layers.{}.input_layernorm.bias".format(idx) not in state_dict: state_dict["llama.layers.{}.input_layernorm.bias".format(idx)] = paddle.zeros( - shape=[self.hidden_size], dtype=paddle.get_default_dtype() + shape=[self.hidden_size], + dtype=paddle.get_default_dtype(), ) state_dict["llama.layers.{}.post_attention_layernorm.bias".format(idx)] = paddle.zeros( [self.hidden_size], dtype=paddle.get_default_dtype() @@ -1315,10 +1363,12 @@ class LlamaInferenceModel(LlamaPretrainedModel): dtype=paddle.get_default_dtype(), ) unfused_state_dict["mlp.gate_proj.bias"] = paddle.zeros( - shape=[self.intermediate_size], dtype=paddle.get_default_dtype() + shape=[self.intermediate_size], + dtype=paddle.get_default_dtype(), ) unfused_state_dict["mlp.up_proj.bias"] = paddle.zeros( - shape=[self.intermediate_size], dtype=paddle.get_default_dtype() + shape=[self.intermediate_size], + dtype=paddle.get_default_dtype(), ) else: unfused_state_dict["self_attn.q_proj.bias"] = state_dict[ @@ -1356,7 +1406,11 @@ class LlamaInferenceModel(LlamaPretrainedModel): self.transformer_block.qkv_biases[idx].set_value(paddle.to_tensor(concated_qkv_biases)) concated_ffn1_bias = np.concatenate( - [unfused_state_dict["mlp.gate_proj.bias"], unfused_state_dict["mlp.up_proj.bias"]], axis=-1 + [ + unfused_state_dict["mlp.gate_proj.bias"], + unfused_state_dict["mlp.up_proj.bias"], + ], + axis=-1, ) self.transformer_block.ffn1_biases[idx].set_value(paddle.to_tensor(concated_ffn1_bias)) @@ -1404,10 +1458,23 @@ class LlamaBlockInferenceModel(LlamaInferenceModel): token_num = paddle.sum(seq_lens_this_time) from paddlenlp_ops import get_padding_offset_v2 + cum_offsets_now = paddle.cast(cum_offsets_now, dtype="int32") + ids_remove_padding, cum_offsets, padding_offset, cu_seqlens_q, cu_seqlens_k = get_padding_offset_v2( - input_ids, cum_offsets_now, token_num, seq_lens_this_time, draft_tokens, seq_lens_encoder + input_ids, + cum_offsets_now, + token_num, + seq_lens_this_time, + draft_tokens, + seq_lens_encoder, + ) + return ( + ids_remove_padding, + padding_offset, + cum_offsets, + cu_seqlens_q, + cu_seqlens_k, ) - return ids_remove_padding, padding_offset, cum_offsets, cu_seqlens_q, cu_seqlens_k def forward( self, @@ -1466,13 +1533,21 @@ class LlamaBlockInferenceModel(LlamaInferenceModel): # whether speculative decoding or not if draft_tokens is None: - ids_remove_padding, padding_offset, cum_offsets, cu_seqlens_q, cu_seqlens_k = self.remove_padding( - input_ids, seq_lens_this_time - ) + ( + ids_remove_padding, + padding_offset, + cum_offsets, + cu_seqlens_q, + cu_seqlens_k, + ) = self.remove_padding(input_ids, seq_lens_this_time) else: - ids_remove_padding, padding_offset, cum_offsets, cu_seqlens_q, cu_seqlens_k = self.remove_padding( - input_ids, seq_lens_this_time, draft_tokens, seq_lens_encoder - ) + ( + ids_remove_padding, + padding_offset, + cum_offsets, + cu_seqlens_q, + cu_seqlens_k, + ) = self.remove_padding(input_ids, seq_lens_this_time, draft_tokens, seq_lens_encoder) kwargs["cu_seqlens_q"] = cu_seqlens_q kwargs["cu_seqlens_k"] = cu_seqlens_k @@ -1509,7 +1584,11 @@ class EagleForLlamaBlockInferenceModel(LlamaBlockInferenceModel): if config.tensor_parallel_degree > 1: self.fc = ColumnParallelLinear( - self.hidden_size * 2, self.hidden_size, has_bias=True, gather_output=True, fuse_matmul_bias=True + self.hidden_size * 2, + self.hidden_size, + has_bias=True, + gather_output=True, + fuse_matmul_bias=True, ) else: self.fc = nn.Linear(self.hidden_size * 2, self.hidden_size, bias_attr=True) @@ -1669,7 +1748,11 @@ class LlamaForCausalLMInferenceModel(GenerationInferenceModel, LlamaPretrainedMo super().__init__(config) self.llama = LlamaInferenceModel(config) if config.tie_word_embeddings: - self.lm_head = LlamaLMHead(config, embedding_weights=self.llama.embed_tokens.weight, transpose_y=True) + self.lm_head = LlamaLMHead( + config, + embedding_weights=self.llama.embed_tokens.weight, + transpose_y=True, + ) self.tie_weights() else: self.lm_head = LlamaLMHead(config) @@ -1867,7 +1950,11 @@ class LlamaForCausalLMBlockInferenceModel(GenerationBlockInferenceModel, LlamaPr self.llama = LlamaBlockInferenceModel(config) if config.tie_word_embeddings: - self.lm_head = LlamaLMHead(config, embedding_weights=self.llama.embed_tokens.weight, transpose_y=True) + self.lm_head = LlamaLMHead( + config, + embedding_weights=self.llama.embed_tokens.weight, + transpose_y=True, + ) self.tie_weights() else: self.lm_head = LlamaLMHead(config) @@ -2108,7 +2195,11 @@ class EagleLlamaForCausalLMBlockInferenceModel(LlamaForCausalLMBlockInferenceMod self.eagle = EagleForLlamaBlockInferenceModel(config) if config.tie_word_embeddings: - self.lm_head = LlamaLMHead(config, embedding_weights=self.llama.embed_tokens.weight, transpose_y=True) + self.lm_head = LlamaLMHead( + config, + embedding_weights=self.llama.embed_tokens.weight, + transpose_y=True, + ) self.tie_weights() else: self.lm_head = LlamaLMHead(config) diff --git a/slm/pipelines/examples/contrastive_training/README.md b/slm/pipelines/examples/contrastive_training/README.md index 3b6b06e91a..99d113e643 100644 --- a/slm/pipelines/examples/contrastive_training/README.md +++ b/slm/pipelines/examples/contrastive_training/README.md @@ -1,6 +1,6 @@ # 向量检索模型训练 -推荐安装 gpu 版本的[PaddlePaddle](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/conda/linux-conda.html)，以 cuda11.8的 paddle 为例，安装命令如下： +推荐安装 gpu 版本的[PaddlePaddle](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/conda/linux-conda.html)，以 cuda 12.6 的 paddle 为例，安装命令如下： ``` # 创建一个名为 paddle_env 的新环境，并激活 @@ -11,9 +11,9 @@ conda activate paddle_env pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html # 安装 paddlepaddle-gpu nightly版本 -pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ +pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ -#安装其他依赖： +# 安装其他依赖： pip install -r slm/pipelines/examples/contrastive_training/requirements.txt ``` @@ -153,6 +153,10 @@ python -u evaluation/eval_mteb.py \ --base_model_name_or_path ${model_path} \ --output_folder eval_results/${model_path} \ --task_name 'DuRetrieval' \ + --document_instruction "" \ + --query_instruction "" \ + --padding_side right \ + --pooling_method "cls" \ --eval_batch_size 8 \ --max_seq_length 2048 \ --task_split dev @@ -164,6 +168,7 @@ model_path=repllama-v1-7b-duretrieval 或 NV-Embed-v1-duretrieval python -u evaluation/eval_mteb.py \ --base_model_name_or_path ${model_path} \ --output_folder eval_results/${model_path} \ + --document_instruction "" \ --query_instruction "query: " \ --task_name 'DuRetrieval' \ --eval_batch_size 8 \ @@ -197,7 +202,7 @@ python -u evaluation/eval_mteb.py \ -| 模型 | 最大序列长度 | +| 模型 | 最大序列长度（单卡80G） | |-----------------------------|--------------| | [RocketQA V1](https://github.com/PaddlePaddle/RocketQA/tree/main/research/RocketQA_NAACL2021) | 512 | | [RocketQA V2](https://github.com/PaddlePaddle/RocketQA/tree/main/research/RocketQAv2_EMNLP2021) | 512 | @@ -206,7 +211,7 @@ python -u evaluation/eval_mteb.py \ | [NV‑Embed‑v1](https://huggingface.co/nvidia/NV-Embed-v1) | 4096 | | [BGE‑EN‑ICL](https://huggingface.co/BAAI/bge-en-icl) | 4096 | | [LLARA‑passage](https://huggingface.co/BAAI/LLARA-passage) | 4096 | -| [Qwen3-Embedding-8B](https://huggingface.co/Qwen/Qwen3-Embedding-8B) | 32K | +| [Qwen3-Embedding-8B](https://huggingface.co/Qwen/Qwen3-Embedding-8B) | 8192 | 可支持配置的参数： - `base_model_name_or_path`: 模型名称或路径 @@ -308,9 +313,9 @@ python shortgpt_prune.py \ - `--layers_path`: 模型对象中指向 transformer 层列表的点分隔路径（例如 repllama 为`"llama.layers"`, llama 为`"model.layers"`）。 #### 性能评估 -剪枝完成后，可以使用 output_model_path 路径下的新模型进行[MTEB 评估](#评估)。 +剪枝完成后，可以使用 `output_model_path` 路径下的新模型进行[MTEB 评估](#评估)。 -在多个检索任务上评估了`RepLLaMA`模型剪枝前后的性能和推理速度。所有实验均在单张 80G A100 GPU 上进行。 +在多个检索任务上评估了 `RepLLaMA` 模型剪枝前后的性能和推理速度。所有实验均在单张 80G A100 GPU 上进行。 | 模型 | 指标 | MSMARCO-Title
(MRR@10) | SciFact
(NDCG@10) | FiQA2018
(NDCG@10)| QuoraRetrieval
(NDCG@10) | NFCorpus
(NDCG@10) | @@ -325,6 +330,26 @@ python shortgpt_prune.py \ ### 模型量化 支持对向量模型进行量化加载，以降低显存占用和推理延迟。 +#### 安装依赖 + +要使用量化功能，需要安装 `paddlenlp_ops` 自定义算子。安装过程大约需要 20 分钟。 + +``` +# 克隆 PaddleNLP 仓库并进入目录 +git clone https://github.com/PaddlePaddle/PaddleNLP.git +cd PaddleNLP + +# 更新 Git 子模块 +git submodule update --init + +# 编译并安装 +cd csrc +bash tools/build_wheel.sh +``` + +编译完成后，脚本会自动将 `paddlenlp_ops` 安装到当前 Python 环境，并生成一个 `.whl` 安装包。您可以将此文件分享并在其他环境中使用 `pip install` 进行安装。 + + #### 使用方法 ```bash python -u evaluation/eval_mteb.py \ @@ -333,7 +358,9 @@ python -u evaluation/eval_mteb.py \ --task_name 'SciFact' \ --eval_batch_size 8 \ --max_seq_length 2048 \ - --task_split dev \ + --task_split 'test' \ + --query_instruction 'query: ' \ + --document_instruction 'passage: ' \ --quant_type weight_only_int8 \ --kv_cache_reuse 1 ``` @@ -343,7 +370,7 @@ python -u evaluation/eval_mteb.py \ #### 性能评估 -在多个检索任务上评估了`RepLLaMA`模型量化加载前后的性能和推理速度。所有实验均在单张 80G A100 GPU 上进行。 +在多个检索任务上评估了 `RepLLaMA` 模型量化加载前后的性能和推理速度。所有实验均在单张 80G A100 GPU 上进行。 | 模型 | 指标 | MSMARCO-Title
(MRR@10) | SciFact
(NDCG@10) | FiQA2018
(NDCG@10)| QuoraRetrieval
(NDCG@10) | NFCorpus
(NDCG@10) | | :--- | :--- | :---: | :---: | :---: | :---: | :---: | @@ -356,22 +383,22 @@ python -u evaluation/eval_mteb.py \ ## Reference -[1] Aditya Kusupati, Gantavya Bhatt, Aniket Rege, Matthew Wallingford, Aditya Sinha, Vivek Ramanujan, William Howard-Snyder, Kaifeng Chen, Sham M. Kakade, Prateek Jain, Ali Farhadi: Matryoshka Representation Learning. NeurIPS 2022. +[1] Yingqi Qu, Yuchen Ding, Jing Liu, Kai Liu, Ruiyang Ren, Wayne Xin Zhao, Daxiang Dong, Hua Wu, Haifeng Wang: RocketQA: An Optimized Training Approach to Dense Passage Retrieval for Open-Domain Question Answering. NAACL 2021 -[2] Xueguang Ma, Liang Wang, Nan Yang, Furu Wei, Jimmy Lin: Fine-Tuning LLaMA for Multi-Stage Text Retrieval. arXiv 2023. +[2] Ruiyang Ren, Yingqi Qu, Jing Liu, Wayne Xin Zhao, Qiaoqiao She, Hua Wu, Haifeng Wang, Ji-Rong Wen: RocketQAv2: A Joint Training Method for Dense Passage Retrieval and Passage Re-ranking. EMNLP 2021 -[3] Shitao Xiao, Zheng Liu, Peitian Zhang, Niklas Muennighof: C-Pack: Packaged Resources To Advance General Chinese Embedding. SIGIR 2024. +[3] Aditya Kusupati, Gantavya Bhatt, Aniket Rege, Matthew Wallingford, Aditya Sinha, Vivek Ramanujan, William Howard-Snyder, Kaifeng Chen, Sham M. Kakade, Prateek Jain, Ali Farhadi: Matryoshka Representation Learning. NeurIPS 2022. -[4] Niklas Muennighoff, Nouamane Tazi, Loic Magne, Nils Reimers: MTEB: Massive Text Embedding Benchmark. EACL 2023. +[4] Xueguang Ma, Liang Wang, Nan Yang, Furu Wei, Jimmy Lin: Fine-Tuning LLaMA for Multi-Stage Text Retrieval. arXiv 2023. -[5] Chankyu Lee, Rajarshi Roy, Mengyao Xu, Jonathan Raiman, Mohammad Shoeybi, Bryan Catanzaro, Wei Ping: NV-Embed: Improved Techniques for Training LLMs as Generalist Embedding Models. ICLR 2025. +[5] Niklas Muennighoff, Nouamane Tazi, Loic Magne, Nils Reimers: MTEB: Massive Text Embedding Benchmark. EACL 2023. -[6] Zheng Liu, Chaofan Li, Shitao Xiao, Yingxia Shao, Defu Lian: Llama2Vec: Unsupervised Adaptation of Large Language Models for Dense Retrieval. ACL 2024. +[6] Shitao Xiao, Zheng Liu, Peitian Zhang, Niklas Muennighof: C-Pack: Packaged Resources To Advance General Chinese Embedding. SIGIR 2024. -[7] Chaofan Li, MingHao Qin, Shitao Xiao, Jianlyu Chen, Kun Luo, Yingxia Shao, Defu Lian, Zheng Liu: Making Text Embedders Few-Shot Learners. ICLR 2025. +[7] Zheng Liu, Chaofan Li, Shitao Xiao, Yingxia Shao, Defu Lian: Llama2Vec: Unsupervised Adaptation of Large Language Models for Dense Retrieval. ACL 2024. -[8] Yingqi Qu, Yuchen Ding, Jing Liu, Kai Liu, Ruiyang Ren, Wayne Xin Zhao, Daxiang Dong, Hua Wu, Haifeng Wang: RocketQA: An Optimized Training Approach to Dense Passage Retrieval for Open-Domain Question Answering. NAACL 2021 +[8] Chankyu Lee, Rajarshi Roy, Mengyao Xu, Jonathan Raiman, Mohammad Shoeybi, Bryan Catanzaro, Wei Ping: NV-Embed: Improved Techniques for Training LLMs as Generalist Embedding Models. ICLR 2025. -[9] Ruiyang Ren, Yingqi Qu, Jing Liu, Wayne Xin Zhao, Qiaoqiao She, Hua Wu, Haifeng Wang, Ji-Rong Wen: RocketQAv2: A Joint Training Method for Dense Passage Retrieval and Passage Re-ranking. EMNLP 2021 +[9] Chaofan Li, MingHao Qin, Shitao Xiao, Jianlyu Chen, Kun Luo, Yingxia Shao, Defu Lian, Zheng Liu: Making Text Embedders Few-Shot Learners. ICLR 2025. [10] Xin Men, Mingyu Xu, Qingyu Zhang, Bingning Wang, Hongyu Lin, Yaojie Lu, Xianpei Han, Weipeng Chen: Shortgpt: Layers in large language models are more redundant than you expect. ACL Findings 2025 diff --git a/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.py b/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.py index 729585adf9..59c2c672ef 100644 --- a/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.py +++ b/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.py @@ -253,4 +253,4 @@ if __name__ == "__main__": output_folder=f"{args.output_folder}/{args.task_name}/{args.quant_type}/{args.pooling_method}", score_function="dot", eval_splits=[args.task_split], - ) + ) \ No newline at end of file diff --git a/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.sh b/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.sh index 52e146381b..b207cee554 100644 --- a/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.sh +++ b/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.sh @@ -17,22 +17,46 @@ # --- Script Configuration --- # Exit immediately if a command exits with a non-zero status. set -e +declare -A query_instructions=( + ["ArguAna"]="Given a claim, find documents that refute the claim." + ["ClimateFEVER"]="Given a claim about climate change, retrieve documents that support or refute the claim." + ["DBPedia"]="Given a query, retrieve relevant entity descriptions from DBPedia." + ["FEVER"]="Given a claim, retrieve documents that support or refute the claim." + ["FiQA2018"]="Given a financial question, retrieve user replies that best answer the question." + ["HotpotQA"]="Given a multi-hop question, retrieve documents that can help answer the question." + ["MSMARCO"]="Given a web search query, retrieve relevant passages that answer the query." + ["NFCorpus"]="Given a question, retrieve relevant documents that best answer the question." + ["Natural Question"]="Given a question, retrieve Wikipedia passages that answer the question." + ["QuoraRetrieval"]="Given a question, retrieve questions that are semantically equivalent to the given question." + ["SCIDOCS"]="Given a scientific paper title, retrieve paper abstracts that are cited by the given paper." + ["SciFact"]="Given a scientific claim, retrieve documents that support or refute the claim." + ["Touche2020"]="Given a question, retrieve detailed and persuasive arguments that answer the question." + ["TREC-COVID"]="Given a query, retrieve documents that answer the query." + ["MSMARCOTITLE"]="Given a web search query, retrieve relevant passages that answer the query." + ["CQADupstackAndroidRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackEnglishRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackGamingRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackGisRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackMathematicaRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackPhysicsRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackProgrammersRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackStatsRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackTexRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackUnixRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackWebmastersRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." + ["CQADupstackWordpressRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question." +) # Define the list of all tasks (datasets) to be evaluated. -# TASKS=( -# "ArguAna" "ClimateFEVER" "DBPedia" "FEVER" "FiQA2018" "HotpotQA" "MSMARCO" "NFCorpus" "NQ" "QuoraRetrieval" -# "SCIDOCS" "SciFact" "Touche2020" "TRECCOVID" "CQADupstackAndroidRetrieval" "CQADupstackEnglishRetrieval" -# "CQADupstackGamingRetrieval" "CQADupstackGisRetrieval" "CQADupstackMathematicaRetrieval" "CQADupstackPhysicsRetrieval" -# "CQADupstackProgrammersRetrieval" "CQADupstackStatsRetrieval" "CQADupstackTexRetrieval" "CQADupstackUnixRetrieval" -# "CQADupstackWebmastersRetrieval" "CQADupstackWordpressRetrieval" "MSMARCOTITLE" -# ) - -TASKS=("ArguAna" "SCIDOCS" "FEVER") - +TASKS=( + "ArguAna" "ClimateFEVER" "DBPedia" "FEVER" "FiQA2018" "HotpotQA" "MSMARCO" "NFCorpus" "NQ" "QuoraRetrieval" + "SCIDOCS" "SciFact" "Touche2020" "TRECCOVID" "CQADupstackAndroidRetrieval" "CQADupstackEnglishRetrieval" + "CQADupstackGamingRetrieval" "CQADupstackGisRetrieval" "CQADupstackMathematicaRetrieval" "CQADupstackPhysicsRetrieval" + "CQADupstackProgrammersRetrieval" "CQADupstackStatsRetrieval" "CQADupstackTexRetrieval" "CQADupstackUnixRetrieval" + "CQADupstackWebmastersRetrieval" "CQADupstackWordpressRetrieval" "MSMARCOTITLE") # You can uncomment the models you wish to evaluate. -# MODELS_TO_RUN=("RocketQA-V1" "RocketQA-V2" "BGE" "RepLLaMA" "NV-Embed-v1" "BGE-EN-ICL" "LLARA-passage") -MODELS_TO_RUN=("BGE") +MODELS_TO_RUN=("RocketQA-V1" "RocketQA-V2" "BGE" "RepLLaMA" "NV-Embed-v1" "BGE-EN-ICL" "LLARA-passage" "Qwen3-Embedding") # =================================================================================== @@ -97,7 +121,8 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " BGE " ]]; then --output_folder en_results/bge-large-en-v1.5_2 \ --task_name "$task" \ --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \ - --document_instruction 'Represent this sentence for searching relevant passages: ' \ + --query_instruction 'Represent this sentence for searching relevant passages: ' \ + --document_instruction "" \ --pooling_method mean \ --max_seq_length 512 \ --eval_batch_size 32 \ @@ -139,10 +164,12 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " NV-Embed-v1 " ]]; then echo "===== Running Evaluation for Model: NV-Embed-v1 =====" for task in "${TASKS[@]}"; do echo "--- Task: $task ---" + query_instruction="${query_instructions[$task]}" python3.10 evaluation/eval_mteb.py \ --base_model_name_or_path nvidia/NV-Embed-v1 \ --output_folder en_results/nv-embed-v1 \ - --query_instruction "Given a claim, find documents that refute the claim" \ + --query_instruction "$query_instruction" \ + --document_instruction "" \ --task_name "$task" \ --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \ --eval_batch_size 8 @@ -157,12 +184,14 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " BGE-EN-ICL " ]]; then echo "===== Running Evaluation for Model: BGE-EN-ICL =====" for task in "${TASKS[@]}"; do echo "--- Task: $task ---" + query_instruction="${query_instructions[$task]}" python3.10 evaluation/eval_mteb.py \ --base_model_name_or_path BAAI/bge-en-icl \ --output_folder en_results/bge-en-icl \ --task_name "$task" \ --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \ - --query_instruction $' Given a scientific claim, retrieve documents that support or refute the claim.\n' \ + --query_instruction " ${query_instruction}\n" \ + --document_instruction "" \ --max_seq_length 512 \ --eval_batch_size 32 \ --dtype "float32" \ @@ -185,6 +214,8 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " LLARA-passage " ]]; then --output_folder en_results/llara-passage \ --task_name "$task" \ --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \ + --query_instruction "" \ + --document_instruction "" \ --eval_batch_size 8 \ --pooling_method last_8 \ --model_flag llara \ @@ -195,5 +226,29 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " LLARA-passage " ]]; then fi +# =================================================================================== +# 👀 8. Qwen3-Embedding +# =================================================================================== +if [[ " ${MODELS_TO_RUN[*]} " =~ " Qwen3-Embedding " ]]; then + echo "===== Running Evaluation for Model: Qwen3-Embedding =====" + for task in "${TASKS[@]}"; do + echo "--- Task: $task ---" + query_instruction="${query_instructions[$task]}" + python3.10 evaluation/eval_mteb.py \ + --base_model_name_or_path Qwen/Qwen3-Embedding-8B \ + --output_folder en_results/qwen3-embedding \ + --task_name "$task" \ + --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \ + --query_instruction "Instruct: ${query_instruction}\nQuery:" \ + --document_instruction "" \ + --eval_batch_size 8 \ + --pooling_method last \ + --model_flag qwen3 \ + --add_bos_token 0 \ + --add_eos_token 1 \ + --max_seq_length 4096 + done +fi + echo "All specified evaluations are complete." \ No newline at end of file diff --git a/slm/pipelines/examples/contrastive_training/evaluation/modelling_quant.py b/slm/pipelines/examples/contrastive_training/evaluation/modelling_quant.py index fb3ee3571e..680aac8070 100644 --- a/slm/pipelines/examples/contrastive_training/evaluation/modelling_quant.py +++ b/slm/pipelines/examples/contrastive_training/evaluation/modelling_quant.py @@ -83,7 +83,7 @@ class DygraphBlockInferenceHiddenPredictor(DygraphBlockInferencePredictor): ) inputs = self.model_inputs - + _, full_hidden_states = self.model( input_ids=inputs["input_ids"], seq_lens_this_time=inputs["seq_lens_this_time"], @@ -245,7 +245,7 @@ class HiddenPredictorWrapper: try_import("paddlenlp_ops") except ImportError: logger.warning("paddlenlp_ops does not exist, please install paddlenlp_ops.") - return + #return tensor_parallel_degree = paddle.distributed.get_world_size() if tensor_parallel_degree > 1: strategy = fleet.DistributedStrategy() diff --git a/slm/pipelines/examples/contrastive_training/shortgpt_prune.py b/slm/pipelines/examples/contrastive_training/shortgpt_prune.py index 3ad3018068..a444625e4d 100644 --- a/slm/pipelines/examples/contrastive_training/shortgpt_prune.py +++ b/slm/pipelines/examples/contrastive_training/shortgpt_prune.py @@ -335,7 +335,7 @@ def main(): prune_order = sorted(range(len(short_model.importances)), key=lambda i: short_model.importances[i]) layers_to_delete = set(prune_order[: args.n_prune_layers]) - + print("\n--- Importance Calculation Complete ---") print(f"Calculated importances: {[f'{v:.2f}' for v in short_model.importances]}") print(f"Pruning order (least to most important): {prune_order}")