[Embedding] Fix readme and env for contrastive_training (#11108)

modified: csrc/gpu/unittest/test_get_padding_offset_v2.py modified: paddlenlp/experimental/transformers/llama/modeling.py modified: slm/pipelines/examples/contrastive_training/README.md modified: slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.py modified: slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.sh modified: slm/pipelines/examples/contrastive_training/evaluation/modelling_quant.py modified: slm/pipelines/examples/contrastive_training/shortgpt_prune.py
2025-09-29 10:53:53 +08:00
parent 937b766f21
commit 3c51eda5d6
7 changed files with 256 additions and 84 deletions
--- a/csrc/gpu/unittest/test_get_padding_offset_v2.py
+++ b/csrc/gpu/unittest/test_get_padding_offset_v2.py
@@ -64,6 +64,5 @@ class GetPaddingOffsetV2Test(unittest.TestCase):
        assert sum(ref_cu_seqlens_q - cu_seqlens_q) == 0, "Check cu_seqlens_q failed."
        assert sum(ref_cu_seqlens_k - cu_seqlens_k) == 0, "Check cu_seqlens_k failed."

-
 if __name__ == "__main__":
    unittest.main()
--- a/paddlenlp/experimental/transformers/llama/modeling.py
+++ b/paddlenlp/experimental/transformers/llama/modeling.py
@@ -150,13 +150,15 @@ class LlamaAvxInferenceModel(LlamaPretrainedModel):
        ln_scale_attrs = [paddle.ParamAttr(name="fusellama.{}.ln_scale".format(i)) for i in range(self.num_layers)]
        qkv_weight_attrs = [
            paddle.ParamAttr(
-                name="fusellama.{}.qkv_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                name="fusellama.{}.qkv_weight".format(i),
+                initializer=paddle.nn.initializer.Constant(value=0),
            )
            for i in range(self.num_layers)
        ]
        out_proj_weight_attrs = [
            paddle.ParamAttr(
-                name="fusellama.{}.out_proj_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                name="fusellama.{}.out_proj_weight".format(i),
+                initializer=paddle.nn.initializer.Constant(value=0),
            )
            for i in range(self.num_layers)
        ]
@@ -165,19 +167,22 @@ class LlamaAvxInferenceModel(LlamaPretrainedModel):
        ]
        gate_weight_attrs = [
            paddle.ParamAttr(
-                name="fusellama.{}.gate_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                name="fusellama.{}.gate_weight".format(i),
+                initializer=paddle.nn.initializer.Constant(value=0),
            )
            for i in range(self.num_layers)
        ]
        up_weight_attrs = [
            paddle.ParamAttr(
-                name="fusellama.{}.up_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                name="fusellama.{}.up_weight".format(i),
+                initializer=paddle.nn.initializer.Constant(value=0),
            )
            for i in range(self.num_layers)
        ]
        ffn2_weight_attrs = [
            paddle.ParamAttr(
-                name="fusellama.{}.ffn2_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                name="fusellama.{}.ffn2_weight".format(i),
+                initializer=paddle.nn.initializer.Constant(value=0),
            )
            for i in range(self.num_layers)
        ]
@@ -328,7 +333,8 @@ class LlamaAvxInferenceModel(LlamaPretrainedModel):
                )  # reshape(3, self.num_attention_heself.hidden_sizeads // self.config.tensor_parallel_degree, head_size, )
            if "llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx) in state_dict.keys():
                concated_ffn1_weight = np.concatenate(
-                    split_fn(state_dict["llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx)]), axis=-1
+                    split_fn(state_dict["llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx)]),
+                    axis=-1,
                )
            else:
                unfused_state_dict["mlp.gate_proj.weight"] = state_dict[
@@ -336,7 +342,11 @@ class LlamaAvxInferenceModel(LlamaPretrainedModel):
                ]
                unfused_state_dict["mlp.up_proj.weight"] = state_dict["llama.layers.{}.mlp.up_proj.weight".format(idx)]
                concated_ffn1_weight = np.concatenate(
-                    [unfused_state_dict["mlp.gate_proj.weight"], unfused_state_dict["mlp.up_proj.weight"]], axis=-1
+                    [
+                        unfused_state_dict["mlp.gate_proj.weight"],
+                        unfused_state_dict["mlp.up_proj.weight"],
+                    ],
+                    axis=-1,
                )
            gate_up_list = split_fn(concated_ffn1_weight)
            gate_weight_tensor = paddle.to_tensor(gate_up_list[0])
@@ -475,13 +485,15 @@ class LlamaInferenceModel(LlamaPretrainedModel):
        ln_scale_attrs = [paddle.ParamAttr(name="fusellama.{}.ln_scale".format(i)) for i in range(self.num_layers)]
        qkv_weight_attrs = [
            paddle.ParamAttr(
-                name="fusellama.{}.qkv_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                name="fusellama.{}.qkv_weight".format(i),
+                initializer=paddle.nn.initializer.Constant(value=0),
            )
            for i in range(self.num_layers)
        ]
        out_proj_weight_attrs = [
            paddle.ParamAttr(
-                name="fusellama.{}.out_proj_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                name="fusellama.{}.out_proj_weight".format(i),
+                initializer=paddle.nn.initializer.Constant(value=0),
            )
            for i in range(self.num_layers)
        ]
@@ -491,26 +503,30 @@ class LlamaInferenceModel(LlamaPretrainedModel):
        if "fp8" in self.quant_type:
            ffn1_0_weight_attrs = [
                paddle.ParamAttr(
-                    name="fusellama.{}.ffn1_0_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                    name="fusellama.{}.ffn1_0_weight".format(i),
+                    initializer=paddle.nn.initializer.Constant(value=0),
                )
                for i in range(self.num_layers)
            ]
            ffn1_1_weight_attrs = [
                paddle.ParamAttr(
-                    name="fusellama.{}.ffn1_1_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                    name="fusellama.{}.ffn1_1_weight".format(i),
+                    initializer=paddle.nn.initializer.Constant(value=0),
                )
                for i in range(self.num_layers)
            ]
        else:
            ffn1_weight_attrs = [
                paddle.ParamAttr(
-                    name="fusellama.{}.ffn1_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                    name="fusellama.{}.ffn1_weight".format(i),
+                    initializer=paddle.nn.initializer.Constant(value=0),
                )
                for i in range(self.num_layers)
            ]
        ffn2_weight_attrs = [
            paddle.ParamAttr(
-                name="fusellama.{}.ffn2_weight".format(i), initializer=paddle.nn.initializer.Constant(value=0)
+                name="fusellama.{}.ffn2_weight".format(i),
+                initializer=paddle.nn.initializer.Constant(value=0),
            )
            for i in range(self.num_layers)
        ]
@@ -784,7 +800,12 @@ class LlamaInferenceModel(LlamaPretrainedModel):
        from paddlenlp_ops import fused_get_rotary_embedding

        new_rope = fused_get_rotary_embedding(
-            input_ids, position_ids, self.head_dim_shape_tensor, position_offset, self.rope_theta, self.use_neox
+            input_ids,
+            position_ids,
+            self.head_dim_shape_tensor,
+            position_offset,
+            self.rope_theta,
+            self.use_neox,
        )

        with dy2st_nocheck_guard_context():
@@ -800,7 +821,7 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                seq_lens=seq_lens,
                rotary_embs=new_rope,
                rotary_emb_dims=1,
-                time_step=paddle.increment(paddle.shape(attention_mask)[-1], -1) if is_decoder else None,
+                time_step=(paddle.increment(paddle.shape(attention_mask)[-1], -1) if is_decoder else None),
            )
        hidden_states = self.norm(hidden_states)

@@ -831,14 +852,18 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                weight_scale_json_path = resolve_file_path(self.quant_model_path, "weight_scales.json")
                if self.config.tensor_parallel_degree > 1 and not self.config.single_card_ptq:
                    act_scale_json_path = resolve_file_path(
-                        self.quant_model_path, f"act_scales_{self.config.tensor_parallel_rank}.json"
+                        self.quant_model_path,
+                        f"act_scales_{self.config.tensor_parallel_rank}.json",
                    )
                    weight_scale_json_path = resolve_file_path(
-                        self.quant_model_path, f"weight_scales_{self.config.tensor_parallel_rank}.json"
+                        self.quant_model_path,
+                        f"weight_scales_{self.config.tensor_parallel_rank}.json",
                    )

                act_scales = ActScalesLoader(
-                    act_scale_json_path, act_scale_map_dict, num_of_layers=self.config.num_hidden_layers
+                    act_scale_json_path,
+                    act_scale_map_dict,
+                    num_of_layers=self.config.num_hidden_layers,
                )

                weight_scales = PerTensorWeightScalesLoader(
@@ -869,13 +894,17 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                    weight_scale_json_path = resolve_file_path(self.quant_model_path, "weight_scales.json")
                    if self.config.tensor_parallel_degree > 1 and not self.config.single_card_ptq:
                        act_scale_json_path = resolve_file_path(
-                            self.quant_model_path, f"act_scales_{self.config.tensor_parallel_rank}.json"
+                            self.quant_model_path,
+                            f"act_scales_{self.config.tensor_parallel_rank}.json",
                        )
                        weight_scale_json_path = resolve_file_path(
-                            self.quant_model_path, f"weight_scales_{self.config.tensor_parallel_rank}.json"
+                            self.quant_model_path,
+                            f"weight_scales_{self.config.tensor_parallel_rank}.json",
                        )
                    act_scale_loader = ActScalesLoader(
-                        act_scale_json_path, act_scale_map_dict, num_of_layers=self.config.num_hidden_layers
+                        act_scale_json_path,
+                        act_scale_map_dict,
+                        num_of_layers=self.config.num_hidden_layers,
                    )
                    weight_scales_loader = WeightScalesLoader(
                        weight_scale_json_path,
@@ -956,7 +985,8 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                cache_scale_json_path = resolve_file_path(self.quant_model_path, "cachekv_scales.json")
                if self.config.tensor_parallel_degree > 1 and not self.config.single_card_ptq:
                    cache_scale_json_path = resolve_file_path(
-                        self.quant_model_path, f"cachekv_scales_{self.config.tensor_parallel_rank}.json"
+                        self.quant_model_path,
+                        f"cachekv_scales_{self.config.tensor_parallel_rank}.json",
                    )
                cache_scales_loader = CacheScaleLoader(
                    cache_scale_json_path,
@@ -1095,7 +1125,9 @@ class LlamaInferenceModel(LlamaPretrainedModel):
            if self.use_weight_only:
                qkv_weight_tensor = paddle.transpose(qkv_weight_tensor, perm=[1, 0])
                qkv_quanted_weight_tensor, qkv_weight_scale_tensor = weight_quantize(
-                    qkv_weight_tensor.cpu(), algo=self.quant_algo, group_size=self.weightonly_group_size
+                    qkv_weight_tensor.cpu(),
+                    algo=self.quant_algo,
+                    group_size=self.weightonly_group_size,
                )
                self.transformer_block.qkv_weights[idx].set_value(qkv_quanted_weight_tensor.cuda())
                self.transformer_block.qkv_weights_scale[idx].set_value(qkv_weight_scale_tensor.cuda())
@@ -1113,7 +1145,9 @@ class LlamaInferenceModel(LlamaPretrainedModel):
            ).cast(paddle.get_default_dtype())
            if self.use_weight_only:
                linear_quanted_weight_tensor, linear_weight_scale_tensor = weight_quantize(
-                    linear_weight_tensor.cpu(), algo=self.quant_algo, group_size=self.weightonly_group_size
+                    linear_weight_tensor.cpu(),
+                    algo=self.quant_algo,
+                    group_size=self.weightonly_group_size,
                )
                self.transformer_block.linear_weights[idx].set_value(linear_quanted_weight_tensor.cuda())
                self.transformer_block.linear_weights_scale[idx].set_value(linear_weight_scale_tensor.cuda())
@@ -1160,7 +1194,8 @@ class LlamaInferenceModel(LlamaPretrainedModel):

            if "llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx) in state_dict.keys():
                concated_ffn1_weight = np.concatenate(
-                    split_fn(state_dict["llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx)]), axis=-1
+                    split_fn(state_dict["llama.layers.{}.mlp.gate_up_fused_proj.weight".format(idx)]),
+                    axis=-1,
                )
            else:
                unfused_state_dict["mlp.gate_proj.weight"] = state_dict[
@@ -1168,13 +1203,19 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                ]
                unfused_state_dict["mlp.up_proj.weight"] = state_dict["llama.layers.{}.mlp.up_proj.weight".format(idx)]
                concated_ffn1_weight = np.concatenate(
-                    [unfused_state_dict["mlp.gate_proj.weight"], unfused_state_dict["mlp.up_proj.weight"]], axis=-1
+                    [
+                        unfused_state_dict["mlp.gate_proj.weight"],
+                        unfused_state_dict["mlp.up_proj.weight"],
+                    ],
+                    axis=-1,
                )
            ffn1_weight_tensor = paddle.to_tensor(concated_ffn1_weight).cast(paddle.get_default_dtype())

            if self.use_weight_only:
                ffn1_quanted_weight_tensor, ffn1_weight_scale_tensor = weight_quantize(
-                    ffn1_weight_tensor.cpu(), algo=self.quant_algo, group_size=self.weightonly_group_size
+                    ffn1_weight_tensor.cpu(),
+                    algo=self.quant_algo,
+                    group_size=self.weightonly_group_size,
                )
                self.transformer_block.ffn1_weights[idx].set_value(ffn1_quanted_weight_tensor.cuda())
                self.transformer_block.ffn1_weights_scale[idx].set_value(ffn1_weight_scale_tensor.cuda())
@@ -1201,7 +1242,10 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                    )
                else:
                    self.transformer_block.ffn1_weights[idx].set_value(
-                        paddle.cast(paddle.to_tensor(concated_ffn1_weight).transpose((1, 0)), w_dtype)
+                        paddle.cast(
+                            paddle.to_tensor(concated_ffn1_weight).transpose((1, 0)),
+                            w_dtype,
+                        )
                    )
            else:
                self.transformer_block.ffn1_weights[idx].set_value(ffn1_weight_tensor)
@@ -1211,7 +1255,9 @@ class LlamaInferenceModel(LlamaPretrainedModel):
            )
            if self.use_weight_only:
                ffn2_quanted_weight_tensor, ffn2_weight_scale_tensor = weight_quantize(
-                    ffn2_weight_tensor.cpu(), algo=self.quant_algo, group_size=self.weightonly_group_size
+                    ffn2_weight_tensor.cpu(),
+                    algo=self.quant_algo,
+                    group_size=self.weightonly_group_size,
                )
                self.transformer_block.ffn2_weights[idx].set_value(ffn2_quanted_weight_tensor.cuda())
                self.transformer_block.ffn2_weights_scale[idx].set_value(ffn2_weight_scale_tensor.cuda())
@@ -1231,7 +1277,8 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                if paddle.is_compiled_with_rocm():
                    self.transformer_block.ffn2_weights[idx].set_value(
                        paddle.cast(
-                            paddle.to_tensor(state_dict["llama.layers.{}.mlp.down_proj.weight".format(idx)]), w_dtype
+                            paddle.to_tensor(state_dict["llama.layers.{}.mlp.down_proj.weight".format(idx)]),
+                            w_dtype,
                        )
                    )
                else:
@@ -1297,7 +1344,8 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                    if self.use_fake_parameter:
                        if "llama.layers.{}.input_layernorm.bias".format(idx) not in state_dict:
                            state_dict["llama.layers.{}.input_layernorm.bias".format(idx)] = paddle.zeros(
-                                shape=[self.hidden_size], dtype=paddle.get_default_dtype()
+                                shape=[self.hidden_size],
+                                dtype=paddle.get_default_dtype(),
                            )
                            state_dict["llama.layers.{}.post_attention_layernorm.bias".format(idx)] = paddle.zeros(
                                [self.hidden_size], dtype=paddle.get_default_dtype()
@@ -1315,10 +1363,12 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                                dtype=paddle.get_default_dtype(),
                            )
                            unfused_state_dict["mlp.gate_proj.bias"] = paddle.zeros(
-                                shape=[self.intermediate_size], dtype=paddle.get_default_dtype()
+                                shape=[self.intermediate_size],
+                                dtype=paddle.get_default_dtype(),
                            )
                            unfused_state_dict["mlp.up_proj.bias"] = paddle.zeros(
-                                shape=[self.intermediate_size], dtype=paddle.get_default_dtype()
+                                shape=[self.intermediate_size],
+                                dtype=paddle.get_default_dtype(),
                            )
                    else:
                        unfused_state_dict["self_attn.q_proj.bias"] = state_dict[
@@ -1356,7 +1406,11 @@ class LlamaInferenceModel(LlamaPretrainedModel):
                    self.transformer_block.qkv_biases[idx].set_value(paddle.to_tensor(concated_qkv_biases))

                    concated_ffn1_bias = np.concatenate(
-                        [unfused_state_dict["mlp.gate_proj.bias"], unfused_state_dict["mlp.up_proj.bias"]], axis=-1
+                        [
+                            unfused_state_dict["mlp.gate_proj.bias"],
+                            unfused_state_dict["mlp.up_proj.bias"],
+                        ],
+                        axis=-1,
                    )

                    self.transformer_block.ffn1_biases[idx].set_value(paddle.to_tensor(concated_ffn1_bias))
@@ -1404,10 +1458,23 @@ class LlamaBlockInferenceModel(LlamaInferenceModel):
        token_num = paddle.sum(seq_lens_this_time)
        from paddlenlp_ops import get_padding_offset_v2

+        cum_offsets_now = paddle.cast(cum_offsets_now, dtype="int32")
+
        ids_remove_padding, cum_offsets, padding_offset, cu_seqlens_q, cu_seqlens_k = get_padding_offset_v2(
-            input_ids, cum_offsets_now, token_num, seq_lens_this_time, draft_tokens, seq_lens_encoder
+            input_ids,
+            cum_offsets_now,
+            token_num,
+            seq_lens_this_time,
+            draft_tokens,
+            seq_lens_encoder,
+        )
+        return (
+            ids_remove_padding,
+            padding_offset,
+            cum_offsets,
+            cu_seqlens_q,
+            cu_seqlens_k,
        )
-        return ids_remove_padding, padding_offset, cum_offsets, cu_seqlens_q, cu_seqlens_k

    def forward(
        self,
@@ -1466,13 +1533,21 @@ class LlamaBlockInferenceModel(LlamaInferenceModel):

            # whether speculative decoding or not
            if draft_tokens is None:
-                ids_remove_padding, padding_offset, cum_offsets, cu_seqlens_q, cu_seqlens_k = self.remove_padding(
-                    input_ids, seq_lens_this_time
-                )
+                (
+                    ids_remove_padding,
+                    padding_offset,
+                    cum_offsets,
+                    cu_seqlens_q,
+                    cu_seqlens_k,
+                ) = self.remove_padding(input_ids, seq_lens_this_time)
            else:
-                ids_remove_padding, padding_offset, cum_offsets, cu_seqlens_q, cu_seqlens_k = self.remove_padding(
-                    input_ids, seq_lens_this_time, draft_tokens, seq_lens_encoder
-                )
+                (
+                    ids_remove_padding,
+                    padding_offset,
+                    cum_offsets,
+                    cu_seqlens_q,
+                    cu_seqlens_k,
+                ) = self.remove_padding(input_ids, seq_lens_this_time, draft_tokens, seq_lens_encoder)

            kwargs["cu_seqlens_q"] = cu_seqlens_q
            kwargs["cu_seqlens_k"] = cu_seqlens_k
@@ -1509,7 +1584,11 @@ class EagleForLlamaBlockInferenceModel(LlamaBlockInferenceModel):

        if config.tensor_parallel_degree > 1:
            self.fc = ColumnParallelLinear(
-                self.hidden_size * 2, self.hidden_size, has_bias=True, gather_output=True, fuse_matmul_bias=True
+                self.hidden_size * 2,
+                self.hidden_size,
+                has_bias=True,
+                gather_output=True,
+                fuse_matmul_bias=True,
            )
        else:
            self.fc = nn.Linear(self.hidden_size * 2, self.hidden_size, bias_attr=True)
@@ -1669,7 +1748,11 @@ class LlamaForCausalLMInferenceModel(GenerationInferenceModel, LlamaPretrainedMo
        super().__init__(config)
        self.llama = LlamaInferenceModel(config)
        if config.tie_word_embeddings:
-            self.lm_head = LlamaLMHead(config, embedding_weights=self.llama.embed_tokens.weight, transpose_y=True)
+            self.lm_head = LlamaLMHead(
+                config,
+                embedding_weights=self.llama.embed_tokens.weight,
+                transpose_y=True,
+            )
            self.tie_weights()
        else:
            self.lm_head = LlamaLMHead(config)
@@ -1867,7 +1950,11 @@ class LlamaForCausalLMBlockInferenceModel(GenerationBlockInferenceModel, LlamaPr

        self.llama = LlamaBlockInferenceModel(config)
        if config.tie_word_embeddings:
-            self.lm_head = LlamaLMHead(config, embedding_weights=self.llama.embed_tokens.weight, transpose_y=True)
+            self.lm_head = LlamaLMHead(
+                config,
+                embedding_weights=self.llama.embed_tokens.weight,
+                transpose_y=True,
+            )
            self.tie_weights()
        else:
            self.lm_head = LlamaLMHead(config)
@@ -2108,7 +2195,11 @@ class EagleLlamaForCausalLMBlockInferenceModel(LlamaForCausalLMBlockInferenceMod

        self.eagle = EagleForLlamaBlockInferenceModel(config)
        if config.tie_word_embeddings:
-            self.lm_head = LlamaLMHead(config, embedding_weights=self.llama.embed_tokens.weight, transpose_y=True)
+            self.lm_head = LlamaLMHead(
+                config,
+                embedding_weights=self.llama.embed_tokens.weight,
+                transpose_y=True,
+            )
            self.tie_weights()
        else:
            self.lm_head = LlamaLMHead(config)
--- a/slm/pipelines/examples/contrastive_training/README.md
+++ b/slm/pipelines/examples/contrastive_training/README.md
@@ -1,6 +1,6 @@
 # 向量检索模型训练

-推荐安装 gpu 版本的[PaddlePaddle](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/conda/linux-conda.html)，以 cuda11.8的 paddle 为例，安装命令如下：
+推荐安装 gpu 版本的[PaddlePaddle](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/conda/linux-conda.html)，以 cuda 12.6 的 paddle 为例，安装命令如下：

 ```
 # 创建一个名为 paddle_env 的新环境，并激活
@@ -11,9 +11,9 @@ conda activate paddle_env
 pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html

 # 安装 paddlepaddle-gpu nightly版本
-pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/
+pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/

-#安装其他依赖：
+# 安装其他依赖：
 pip install -r slm/pipelines/examples/contrastive_training/requirements.txt
 ```

@@ -153,6 +153,10 @@ python -u evaluation/eval_mteb.py \
    --base_model_name_or_path ${model_path} \
    --output_folder eval_results/${model_path} \
    --task_name 'DuRetrieval' \
+    --document_instruction "" \
+    --query_instruction "" \
+    --padding_side right \
+    --pooling_method "cls" \
    --eval_batch_size 8 \
    --max_seq_length 2048 \
    --task_split dev
@@ -164,6 +168,7 @@ model_path=repllama-v1-7b-duretrieval 或 NV-Embed-v1-duretrieval
 python -u evaluation/eval_mteb.py \
    --base_model_name_or_path ${model_path} \
    --output_folder eval_results/${model_path} \
+    --document_instruction "" \
    --query_instruction "query: " \
    --task_name 'DuRetrieval' \
    --eval_batch_size 8 \
@@ -197,7 +202,7 @@ python -u evaluation/eval_mteb.py \
 <!-- |  **模&nbsp;型**         | [RocketQA&nbsp;V1](https://github.com/PaddlePaddle/RocketQA/tree/main/research/RocketQA_NAACL2021) | [RocketQA&nbsp;V2](https://github.com/PaddlePaddle/RocketQA/tree/main/research/RocketQAv2_EMNLP2021) | [BGE‑Large‑en‑v1.5](https://huggingface.co/BAAI/bge-large-en-v1.5) | [RepLLaMA‑passage](https://huggingface.co/castorini/repllama-v1-7b-lora-passage) | [NV‑Embed‑v1](https://huggingface.co/nvidia/NV-Embed-v1) | [BGE‑EN‑ICL](https://huggingface.co/BAAI/bge-en-icl) | [LLARA‑passage](https://huggingface.co/BAAI/LLARA-passage) |
 |--------------|-------------|-------------|-------------------|-----------------------------|-------------|------------------------|---------------|
 | **最&nbsp;大&nbsp;序&nbsp;列&nbsp;长&nbsp;度**  | 512         |     512     |        512        |            4096             |    4096     |          4096          |     4096      | -->
-| 模型                        | 最大序列长度 |
+| 模型                        | 最大序列长度（单卡80G） |
 |-----------------------------|--------------|
 | [RocketQA&nbsp;V1](https://github.com/PaddlePaddle/RocketQA/tree/main/research/RocketQA_NAACL2021)    |     512      |
 | [RocketQA&nbsp;V2](https://github.com/PaddlePaddle/RocketQA/tree/main/research/RocketQAv2_EMNLP2021)  |     512      |
@@ -206,7 +211,7 @@ python -u evaluation/eval_mteb.py \
 | [NV‑Embed‑v1](https://huggingface.co/nvidia/NV-Embed-v1)                                              |     4096     |
 | [BGE‑EN‑ICL](https://huggingface.co/BAAI/bge-en-icl)                                                  |     4096     |
 | [LLARA‑passage](https://huggingface.co/BAAI/LLARA-passage)                                            |     4096     |
-| [Qwen3-Embedding-8B](https://huggingface.co/Qwen/Qwen3-Embedding-8B)                                  |     32K     |
+| [Qwen3-Embedding-8B](https://huggingface.co/Qwen/Qwen3-Embedding-8B)                                  |     8192     |

 可支持配置的参数：
 - `base_model_name_or_path`: 模型名称或路径
@@ -308,9 +313,9 @@ python shortgpt_prune.py \
 - `--layers_path`: 模型对象中指向 transformer 层列表的点分隔路径（例如 repllama 为`"llama.layers"`, llama 为`"model.layers"`）。

 #### 性能评估
-剪枝完成后，可以使用 output_model_path 路径下的新模型进行[MTEB 评估](#评估)。
+剪枝完成后，可以使用 `output_model_path` 路径下的新模型进行[MTEB 评估](#评估)。

-在多个检索任务上评估了`RepLLaMA`模型剪枝前后的性能和推理速度。所有实验均在单张 80G A100 GPU 上进行。
+在多个检索任务上评估了 `RepLLaMA` 模型剪枝前后的性能和推理速度。所有实验均在单张 80G A100 GPU 上进行。


 | 模型 | 指标 | MSMARCO-Title<br>(MRR@10) | SciFact<br>(NDCG@10) | FiQA2018<br>(NDCG@10)| QuoraRetrieval<br>(NDCG@10) | NFCorpus<br>(NDCG@10) |
@@ -325,6 +330,26 @@ python shortgpt_prune.py \
 ### 模型量化
 支持对向量模型进行量化加载，以降低显存占用和推理延迟。

+#### 安装依赖
+
+要使用量化功能，需要安装 `paddlenlp_ops` 自定义算子。安装过程大约需要 20 分钟。
+
+```
+# 克隆 PaddleNLP 仓库并进入目录
+git clone https://github.com/PaddlePaddle/PaddleNLP.git
+cd PaddleNLP
+
+# 更新 Git 子模块
+git submodule update --init
+
+# 编译并安装
+cd csrc
+bash tools/build_wheel.sh
+```
+
+编译完成后，脚本会自动将 `paddlenlp_ops` 安装到当前 Python 环境，并生成一个 `.whl` 安装包。您可以将此文件分享并在其他环境中使用 `pip install` 进行安装。
+
+
 #### 使用方法
 ```bash
 python -u evaluation/eval_mteb.py \
@@ -333,7 +358,9 @@ python -u evaluation/eval_mteb.py \
    --task_name 'SciFact' \
    --eval_batch_size 8 \
    --max_seq_length 2048 \
-    --task_split dev \
+    --task_split 'test' \
+    --query_instruction 'query: ' \
+    --document_instruction 'passage: ' \
    --quant_type weight_only_int8 \
    --kv_cache_reuse 1
 ```
@@ -343,7 +370,7 @@ python -u evaluation/eval_mteb.py \


 #### 性能评估
-在多个检索任务上评估了`RepLLaMA`模型量化加载前后的性能和推理速度。所有实验均在单张 80G A100 GPU 上进行。
+在多个检索任务上评估了 `RepLLaMA` 模型量化加载前后的性能和推理速度。所有实验均在单张 80G A100 GPU 上进行。

 | 模型 | 指标 | MSMARCO-Title<br>(MRR@10) | SciFact<br>(NDCG@10) | FiQA2018<br>(NDCG@10)| QuoraRetrieval<br>(NDCG@10) | NFCorpus<br>(NDCG@10) |
 | :--- | :--- | :---: | :---: | :---: | :---: | :---: |
@@ -356,22 +383,22 @@ python -u evaluation/eval_mteb.py \

 ## Reference

-[1] Aditya Kusupati, Gantavya Bhatt, Aniket Rege, Matthew Wallingford, Aditya Sinha, Vivek Ramanujan, William Howard-Snyder, Kaifeng Chen, Sham M. Kakade, Prateek Jain, Ali Farhadi: Matryoshka Representation Learning. NeurIPS 2022.
+[1] Yingqi Qu, Yuchen Ding, Jing Liu, Kai Liu, Ruiyang Ren, Wayne Xin Zhao, Daxiang Dong, Hua Wu, Haifeng Wang: RocketQA: An Optimized Training Approach to Dense Passage Retrieval for Open-Domain Question Answering. NAACL 2021

-[2] Xueguang Ma, Liang Wang, Nan Yang, Furu Wei, Jimmy Lin: Fine-Tuning LLaMA for Multi-Stage Text Retrieval. arXiv 2023.
+[2] Ruiyang Ren, Yingqi Qu, Jing Liu, Wayne Xin Zhao, Qiaoqiao She, Hua Wu, Haifeng Wang, Ji-Rong Wen: RocketQAv2: A Joint Training Method for Dense Passage Retrieval and Passage Re-ranking. EMNLP 2021

-[3] Shitao Xiao, Zheng Liu, Peitian Zhang, Niklas Muennighof: C-Pack: Packaged Resources To Advance General Chinese Embedding. SIGIR 2024.
+[3] Aditya Kusupati, Gantavya Bhatt, Aniket Rege, Matthew Wallingford, Aditya Sinha, Vivek Ramanujan, William Howard-Snyder, Kaifeng Chen, Sham M. Kakade, Prateek Jain, Ali Farhadi: Matryoshka Representation Learning. NeurIPS 2022.

-[4] Niklas Muennighoff, Nouamane Tazi, Loic Magne, Nils Reimers: MTEB: Massive Text Embedding Benchmark. EACL 2023.
+[4] Xueguang Ma, Liang Wang, Nan Yang, Furu Wei, Jimmy Lin: Fine-Tuning LLaMA for Multi-Stage Text Retrieval. arXiv 2023.

-[5] Chankyu Lee, Rajarshi Roy, Mengyao Xu, Jonathan Raiman, Mohammad Shoeybi, Bryan Catanzaro, Wei Ping: NV-Embed: Improved Techniques for Training LLMs as Generalist Embedding Models. ICLR 2025.
+[5] Niklas Muennighoff, Nouamane Tazi, Loic Magne, Nils Reimers: MTEB: Massive Text Embedding Benchmark. EACL 2023.

-[6] Zheng Liu, Chaofan Li, Shitao Xiao, Yingxia Shao, Defu Lian: Llama2Vec: Unsupervised Adaptation of Large Language Models for Dense Retrieval. ACL 2024.
+[6] Shitao Xiao, Zheng Liu, Peitian Zhang, Niklas Muennighof: C-Pack: Packaged Resources To Advance General Chinese Embedding. SIGIR 2024.

-[7] Chaofan Li, MingHao Qin, Shitao Xiao, Jianlyu Chen, Kun Luo, Yingxia Shao, Defu Lian, Zheng Liu: Making Text Embedders Few-Shot Learners. ICLR 2025.
+[7] Zheng Liu, Chaofan Li, Shitao Xiao, Yingxia Shao, Defu Lian: Llama2Vec: Unsupervised Adaptation of Large Language Models for Dense Retrieval. ACL 2024.

-[8] Yingqi Qu, Yuchen Ding, Jing Liu, Kai Liu, Ruiyang Ren, Wayne Xin Zhao, Daxiang Dong, Hua Wu, Haifeng Wang: RocketQA: An Optimized Training Approach to Dense Passage Retrieval for Open-Domain Question Answering. NAACL 2021
+[8] Chankyu Lee, Rajarshi Roy, Mengyao Xu, Jonathan Raiman, Mohammad Shoeybi, Bryan Catanzaro, Wei Ping: NV-Embed: Improved Techniques for Training LLMs as Generalist Embedding Models. ICLR 2025.

-[9] Ruiyang Ren, Yingqi Qu, Jing Liu, Wayne Xin Zhao, Qiaoqiao She, Hua Wu, Haifeng Wang, Ji-Rong Wen: RocketQAv2: A Joint Training Method for Dense Passage Retrieval and Passage Re-ranking. EMNLP 2021
+[9] Chaofan Li, MingHao Qin, Shitao Xiao, Jianlyu Chen, Kun Luo, Yingxia Shao, Defu Lian, Zheng Liu: Making Text Embedders Few-Shot Learners. ICLR 2025.

 [10] Xin Men, Mingyu Xu, Qingyu Zhang, Bingning Wang, Hongyu Lin, Yaojie Lu, Xianpei Han, Weipeng Chen: Shortgpt: Layers in large language models are more redundant than you expect. ACL Findings 2025
--- a/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.sh
+++ b/slm/pipelines/examples/contrastive_training/evaluation/eval_mteb.sh
@@ -17,22 +17,46 @@
 # --- Script Configuration ---
 # Exit immediately if a command exits with a non-zero status.
 set -e
+declare -A query_instructions=(
+    ["ArguAna"]="Given a claim, find documents that refute the claim."
+    ["ClimateFEVER"]="Given a claim about climate change, retrieve documents that support or refute the claim."
+    ["DBPedia"]="Given a query, retrieve relevant entity descriptions from DBPedia."
+    ["FEVER"]="Given a claim, retrieve documents that support or refute the claim."
+    ["FiQA2018"]="Given a financial question, retrieve user replies that best answer the question."
+    ["HotpotQA"]="Given a multi-hop question, retrieve documents that can help answer the question."
+    ["MSMARCO"]="Given a web search query, retrieve relevant passages that answer the query."
+    ["NFCorpus"]="Given a question, retrieve relevant documents that best answer the question."
+    ["Natural Question"]="Given a question, retrieve Wikipedia passages that answer the question."
+    ["QuoraRetrieval"]="Given a question, retrieve questions that are semantically equivalent to the given question."
+    ["SCIDOCS"]="Given a scientific paper title, retrieve paper abstracts that are cited by the given paper."
+    ["SciFact"]="Given a scientific claim, retrieve documents that support or refute the claim."
+    ["Touche2020"]="Given a question, retrieve detailed and persuasive arguments that answer the question."
+    ["TREC-COVID"]="Given a query, retrieve documents that answer the query."
+    ["MSMARCOTITLE"]="Given a web search query, retrieve relevant passages that answer the query."
+    ["CQADupstackAndroidRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackEnglishRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackGamingRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackGisRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackMathematicaRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackPhysicsRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackProgrammersRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackStatsRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackTexRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackUnixRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackWebmastersRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+    ["CQADupstackWordpressRetrieval"]="Given a question, retrieve detailed question descriptions from Stackexchange that are duplicates to the given question."
+)

 # Define the list of all tasks (datasets) to be evaluated.
-# TASKS=(
-#     "ArguAna" "ClimateFEVER" "DBPedia" "FEVER" "FiQA2018" "HotpotQA" "MSMARCO" "NFCorpus" "NQ" "QuoraRetrieval" 
-#     "SCIDOCS" "SciFact" "Touche2020" "TRECCOVID" "CQADupstackAndroidRetrieval" "CQADupstackEnglishRetrieval" 
-#     "CQADupstackGamingRetrieval" "CQADupstackGisRetrieval" "CQADupstackMathematicaRetrieval" "CQADupstackPhysicsRetrieval" 
-#     "CQADupstackProgrammersRetrieval" "CQADupstackStatsRetrieval" "CQADupstackTexRetrieval" "CQADupstackUnixRetrieval" 
-#     "CQADupstackWebmastersRetrieval" "CQADupstackWordpressRetrieval" "MSMARCOTITLE"
-# )
-
-TASKS=("ArguAna" "SCIDOCS" "FEVER")
-
+TASKS=(
+    "ArguAna" "ClimateFEVER" "DBPedia" "FEVER" "FiQA2018" "HotpotQA" "MSMARCO" "NFCorpus" "NQ" "QuoraRetrieval" 
+    "SCIDOCS" "SciFact" "Touche2020" "TRECCOVID" "CQADupstackAndroidRetrieval" "CQADupstackEnglishRetrieval" 
+    "CQADupstackGamingRetrieval" "CQADupstackGisRetrieval" "CQADupstackMathematicaRetrieval" "CQADupstackPhysicsRetrieval" 
+    "CQADupstackProgrammersRetrieval" "CQADupstackStatsRetrieval" "CQADupstackTexRetrieval" "CQADupstackUnixRetrieval" 
+    "CQADupstackWebmastersRetrieval" "CQADupstackWordpressRetrieval" "MSMARCOTITLE")

 # You can uncomment the models you wish to evaluate.
-# MODELS_TO_RUN=("RocketQA-V1" "RocketQA-V2" "BGE" "RepLLaMA" "NV-Embed-v1" "BGE-EN-ICL" "LLARA-passage")
-MODELS_TO_RUN=("BGE") 
+MODELS_TO_RUN=("RocketQA-V1" "RocketQA-V2" "BGE" "RepLLaMA" "NV-Embed-v1" "BGE-EN-ICL" "LLARA-passage" "Qwen3-Embedding")


 # ===================================================================================
@@ -97,7 +121,8 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " BGE " ]]; then
              --output_folder en_results/bge-large-en-v1.5_2 \
              --task_name "$task" \
              --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \
-              --document_instruction 'Represent this sentence for searching relevant passages: ' \
+              --query_instruction 'Represent this sentence for searching relevant passages: ' \
+              --document_instruction "" \
              --pooling_method mean \
              --max_seq_length 512 \
              --eval_batch_size 32 \
@@ -139,10 +164,12 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " NV-Embed-v1 " ]]; then
    echo "===== Running Evaluation for Model: NV-Embed-v1 ====="
    for task in "${TASKS[@]}"; do
        echo "--- Task: $task ---"
+        query_instruction="${query_instructions[$task]}"
        python3.10 evaluation/eval_mteb.py \
              --base_model_name_or_path nvidia/NV-Embed-v1 \
              --output_folder en_results/nv-embed-v1 \
-              --query_instruction "Given a claim, find documents that refute the claim" \
+              --query_instruction "$query_instruction" \
+              --document_instruction "" \
              --task_name "$task" \
              --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \
              --eval_batch_size 8
@@ -157,12 +184,14 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " BGE-EN-ICL " ]]; then
    echo "===== Running Evaluation for Model: BGE-EN-ICL ====="
    for task in "${TASKS[@]}"; do
        echo "--- Task: $task ---"
+        query_instruction="${query_instructions[$task]}"
        python3.10 evaluation/eval_mteb.py \
              --base_model_name_or_path BAAI/bge-en-icl \
              --output_folder en_results/bge-en-icl \
              --task_name "$task" \
              --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \
-              --query_instruction $'<instruct> Given a scientific claim, retrieve documents that support or refute the claim.\n<query>' \
+              --query_instruction "<instruct> ${query_instruction}\n<query>" \
+              --document_instruction "" \
              --max_seq_length 512 \
              --eval_batch_size 32 \
              --dtype "float32" \
@@ -185,6 +214,8 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " LLARA-passage " ]]; then
              --output_folder en_results/llara-passage \
              --task_name "$task" \
              --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \
+              --query_instruction "" \
+              --document_instruction "" \
              --eval_batch_size 8 \
              --pooling_method last_8 \
              --model_flag llara \
@@ -195,5 +226,29 @@ if [[ " ${MODELS_TO_RUN[*]} " =~ " LLARA-passage " ]]; then
 fi


+# ===================================================================================
+# 👀 8. Qwen3-Embedding
+# ===================================================================================
+if [[ " ${MODELS_TO_RUN[*]} " =~ " Qwen3-Embedding " ]]; then
+    echo "===== Running Evaluation for Model: Qwen3-Embedding ====="
+    for task in "${TASKS[@]}"; do
+        echo "--- Task: $task ---"
+        query_instruction="${query_instructions[$task]}"
+        python3.10 evaluation/eval_mteb.py \
+              --base_model_name_or_path Qwen/Qwen3-Embedding-8B \
+              --output_folder en_results/qwen3-embedding \
+              --task_name "$task" \
+              --task_split $([[ "$task" == *"MSMARCO"* ]] && echo "dev" || echo "test") \
+              --query_instruction "Instruct: ${query_instruction}\nQuery:" \
+              --document_instruction "" \
+              --eval_batch_size 8 \
+              --pooling_method last \
+              --model_flag qwen3 \
+              --add_bos_token 0 \
+              --add_eos_token 1 \
+              --max_seq_length 4096 
+    done
+fi
+

 echo "All specified evaluations are complete."
--- a/slm/pipelines/examples/contrastive_training/evaluation/modelling_quant.py
+++ b/slm/pipelines/examples/contrastive_training/evaluation/modelling_quant.py
@@ -245,7 +245,7 @@ class HiddenPredictorWrapper:
            try_import("paddlenlp_ops")
        except ImportError:
            logger.warning("paddlenlp_ops does not exist, please install paddlenlp_ops.")
-            return
+            #return
        tensor_parallel_degree = paddle.distributed.get_world_size()
        if tensor_parallel_degree > 1:
            strategy = fleet.DistributedStrategy()