[PIR] Update simcse to apply pir (#10396)

This commit is contained in:
Hanlin Tang
2025-05-21 11:47:08 +08:00
committed by GitHub
parent 868f6eeed5
commit af16c95c07
3 changed files with 17 additions and 3 deletions

View File

@@ -149,6 +149,12 @@ simcse/
<a name="模型训练"></a>
下载数据集并解压到当前目录:
```shell
wget https://bj.bcebos.com/v1/paddlenlp/data/literature_search_data.zip
unzip literature_search_data.zip
```
## 5. 模型训练
**语义索引预训练模型下载链接:**

View File

@@ -12,4 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
python predict.py --model_dir=../../output
python deploy/python/predict.py --model_dir=./output

View File

@@ -16,12 +16,17 @@ import argparse
import os
import sys
import numpy as np
import paddle
from paddle import inference
from scipy import spatial
from paddlenlp.data import Pad, Tuple
from paddlenlp.transformers import AutoTokenizer
from paddlenlp.utils.env import (
PADDLE_INFERENCE_MODEL_SUFFIX,
PADDLE_INFERENCE_WEIGHTS_SUFFIX,
)
from paddlenlp.utils.log import logger
sys.path.append(".")
@@ -90,8 +95,8 @@ class Predictor(object):
self.max_seq_length = max_seq_length
self.batch_size = batch_size
model_file = model_dir + "/inference.get_pooled_embedding.pdmodel"
params_file = model_dir + "/inference.get_pooled_embedding.pdiparams"
model_file = model_dir + f"/inference{PADDLE_INFERENCE_MODEL_SUFFIX}"
params_file = model_dir + f"/inference{PADDLE_INFERENCE_WEIGHTS_SUFFIX}"
if not os.path.exists(model_file):
raise ValueError("not find model file path {}".format(model_file))
if not os.path.exists(params_file):
@@ -238,6 +243,9 @@ class Predictor(object):
if args.benchmark:
self.autolog.times.end(stamp=True)
query_logits = np.atleast_2d(query_logits)
title_logits = np.atleast_2d(title_logits)
result = [float(1 - spatial.distance.cosine(arr1, arr2)) for arr1, arr2 in zip(query_logits, title_logits)]
return result