[PIR] Update simcse to apply pir (#10396)
This commit is contained in:
@@ -149,6 +149,12 @@ simcse/
|
||||
|
||||
<a name="模型训练"></a>
|
||||
|
||||
下载数据集并解压到当前目录:
|
||||
```shell
|
||||
wget https://bj.bcebos.com/v1/paddlenlp/data/literature_search_data.zip
|
||||
unzip literature_search_data.zip
|
||||
```
|
||||
|
||||
## 5. 模型训练
|
||||
|
||||
**语义索引预训练模型下载链接:**
|
||||
|
||||
@@ -12,4 +12,4 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
python predict.py --model_dir=../../output
|
||||
python deploy/python/predict.py --model_dir=./output
|
||||
@@ -16,12 +16,17 @@ import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
from paddle import inference
|
||||
from scipy import spatial
|
||||
|
||||
from paddlenlp.data import Pad, Tuple
|
||||
from paddlenlp.transformers import AutoTokenizer
|
||||
from paddlenlp.utils.env import (
|
||||
PADDLE_INFERENCE_MODEL_SUFFIX,
|
||||
PADDLE_INFERENCE_WEIGHTS_SUFFIX,
|
||||
)
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
sys.path.append(".")
|
||||
@@ -90,8 +95,8 @@ class Predictor(object):
|
||||
self.max_seq_length = max_seq_length
|
||||
self.batch_size = batch_size
|
||||
|
||||
model_file = model_dir + "/inference.get_pooled_embedding.pdmodel"
|
||||
params_file = model_dir + "/inference.get_pooled_embedding.pdiparams"
|
||||
model_file = model_dir + f"/inference{PADDLE_INFERENCE_MODEL_SUFFIX}"
|
||||
params_file = model_dir + f"/inference{PADDLE_INFERENCE_WEIGHTS_SUFFIX}"
|
||||
if not os.path.exists(model_file):
|
||||
raise ValueError("not find model file path {}".format(model_file))
|
||||
if not os.path.exists(params_file):
|
||||
@@ -238,6 +243,9 @@ class Predictor(object):
|
||||
|
||||
if args.benchmark:
|
||||
self.autolog.times.end(stamp=True)
|
||||
|
||||
query_logits = np.atleast_2d(query_logits)
|
||||
title_logits = np.atleast_2d(title_logits)
|
||||
result = [float(1 - spatial.distance.cosine(arr1, arr2)) for arr1, arr2 in zip(query_logits, title_logits)]
|
||||
return result
|
||||
|
||||
|
||||
Reference in New Issue
Block a user