Merge branch 'develop' into ppdiffusers

* add fid

* update doc

* update import

* add code annotation

* update docs

* add hack for bn

.copyright.hook

@@ -0,0 +1,134 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import io
+import re
+import sys
+import os
+import datetime
+
+COPYRIGHT = '''Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.'''
+
+def _generate_copyright(comment_mark):
+    copyright=COPYRIGHT.split(os.linesep)
+    header = copyright[0].rstrip()
+
+    p = re.search('(\d{4})', header).group(0)
+    now = datetime.datetime.now()
+
+    header = header.replace(p,str(now.year))
+
+    ans=[comment_mark + " " + header + os.linesep]
+    for idx, line in enumerate(copyright[1:]):
+        ans.append(comment_mark + " " + line.rstrip() + os.linesep)
+
+    return ans
+
+def _get_comment_mark(path):
+    lang_type=re.compile(r"\.(py|sh)$")
+    if lang_type.search(path) is not None:
+        return "#"
+
+    lang_type=re.compile(r"\.(h|c|hpp|cc|cpp|cu|go|cuh|proto)$")
+    if lang_type.search(path) is not None:
+        return "//"
+
+    return None
+
+
+RE_ENCODE = re.compile(r"^[ \t\v]*#.*?coding[:=]", re.IGNORECASE)
+RE_COPYRIGHT = re.compile(r".*Copyright( \(c\))* \d{4}", re.IGNORECASE)
+RE_SHEBANG = re.compile(r"^[ \t\v]*#[ \t]?\!")
+
+def _check_copyright(path):
+    head=[]
+    try:
+        with open(path) as f:
+            head = [next(f) for x in range(4)]
+    except StopIteration:
+        pass
+
+    for idx, line in enumerate(head):
+        if RE_COPYRIGHT.search(line) is not None:
+            return True
+
+    return False
+
+def generate_copyright(path, comment_mark):
+    original_contents = io.open(path, encoding="utf-8").readlines()
+    head = original_contents[0:4]
+
+    insert_line_no=0
+    for i, line in enumerate(head):
+        if RE_ENCODE.search(line) or RE_SHEBANG.search(line):
+            insert_line_no=i+1
+
+    copyright = _generate_copyright(comment_mark)
+    if insert_line_no == 0:
+        new_contents = copyright
+        if len(original_contents) > 0 and len(original_contents[0].strip()) != 0:
+            new_contents.append(os.linesep)
+        new_contents.extend(original_contents)
+    else:
+        new_contents=original_contents[0:insert_line_no]
+        new_contents.append(os.linesep)
+        new_contents.extend(copyright)
+        if len(original_contents) > insert_line_no and len(original_contents[insert_line_no].strip()) != 0:
+            new_contents.append(os.linesep)
+        new_contents.extend(original_contents[insert_line_no:])
+    new_contents="".join(new_contents)
+
+    with io.open(path, 'w') as output_file:
+        output_file.write(new_contents)
+
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        description='Checker for copyright declaration.')
+    parser.add_argument('filenames', nargs='*', help='Filenames to check')
+    args = parser.parse_args(argv)
+
+    retv = 0
+    for path in args.filenames:
+        comment_mark = _get_comment_mark(path)
+        if comment_mark is None:
+            print("warning:Unsupported file", path, file=sys.stderr)
+            continue
+
+        if _check_copyright(path):
+            continue
+
+        generate_copyright(path, comment_mark)
+
+
+if __name__ == '__main__':
+    exit(main())

.github/ISSUE_TEMPLATE/----.md

@@ -1,15 +0,0 @@
----
-name: 问题反馈
-about: PaddleNLP问题反馈
-title: ''
-labels: ''
-assignees: ''
-
----
-
-欢迎您反馈PaddleNLP使用问题，非常感谢您对PaddleNLP的贡献！
-在留下您的问题时，辛苦您同步提供如下信息：
-- 版本、环境信息
-1）PaddleNLP和PaddlePaddle版本：请提供您的PaddleNLP和PaddlePaddle版本号，例如PaddleNLP 2.0.4，PaddlePaddle2.1.1
-2）系统环境：请您描述系统类型，例如Linux/Windows/MacOS/，python版本
-- 复现信息：如为报错，请给出复现环境、复现步骤

.github/ISSUE_TEMPLATE/ask-question.yml

@@ -0,0 +1,23 @@
+name: 🐛 Ask Question
+description: 请描述您使用PaddleNLP时遇到的问题
+title: "[Question]: "
+labels: 
+  - question
+body:
+- type: markdown
+  attributes:
+    value: >
+      #### 你可以在这里提出一个使用/咨询问题，提问之前请确保：
+      
+      - 1）已经百度/谷歌搜索过你的问题，但是没有找到解答；
+      
+      - 2）已经在官网查询过[API文档](https://www.paddlepaddle.org.cn/documentation/docs/zh/api/index_cn.html)与[FAQ](https://www.paddlepaddle.org.cn/documentation/docs/zh/faq/index_cn.html)，但是没有找到解答；
+      
+      - 3）已经在[历史issue](https://github.com/PaddlePaddle/Paddle/issues)中搜索过，没有找到同类issue或issue未被解答。
+      
+- type: textarea
+  id: question
+  attributes:
+    label: 请提出你的问题
+  validations:
+    required: true

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,45 @@
+name: 🐛 Bug Report
+description: PaddleNLP问题反馈
+title: "[Bug]: "
+labels: bug
+body: 
+  - type: textarea
+    id: environment
+    attributes:
+      label: 软件环境
+      description: |
+        请使用以下命令给出您本地Paddle相关包信息
+          ```sh
+          pip list | grep paddle
+          
+          ```
+      value: |
+          - paddlepaddle:
+          - paddlepaddle-gpu: 
+          - paddlenlp: 
+      render: Markdown
+    validations:
+      required: true
+  - type: checkboxes
+    id: dumplicated-problem
+    attributes:
+      label: 重复问题
+      description: 是否已在issues中搜索相关问题
+      options:
+      - label: I have searched the existing issues
+        required: true
+  - type: textarea
+    id: descripton
+    attributes:
+      label: 错误描述
+      description: 给出错误详细描述，以便能够更好的追踪相关问题
+      render: Markdown
+    validations:
+      required: true
+  - type: textarea
+    id: mvp-code
+    attributes:
+      label: 稳定复现步骤 & 代码
+      description: 请给出稳定复现该问题的步骤 & 代码，以便相关人员能够快速定位到具体问题。
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/docs-report.yml

@@ -0,0 +1,32 @@
+name: 🐛 Docs Report
+description: PaddleNLP文档反馈
+title: "[Docs]: "
+labels: 
+  - documentation
+
+body: 
+  - type: textarea
+    id: environment
+    attributes:
+      label: 软件环境
+      description: |
+        请使用以下命令给出您本地Paddle相关包信息
+          ```sh
+          pip list | grep paddle
+          
+          ```
+      value: |
+          - paddlepaddle:
+          - paddlepaddle-gpu: 
+          - paddlenlp: 
+      render: Markdown
+    validations:
+      required: true
+  - type: textarea
+    id: description
+    attributes:
+      label: 详细描述
+      description: 请详细描述您想要反馈的具体问题
+      render: Markdown
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,30 @@
+name: "\U0001F680 Feature request"
+description: 请详细描述您所需功能
+labels: [ "feature" ]
+body:
+  - type: textarea
+    id: feature-request
+    validations:
+      required: true
+    attributes:
+      label: Feature request
+      description: |
+        对特性提案的清晰而简明的描述。如果论文和代码存在，请提供链接。 
+
+  - type: textarea
+    id: motivation
+    validations:
+      required: true
+    attributes:
+      label: Motivation
+      description: |
+        请概述这项建议的动机。您的特性要求与问题有关吗?
+        
+  - type: textarea
+    id: contribution
+    validations:
+      required: true
+    attributes:
+      label: Your contribution
+      description: |
+        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md)

.github/ISSUE_TEMPLATE/new-model.yaml

@@ -0,0 +1,28 @@
+name: "\U0001F31F 添加新模型"
+description: 请为新模型提交一份说明
+labels: [ "New model" ]
+
+body:
+  - type: textarea
+    id: description-request
+    validations:
+      required: true
+    attributes:
+      label: 简要描述
+      description: |
+        请简要描述模型的类型、解决的问题等。
+
+  - type: checkboxes
+    id: information-tasks
+    attributes:
+      label: 是否已开源
+      options:
+        - label: 已开源
+        - label: 未开源
+
+  - type: textarea
+    id: additional-info
+    attributes:
+      label: 模型详细信息
+      description: |
+        请给出新模型相关信息，如论文地址、现存代码地址等。

.github/ISSUE_TEMPLATE/others.yml

@@ -0,0 +1,23 @@
+name: 🧩 其他 Others
+description: 提出其他问题。
+labels: [others]
+
+body:
+- type: markdown
+  attributes:
+    value: >
+      #### 你可以在这里提出任何前面几类模板不适用的问题，包括但不限于：优化性建议、框架使用体验反馈、版本兼容性问题、报错信息不清楚等。
+
+- type: textarea
+  id: others
+  attributes:
+    label: 问题描述
+  validations:
+    required: true
+    
+- type: markdown
+  attributes:
+    value: >
+      感谢你的贡献 🎉！
+
+

.github/workflows/pypi.yml

@@ -0,0 +1,60 @@
+name: PyPI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r docs/requirements.txt
+          pip install -r ppdiffusers/requirements.txt
+
+  ppdiffusers:
+    name: Pack
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+          make install-ppdiffusers
+
+      - name: Should Deploy to Pypi Server
+        id: should-deploy
+        run: python scripts/should_deploy.py  --name ppdiffusers >> $GITHUB_OUTPUT
+      - name: Check Branch
+        id: check-branch
+        run: |
+          if [[ ${{ github.ref }} =~ ^refs/heads/(develop)$ ]]; then
+              echo "match=true" >> $GITHUB_OUTPUT
+          fi  # See: https://stackoverflow.com/a/58869470/1123955
+
+      - name: Is A Publish Branch
+        if: steps.check-branch.outputs.match == 'true' && steps.should-deploy.outputs.should_deploy == 'true'
+        env:
+          TWINE_USERNAME: paddle-dev
+          TWINE_PASSWORD: ${{ secrets.paddlenlp }}
+        run: |
+          make deploy-ppdiffusers
+
+      - name: Should Not Deploy To Pypi Server
+        if: steps.should-deploy.outputs.should_deploy != 'true'
+        run: echo 'should not deploy pypi server'
+
+      - name: Is Not A Publish Branch
+        if: steps.check-branch.outputs.match != 'true'
+        run: echo 'Not A Publish Branch'

.gitignore

@@ -123,3 +123,4 @@ FETCH_HEAD
 
 # vscode
 .vscode
+./ppdiffusers/ppdiffusers/version.py

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
--   repo: https://github.com/PaddlePaddle/mirrors-yapf.git
-    sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37
+-   repo: https://github.com/google/yapf
+    rev: v0.32.0
     hooks:
     -   id: yapf
         files: \.py$
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    sha: a11d9314b22d8f8c7556443875b731ef05965464
+    rev: v4.1.0
     hooks:
     -   id: check-merge-conflict
     -   id: check-symlinks
@@ -16,7 +16,7 @@ repos:
     -   id: trailing-whitespace
         files: \.md$
 -   repo: https://github.com/Lucas-C/pre-commit-hooks
-    sha: v1.0.1
+    rev: v1.1.14
     hooks:
     -   id: forbid-crlf
         files: \.md$
@@ -26,3 +26,10 @@ repos:
         files: \.md$
     -   id: remove-tabs
         files: \.md$
+-   repo: local
+    hooks:
+    -   id: copyright_checker
+        name: copyright_checker
+        entry: python .copyright.hook
+        language: system
+        files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|xpu|kps|py|sh)$

CODE_OF_CONDUCT.md

@@ -0,0 +1,88 @@
+
+# 贡献者公约
+
+## 我们的承诺
+
+身为社区成员、贡献者和领袖，我们承诺使社区参与者不受骚扰，无论其年龄、体型、可见或不可见的缺陷、族裔、性征、性别认同和表达、经验水平、教育程度、社会与经济地位、国籍、相貌、种族、种姓、肤色、宗教信仰、性倾向或性取向如何。
+
+我们承诺以有助于建立开放、友善、多样化、包容、健康社区的方式行事和互动。
+
+## 我们的准则
+
+有助于为我们的社区创造积极环境的行为例子包括但不限于：
+
+* 表现出对他人的同情和善意
+* 尊重不同的主张、观点和感受
+* 提出和大方接受建设性意见
+* 承担责任并向受我们错误影响的人道歉
+* 注重社区共同诉求，而非个人得失
+
+不当行为例子包括：
+
+* 使用情色化的语言或图像，及性引诱或挑逗
+* 嘲弄、侮辱或诋毁性评论，以及人身或政治攻击
+* 公开或私下的骚扰行为
+* 未经他人明确许可，公布他人的私人信息，如物理或电子邮件地址
+* 其他有理由认定为违反职业操守的不当行为
+
+## 责任和权力
+
+社区领袖有责任解释和落实我们所认可的行为准则，并妥善公正地对他们认为不当、威胁、冒犯或有害的任何行为采取纠正措施。
+
+社区领导有权力和责任删除、编辑或拒绝或拒绝与本行为准则不相符的评论（comment）、提交（commits）、代码、维基（wiki）编辑、议题（issues）或其他贡献，并在适当时机知采取措施的理由。
+
+## 适用范围
+
+本行为准则适用于所有社区场合，也适用于在公共场所代表社区时的个人。
+
+代表社区的情形包括使用官方电子邮件地址、通过官方社交媒体帐户发帖或在线上或线下活动中担任指定代表。
+
+## 监督
+
+辱骂、骚扰或其他不可接受的行为可通过 paddlenlp@baidu.com 向负责监督的社区领袖报告。
+所有投诉都将得到及时和公平的审查和调查。
+
+所有社区领袖都有义务尊重任何事件报告者的隐私和安全。
+
+## 处理方针
+
+社区领袖将遵循下列社区处理方针来明确他们所认定违反本行为准则的行为的处理方式：
+
+### 1. 纠正
+
+**社区影响**：使用不恰当的语言或其他在社区中被认定为不符合职业道德或不受欢迎的行为。
+
+**处理意见**：由社区领袖发出非公开的书面警告，明确说明违规行为的性质，并解释举止如何不妥。或将要求公开道歉。
+
+### 2. 警告
+
+**社区影响**：单个或一系列违规行为。
+
+**处理意见**：警告并对连续性行为进行处理。在指定时间内，不得与相关人员互动，包括主动与行为准则执行者互动。这包括避免在社区场所和外部渠道中的互动。违反这些条款可能会导致临时或永久封禁。
+
+### 3. 临时封禁
+
+**社区影响**: 严重违反社区准则，包括持续的不当行为。
+
+**处理意见**: 在指定时间内，暂时禁止与社区进行任何形式的互动或公开交流。在此期间，不得与相关人员进行公开或私下互动，包括主动与行为准则执行者互动。违反这些条款可能会导致永久封禁。
+
+### 4. 永久封禁
+
+**社区影响**：行为模式表现出违反社区准则，包括持续的不当行为、骚扰个人或攻击或贬低某个类别的个体。
+
+**处理意见**：永久禁止在社区内进行任何形式的公开互动。
+
+## 参见
+
+本行为准则改编自 [Contributor Covenant][homepage] 2.1 版, 参见 [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]。
+
+社区处理方针灵感来源于 [Mozilla's code of conduct enforcement ladder][Mozilla CoC]。
+
+有关本行为准则的常见问题的答案，参见 [https://www.contributor-covenant.org/faq][FAQ]。
+其他语言翻译参见 [https://www.contributor-covenant.org/translations][translations]。
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CODE_OF_CONDUCT_en.md

@@ -0,0 +1,133 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+paddlenlp@baidu.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

Makefile

@@ -0,0 +1,10 @@
+
+
+.PHONY: deploy-ppdiffusers
+deploy-ppdiffusers:
+	cd ppdiffusers && make
+
+.PHONY: install-ppdiffusers
+install-ppdiffusers:
+	cd ppdiffusers && make install
+

README_cn.md

@@ -1,7 +1,7 @@
 **简体中文**🀄 | [English🌎](./README_en.md)
 
 <p align="center">
-  <img src="./docs/imgs/paddlenlp.png" align="middle"  width="500" />
+  <img src="https://user-images.githubusercontent.com/1371212/175816733-8ec25eb0-9af3-4380-9218-27c154518258.png" align="middle"  width="500" />
 </p>
 
 ------------------------------------------------------------------------------------------
@@ -30,15 +30,30 @@
 **PaddleNLP**是一款**简单易用**且**功能强大**的自然语言处理开发库。聚合业界**优质预训练模型**并提供**开箱即用**的开发体验，覆盖NLP多场景的模型库搭配**产业实践范例**可满足开发者**灵活定制**的需求。
 
 ## News 📢
+* 🔥 **2022.10.27 发布 [PaddleNLP v2.4.2](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.4.2)**
+  * NLG能力扩充：新增📄[**基于Pegasus的中文文本摘要方案**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/applications/text_summarization/pegasus)，效果领先；新增❓[**问题生成解决方案**](./examples/question_generation)，提供基于业界领先模型UNIMO-Text和大规模多领域问题生成数据集训练的通用问题生成预训练模型。均支持Taskflow一键调用，支持FasterGeneration高性能推理，训练推理部署全流程打通。
+  * 发布 🖼[**PPDiffusers**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/ppdiffusers)：支持跨模态（如图像与语音）训练和推理的扩散模型（Diffusion Model）工具箱，可快速体验、二次开发 **Stable Diffusion**，持续支持更多模型。
 
-* 🔥 2022.5.16 [**PaddleNLP v2.3**](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.3.0)全新发布！🎉
-  * 💎 发布通用信息抽取技术[**UIE**](./model_zoo/uie)，单模型支持实体识别、关系和事件抽取、情感分析等多种开放域信息抽取任务，不限领域和抽取目标，支持**一键抽取**与全流程**小样本**高效定制开发。
-  * 😊 发布文心大模型[**ERNIE 3.0**](./model_zoo/ernie-3.0)轻量级模型，在[CLUE](https://www.cluebenchmarks.com/)上实现同规模结构效果最佳，并提供**🗜️无损压缩**和**⚙️全场景部署**方案。
-  * 🏥 发布中文医疗领域预训练模型[**ERNIE-Health**](./model_zoo/ernie-health)，[CBLUE](https://github.com/CBLUEbenchmark/CBLUE)中文医疗信息处理评测冠军模型。
-  * 💬 发布大规模百亿开放域对话预训练模型[**PLATO-XL**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/model_zoo/plato-xl) ，配合⚡**FasterGeneration**⚡快速实现高性能GPU并行推理加速。
+* 🔥 **2022.10.14 发布 [PaddleNLP v2.4.1](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.4.1)**
+  * 🧾 发布多语言跨模态布局增强文档智能大模型 [**ERNIE-Layout**](./model_zoo/ernie-layout/)，刷新11项任务SOTA。同步发布基于ERNIE-Layout的**文档抽取问答模型DocPrompt** 🔖，精准理解文档图片布局与语义信息，轻松应对各类业务场景。
 
-* 📬 2022.12.12 **PaddleNLP v2.2**发布！新增开箱即用的NLP能力[Taskflow](./docs/model_zoo/taskflow.md)！配套语义检索、智能问答、评论观点抽取[产业案例](./applications)，快速搭建端到端NLP系统！配套视频课程[直通车](https://aistudio.baidu.com/aistudio/course/introduce/24902)！
+* 🔥 **2022.9.6 发布 [PaddleNLP v2.4](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.4.0)**
+  * 💎 NLP工具：**[NLP 流水线系统 Pipelines](./pipelines)** 发布，支持快速搭建搜索引擎、问答系统，可扩展支持各类NLP系统，让解决 NLP 任务像搭积木一样便捷、灵活、高效！
+  * 💢 产业应用：新增 **[文本分类全流程应用方案](./applications/text_classification)** ，覆盖多分类、多标签、层次分类各类场景，支持 **小样本学习** 和 **TrustAI** 可信计算模型训练与调优；[**通用信息抽取 UIE 能力升级**](./model_zoo/uie)，发布 **UIE-M**，支持中英文混合抽取，新增**UIE 数据蒸馏**方案，打破 UIE 推理瓶颈，推理速度提升 100 倍以上；
+  * 🍭 AIGC 内容生成：新增代码生成 SOTA 模型[**CodeGen**](./examples/code_generation/codegen)，支持多种编程语言代码生成；集成[**文图生成潮流模型**](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/docs/model_zoo/taskflow.md#%E6%96%87%E5%9B%BE%E7%94%9F%E6%88%90) DALL·E Mini、Disco Diffusion、Stable Diffusion，更多趣玩模型等你来玩；新增[**中文文本摘要应用**](./applications/text_summarization)，基于大规模语料的中文摘要模型首次发布，可支持 Taskflow 一键调用和定制训练；
+  * 💪 框架升级：[**模型自动压缩 API**](./docs/compression.md) 发布，自动对模型进行裁减和量化，大幅降低模型压缩技术使用门槛；[**小样本 Prompt**](./applications/text_classification/multi_class/few-shot)能力发布，集成 PET、P-Tuning、RGL 等经典算法。
 
+
+## 社区交流
+
+- 微信扫描二维码并填写问卷，回复小助手关键词（NLP）之后，即可加入交流群领取福利
+
+  - 与众多社区开发者以及官方团队深度交流。
+  - 10G重磅NLP学习大礼包！
+
+  <div align="center">
+  <img src="https://user-images.githubusercontent.com/11793384/200770385-a2c02093-0e1d-4189-b324-8b0251cba316.jpg" width="150" height="150" />
+</div>
 ## 特性
 
 #### <a href=#开箱即用的nlp工具集> 📦 开箱即用的NLP工具集 </a>
@@ -56,6 +71,14 @@ Taskflow提供丰富的**📦开箱即用**的产业级NLP预置模型，覆盖
 
 ![taskflow1](https://user-images.githubusercontent.com/11793384/159693816-fda35221-9751-43bb-b05c-7fc77571dd76.gif)
 
+Taskflow最新集成了文生图的趣玩应用，三行代码体验 **Stable Diffusion**
+```python
+from paddlenlp import Taskflow
+text_to_image = Taskflow("text_to_image", model="CompVis/stable-diffusion-v1-4")
+image_list = text_to_image('"In the morning light,Chinese ancient buildings in the mountains,Magnificent and fantastic John Howe landscape,lake,clouds,farm,Fairy tale,light effect,Dream,Greg Rutkowski,James Gurney,artstation"')
+```
+<img width="300" alt="image" src="https://user-images.githubusercontent.com/16698950/194882669-f7cc7c98-d63a-45f4-99c1-0514c6712368.png">
+
 更多使用方法可参考[Taskflow文档](./docs/model_zoo/taskflow.md)。
 
 ### 丰富完备的中文模型库
@@ -75,6 +98,19 @@ electra = AutoModel.from_pretrained('chinese-electra-small')
 gpt = AutoModelForPretraining.from_pretrained('gpt-cpm-large-cn')
 ```
 
+针对预训练模型计算瓶颈，可以使用API一键使用文心ERNIE-Tiny全系列轻量化模型，降低预训练模型部署难度。
+
+```python
+# 6L768H
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+# 6L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-mini-zh')
+# 4L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-micro-zh')
+# 4L312H
+ernie = AutoModel.from_pretrained('ernie-3.0-nano-zh')
+```
+
 对预训练模型应用范式如语义表示、文本分类、句对匹配、序列标注、问答等，提供统一的API体验。
 
 ```python
@@ -174,7 +210,7 @@ PaddleNLP针对信息抽取、语义检索、智能问答、情感分析等高
 </div>
 
 
-更多使用说明请参考[智能问答系统](./applications/question_answering)与[文档智能问答](./applications/doc_vqa)
+更多使用说明请参考[智能问答系统](./applications/question_answering)与[文档智能问答](./applications/document_intelligence/doc_vqa)
 
 #### 💌 评论观点抽取与情感分析
 
@@ -198,7 +234,7 @@ PaddleNLP针对信息抽取、语义检索、智能问答、情感分析等高
 
 ### 高性能分布式训练与推理
 
-#### ⚡ FasterTokenizers：高性能文本处理库
+#### ⚡ FasterTokenizer：高性能文本处理库
 
 <div align="center">
     <img src="https://user-images.githubusercontent.com/11793384/168407921-b4395b1d-44bd-41a0-8c58-923ba2b703ef.png" width="400">
@@ -208,7 +244,7 @@ PaddleNLP针对信息抽取、语义检索、智能问答、情感分析等高
 AutoTokenizer.from_pretrained("ernie-3.0-medium-zh", use_faster=True)
 ```
 
-为了实现更极致的模型部署性能，安装FastTokenizers后只需在`AutoTokenizer` API上打开 `use_faster=True`选项，即可调用C++实现的高性能分词算子，轻松获得超Python百余倍的文本处理加速，更多使用说明可参考[FasterTokenizers文档](./faster_tokenizers)。
+为了实现更极致的模型部署性能，安装FastTokenizers后只需在`AutoTokenizer` API上打开 `use_faster=True`选项，即可调用C++实现的高性能分词算子，轻松获得超Python百余倍的文本处理加速，更多使用说明可参考[FasterTokenizer文档](./faster_tokenizer)。
 
 #### ⚡️ FasterGeneration：高性能生成加速库
 
@@ -294,21 +330,11 @@ PaddleNLP提供[一键预测功能](./docs/model_zoo/taskflow.md)，无需训练
 PaddleNLP提供全流程的文本领域API，可大幅提升NLP任务建模的效率：
 
 - 支持[千言](https://www.luge.ai)等丰富中文数据集加载的[Dataset API](https://paddlenlp.readthedocs.io/zh/latest/data_prepare/dataset_list.html)。
-- 提供🤗Hugging Face Style的API，支持 **500+** 优质预训练模型加载的[Transformers API](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/index.html)。  
+- 提供🤗Hugging Face Style的API，支持 **500+** 优质预训练模型加载的[Transformers API](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/index.html)。
 - 提供30+多语言词向量的[Embedding API](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/embeddings.html)
 
 更多使用方法请参考[API文档](https://paddlenlp.readthedocs.io/zh/latest/)。
 
-## 社区交流
-
-- 微信扫描二维码并填写问卷之后，加入交流群领取福利
-  - 获取5月18-19日每晚20:30《产业级通用信息抽取技术UIE+ERNIE轻量级模型》直播课链接。
-  - 10G重磅NLP学习大礼包！
-
-  <div align="center">
-  <img src="https://user-images.githubusercontent.com/11793384/168411900-d9f3d777-99ab-4b5c-8cdc-ef747a48b864.jpg" width="150" height="150" />
-  </div>
-
 
 ## Citation
 

README_en.md

@@ -1,7 +1,7 @@
 [简体中文🀄](./README_cn.md) |  **English**🌎
 
 <p align="center">
-  <img src="./docs/imgs/paddlenlp.png" align="middle"  width="500" />
+  <img src="https://user-images.githubusercontent.com/1371212/175816733-8ec25eb0-9af3-4380-9218-27c154518258.png" align="middle"  width="500" />
 </p>
 
 ------------------------------------------------------------------------------------------
@@ -30,11 +30,22 @@
 
 ## News 📢
 
-* 🔥 2022.5.16 PaddleNLP [v2.3](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.3.0) Released!🎉
-  * 💎 Release [**UIE** (Universal Information Extraction)](./model_zoo/uie) technique, single model supports multiple **open-domain** IE tasks. Super easy to use and finetune with few examples via [Taskflow](./docs/model_zoo/taskflow.md).
-  * 😊 Release [**ERNIE 3.0**](./model_zoo/ernie-3.0) light-weight model achieved better results compared to ERNIE 2.0 on [CLUE](https://www.cluebenchmarks.com/), also including **🗜️lossless model compression** and **⚙️end-to-end deployment**.
-  * 🏥 Release [**ERNIE-Health**](./model_zoo/ernie-health), a **SOTA** biomedical pretrained model on [CBLUE](https://github.com/CBLUEbenchmark/CBLUE).
-  * 💬 Release [**PLATO-XL**](./model_zoo/plato-xl) with ⚡**FasterGeneration**⚡, the *11B open-domain SOTA chatbot model* can be deployed on multi-GPU and do parallel inference easily.
+* 🔥 **2022.10.27 [PaddleNLP v2.4.2](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.4.2) Released!**
+  * NLG Upgrade: 📄 Release [**Solution of Text Summarization**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/applications/text_summarization/pegasus) based on Pegasus;❓ Release [**Solution of Problem Generation**](./examples/question_generation), providing **general problem generation pre-trained model** based on Baidu's UNIMO Text and large-scale multi domain problem generation dataset。Supporting high-performance inference ability based on FasterGeneration , and covering the whole process of training , inference and deployment.
+* 🔥 **2022.10.14 [PaddleNLP v2.4.1](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.4.1) Released!**
+  * 🧾 Release multilingual/cross-lingual pre-trained models [**ERNIE-Layout**](./model_zoo/ernie-layout/) which achieves new SOTA results in 11 downstream tasks. **DocPrompt** 🔖 based on ERNIE-Layout is also released which has the ability for multilingual document information extraction and question ansering.
+* 🔥 **2022.9.6 [PaddleNLPv2.4](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.4.0) Released!**
+
+  * 💎 NLP Tool：**[Pipelines](./pipelines)** released. Supports for fast construction of search engine and question answering systems, and it is expandable to all kinds of NLP systems. Building end-to-end pipelines for NLP tasks like playing Lego!
+
+  * 💢 Industrial application：Release **[Complete Solution of Text Classification](./applications/text_classification)** covering various scenarios of text classification: multi-class, multi-label and hierarchical, it also supports for **few-shot learning** and the training and optimization of **TrustAI**. Upgrade for [**Universal Information Extraction**](./model_zoo/uie) and release **UIE-M**, support both Chinese and English information extraction in a single model; release the data distillation solution for UIE to break the bottleneck of time-consuming of inference.
+
+  * 🍭 AIGC: Release code generation SOTA model [**CodeGen**](./examples/code_generation/codegen), supports for multiple programming languages code generation. Integrate [**Text to Image Model**](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/docs/model_zoo/taskflow.md#%E6%96%87%E5%9B%BE%E7%94%9F%E6%88%90) DALL·E Mini, Disco Diffusion, Stable Diffusion, let's play and have some fun! Release [**Chinese Text Summarization Application**](./applications/text_summarization), first release of chinese text summarization model pretrained on a large scale of corpus, it can be use via Taskflow API and support for finetuning on your own data.
+
+  * 💪 Framework upgrade: Release [**Auto Model Compression API**](./docs/compression.md), supports for pruning and quantization automatically, lower the barriers of model compression; Release [**Few-shot Prompt**](./applications/text_classification/multi_class/few-shot), includes the algorithms such as PET, P-Tuning and RGL.
+
+
+
 
 ## Features
 
@@ -57,9 +68,9 @@ For more usage please refer to [Taskflow Docs](./docs/model_zoo/taskflow.md).
 
 ### Awesome Chinese Model Zoo
 
-#### 🀄 Comprehensive Chinese Transformer Models  
+#### 🀄 Comprehensive Chinese Transformer Models
 
-We provide **45+** network architectures and over **500+** pretrained models. Not only includes all the SOTA model like ERNIE, PLATO and SKEP released by Baidu, but also integrates most of the high-quality Chinese pretrained model developed by other organizations. Use `AutoModel` API to **⚡SUPER FAST⚡** download pretrained mdoels of different architecture. We welcome all developers to contribute your Transformer models to PaddleNLP!
+We provide **45+** network architectures and over **500+** pretrained models. Not only includes all the SOTA model like ERNIE, PLATO and SKEP released by Baidu, but also integrates most of the high-quality Chinese pretrained model developed by other organizations. Use `AutoModel` API to **⚡SUPER FAST⚡** download pretrained models of different architecture. We welcome all developers to contribute your Transformer models to PaddleNLP!
 
 ```python
 from paddlenlp.transformers import *
@@ -72,6 +83,17 @@ electra = AutoModel.from_pretrained('chinese-electra-small')
 gpt = AutoModelForPretraining.from_pretrained('gpt-cpm-large-cn')
 ```
 
+Due to the computation limitation, you can use the ERNIE-Tiny light models to accelerate the deployment of pretrained models.
+```python
+# 6L768H
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+# 6L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-mini-zh')
+# 4L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-micro-zh')
+# 4L312H
+ernie = AutoModel.from_pretrained('ernie-3.0-nano-zh')
+```
 Unified API experience for NLP task like semantic representation, text classification, sentence matching, sequence labeling, question answering, etc.
 
 ```python
@@ -172,7 +194,7 @@ We provide question answering pipeline which can support FAQ system, Document-le
 </div>
 
 
-For more details please refer to [Question Answering](./applications/question_answering) and [Document VQA](./applications/doc_vqa).
+For more details please refer to [Question Answering](./applications/question_answering) and [Document VQA](./applications/document_intelligence/doc_vqa).
 
 
 #### 💌 Opinion Extraction and Sentiment Analysis
@@ -199,7 +221,7 @@ For more details please refer to [Speech Command Analysis](./applications/speech
 
 ### High Performance Distributed Training and Inference
 
-#### ⚡ FasterTokenizers: High Performance Text Preprocessing Library
+#### ⚡ FasterTokenizer: High Performance Text Preprocessing Library
 
 <div align="center">
     <img src="https://user-images.githubusercontent.com/11793384/168407921-b4395b1d-44bd-41a0-8c58-923ba2b703ef.png" width="400">
@@ -209,7 +231,7 @@ For more details please refer to [Speech Command Analysis](./applications/speech
 AutoTokenizer.from_pretrained("ernie-3.0-medium-zh", use_faster=True)
 ```
 
-Set `use_faster=True` to use C++ Tokenizer kernel to achieve 100x faster on text pre-processing. For more usage please refer to [FasterTokenizers](./faster_tokenizers).
+Set `use_faster=True` to use C++ Tokenizer kernel to achieve 100x faster on text pre-processing. For more usage please refer to [FasterTokenizer](./faster_tokenizer).
 
 #### ⚡ FasterGeneration: High Perforance Generation Library
 
@@ -295,10 +317,6 @@ Please find all PaddleNLP API Reference from our [readthedocs](https://paddlenlp
 
 ## Community
 
-### Special Interest Group (SIG)
-
-Welcome to join [PaddleNLP SIG](https://iwenjuan.baidu.com/?code=bkypg8) for contribution, eg. Dataset, Models and Toolkit.
-
 ### Slack
 
 To connect with other users and contributors, welcome to join our [Slack channel](https://paddlenlp.slack.com/).
@@ -307,9 +325,11 @@ To connect with other users and contributors, welcome to join our [Slack channel
 
 Scan the QR code below with your Wechat⬇️. You can access to official technical exchange group. Look forward to your participation.
 
- <div align="center">
- <img src="https://user-images.githubusercontent.com/11793384/168411900-d9f3d777-99ab-4b5c-8cdc-ef747a48b864.jpg" width="150" height="150" />
- </div>
+<div align="center">
+<img src="https://user-images.githubusercontent.com/11793384/200770385-a2c02093-0e1d-4189-b324-8b0251cba316.jpg" width="150" height="150" />
+</div>
+
+
 
 ## Citation
 
@@ -325,7 +345,7 @@ If you find PaddleNLP useful in your research, please consider cite
 
 ## Acknowledge
 
-We have borrowed from Hugging Face's [Transformer](https://github.com/huggingface/transformers)🤗 excellent design on pretrained models usage, and we would like to express our gratitude to the authors of Hugging Face and its open source community.
+We have borrowed from Hugging Face's [Transformers](https://github.com/huggingface/transformers)🤗 excellent design on pretrained models usage, and we would like to express our gratitude to the authors of Hugging Face and its open source community.
 
 ## License
 

VERSION

@@ -0,0 +1 @@
+4.2.3

applications/doc_vqa/Extraction/run_test.sh

@@ -1,24 +0,0 @@
-export CUDA_VISIBLE_DEVICES=0
-
-QUESTION=$1
-
-python3 change_to_mrc.py ${QUESTION}
-
-python3  ./run_docvqa.py \
-    --model_name_or_path "layoutxlm-base-uncased" \
-    --max_seq_len 512 \
-    --do_test true \
-	--test_file "data/demo_test.json" \
-	--num_train_epochs 100 \
-    --eval_steps 6000 \
-    --save_steps 6000 \
-    --output_dir "output/" \
-    --save_path "data/decode_res.json" \
-	--init_checkpoint "./checkpoints/layoutxlm/" \
-    --learning_rate 3e-5 \
-    --warmup_steps 12000 \
-    --per_gpu_train_batch_size 4 \
-    --per_gpu_eval_batch_size 1 \
-    --seed 2048
-
-python3 view.py

applications/doc_vqa/Extraction/run_train.sh

@@ -1,18 +0,0 @@
-export CUDA_VISIBLE_DEVICES=0
-
-python3 ./run_docvqa.py \
-    --model_name_or_path "layoutxlm-base-uncased" \
-    --max_seq_len 512 \
-    --train_file "data/train.json" \
-    --init_checkpoint "checkpoints/base_model" \
-	--do_train true \
-    --num_train_epochs 50 \
-    --eval_steps 24000 \
-    --save_steps 40 \
-    --output_dir "output" \
-    --save_path "data/decode_res.json" \
-    --learning_rate 3e-5 \
-    --warmup_steps 40 \
-    --per_gpu_train_batch_size 4 \
-    --per_gpu_eval_batch_size 4 \
-    --seed 2048

applications/doc_vqa/Rerank/change_to_rerank.py

@@ -1,19 +0,0 @@
-import sys
-import json
-
-question = sys.argv[1]
-
-with open('../OCR_process/demo_ocr_res.json', 'r', encoding='utf8') as f:
-    paras = []
-    for line in f:
-        line = json.loads(line.strip())
-        document = line['document']
-        para = []
-        for token in document:
-            token = token.replace('▁', '')
-            para.append(token)
-        paras.append(''.join(para))
-
-with open('./data/demo.tsv', 'w', encoding='utf8') as f:
-    for para in paras:
-        f.write('{}\t\t{}\t0\n'.format(question, para))

applications/doc_vqa/run_test.sh

@@ -1,20 +0,0 @@
-export CUDA_VISIBLE_DEVICES=0
-
-QUESTION=$1
-
-# Question: NFC咋开门
-
-if [ $# != 1 ];then
-    echo "USAGE: sh script/run_cross_encoder_test.sh \$QUESTION"
-    exit 1
-fi
-
-# compute scores for QUESTION and OCR parsing results  with Rerank module
-cd Rerank
-bash run_test.sh ${QUESTION}
-cd ..
-
-# extraction answer for QUESTION from the top1 of rank
-cd Extraction
-bash run_test.sh ${QUESTION}
-cd ..

applications/document_intelligence/README.md

@@ -0,0 +1,188 @@
+# 文档智能应用
+
+**目录**
+- [1. 文档智能应用简介](#文档智能应用简介)
+- [2. 技术特色介绍](#技术特色介绍)
+  - [2.1 多语言跨模态训练基座](#多语言跨模态训练基座)
+  - [2.2 多场景覆盖](#多场景覆盖)
+- [3. 快速开始](#快速开始)
+  - [3.1 开箱即用](#开箱即用)
+  - [3.2 产业级流程方案](#产业级流程方案)
+
+## 1. 文档智能应用简介
+
+文档智能（DI, Document Intelligence）主要指**对于网页、数字文档或扫描文档所包含的文本以及丰富的排版格式等信息，通过人工智能技术进行理解、分类、提取以及信息归纳**的过程。文档智能技术广泛应用于金融、保险、能源、物流、医疗等行业，常见的应用场景包括财务报销单、招聘简历、企业财报、合同文书、动产登记证、法律判决书、物流单据等多模态文档的关键信息抽取、文档解析、文档比对等。
+
+在实际应用中，需要解决文档格式繁杂、布局多样、信息模态多样、需求开放、业务数据少等多重难题。针对文档智能领域的痛点和难点，PaddleNLP将持续开源一系列产业实践范例，解决开发者们实际应用难题。
+
+<div align="center">
+    <img width="1000" height="270" alt="文档智能技术一般流程" src="https://user-images.githubusercontent.com/40840292/196361583-6b1c66d1-6a9b-4193-949a-71e2d420a82a.png">
+</div>
+
+<a name="技术特色介绍"></a>
+
+## 2. 技术特色介绍
+
+<a name="多语言跨模态训练基座"></a>
+
+### 2.1 多语言跨模态训练基座
+
+近期，百度文心文档智能，基于多语言跨模态布局增强的文档智能大模型[ERNIE-Layout](http://arxiv.org/abs/2210.06155)，刷新了五类11项文档智能任务效果。依托文心ERNIE大模型，基于布局知识增强技术，融合文本、图像、布局等信息进行联合建模，能够对多模态文档（如文档图片、PDF 文件、扫描件等）进行深度理解与分析，为各类上层应用提供SOTA模型底座。
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/196373896-597f6178-4c78-41a1-bb12-796546644b32.png width="600"/>
+</div>
+
+<a name="多场景覆盖"></a>
+
+### 2.2 多场景覆盖
+
+以下是文档智能技术的一些应用场景展示：
+
+- 发票抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/196118171-fd3e49a0-b9f1-4536-a904-c48f709a2dec.png height=350 width=1000 hspace='10'/>
+</div>
+
+- 海报抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195610368-04230855-62de-439e-b708-2c195b70461f.png height=600 width=1000 hspace='15'/>
+</div>
+
+- 网页抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195611613-bdbe692e-d7f2-4a2b-b548-1a933463b0b9.png height=350 width=1000 hspace='10'/>
+</div>
+
+
+- 表格抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195610692-8367f1c8-32c2-4b5d-9514-a149795cf609.png height=350 width=1000 hspace='10'/>
+</div>
+
+
+- 试卷抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195823294-d891d95a-2ef8-4519-be59-0fedb96c00de.png height=700 width=1000 hspace='10'/>
+</div>
+
+
+- 英文票据多语种（中、英、日、泰、西班牙、俄语）抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195610820-7fb88608-b317-45fc-a6ab-97bf3b20a4ac.png height=400 width=1000 hspace='15'/>
+</div>
+
+- 中文票据多语种（中简、中繁、英、日、法语）抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195611075-9323ce9f-134b-4657-ab1c-f4892075d909.png height=350 width=1000 hspace='15'/>
+</div>
+
+- Demo图片可在此[下载](https://bj.bcebos.com/paddlenlp/taskflow/document_intelligence/demo.zip)
+
+<a name="快速开始"></a>
+
+## 3. 快速开始
+
+<a name="开箱即用"></a>
+
+### 3.1 开箱即用
+
+开源DocPrompt开放文档抽取问答模型，以ERNIE-Layout为底座，可精准理解图文信息，推理学习附加知识，准备捕捉图片、PDF等多模态文档中的每个细节。
+
+🧾 通过[Huggingface网页](https://huggingface.co/spaces/PaddlePaddle/ERNIE-Layout)体验DocPrompt功能：
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195749427-864d7744-1fd1-455e-99c6-53a260776483.jpg height=700 width=1100 hspace='10'/>
+</div>
+
+#### Taskflow
+
+通过``paddlenlp.Taskflow``三行代码调用DocPrompt功能，具备多语言文档抽取问答能力，部分应用场景展示如下：
+
+- 输入格式
+
+```
+[
+  {"doc": "./invoice.jpg", "prompt": ["发票号码是多少?", "校验码是多少?"]},
+  {"doc": "./resume.png", "prompt": ["五百丁本次想要担任的是什么职位?", "五百丁是在哪里上的大学?", "大学学的是什么专业?"]}
+]
+```
+
+默认使用PaddleOCR进行OCR识别，同时支持用户通过``word_boxes``传入自己的OCR结果，格式为``List[str, List[float, float, float, float]]``。
+
+```
+[
+  {"doc": doc_path, "prompt": prompt, "word_boxes": word_boxes}
+]
+```
+
+- 支持单条、批量预测
+
+  - 支持本地图片路径输入
+
+  <div align="center">
+      <img src=https://user-images.githubusercontent.com/40840292/194748579-f9e8aa86-7f65-4827-bfae-824c037228b3.png height=800 hspace='20'/>
+  </div>
+
+  ```python
+  >>> from pprint import pprint
+  >>> from paddlenlp import Taskflow
+
+  >>> docprompt = Taskflow("document_intelligence")
+  >>> pprint(docprompt([{"doc": "./resume.png", "prompt": ["五百丁本次想要担任的是什么职位?", "五百丁是在哪里上的大学?", "大学学的是什么专业?"]}]))
+  [{'prompt': '五百丁本次想要担任的是什么职位?',
+    'result': [{'end': 7, 'prob': 1.0, 'start': 4, 'value': '客户经理'}]},
+  {'prompt': '五百丁是在哪里上的大学?',
+    'result': [{'end': 37, 'prob': 1.0, 'start': 31, 'value': '广州五百丁学院'}]},
+  {'prompt': '大学学的是什么专业?',
+    'result': [{'end': 44, 'prob': 0.82, 'start': 38, 'value': '金融学(本科）'}]}]
+  ```
+
+  - http图片链接输入
+
+  <div align="center">
+      <img src=https://user-images.githubusercontent.com/40840292/194748592-e20b2a5f-d36b-46fb-8057-86755d188af0.jpg height=400 hspace='10'/>
+  </div>
+
+  ```python
+  >>> from pprint import pprint
+  >>> from paddlenlp import Taskflow
+
+  >>> docprompt = Taskflow("document_intelligence")
+  >>> pprint(docprompt([{"doc": "https://bj.bcebos.com/paddlenlp/taskflow/document_intelligence/images/invoice.jpg", "prompt": ["发票号码是多少?", "校验码是多少?"]}]))
+  [{'prompt': '发票号码是多少?',
+    'result': [{'end': 2, 'prob': 0.74, 'start': 2, 'value': 'No44527206'}]},
+  {'prompt': '校验码是多少?',
+    'result': [{'end': 233,
+                'prob': 1.0,
+                'start': 231,
+                'value': '01107 555427109891646'}]}]
+  ```
+
+- 可配置参数说明
+  * `batch_size`：批处理大小，请结合机器情况进行调整，默认为1。
+  * `lang`：选择PaddleOCR的语言，`ch`可在中英混合的图片中使用，`en`在英文图片上的效果更好，默认为`ch`。
+  * `topn`: 如果模型识别出多个结果，将返回前n个概率值最高的结果，默认为1。
+
+<a name="产业级流程方案"></a>
+
+### 3.2 产业级流程方案
+
+针对文档智能领域的痛点和难点，PaddleNLP将持续开源一系列文档智能产业实践范例，解决开发者们实际应用难题。
+
+- 👉 [汽车说明书跨模态智能问答](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/applications/document_intelligence/doc_vqa#readme)
+
+更多：百度TextMind智能文档分析平台可提供包括文档信息抽取、文本内容审查、企业文档管理、文档格式解析、文档内容比对等全方位一站式的文档智能服务，已形成一套完整的企业文档场景化解决方案，满足银行、券商、法律、能源、传媒、通信、物流等不同行业和场景的文档处理需求，以AI助力企业的办公智能化升级和数字化转型。欢迎深度交流与商业合作，了解详情：https://ai.baidu.com/tech/nlp/Textanalysis
+
+## References
+
+- [文档智能：数据集、模型和应用](http://jcip.cipsc.org.cn/CN/abstract/abstract3331.shtml)
+
+- [ERNIE-Layout: Layout-Knowledge Enhanced Multi-modal Pre-training for Document Understanding](http://arxiv.org/abs/2210.06155)

applications/document_intelligence/doc_vqa/Extraction/change_to_mrc.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import sys
 import json
 import numpy as np

applications/document_intelligence/doc_vqa/Extraction/docvqa.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 import cv2
 import sys
@@ -14,6 +28,7 @@ sys.path.insert(0, "../")
 
 
 class DocVQAExample(object):
+
     def __init__(self,
                  question,
                  doc_tokens,
@@ -48,6 +63,7 @@ class DocVQAFeatures(object):
 
 
 class DocVQA(Dataset):
+
     def __init__(self,
                  args,
                  tokenizer,
@@ -264,7 +280,8 @@ class DocVQA(Dataset):
                 input_mask=spans_input_mask,
                 segment_ids=spans_segment_ids,
                 boxes=spans_boxes_tokens,
-                label=label_ids, )
+                label=label_ids,
+            )
             features.append(feature)
         return features
 
@@ -292,8 +309,9 @@ class DocVQA(Dataset):
                 scale_y = 1000 / max(width, height)
 
             scaled_doc_boxes = [[
-                round((b[0] - x_min) * scale_x), round(
-                    (b[2] - y_min) * scale_y), round((b[1] - x_min) * scale_x),
+                round((b[0] - x_min) * scale_x),
+                round((b[2] - y_min) * scale_y),
+                round((b[1] - x_min) * scale_x),
                 round((b[3] - y_min) * scale_y)
             ] for b in doc_boxes]
 
@@ -308,11 +326,10 @@ class DocVQA(Dataset):
                     if pos > 1000:
                         print(width, height, box, oribox)
 
-            example = DocVQAExample(
-                question=question,
-                doc_tokens=doc_tokens,
-                doc_boxes=scaled_doc_boxes,
-                labels=labels)
+            example = DocVQAExample(question=question,
+                                    doc_tokens=doc_tokens,
+                                    doc_boxes=scaled_doc_boxes,
+                                    labels=labels)
             examples.append(example)
         return examples
 
@@ -338,19 +355,21 @@ class DocVQA(Dataset):
             max_span_num=self.max_span_num,
             max_query_length=self.max_query_length)
 
-        all_input_ids = paddle.to_tensor(
-            [f.input_ids for f in features], dtype="int64")
-        all_input_mask = paddle.to_tensor(
-            [f.input_mask for f in features], dtype="int64")
-        all_segment_ids = paddle.to_tensor(
-            [f.segment_ids for f in features], dtype="int64")
-        all_bboxes = paddle.to_tensor(
-            [f.boxes for f in features], dtype="int64")
-        all_labels = paddle.to_tensor(
-            [f.label for f in features], dtype="int64")
+        all_input_ids = paddle.to_tensor([f.input_ids for f in features],
+                                         dtype="int64")
+        all_input_mask = paddle.to_tensor([f.input_mask for f in features],
+                                          dtype="int64")
+        all_segment_ids = paddle.to_tensor([f.segment_ids for f in features],
+                                           dtype="int64")
+        all_bboxes = paddle.to_tensor([f.boxes for f in features],
+                                      dtype="int64")
+        all_labels = paddle.to_tensor([f.label for f in features],
+                                      dtype="int64")
         self.sample_list = [
-            np.array(all_input_ids), np.array(all_input_mask),
-            np.array(all_segment_ids), np.array(all_bboxes),
+            np.array(all_input_ids),
+            np.array(all_input_mask),
+            np.array(all_segment_ids),
+            np.array(all_bboxes),
             np.array(all_labels)
         ]
 

applications/document_intelligence/doc_vqa/Extraction/model.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import numpy as np
 
 import paddle
@@ -7,6 +21,7 @@ from paddlenlp.transformers import LayoutXLMPretrainedModel
 
 
 class Crf_decoding(paddle.fluid.dygraph.Layer):
+
     def __init__(self, param_attr, size=None, is_test=True, dtype='float32'):
         super(Crf_decoding, self).__init__()
 
@@ -38,16 +53,20 @@ class Crf_decoding(paddle.fluid.dygraph.Layer):
         }
         if length is not None:
             this_inputs['Length'] = [length]
-        self._helper.append_op(
-            type='crf_decoding',
-            inputs=this_inputs,
-            outputs={"ViterbiPath": [viterbi_path]},
-            attrs={"is_test": self._is_test, })
+        self._helper.append_op(type='crf_decoding',
+                               inputs=this_inputs,
+                               outputs={"ViterbiPath": [viterbi_path]},
+                               attrs={
+                                   "is_test": self._is_test,
+                               })
         return viterbi_path
 
 
 class Chunk_eval(paddle.fluid.dygraph.Layer):
-    def __init__(self, num_chunk_types, chunk_scheme,
+
+    def __init__(self,
+                 num_chunk_types,
+                 chunk_scheme,
                  excluded_chunk_types=None):
         super(Chunk_eval, self).__init__()
         self.num_chunk_types = num_chunk_types
@@ -73,27 +92,30 @@ class Chunk_eval(paddle.fluid.dygraph.Layer):
         if seq_length is not None:
             this_input["SeqLength"] = [seq_length]
 
-        self._helper.append_op(
-            type='chunk_eval',
-            inputs=this_input,
-            outputs={
-                "Precision": [precision],
-                "Recall": [recall],
-                "F1-Score": [f1_score],
-                "NumInferChunks": [num_infer_chunks],
-                "NumLabelChunks": [num_label_chunks],
-                "NumCorrectChunks": [num_correct_chunks]
-            },
-            attrs={
-                "num_chunk_types": self.num_chunk_types,
-                "chunk_scheme": self.chunk_scheme,
-                "excluded_chunk_types": self.excluded_chunk_types or []
-            })
+        self._helper.append_op(type='chunk_eval',
+                               inputs=this_input,
+                               outputs={
+                                   "Precision": [precision],
+                                   "Recall": [recall],
+                                   "F1-Score": [f1_score],
+                                   "NumInferChunks": [num_infer_chunks],
+                                   "NumLabelChunks": [num_label_chunks],
+                                   "NumCorrectChunks": [num_correct_chunks]
+                               },
+                               attrs={
+                                   "num_chunk_types":
+                                   self.num_chunk_types,
+                                   "chunk_scheme":
+                                   self.chunk_scheme,
+                                   "excluded_chunk_types":
+                                   self.excluded_chunk_types or []
+                               })
         return (precision, recall, f1_score, num_infer_chunks, num_label_chunks,
                 num_correct_chunks)
 
 
 class Linear_chain_crf(paddle.fluid.dygraph.Layer):
+
     def __init__(self, param_attr, size=None, is_test=False, dtype='float32'):
         super(Linear_chain_crf, self).__init__()
 
@@ -131,20 +153,22 @@ class Linear_chain_crf(paddle.fluid.dygraph.Layer):
         }
         if length is not None:
             this_inputs['Length'] = [length]
-        self._helper.append_op(
-            type='linear_chain_crf',
-            inputs=this_inputs,
-            outputs={
-                "Alpha": [alpha],
-                "EmissionExps": [emission_exps],
-                "TransitionExps": transition_exps,
-                "LogLikelihood": log_likelihood
-            },
-            attrs={"is_test": self._is_test, })
+        self._helper.append_op(type='linear_chain_crf',
+                               inputs=this_inputs,
+                               outputs={
+                                   "Alpha": [alpha],
+                                   "EmissionExps": [emission_exps],
+                                   "TransitionExps": transition_exps,
+                                   "LogLikelihood": log_likelihood
+                               },
+                               attrs={
+                                   "is_test": self._is_test,
+                               })
         return log_likelihood
 
 
 class LayoutXLMForTokenClassification_with_CRF(LayoutXLMPretrainedModel):
+
     def __init__(self, layoutxlm, num_classes, dropout=None):
         super(LayoutXLMForTokenClassification_with_CRF, self).__init__()
         self.num_classes = num_classes
@@ -152,8 +176,8 @@ class LayoutXLMForTokenClassification_with_CRF(LayoutXLMPretrainedModel):
             self.layoutxlm = LayoutXLMModel(**layoutxlm)
         else:
             self.layoutxlm = layoutxlm
-        self.dropout = nn.Dropout(dropout if dropout is not None else
-                                  self.layoutxlm.config["hidden_dropout_prob"])
+        self.dropout = nn.Dropout(dropout if dropout is not None else self.
+                                  layoutxlm.config["hidden_dropout_prob"])
         self.emission_classifier = nn.Linear(
             self.layoutxlm.config["hidden_size"], self.num_classes)
         self.emission_classifier.apply(self.init_weights)
@@ -198,7 +222,8 @@ class LayoutXLMForTokenClassification_with_CRF(LayoutXLMPretrainedModel):
             attention_mask=attention_mask,
             token_type_ids=token_type_ids,
             position_ids=position_ids,
-            head_mask=head_mask, )
+            head_mask=head_mask,
+        )
         seq_length = input_ids.shape[1]
         # sequence out and image out
         sequence_logits, image_output = outputs[0][:, :seq_length], outputs[
@@ -208,8 +233,9 @@ class LayoutXLMForTokenClassification_with_CRF(LayoutXLMPretrainedModel):
         labels = labels.reshape([-1, seq_length, 1])
 
         # standard crf loss
-        crf_cost = self.linear_chain_crf(
-            input=emission, label=labels, length=length)
+        crf_cost = self.linear_chain_crf(input=emission,
+                                         label=labels,
+                                         length=length)
         crf_decode = self.crf_decoding(input=emission, length=length)
         if is_train:
             return [crf_cost]

applications/document_intelligence/doc_vqa/Extraction/run_docvqa.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 import sys
 import copy
@@ -72,7 +86,8 @@ def main(args):
         format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
         datefmt="%m/%d/%Y %H:%M:%S",
         level=logging.INFO
-        if paddle.distributed.get_rank() == 0 else logging.WARN, )
+        if paddle.distributed.get_rank() == 0 else logging.WARN,
+    )
 
     ch = logging.StreamHandler()
     ch.setLevel(logging.DEBUG)
@@ -90,14 +105,13 @@ def main(args):
     if args.do_test:
         model = LayoutXLMForTokenClassification_with_CRF.from_pretrained(
             args.init_checkpoint)
-        evaluate(
-            args,
-            model,
-            tokenizer,
-            label2id_map,
-            id2label_map,
-            pad_token_label_id,
-            global_step=0)
+        evaluate(args,
+                 model,
+                 tokenizer,
+                 label2id_map,
+                 id2label_map,
+                 pad_token_label_id,
+                 global_step=0)
         exit(0)
 
     if args.init_checkpoint:
@@ -113,14 +127,13 @@ def main(args):
     if paddle.distributed.get_world_size() > 1:
         model = paddle.DataParallel(model)
 
-    train_dataset = DocVQA(
-        args,
-        tokenizer,
-        label2id_map,
-        max_seq_len=args.max_seq_len,
-        max_query_length=args.max_query_length,
-        max_doc_length=args.max_doc_length,
-        max_span_num=args.max_span_num)
+    train_dataset = DocVQA(args,
+                           tokenizer,
+                           label2id_map,
+                           max_seq_len=args.max_seq_len,
+                           max_query_length=args.max_query_length,
+                           max_doc_length=args.max_doc_length,
+                           max_span_num=args.max_span_num)
 
     train_sampler = paddle.io.DistributedBatchSampler(
         train_dataset, batch_size=args.per_gpu_train_batch_size, shuffle=False)
@@ -128,12 +141,11 @@ def main(args):
     args.train_batch_size = args.per_gpu_train_batch_size * max(
         1, paddle.distributed.get_world_size())
 
-    train_dataloader = paddle.io.DataLoader(
-        train_dataset,
-        batch_sampler=train_sampler,
-        num_workers=0,
-        use_shared_memory=True,
-        collate_fn=None)
+    train_dataloader = paddle.io.DataLoader(train_dataset,
+                                            batch_sampler=train_sampler,
+                                            num_workers=0,
+                                            use_shared_memory=True,
+                                            collate_fn=None)
 
     t_total = len(train_dataloader) * args.num_train_epochs
     # build linear decay with warmup lr sch
@@ -149,11 +161,10 @@ def main(args):
             start_lr=0,
             end_lr=args.learning_rate)
 
-    optimizer = paddle.optimizer.AdamW(
-        learning_rate=lr_scheduler,
-        parameters=model.parameters(),
-        epsilon=args.adam_epsilon,
-        weight_decay=args.weight_decay)
+    optimizer = paddle.optimizer.AdamW(learning_rate=lr_scheduler,
+                                       parameters=model.parameters(),
+                                       epsilon=args.adam_epsilon,
+                                       weight_decay=args.weight_decay)
 
     logger.info("***** Running training *****")
     logger.info("  Num examples = %d", len(train_dataset))
@@ -162,7 +173,8 @@ def main(args):
                 args.per_gpu_train_batch_size)
     logger.info(
         "  Total train batch size (w. parallel, distributed) = %d",
-        args.train_batch_size * paddle.distributed.get_world_size(), )
+        args.train_batch_size * paddle.distributed.get_world_size(),
+    )
     logger.info("  Total optimization steps = %d", t_total)
 
     global_step = 0
@@ -176,22 +188,21 @@ def main(args):
             input_ids, input_mask, segment_ids, bboxes, labels = batch
             if input_ids.shape[0] != args.per_gpu_train_batch_size:
                 continue
-            outputs = model(
-                input_ids=input_ids,
-                bbox=bboxes,
-                attention_mask=input_mask,
-                token_type_ids=segment_ids,
-                labels=labels,
-                is_train=True)
-            # model outputs are always tuple in ppnlp (see doc)
+            outputs = model(input_ids=input_ids,
+                            bbox=bboxes,
+                            attention_mask=input_mask,
+                            token_type_ids=segment_ids,
+                            labels=labels,
+                            is_train=True)
+            # model outputs are always tuple in paddlenlp (see doc)
             loss = outputs[0]
             loss = loss.mean()
             if global_step % 50 == 0:
                 logger.info(
-                    "[epoch {}/{}][iter: {}/{}] lr: {:.5f}, train loss: {:.5f}, ".
-                    format(epoch_id, args.num_train_epochs, step,
-                           len(train_dataloader),
-                           lr_scheduler.get_lr(), loss.numpy()[0]))
+                    "[epoch {}/{}][iter: {}/{}] lr: {:.5f}, train loss: {:.5f}, "
+                    .format(epoch_id, args.num_train_epochs, step,
+                            len(train_dataloader), lr_scheduler.get_lr(),
+                            loss.numpy()[0]))
 
             loss.backward()
             tr_loss += loss.item()
@@ -307,9 +318,9 @@ def decode(tokenizer, res):
     save_f1 = []
     for i in range(len(res)):
         input_ids, label_ids, predict_ids, bbox = res[i]
-        remove_pos = len(' '.join([str(x) for x in input_ids]).split('2 6 ')[0]
-                         .strip(' ').split(
-                             ' ')) + 2  # remove the question bbox and sep bbox
+        remove_pos = len(' '.join(
+            [str(x) for x in input_ids]).split('2 6 ')[0].strip(' ').split(
+                ' ')) + 2  # remove the question bbox and sep bbox
         start_pos = input_ids.index(sep_id)
         query_text = []
         for idx in range(1, start_pos):
@@ -356,24 +367,22 @@ def evaluate(args,
              prefix="",
              global_step=0):
 
-    eval_dataset = DocVQA(
-        args,
-        tokenizer,
-        label2id_map,
-        max_seq_len=512,
-        max_query_length=20,
-        max_doc_length=512,
-        max_span_num=1)
+    eval_dataset = DocVQA(args,
+                          tokenizer,
+                          label2id_map,
+                          max_seq_len=512,
+                          max_query_length=20,
+                          max_doc_length=512,
+                          max_span_num=1)
 
     args.eval_batch_size = args.per_gpu_eval_batch_size * max(
         1, paddle.distributed.get_world_size())
 
-    eval_dataloader = paddle.io.DataLoader(
-        eval_dataset,
-        batch_size=args.eval_batch_size,
-        num_workers=0,
-        use_shared_memory=True,
-        collate_fn=None)
+    eval_dataloader = paddle.io.DataLoader(eval_dataset,
+                                           batch_size=args.eval_batch_size,
+                                           num_workers=0,
+                                           use_shared_memory=True,
+                                           collate_fn=None)
 
     # Eval!
     logger.info("***** Running evaluation %s *****", prefix)
@@ -387,13 +396,12 @@ def evaluate(args,
 
             if input_ids.shape[0] != args.eval_batch_size:
                 continue
-            outputs = model(
-                input_ids=input_ids,
-                bbox=bboxes,
-                attention_mask=input_mask,
-                token_type_ids=segment_ids,
-                labels=labels,
-                is_train=False)
+            outputs = model(input_ids=input_ids,
+                            bbox=bboxes,
+                            attention_mask=input_mask,
+                            token_type_ids=segment_ids,
+                            labels=labels,
+                            is_train=False)
             labels = labels.numpy()
             crf_decode = outputs[1].numpy()
             bboxes = bboxes.squeeze().numpy()
@@ -401,7 +409,8 @@ def evaluate(args,
 
             for index in range(input_ids.shape[0]):
                 res.append([
-                    list(input_ids[index]), list(labels[index]),
+                    list(input_ids[index]),
+                    list(labels[index]),
                     list(crf_decode[index]), bboxes[index]
                 ])
 

applications/document_intelligence/doc_vqa/Extraction/run_test.sh

@@ -0,0 +1,38 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export CUDA_VISIBLE_DEVICES=0
+
+QUESTION=$1
+
+python3 change_to_mrc.py ${QUESTION}
+
+python3  ./run_docvqa.py \
+    --model_name_or_path "layoutxlm-base-uncased" \
+    --max_seq_len 512 \
+    --do_test true \
+	--test_file "data/demo_test.json" \
+	--num_train_epochs 100 \
+    --eval_steps 6000 \
+    --save_steps 6000 \
+    --output_dir "output/" \
+    --save_path "data/decode_res.json" \
+	--init_checkpoint "./checkpoints/layoutxlm/" \
+    --learning_rate 3e-5 \
+    --warmup_steps 12000 \
+    --per_gpu_train_batch_size 4 \
+    --per_gpu_eval_batch_size 1 \
+    --seed 2048
+
+python3 view.py

applications/document_intelligence/doc_vqa/Extraction/run_train.sh

@@ -0,0 +1,32 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export CUDA_VISIBLE_DEVICES=0
+
+python3 ./run_docvqa.py \
+    --model_name_or_path "layoutxlm-base-uncased" \
+    --max_seq_len 512 \
+    --train_file "data/train.json" \
+    --init_checkpoint "checkpoints/base_model" \
+	--do_train true \
+    --num_train_epochs 50 \
+    --eval_steps 24000 \
+    --save_steps 40 \
+    --output_dir "output" \
+    --save_path "data/decode_res.json" \
+    --learning_rate 3e-5 \
+    --warmup_steps 40 \
+    --per_gpu_train_batch_size 4 \
+    --per_gpu_eval_batch_size 4 \
+    --seed 2048

applications/document_intelligence/doc_vqa/Extraction/view.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import cv2
 import json
 import numpy as np

applications/document_intelligence/doc_vqa/OCR_process/ocr_process.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 import json
 from paddleocr import PaddleOCR
@@ -8,6 +22,7 @@ import sys
 from multiprocessing import Process, Queue
 
 from paddlenlp.transformers import LayoutXLMTokenizer
+
 tokenizer = LayoutXLMTokenizer.from_pretrained("layoutxlm-base-uncased")
 
 
@@ -38,8 +53,8 @@ def merge_bbox(tok_bboxes):
         height_m += y_max - y_min
         width_m += x_max - x_min
     height_m = height_m / len(tok_bboxes)
-    if (height_g - height_m
-        ) < 0.5 * height_m and width_g - width_m < 0.1 * width_m:
+    if (height_g -
+            height_m) < 0.5 * height_m and width_g - width_m < 0.1 * width_m:
         return False, [min_gx, max_gx, min_gy, max_gy]
     else:
         return True, tok_bboxes[0]
@@ -123,20 +138,20 @@ def xlm_parse(ocr_res, tokenizer):
                     for j in range(i, len(span_text)):
                         if span_text[j].lower() == span_tokens[tid + 1][0]:
                             break
-                        if span_text[j].lower() == "，" and span_tokens[tid + 1][
-                                0] == ",":
+                        if span_text[j].lower() == "，" and span_tokens[
+                                tid + 1][0] == ",":
                             break
-                        if span_text[j].lower() == "；" and span_tokens[tid + 1][
-                                0] == ";":
+                        if span_text[j].lower() == "；" and span_tokens[
+                                tid + 1][0] == ";":
                             break
-                        if span_text[j].lower() == "）" and span_tokens[tid + 1][
-                                0] == ")":
+                        if span_text[j].lower() == "）" and span_tokens[
+                                tid + 1][0] == ")":
                             break
-                        if span_text[j].lower() == "（" and span_tokens[tid + 1][
-                                0] == "(":
+                        if span_text[j].lower() == "（" and span_tokens[
+                                tid + 1][0] == "(":
                             break
-                        if span_text[j].lower() == "￥" and span_tokens[tid + 1][
-                                0] == "¥":
+                        if span_text[j].lower() == "￥" and span_tokens[
+                                tid + 1][0] == "¥":
                             break
 
                     tok_len = j - i
@@ -213,7 +228,8 @@ def tokenize_ocr_res(ocr_reses):
 
                 shift = tok_x_max
                 new_token_boxes.append(
-                    [round(tok_x_min), round(tok_x_max), tok_y_min, tok_y_max])
+                    [round(tok_x_min),
+                     round(tok_x_max), tok_y_min, tok_y_max])
                 new_tokens.append(char)
             new_res.append({
                 "text": para["text"],
@@ -251,8 +267,8 @@ def process_input(ocr_reses, tokenizer, save_ocr_path):
 def ocr_preprocess(img_dir):
     ocr = PaddleOCR(use_angle_cls=True, lang="ch", use_gpu=True)
     ocr_reses = []
-    img_names = sorted(
-        os.listdir(img_dir), key=lambda x: int(x.split("_")[1].split(".")[0]))
+    img_names = sorted(os.listdir(img_dir),
+                       key=lambda x: int(x.split("_")[1].split(".")[0]))
     for img_name in img_names:
         img_path = os.path.join(img_dir, img_name)
         parsing_res = ocr.ocr(img_path, cls=True)

applications/document_intelligence/doc_vqa/Rerank/change_to_rerank.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import json
+
+question = sys.argv[1]
+
+with open('../OCR_process/demo_ocr_res.json', 'r', encoding='utf8') as f:
+    paras = []
+    for line in f:
+        line = json.loads(line.strip())
+        document = line['document']
+        para = []
+        for token in document:
+            token = token.replace('▁', '')
+            para.append(token)
+        paras.append(''.join(para))
+
+with open('./data/demo.tsv', 'w', encoding='utf8') as f:
+    for para in paras:
+        f.write('{}\t\t{}\t0\n'.format(question, para))

applications/document_intelligence/doc_vqa/Rerank/run_test.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
 
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 export CUDA_VISIBLE_DEVICES=0
 
 QUESTION=$1

applications/document_intelligence/doc_vqa/Rerank/run_train.sh

@@ -1,4 +1,19 @@
 #!/bin/bash
+
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 export CUDA_VISIBLE_DEVICES=0
 
 if [ $# != 4 ];then

applications/document_intelligence/doc_vqa/Rerank/src/batching.py

@@ -52,8 +52,8 @@ def pad_batch_data(insts,
 
     if return_input_mask:
         # This is used to avoid attention on paddings.
-        input_mask_data = np.array([[1] * len(inst) + [0] *
-                                    (max_len - len(inst)) for inst in insts])
+        input_mask_data = np.array(
+            [[1] * len(inst) + [0] * (max_len - len(inst)) for inst in insts])
         input_mask_data = np.expand_dims(input_mask_data, axis=-1)
         return_list += [input_mask_data.astype("float32")]
 

applications/document_intelligence/doc_vqa/Rerank/src/cross_encoder.py

@@ -37,30 +37,32 @@ def create_model(args,
                  ernie_config,
                  is_prediction=False,
                  task_name=""):
-    pyreader = fluid.layers.py_reader(
-        capacity=50,
-        shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1],
-                [-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1],
-                [-1, args.max_seq_len, 1], [-1, 1], [-1, 1]],
-        dtypes=[
-            'int64', 'int64', 'int64', 'int64', 'float32', 'int64', 'int64'
-        ],
-        lod_levels=[0, 0, 0, 0, 0, 0, 0],
-        name=task_name + "_" + pyreader_name,
-        use_double_buffer=True)
+    pyreader = fluid.layers.py_reader(capacity=50,
+                                      shapes=[[-1, args.max_seq_len, 1],
+                                              [-1, args.max_seq_len, 1],
+                                              [-1, args.max_seq_len, 1],
+                                              [-1, args.max_seq_len, 1],
+                                              [-1, args.max_seq_len, 1],
+                                              [-1, 1], [-1, 1]],
+                                      dtypes=[
+                                          'int64', 'int64', 'int64', 'int64',
+                                          'float32', 'int64', 'int64'
+                                      ],
+                                      lod_levels=[0, 0, 0, 0, 0, 0, 0],
+                                      name=task_name + "_" + pyreader_name,
+                                      use_double_buffer=True)
 
     (src_ids, sent_ids, pos_ids, task_ids, input_mask, labels,
      qids) = fluid.layers.read_file(pyreader)
 
     def _model(is_noise=False):
-        ernie = ErnieModel(
-            src_ids=src_ids,
-            position_ids=pos_ids,
-            sentence_ids=sent_ids,
-            task_ids=task_ids,
-            input_mask=input_mask,
-            config=ernie_config,
-            is_noise=is_noise)
+        ernie = ErnieModel(src_ids=src_ids,
+                           position_ids=pos_ids,
+                           sentence_ids=sent_ids,
+                           task_ids=task_ids,
+                           input_mask=input_mask,
+                           config=ernie_config,
+                           is_noise=is_noise)
 
         cls_feats = ernie.get_pooled_output()
         if not is_noise:
@@ -93,8 +95,9 @@ def create_model(args,
         ce_loss, probs = fluid.layers.softmax_with_cross_entropy(
             logits=logits, label=labels, return_softmax=True)
         loss = fluid.layers.mean(x=ce_loss)
-        accuracy = fluid.layers.accuracy(
-            input=probs, label=labels, total=num_seqs)
+        accuracy = fluid.layers.accuracy(input=probs,
+                                         label=labels,
+                                         total=num_seqs)
         graph_vars = {
             "loss": loss,
             "probs": probs,
@@ -233,8 +236,8 @@ def evaluate(exe,
             % (eval_phase, cost, ret, total_num_seqs, elapsed_time)
     elif metric == "acc_and_f1_and_mrr":
         ret_a = acc_and_f1(preds, labels)
-        preds = sorted(
-            zip(qids, scores, labels), key=lambda elem: (elem[0], -elem[1]))
+        preds = sorted(zip(qids, scores, labels),
+                       key=lambda elem: (elem[0], -elem[1]))
         ret_b = evaluate_mrr(preds)
         evaluate_info = "[%s evaluation] ave loss: %f, acc: %f, f1: %f, mrr: %f, data_num: %d, elapsed time: %f s" \
             % (eval_phase, cost, ret_a['acc'], ret_a['f1'], ret_b, total_num_seqs, elapsed_time)

applications/document_intelligence/doc_vqa/Rerank/src/index_search.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import sys
 import time
 import faiss
@@ -43,8 +57,8 @@ def search(index, emb_file, qid_list, outfile, top_k):
     with open(outfile, 'w') as out:
         for batch_vec in read_embed(emb_file):
             q_emb_matrix = np.array(batch_vec)
-            res_dist, res_p_id = index.search(
-                q_emb_matrix.astype('float32'), top_k)
+            res_dist, res_p_id = index.search(q_emb_matrix.astype('float32'),
+                                              top_k)
             for i in range(len(q_emb_matrix)):
                 qid = qid_list[q_idx]
                 for j in range(top_k):

applications/document_intelligence/doc_vqa/Rerank/src/merge.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import sys
 
 shift = int(sys.argv[1])

applications/document_intelligence/doc_vqa/Rerank/src/model/ernie.py

@@ -32,6 +32,7 @@ log = logging.getLogger(__name__)
 
 
 class ErnieConfig(object):
+
     def __init__(self, config_path):
         self._config_dict = self._parse(config_path)
 
@@ -55,6 +56,7 @@ class ErnieConfig(object):
 
 
 class ErnieModel(object):
+
     def __init__(self,
                  src_ids,
                  position_ids,
@@ -109,26 +111,23 @@ class ErnieModel(object):
             input=src_ids,
             size=[self._voc_size, self._emb_size],
             dtype=self._emb_dtype,
-            param_attr=fluid.ParamAttr(
-                name=model_name + self._word_emb_name,
-                initializer=self._param_initializer),
+            param_attr=fluid.ParamAttr(name=model_name + self._word_emb_name,
+                                       initializer=self._param_initializer),
             is_sparse=False)
 
         position_emb_out = fluid.layers.embedding(
             input=position_ids,
             size=[self._max_position_seq_len, self._emb_size],
             dtype=self._emb_dtype,
-            param_attr=fluid.ParamAttr(
-                name=model_name + self._pos_emb_name,
-                initializer=self._param_initializer))
+            param_attr=fluid.ParamAttr(name=model_name + self._pos_emb_name,
+                                       initializer=self._param_initializer))
 
         sent_emb_out = fluid.layers.embedding(
             sentence_ids,
             size=[self._sent_types, self._emb_size],
             dtype=self._emb_dtype,
-            param_attr=fluid.ParamAttr(
-                name=model_name + self._sent_emb_name,
-                initializer=self._param_initializer))
+            param_attr=fluid.ParamAttr(name=model_name + self._sent_emb_name,
+                                       initializer=self._param_initializer))
 
         emb_out = emb_out + position_emb_out
         emb_out = emb_out + sent_emb_out
@@ -138,25 +137,28 @@ class ErnieModel(object):
                 task_ids,
                 size=[self._task_types, self._emb_size],
                 dtype=self._emb_dtype,
-                param_attr=fluid.ParamAttr(
-                    name=model_name + self._task_emb_name,
-                    initializer=self._param_initializer))
+                param_attr=fluid.ParamAttr(name=model_name +
+                                           self._task_emb_name,
+                                           initializer=self._param_initializer))
 
             emb_out = emb_out + task_emb_out
 
-        emb_out = pre_process_layer(
-            emb_out,
-            'nd',
-            self._prepostprocess_dropout,
-            name=model_name + 'pre_encoder')
+        emb_out = pre_process_layer(emb_out,
+                                    'nd',
+                                    self._prepostprocess_dropout,
+                                    name=model_name + 'pre_encoder')
 
-        self_attn_mask = fluid.layers.matmul(
-            x=input_mask, y=input_mask, transpose_y=True)
+        self_attn_mask = fluid.layers.matmul(x=input_mask,
+                                             y=input_mask,
+                                             transpose_y=True)
 
-        self_attn_mask = fluid.layers.scale(
-            x=self_attn_mask, scale=10000.0, bias=-1.0, bias_after_scale=False)
-        n_head_self_attn_mask = fluid.layers.stack(
-            x=[self_attn_mask] * self._n_head, axis=1)
+        self_attn_mask = fluid.layers.scale(x=self_attn_mask,
+                                            scale=10000.0,
+                                            bias=-1.0,
+                                            bias_after_scale=False)
+        n_head_self_attn_mask = fluid.layers.stack(x=[self_attn_mask] *
+                                                   self._n_head,
+                                                   axis=1)
         n_head_self_attn_mask.stop_gradient = True
 
         self._enc_out, self.checkpoints = encoder(
@@ -183,21 +185,25 @@ class ErnieModel(object):
 
     def get_cls_output(self):
         """Get the first feature of each sequence for classification"""
-        cls_output = fluid.layers.slice(
-            input=self._enc_out, axes=[1], starts=[0], ends=[1])
+        cls_output = fluid.layers.slice(input=self._enc_out,
+                                        axes=[1],
+                                        starts=[0],
+                                        ends=[1])
         cls_output = fluid.layers.squeeze(cls_output, axes=[1])
         return cls_output
 
     def get_pooled_output(self):
         """Get the first feature of each sequence for classification"""
-        next_sent_feat = fluid.layers.slice(
-            input=self._enc_out, axes=[1], starts=[0], ends=[1])
+        next_sent_feat = fluid.layers.slice(input=self._enc_out,
+                                            axes=[1],
+                                            starts=[0],
+                                            ends=[1])
         next_sent_feat = fluid.layers.fc(
             input=next_sent_feat,
             size=self._emb_size,
             act="tanh",
-            param_attr=fluid.ParamAttr(
-                name="pooled_fc.w_0", initializer=self._param_initializer),
+            param_attr=fluid.ParamAttr(name="pooled_fc.w_0",
+                                       initializer=self._param_initializer),
             bias_attr="pooled_fc.b_0")
         return next_sent_feat
 
@@ -208,8 +214,8 @@ class ErnieModel(object):
 
         # extract the first token feature in each sentence
         self.next_sent_feat = self.get_pooled_output()
-        reshaped_emb_out = fluid.layers.reshape(
-            x=self._enc_out, shape=[-1, self._emb_size])
+        reshaped_emb_out = fluid.layers.reshape(x=self._enc_out,
+                                                shape=[-1, self._emb_size])
         # extract masked tokens' feature
         mask_feat = fluid.layers.gather(input=reshaped_emb_out, index=mask_pos)
 
@@ -218,9 +224,8 @@ class ErnieModel(object):
             input=mask_feat,
             size=self._emb_size,
             act=self._hidden_act,
-            param_attr=fluid.ParamAttr(
-                name='mask_lm_trans_fc.w_0',
-                initializer=self._param_initializer),
+            param_attr=fluid.ParamAttr(name='mask_lm_trans_fc.w_0',
+                                       initializer=self._param_initializer),
             bias_attr=fluid.ParamAttr(name='mask_lm_trans_fc.b_0'))
 
         # transform: layer norm
@@ -246,11 +251,10 @@ class ErnieModel(object):
                 y=fluid.default_main_program().global_block().var(
                     self._word_emb_name),
                 transpose_y=True)
-            fc_out += fluid.layers.create_parameter(
-                shape=[self._voc_size],
-                dtype=self._emb_dtype,
-                attr=mask_lm_out_bias_attr,
-                is_bias=True)
+            fc_out += fluid.layers.create_parameter(shape=[self._voc_size],
+                                                    dtype=self._emb_dtype,
+                                                    attr=mask_lm_out_bias_attr,
+                                                    is_bias=True)
 
         else:
             fc_out = fluid.layers.fc(input=mask_trans_feat,
@@ -260,8 +264,8 @@ class ErnieModel(object):
                                          initializer=self._param_initializer),
                                      bias_attr=mask_lm_out_bias_attr)
 
-        mask_lm_loss = fluid.layers.softmax_with_cross_entropy(
-            logits=fc_out, label=mask_label)
+        mask_lm_loss = fluid.layers.softmax_with_cross_entropy(logits=fc_out,
+                                                               label=mask_label)
         mean_mask_lm_loss = fluid.layers.mean(mask_lm_loss)
 
         return mean_mask_lm_loss

applications/document_intelligence/doc_vqa/Rerank/src/model/transformer_encoder.py

@@ -54,23 +54,20 @@ def multi_head_attention(queries,
         q = layers.fc(input=queries,
                       size=d_key * n_head,
                       num_flatten_dims=2,
-                      param_attr=fluid.ParamAttr(
-                          name=name + '_query_fc.w_0',
-                          initializer=param_initializer),
+                      param_attr=fluid.ParamAttr(name=name + '_query_fc.w_0',
+                                                 initializer=param_initializer),
                       bias_attr=name + '_query_fc.b_0')
         k = layers.fc(input=keys,
                       size=d_key * n_head,
                       num_flatten_dims=2,
-                      param_attr=fluid.ParamAttr(
-                          name=name + '_key_fc.w_0',
-                          initializer=param_initializer),
+                      param_attr=fluid.ParamAttr(name=name + '_key_fc.w_0',
+                                                 initializer=param_initializer),
                       bias_attr=name + '_key_fc.b_0')
         v = layers.fc(input=values,
                       size=d_value * n_head,
                       num_flatten_dims=2,
-                      param_attr=fluid.ParamAttr(
-                          name=name + '_value_fc.w_0',
-                          initializer=param_initializer),
+                      param_attr=fluid.ParamAttr(name=name + '_value_fc.w_0',
+                                                 initializer=param_initializer),
                       bias_attr=name + '_value_fc.b_0')
         return q, k, v
 
@@ -84,8 +81,9 @@ def multi_head_attention(queries,
         hidden_size = x.shape[-1]
         # The value 0 in shape attr means copying the corresponding dimension
         # size of the input as the output dimension size.
-        reshaped = layers.reshape(
-            x=x, shape=[0, 0, n_head, hidden_size // n_head], inplace=True)
+        reshaped = layers.reshape(x=x,
+                                  shape=[0, 0, n_head, hidden_size // n_head],
+                                  inplace=True)
 
         # permuate the dimensions into:
         # [batch_size, n_head, max_sequence_len, hidden_size_per_head]
@@ -103,10 +101,9 @@ def multi_head_attention(queries,
         trans_x = layers.transpose(x, perm=[0, 2, 1, 3])
         # The value 0 in shape attr means copying the corresponding dimension
         # size of the input as the output dimension size.
-        return layers.reshape(
-            x=trans_x,
-            shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]],
-            inplace=True)
+        return layers.reshape(x=trans_x,
+                              shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]],
+                              inplace=True)
 
     def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate):
         """
@@ -118,11 +115,10 @@ def multi_head_attention(queries,
             product += attn_bias
         weights = layers.softmax(product)
         if dropout_rate:
-            weights = layers.dropout(
-                weights,
-                dropout_prob=dropout_rate,
-                dropout_implementation="upscale_in_train",
-                is_test=False)
+            weights = layers.dropout(weights,
+                                     dropout_prob=dropout_rate,
+                                     dropout_implementation="upscale_in_train",
+                                     is_test=False)
         out = layers.matmul(weights, v)
         return out
 
@@ -133,11 +129,9 @@ def multi_head_attention(queries,
         # v, which is the cache input for next time step, reshape the cache
         # input from the previous time step first.
         k = cache["k"] = layers.concat(
-            [layers.reshape(
-                cache["k"], shape=[0, 0, d_model]), k], axis=1)
+            [layers.reshape(cache["k"], shape=[0, 0, d_model]), k], axis=1)
         v = cache["v"] = layers.concat(
-            [layers.reshape(
-                cache["v"], shape=[0, 0, d_model]), v], axis=1)
+            [layers.reshape(cache["v"], shape=[0, 0, d_model]), v], axis=1)
 
     q = __split_heads(q, n_head)
     k = __split_heads(k, n_head)
@@ -180,21 +174,23 @@ def positionwise_feed_forward(x,
                            initializer=param_initializer),
                        bias_attr=name + '_fc_0.b_0')
     if dropout_rate:
-        hidden = layers.dropout(
-            hidden,
-            dropout_prob=dropout_rate,
-            dropout_implementation="upscale_in_train",
-            is_test=False)
+        hidden = layers.dropout(hidden,
+                                dropout_prob=dropout_rate,
+                                dropout_implementation="upscale_in_train",
+                                is_test=False)
     out = layers.fc(input=hidden,
                     size=d_hid,
                     num_flatten_dims=2,
-                    param_attr=fluid.ParamAttr(
-                        name=name + '_fc_1.w_0', initializer=param_initializer),
+                    param_attr=fluid.ParamAttr(name=name + '_fc_1.w_0',
+                                               initializer=param_initializer),
                     bias_attr=name + '_fc_1.b_0')
     return out
 
 
-def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.,
+def pre_post_process_layer(prev_out,
+                           out,
+                           process_cmd,
+                           dropout_rate=0.,
                            name=''):
     """
     Add residual connection, layer normalization and droput to the out tensor
@@ -222,11 +218,10 @@ def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.,
                 out = layers.cast(x=out, dtype="float16")
         elif cmd == "d":  # add dropout
             if dropout_rate:
-                out = layers.dropout(
-                    out,
-                    dropout_prob=dropout_rate,
-                    dropout_implementation="upscale_in_train",
-                    is_test=False)
+                out = layers.dropout(out,
+                                     dropout_prob=dropout_rate,
+                                     dropout_implementation="upscale_in_train",
+                                     is_test=False)
     return out
 
 
@@ -255,46 +250,42 @@ def encoder_layer(enc_input,
     with the post_process_layer to add residual connection, layer normalization
     and droput.
     """
-    attn_output = multi_head_attention(
-        pre_process_layer(
-            enc_input,
-            preprocess_cmd,
-            prepostprocess_dropout,
-            name=name + '_pre_att'),
-        None,
-        None,
-        attn_bias,
-        d_key,
-        d_value,
-        d_model,
-        n_head,
-        attention_dropout,
-        param_initializer=param_initializer,
-        name=name + '_multi_head_att')
-    attn_output = post_process_layer(
-        enc_input,
+    attn_output = multi_head_attention(pre_process_layer(enc_input,
+                                                         preprocess_cmd,
+                                                         prepostprocess_dropout,
+                                                         name=name +
+                                                         '_pre_att'),
+                                       None,
+                                       None,
+                                       attn_bias,
+                                       d_key,
+                                       d_value,
+                                       d_model,
+                                       n_head,
+                                       attention_dropout,
+                                       param_initializer=param_initializer,
+                                       name=name + '_multi_head_att')
+    attn_output = post_process_layer(enc_input,
+                                     attn_output,
+                                     postprocess_cmd,
+                                     prepostprocess_dropout,
+                                     name=name + '_post_att')
+    ffd_output = positionwise_feed_forward(pre_process_layer(
         attn_output,
-        postprocess_cmd,
+        preprocess_cmd,
         prepostprocess_dropout,
-        name=name + '_post_att')
-    ffd_output = positionwise_feed_forward(
-        pre_process_layer(
-            attn_output,
-            preprocess_cmd,
-            prepostprocess_dropout,
-            name=name + '_pre_ffn'),
-        d_inner_hid,
-        d_model,
-        relu_dropout,
-        hidden_act,
-        param_initializer=param_initializer,
-        name=name + '_ffn')
-    return post_process_layer(
-        attn_output,
-        ffd_output,
-        postprocess_cmd,
-        prepostprocess_dropout,
-        name=name + '_post_ffn'), ffd_output
+        name=name + '_pre_ffn'),
+                                           d_inner_hid,
+                                           d_model,
+                                           relu_dropout,
+                                           hidden_act,
+                                           param_initializer=param_initializer,
+                                           name=name + '_ffn')
+    return post_process_layer(attn_output,
+                              ffd_output,
+                              postprocess_cmd,
+                              prepostprocess_dropout,
+                              name=name + '_post_ffn'), ffd_output
 
 
 def encoder(enc_input,
@@ -320,28 +311,26 @@ def encoder(enc_input,
     """
     checkpoints = []
     for i in range(n_layer):
-        enc_output, cp = encoder_layer(
-            enc_input,
-            attn_bias,
-            n_head,
-            d_key,
-            d_value,
-            d_model,
-            d_inner_hid,
-            prepostprocess_dropout,
-            attention_dropout,
-            relu_dropout,
-            hidden_act,
-            preprocess_cmd,
-            postprocess_cmd,
-            param_initializer=param_initializer,
-            name=name + '_layer_' + str(i))
+        enc_output, cp = encoder_layer(enc_input,
+                                       attn_bias,
+                                       n_head,
+                                       d_key,
+                                       d_value,
+                                       d_model,
+                                       d_inner_hid,
+                                       prepostprocess_dropout,
+                                       attention_dropout,
+                                       relu_dropout,
+                                       hidden_act,
+                                       preprocess_cmd,
+                                       postprocess_cmd,
+                                       param_initializer=param_initializer,
+                                       name=name + '_layer_' + str(i))
         checkpoints.append(cp)
         enc_input = enc_output
-    enc_output = pre_process_layer(
-        enc_output,
-        preprocess_cmd,
-        prepostprocess_dropout,
-        name=model_name + "post_encoder")
+    enc_output = pre_process_layer(enc_output,
+                                   preprocess_cmd,
+                                   prepostprocess_dropout,
+                                   name=model_name + "post_encoder")
 
     return enc_output, checkpoints

applications/document_intelligence/doc_vqa/Rerank/src/optimization.py

@@ -96,11 +96,11 @@ def optimization(loss,
                 learning_rate=scheduled_lr)
         else:
             optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr)
-        optimizer._learning_rate_map[fluid.default_main_program(
-        )] = scheduled_lr
+        optimizer._learning_rate_map[
+            fluid.default_main_program()] = scheduled_lr
 
-    fluid.clip.set_gradient_clip(
-        clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0))
+    fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByGlobalNorm(
+        clip_norm=1.0))
 
     def exclude_from_weight_decay(name):
         if name.find("layer_norm") > -1:
@@ -119,8 +119,8 @@ def optimization(loss,
 
     if dist_strategy is not None:
         # use fleet api
-        optimizer = fleet.distributed_optimizer(
-            optimizer, strategy=dist_strategy)
+        optimizer = fleet.distributed_optimizer(optimizer,
+                                                strategy=dist_strategy)
 
     _, param_grads = optimizer.minimize(loss)
 

applications/document_intelligence/doc_vqa/Rerank/src/reader_ce.py

@@ -39,6 +39,7 @@ if six.PY3:
 
 
 def csv_reader(fd, delimiter='\t', trainer_id=0, trainer_num=1):
+
     def gen():
         for i, line in enumerate(fd):
             if i % trainer_num == trainer_id:
@@ -52,6 +53,7 @@ def csv_reader(fd, delimiter='\t', trainer_id=0, trainer_num=1):
 
 
 class BaseReader(object):
+
     def __init__(self,
                  vocab_path,
                  label_map_config=None,
@@ -65,8 +67,8 @@ class BaseReader(object):
                  for_cn=True,
                  task_id=0):
         self.max_seq_len = max_seq_len
-        self.tokenizer = tokenization.FullTokenizer(
-            vocab_file=vocab_path, do_lower_case=do_lower_case)
+        self.tokenizer = tokenization.FullTokenizer(vocab_file=vocab_path,
+                                                    do_lower_case=do_lower_case)
         self.vocab = self.tokenizer.vocab
         self.pad_id = self.vocab["[PAD]"]
         self.cls_id = self.vocab["[CLS]"]
@@ -180,10 +182,9 @@ class BaseReader(object):
         if self.is_inference:
             Record = namedtuple('Record',
                                 ['token_ids', 'text_type_ids', 'position_ids'])
-            record = Record(
-                token_ids=token_ids,
-                text_type_ids=text_type_ids,
-                position_ids=position_ids)
+            record = Record(token_ids=token_ids,
+                            text_type_ids=text_type_ids,
+                            position_ids=position_ids)
         else:
             if self.label_map:
                 label_id = self.label_map[example.label]
@@ -198,12 +199,11 @@ class BaseReader(object):
             if "qid" in example._fields:
                 qid = example.qid
 
-            record = Record(
-                token_ids=token_ids,
-                text_type_ids=text_type_ids,
-                position_ids=position_ids,
-                label_id=label_id,
-                qid=qid)
+            record = Record(token_ids=token_ids,
+                            text_type_ids=text_type_ids,
+                            position_ids=position_ids,
+                            label_id=label_id,
+                            qid=qid)
         return record
 
     def _prepare_batch_data(self, examples, batch_size, phase=None):
@@ -247,11 +247,10 @@ class BaseReader(object):
             #            examples = examples[trainer_id: (len(examples) //trainer_num) * trainer_num : trainer_num]
             self.num_examples_per_node = self.total_num // trainer_num
             self.num_examples = self.num_examples_per_node * trainer_num
-            examples = self._read_tsv(
-                input_file,
-                trainer_id=trainer_id,
-                trainer_num=trainer_num,
-                num_examples=self.num_examples_per_node)
+            examples = self._read_tsv(input_file,
+                                      trainer_id=trainer_id,
+                                      trainer_num=trainer_num,
+                                      num_examples=self.num_examples_per_node)
             log.info('apply sharding %d/%d' % (trainer_id, trainer_num))
         else:
             examples = self._read_tsv(input_file)
@@ -265,8 +264,9 @@ class BaseReader(object):
                 if shuffle:
                     np.random.shuffle(examples)
 
-                for batch_data in self._prepare_batch_data(
-                        examples, batch_size, phase=phase):
+                for batch_data in self._prepare_batch_data(examples,
+                                                           batch_size,
+                                                           phase=phase):
                     if len(all_dev_batches) < dev_count:
                         all_dev_batches.append(batch_data)
                     if len(all_dev_batches) == dev_count:
@@ -286,6 +286,7 @@ class BaseReader(object):
 
 
 class ClassifyReader(BaseReader):
+
     def _read_tsv(self,
                   input_file,
                   quotechar=None,
@@ -294,8 +295,9 @@ class ClassifyReader(BaseReader):
                   num_examples=0):
         """Reads a tab separated value file."""
         with open(input_file, 'r', encoding='utf8') as f:
-            reader = csv_reader(
-                f, trainer_id=trainer_id, trainer_num=trainer_num)
+            reader = csv_reader(f,
+                                trainer_id=trainer_id,
+                                trainer_num=trainer_num)
             #            headers = next(reader)
             headers = 'query\ttitle\tpara\tlabel'.split('\t')
             text_indices = [
@@ -335,14 +337,15 @@ class ClassifyReader(BaseReader):
                 batch_qids = np.array([]).astype("int64").reshape([-1, 1])
 
         # padding
-        padded_token_ids, input_mask = pad_batch_data(
-            batch_token_ids, pad_idx=self.pad_id, return_input_mask=True)
-        padded_text_type_ids = pad_batch_data(
-            batch_text_type_ids, pad_idx=self.pad_id)
-        padded_position_ids = pad_batch_data(
-            batch_position_ids, pad_idx=self.pad_id)
-        padded_task_ids = np.ones_like(
-            padded_token_ids, dtype="int64") * self.task_id
+        padded_token_ids, input_mask = pad_batch_data(batch_token_ids,
+                                                      pad_idx=self.pad_id,
+                                                      return_input_mask=True)
+        padded_text_type_ids = pad_batch_data(batch_text_type_ids,
+                                              pad_idx=self.pad_id)
+        padded_position_ids = pad_batch_data(batch_position_ids,
+                                             pad_idx=self.pad_id)
+        padded_task_ids = np.ones_like(padded_token_ids,
+                                       dtype="int64") * self.task_id
 
         return_list = [
             padded_token_ids, padded_text_type_ids, padded_position_ids,

applications/document_intelligence/doc_vqa/Rerank/src/tokenization.py

@@ -364,8 +364,8 @@ def _is_punctuation(char):
     # Characters such as "^", "$", and "`" are not in the Unicode
     # Punctuation class but we treat them as punctuation anyways, for
     # consistency.
-    if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or
-        (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):
+    if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64)
+            or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):
         return True
     cat = unicodedata.category(char)
     if cat.startswith("P"):

applications/document_intelligence/doc_vqa/Rerank/src/train_ce.py

@@ -63,17 +63,16 @@ def main(args):
         dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
     exe = fluid.Executor(place)
 
-    reader = reader_ce.ClassifyReader(
-        vocab_path=args.vocab_path,
-        label_map_config=args.label_map_config,
-        max_seq_len=args.max_seq_len,
-        total_num=args.train_data_size,
-        do_lower_case=args.do_lower_case,
-        in_tokens=args.in_tokens,
-        random_seed=args.random_seed,
-        tokenizer=args.tokenizer,
-        for_cn=args.for_cn,
-        task_id=args.task_id)
+    reader = reader_ce.ClassifyReader(vocab_path=args.vocab_path,
+                                      label_map_config=args.label_map_config,
+                                      max_seq_len=args.max_seq_len,
+                                      total_num=args.train_data_size,
+                                      do_lower_case=args.do_lower_case,
+                                      in_tokens=args.in_tokens,
+                                      random_seed=args.random_seed,
+                                      tokenizer=args.tokenizer,
+                                      for_cn=args.for_cn,
+                                      task_id=args.task_id)
 
     if not (args.do_train or args.do_val or args.do_test):
         raise ValueError("For args `do_train`, `do_val` and `do_test`, at "
@@ -195,11 +194,13 @@ def main(args):
                 "WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
                 "both are set! Only arg 'init_checkpoint' is made valid.")
         if args.init_checkpoint:
-            init_checkpoint(
-                exe, args.init_checkpoint, main_program=startup_prog)
+            init_checkpoint(exe,
+                            args.init_checkpoint,
+                            main_program=startup_prog)
         elif args.init_pretraining_params:
-            init_pretraining_params(
-                exe, args.init_pretraining_params, main_program=startup_prog)
+            init_pretraining_params(exe,
+                                    args.init_pretraining_params,
+                                    main_program=startup_prog)
     elif args.do_val or args.do_test:
         if not args.init_checkpoint:
             raise ValueError("args 'init_checkpoint' should be set if"
@@ -236,13 +237,12 @@ def main(args):
                     train_exe.run(fetch_list=[], program=train_program)
 
                 else:
-                    outputs = evaluate(
-                        train_exe,
-                        train_program,
-                        train_pyreader,
-                        graph_vars,
-                        "train",
-                        metric=args.metric)
+                    outputs = evaluate(train_exe,
+                                       train_program,
+                                       train_pyreader,
+                                       graph_vars,
+                                       "train",
+                                       metric=args.metric)
 
                     if args.verbose:
                         verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size(
@@ -307,12 +307,11 @@ def main(args):
     # final eval on dianostic, hack for glue-ax
     if args.diagnostic:
         test_pyreader.decorate_tensor_provider(
-            reader.data_generator(
-                args.diagnostic,
-                batch_size=args.batch_size,
-                epoch=1,
-                dev_count=1,
-                shuffle=False))
+            reader.data_generator(args.diagnostic,
+                                  batch_size=args.batch_size,
+                                  epoch=1,
+                                  dev_count=1,
+                                  shuffle=False))
 
         log.info("Final diagnostic")
         qids, preds, probs = predict(test_exe, test_prog, test_pyreader,
@@ -332,22 +331,20 @@ def evaluate_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars,
     # evaluate dev set
     for ds in args.dev_set.split(','):
         test_pyreader.decorate_tensor_provider(
-            reader.data_generator(
-                ds,
-                batch_size=args.predict_batch_size,
-                epoch=1,
-                dev_count=1,
-                shuffle=False))
+            reader.data_generator(ds,
+                                  batch_size=args.predict_batch_size,
+                                  epoch=1,
+                                  dev_count=1,
+                                  shuffle=False))
         log.info("validation result of dataset {}:".format(ds))
-        evaluate_info = evaluate(
-            exe,
-            test_prog,
-            test_pyreader,
-            graph_vars,
-            "dev",
-            metric=args.metric)
-        log.info(evaluate_info + ', file: {}, epoch: {}, steps: {}'.format(
-            ds, epoch, steps))
+        evaluate_info = evaluate(exe,
+                                 test_prog,
+                                 test_pyreader,
+                                 graph_vars,
+                                 "dev",
+                                 metric=args.metric)
+        log.info(evaluate_info +
+                 ', file: {}, epoch: {}, steps: {}'.format(ds, epoch, steps))
 
 
 def predict_wrapper(args,
@@ -364,12 +361,11 @@ def predict_wrapper(args,
 
     for test_f, save_f in zip(test_sets, save_dirs):
         test_pyreader.decorate_tensor_provider(
-            reader.data_generator(
-                test_f,
-                batch_size=args.predict_batch_size,
-                epoch=1,
-                dev_count=1,
-                shuffle=False))
+            reader.data_generator(test_f,
+                                  batch_size=args.predict_batch_size,
+                                  epoch=1,
+                                  dev_count=1,
+                                  shuffle=False))
 
         if epoch is not None or steps is not None:
             save_path = save_f + '.' + str(epoch) + '.' + str(steps)

applications/document_intelligence/doc_vqa/Rerank/src/utils/args.py

@@ -31,7 +31,8 @@ log = logging.getLogger(__name__)
 
 def prepare_logger(logger, debug=False, save_to_file=None):
     formatter = logging.Formatter(
-        fmt='[%(levelname)s] %(asctime)s [%(filename)12s:%(lineno)5d]:\t%(message)s'
+        fmt=
+        '[%(levelname)s] %(asctime)s [%(filename)12s:%(lineno)5d]:\t%(message)s'
     )
     console_hdl = logging.StreamHandler()
     console_hdl.setFormatter(formatter)
@@ -51,19 +52,24 @@ def str2bool(v):
 
 
 class ArgumentGroup(object):
+
     def __init__(self, parser, title, des):
         self._group = parser.add_argument_group(title=title, description=des)
 
-    def add_arg(self, name, type, default, help, positional_arg=False,
+    def add_arg(self,
+                name,
+                type,
+                default,
+                help,
+                positional_arg=False,
                 **kwargs):
         prefix = "" if positional_arg else "--"
         type = str2bool if type == bool else type
-        self._group.add_argument(
-            prefix + name,
-            default=default,
-            type=type,
-            help=help + ' Default: %(default)s.',
-            **kwargs)
+        self._group.add_argument(prefix + name,
+                                 default=default,
+                                 type=type,
+                                 help=help + ' Default: %(default)s.',
+                                 **kwargs)
 
 
 def print_arguments(args):

applications/document_intelligence/doc_vqa/Rerank/src/utils/init.py

@@ -44,11 +44,10 @@ def init_checkpoint(exe, init_checkpoint_path, main_program):
         #    print ("Var exists: [%s]" % (var.name))
         return os.path.exists(os.path.join(init_checkpoint_path, var.name))
 
-    fluid.io.load_vars(
-        exe,
-        init_checkpoint_path,
-        main_program=main_program,
-        predicate=existed_persitables)
+    fluid.io.load_vars(exe,
+                       init_checkpoint_path,
+                       main_program=main_program,
+                       predicate=existed_persitables)
     log.info("Load model from {}".format(init_checkpoint_path))
 
 
@@ -66,10 +65,9 @@ def init_pretraining_params(exe, pretraining_params_path, main_program):
         #    print ("Var exists: [%s]" % (var.name))
         return os.path.exists(os.path.join(pretraining_params_path, var.name))
 
-    fluid.io.load_vars(
-        exe,
-        pretraining_params_path,
-        main_program=main_program,
-        predicate=existed_params)
-    log.info("Load pretraining parameters from {}.".format(
-        pretraining_params_path))
+    fluid.io.load_vars(exe,
+                       pretraining_params_path,
+                       main_program=main_program,
+                       predicate=existed_params)
+    log.info(
+        "Load pretraining parameters from {}.".format(pretraining_params_path))

applications/document_intelligence/doc_vqa/run_test.sh

@@ -0,0 +1,34 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export CUDA_VISIBLE_DEVICES=0
+
+QUESTION=$1
+
+# Question: NFC咋开门
+
+if [ $# != 1 ];then
+    echo "USAGE: sh script/run_cross_encoder_test.sh \$QUESTION"
+    exit 1
+fi
+
+# compute scores for QUESTION and OCR parsing results  with Rerank module
+cd Rerank
+bash run_test.sh ${QUESTION}
+cd ..
+
+# extraction answer for QUESTION from the top1 of rank
+cd Extraction
+bash run_test.sh ${QUESTION}
+cd ..

applications/document_intelligence/docprompt

@@ -0,0 +1 @@
+../../model_zoo/ernie-layout

applications/experimental/pipelines/README.md

@@ -1,26 +0,0 @@
-## PaddleNLP智能文本产线
-
-本项目基于百度领先NLP技术与飞桨PaddleNLP模型库，为开发者提供低成本搭建端到端NLP应用系统的能力。
-
-## 效果速览
-本项目展示了端到端智能问答系统和端到端语义检索系统的效果速览。
-
-### 智能问答系统
-我们针对中国重点城市的基本信息搭建了百科知识智能问答系统, 您可以根据实际需求基于自己的业务数据快速搭智能问答系统。[点击这里](./examples/question-answering/)快速构建端到端智能问答系统。
-
-<div align="center">
-    <img src="https://user-images.githubusercontent.com/11793384/168514868-1babe981-c675-4f89-9168-dd0a3eede315.gif" width="500">
-</div>
-
-### 语义检索系统
-我们针对[DuReader](https://github.com/baidu/DuReader)数据集搭建了语义检索系统, 您可以根据实际需求基于自己的业务数据快速搭语义检索系统。[点击这里](./examples/semantic-search/)快速构建端到端语义检索系统。
-
-<div align="center">
-    <img src="https://user-images.githubusercontent.com/11793384/168514909-8817d79a-72c4-4be1-8080-93d1f682bb46.gif" width="500">
-</div>
-
-## Acknowledge
-
-我们借鉴了 Deepset.ai [Haystack](https://github.com/deepset-ai/haystack) 优秀的框架设计，在此对[Haystack](https://github.com/deepset-ai/haystack)作者及其开源社区表示感谢。
-
-We learn form the excellent framework design of Deepset.ai [Haystack](https://github.com/deepset-ai/haystack), and we would like to express our thanks to the authors of Haystack and their open source community.

applications/experimental/pipelines/pipelines/nodes/file_converter/docx.py

@@ -1,73 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-# Copyright 2021 deepset GmbH. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Dict, Optional, Any, List
-
-import logging
-from pathlib import Path
-import docx
-
-from pipelines.nodes.file_converter import BaseConverter
-
-logger = logging.getLogger(__name__)
-
-
-class DocxToTextConverter(BaseConverter):
-    def convert(
-            self,
-            file_path: Path,
-            meta: Optional[Dict[str, str]]=None,
-            remove_numeric_tables: Optional[bool]=None,
-            valid_languages: Optional[List[str]]=None,
-            encoding: Optional[str]=None, ) -> List[Dict[str, Any]]:
-        """
-        Extract text from a .docx file.
-        Note: As docx doesn't contain "page" information, we actually extract and return a list of paragraphs here.
-        For compliance with other converters we nevertheless opted for keeping the methods name.
-
-        :param file_path: Path to the .docx file you want to convert
-        :param meta: dictionary of meta data key-value pairs to append in the returned document.
-        :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
-                                      The tabular structures in documents might be noise for the reader model if it
-                                      does not have table parsing capability for finding answers. However, tables
-                                      may also have long strings that could possible candidate for searching answers.
-                                      The rows containing strings are thus retained in this option.
-        :param valid_languages: validate languages from a list of languages specified in the ISO 639-1
-                                (https://en.wikipedia.org/wiki/ISO_639-1) format.
-                                This option can be used to add test for encoding errors. If the extracted text is
-                                not one of the valid languages, then it might likely be encoding error resulting
-                                in garbled text.
-        :param encoding: Not applicable
-        """
-        if remove_numeric_tables is None:
-            remove_numeric_tables = self.remove_numeric_tables
-        if valid_languages is None:
-            valid_languages = self.valid_languages
-        if remove_numeric_tables is True:
-            raise Exception(
-                "'remove_numeric_tables' is not supported by DocxToTextConverter."
-            )
-        if valid_languages is True:
-            raise Exception(
-                "Language validation using 'valid_languages' is not supported by DocxToTextConverter."
-            )
-
-        file = docx.Document(file_path)  # Creating word reader object.
-        paragraphs = [para.text for para in file.paragraphs]
-        documents = []
-        for page in paragraphs:
-            document = {"content": page, "content_type": "text", "meta": meta}
-            documents.append(document)
-        return documents

applications/experimental/pipelines/pipelines/nodes/ranker/ernie_ranker.py

@@ -1,127 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import List, Optional, Union
-import logging
-from pathlib import Path
-
-import paddle
-from paddlenlp.transformers import ErnieCrossEncoder, ErnieTokenizer
-
-from pipelines.schema import Document
-from pipelines.nodes.ranker import BaseRanker
-from pipelines.utils.common_utils import initialize_device_settings
-
-logger = logging.getLogger(__name__)
-
-
-class ErnieRanker(BaseRanker):
-    """
-    Re-Ranking can be used on top of a retriever to boost the performance for document search. This is particularly useful if the retriever has a high recall but is bad in sorting the documents by relevance.
-
-    Usage example:
-    ...
-    retriever = ElasticsearchRetriever(document_store=document_store)
-    ranker = SentenceTransformersRanker(model_name_or_path="rocketqa-zh-dureader-cross-encoder")
-    p = Pipeline()
-    p.add_node(component=retriever, name="ESRetriever", inputs=["Query"])
-    p.add_node(component=ranker, name="Ranker", inputs=["ESRetriever"])
-    """
-
-    def __init__(
-            self,
-            model_name_or_path: Union[str, Path],
-            top_k: int=10,
-            use_gpu: bool=True, ):
-        """
-        :param model_name_or_path: Directory of a saved model or the name of a public model e.g.
-        'rocketqa-zh-dureader-cross-encoder'.
-        :param top_k: The maximum number of documents to return
-        :param use_gpu: Whether to use all available GPUs or the CPU. Falls back on CPU if no GPU is available.
-        """
-
-        # save init parameters to enable export of component config as YAML
-        self.set_config(
-            model_name_or_path=model_name_or_path,
-            top_k=top_k, )
-
-        self.top_k = top_k
-
-        self.devices, _ = initialize_device_settings(
-            use_cuda=use_gpu, multi_gpu=True)
-
-        self.transformer_model = ErnieCrossEncoder(model_name_or_path)
-        self.tokenizer = ErnieTokenizer.from_pretrained(model_name_or_path)
-        self.transformer_model.eval()
-
-        if len(self.devices) > 1:
-            self.model = paddle.DataParallel(self.transformer_model)
-
-    def predict_batch(self,
-                      query_doc_list: List[dict],
-                      top_k: int=None,
-                      batch_size: int=None):
-        """
-        Use loaded Ranker model to, for a list of queries, rank each query's supplied list of Document.
-
-        Returns list of dictionary of query and list of document sorted by (desc.) similarity with query
-
-        :param query_doc_list: List of dictionaries containing queries with their retrieved documents
-        :param top_k: The maximum number of answers to return for each query
-        :param batch_size: Number of samples the model receives in one batch for inference
-        :return: List of dictionaries containing query and ranked list of Document
-        """
-        raise NotImplementedError
-
-    def predict(self,
-                query: str,
-                documents: List[Document],
-                top_k: Optional[int]=None) -> List[Document]:
-        """
-        Use loaded ranker model to re-rank the supplied list of Document.
-
-        Returns list of Document sorted by (desc.) similarity with the query.
-
-        :param query: Query string
-        :param documents: List of Document to be re-ranked
-        :param top_k: The maximum number of documents to return
-        :return: List of Document
-        """
-        if top_k is None:
-            top_k = self.top_k
-
-        features = self.tokenizer(
-            [query for doc in documents], [doc.content for doc in documents],
-            max_seq_len=256,
-            pad_to_max_seq_len=True,
-            truncation_strategy="longest_first")
-
-        tensors = {k: paddle.to_tensor(v) for (k, v) in features.items()}
-
-        with paddle.no_grad():
-            similarity_scores = self.transformer_model.matching(
-                **tensors).numpy()
-
-        for doc, rank_score in zip(documents, similarity_scores):
-            doc.rank_score = rank_score
-            doc.score = rank_score
-
-        sorted_scores_and_documents = sorted(
-            zip(similarity_scores, documents),
-            key=lambda similarity_document_tuple: similarity_document_tuple[0],
-            reverse=True, )
-
-        # rank documents according to scores
-        sorted_documents = [doc for _, doc in sorted_scores_and_documents]
-        return sorted_documents[:top_k]

applications/experimental/pipelines/pipelines/utils/__init__.py

@@ -1,16 +0,0 @@
-from pipelines.utils.preprocessing import convert_files_to_dicts, tika_convert_files_to_dicts
-from pipelines.utils.import_utils import fetch_archive_from_http
-from pipelines.utils.cleaning import clean_wiki_text
-from pipelines.utils.doc_store import (
-    launch_es,
-    launch_milvus,
-    launch_opensearch,
-    launch_weaviate,
-    stop_opensearch,
-    stop_service, )
-from pipelines.utils.export_utils import (
-    print_answers,
-    print_documents,
-    print_questions,
-    export_answers_to_csv,
-    convert_labels_to_squad, )

applications/experimental/pipelines/requirements.txt

@@ -1,14 +0,0 @@
-paddlepaddle-gpu
-paddlenlp
-paddleocr
-requests
-pydantic
-mmh3
-more_itertools
-elasticsearch>=7.7,<=7.10
-sqlalchemy>=1.4.2,<2
-sqlalchemy_utils
-langdetect
-python-docx
-nltk
-pdfplumber

applications/experimental/pipelines/rest_api/controller/__init__.py

@@ -1 +0,0 @@
-from rest_api.pipeline import custom_component  # this import is required for the Custom Components to be registered

applications/experimental/pipelines/rest_api/controller/errors/http_error.py

@@ -1,7 +0,0 @@
-from fastapi import HTTPException
-from starlette.requests import Request
-from starlette.responses import JSONResponse
-
-
-async def http_error_handler(_: Request, exc: HTTPException) -> JSONResponse:
-    return JSONResponse({"errors": [exc.detail]}, status_code=exc.status_code)

applications/experimental/pipelines/rest_api/pipeline/custom_component.py

@@ -1,15 +0,0 @@
-"""
-Pipelines allow putting together Components to build a graph.
-
-In addition to the standard pipelines Components, custom user-defined Components
-can be used in a Pipeline YAML configuration.
-
-The classes for the Custom Components must be defined in this file.
-"""
-
-from pipelines.nodes.base import BaseComponent
-
-
-class SampleComponent(BaseComponent):
-    def run(self, **kwargs):
-        raise NotImplementedError

applications/experimental/pipelines/rest_api/setup.py

@@ -1,37 +0,0 @@
-from setuptools import setup, find_packages
-import logging
-from pathlib import Path
-
-VERSION = "0.0.1a0"
-
-setup(
-    name="pipelines-rest-api",
-    version=VERSION,
-    description="Demo REST API server for pipelines",
-    author="paddlenlp",
-    author_email="paddlenlp@baidu.com",
-    url=" https://github.com/PaddlePaddle/pipelines/tree/master/rest_api",
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Intended Audience :: Science/Research",
-        "Topic :: Scientific/Engineering :: Artificial Intelligence",
-        "Operating System :: OS Independent",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-    ],
-    packages=find_packages(),
-    python_requires=">=3.7, <4",
-    install_requires=[
-        # The link below cannot be translated properly into setup.cfg
-        # because it looks into the parent folder.
-        # TODO check if this is still a limitation later on
-        f"pipelines @ file://localhost/{Path(__file__).parent.parent}#egg=pipelines",
-        "fastapi<1",
-        "uvicorn<1",
-        "gunicorn<21",
-        "python-multipart<1",  # optional FastAPI dependency for form data
-    ], )

applications/experimental/pipelines/setup.py

@@ -1,47 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-import setuptools
-import sys
-import pipelines
-
-long_description = "PIPELINES: An End to End Natural Language Proceessing Development Kit Based on ERNIE"
-
-with open("requirements.txt") as fin:
-    REQUIRED_PACKAGES = fin.read()
-
-setuptools.setup(
-    name="pipelines",
-    version=pipelines.__version__,
-    author="PaddlePaddle Speech and Language Team",
-    author_email="paddlenlp@baidu.com",
-    description=long_description,
-    long_description=long_description,
-    long_description_content_type="text/plain",
-    url="https://github.com/PaddlePaddle/PaddleNLP",
-    packages=setuptools.find_packages(
-        where='.',
-        exclude=('examples*', 'tests*', 'docs*', 'ui*', 'rest_api*')),
-    setup_requires=['cython', 'numpy'],
-    install_requires=REQUIRED_PACKAGES,
-    python_requires='>=3.7',
-    classifiers=[
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'License :: OSI Approved :: Apache Software License',
-        'Operating System :: OS Independent',
-    ],
-    license='Apache 2.0')

applications/experimental/pipelines/ui/Dockerfile

@@ -1,22 +0,0 @@
-FROM python:3.7.4-stretch
-
-WORKDIR /home/user
-
-RUN apt-get update && apt-get install -y curl git pkg-config cmake
-
-# copy code
-RUN mkdir ui/
-COPY setup.py /home/user/ui
-COPY utils.py /home/user/ui
-COPY webapp.py /home/user/ui
-COPY eval_labels_example.csv /home/user/
-
-# install as a package
-RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
-RUN pip install --upgrade pip
-RUN pip install ui/
-
-EXPOSE 8501
-
-# cmd for running the API
-CMD ["python", "-m", "streamlit", "run", "ui/webapp.py"]

applications/experimental/pipelines/ui/setup.py

@@ -1,31 +0,0 @@
-from setuptools import setup, find_packages
-import logging
-from pathlib import Path
-
-VERSION = "0.0.1a0"
-
-setup(
-    name="pipelines-ui",
-    version=VERSION,
-    description="Demo UI for pipelines (https://github.com/PaddlePaddle/pipelines)",
-    author="paddlenlp",
-    author_email="paddlenlp@baidu.com",
-    url=" https://github.com/PaddlePaddle/pipelines/tree/master/ui",
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Intended Audience :: Science/Research",
-        "Topic :: Scientific/Engineering :: Artificial Intelligence",
-        "Operating System :: OS Independent",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-    ],
-    packages=find_packages(),
-    python_requires=">=3.7, <4",
-    install_requires=[
-        "streamlit>=1.2.0, <=1.7.0", "st-annotated-text>=2.0.0, <3",
-        "markdown>=3.3.4, <4"
-    ], )

applications/experimental/pipelines/utils/offline_ann.py

@@ -1,46 +0,0 @@
-import paddle
-from pipelines.utils import convert_files_to_dicts
-from pipelines.document_stores import ElasticsearchDocumentStore
-from pipelines.nodes import DensePassageRetriever
-from pipelines.utils import launch_es
-
-
-def offline_ann():
-
-    launch_es()
-
-    document_store = ElasticsearchDocumentStore(
-        host="127.0.0.1",
-        port="9200",
-        username="",
-        password="",
-        index="baike_cities")
-
-    # 365 个城市百科数据作为 ANN 建库数据
-    doc_dir = "data/baike/"
-
-    # 将每篇文档按照段落进行切分
-    dicts = convert_files_to_dicts(dir_path=doc_dir, split_paragraphs=True)
-
-    print(dicts[:3])
-
-    # 文档数据写入数据库
-    document_store.write_documents(dicts)
-
-    ### 语义索引模型
-    retriever = DensePassageRetriever(
-        document_store=document_store,
-        query_embedding_model="rocketqa-zh-dureader-query-encoder",
-        passage_embedding_model="rocketqa-zh-dureader-query-encoder",
-        max_seq_len_query=64,
-        max_seq_len_passage=256,
-        batch_size=16,
-        use_gpu=True,
-        embed_title=False, )
-
-    # 建立索引库
-    document_store.update_embeddings(retriever)
-
-
-if __name__ == "__main__":
-    offline_ann()

applications/neural_search/README.md

@@ -3,17 +3,20 @@
 ## 1. 场景概述
 
 检索系统存在于我们日常使用的很多产品中，比如商品搜索系统、学术文献检索系等等，本方案提供了检索系统完整实现。限定场景是用户通过输入检索词 Query，快速在海量数据中查找相似文档。
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/12107462/191490721-90a8f526-ad64-4f2b-b9b4-34ab06c6749b.png" width="500px">
+</div>
 
-所谓语义检索（也称基于向量的检索），是指检索系统不再拘泥于用户 Query 字面本身，而是能精准捕捉到用户 Query 后面的真正意图并以此来搜索，从而更准确地向用户返回最符合的结果。通过使用最先进的语义索引模型找到文本的向量表示，在高维向量空间中对它们进行索引，并度量查询向量与索引文档的相似程度，从而解决了关键词索引带来的缺陷。
+所谓语义检索（也称基于向量的检索，如上图所示），是指检索系统不再拘泥于用户 Query 字面本身，而是能精准捕捉到用户 Query 后面的真正意图并以此来搜索，从而更准确地向用户返回最符合的结果。通过使用最先进的语义索引模型找到文本的向量表示，在高维向量空间中对它们进行索引，并度量查询向量与索引文档的相似程度，从而解决了关键词索引带来的缺陷。
 
-例如下面两组文本 Pair，如果基于关键词去计算相似度，两组的相似度是相同的。而从实际语义上看，第一组相似度高于第二组。  
+例如下面两组文本 Pair，如果基于关键词去计算相似度，两组的相似度是相同的。而从实际语义上看，第一组相似度高于第二组。
 
 ```
 车头如何放置车牌    前牌照怎么装
 车头如何放置车牌    后牌照怎么装
 ```
 
-语义检索系统的关键就在于，采用语义而非关键词方式进行召回，达到更精准、更广泛得召回相似结果的目的。
+语义检索系统的关键就在于，采用语义而非关键词方式进行召回，达到更精准、更广泛得召回相似结果的目的。想快速体验搜索的效果，请参考[Pipelines的实现](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/pipelines)
 
 ## 2. 产品功能介绍
 
@@ -33,8 +36,9 @@
         + 兼具无监督数据 和 有监督数据：融合模型
     + 进一步优化方案: 面向领域的预训练 Domain-adaptive Pretraining
 + 性能快
-    + 基于 Paddle Inference 快速抽取向量
-    + 基于 Milvus 快速查询和高性能建库
+    + Paddle Inference 快速抽取向量
+    + Milvus 快速查询和高性能建库
+    + Paddle Serving服务化部署
 
 ###  2.2 功能架构
 
@@ -43,9 +47,11 @@
 #### 2.2.1 整体介绍
 
 
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/12107462/191469309-42a54a67-a3a3-4e43-b6b1-b12be81ddf3d.png" width="800px">
+</div>
 
-![系统流程图](./img/system_pipeline.png)
-以上是nerual_search的系统流程图，其中左侧为召回环节，核心是语义向量抽取模块；右侧是排序环节，核心是排序模型。图中红色虚线框表示在线计算，黑色虚线框表示离线批量处理。下面我们分别介绍召回中的语义向量抽取模块，以及排序模型。
+以上是nerual_search的系统流程图，其中左侧为召回环节，核心是语义向量抽取模块；右侧是排序环节，核心是排序模型。召回环节需要用户通过自己的语料构建向量索引库，用户发起query了之后，就可以检索出相似度最高的向量，然后找出该向量对应的文本；排序环节主要是对召回的文本进行重新排序。下面我们分别介绍召回中的语义向量抽取模块，以及排序模型。
 
 
 #### 2.2.2 召回模块
@@ -68,7 +74,9 @@
 
 #### 2.2.3 排序模块
 
-排序模块基于前沿的预训练模型 ERNIE-Gram，训练 Pair-wise 语义匹配模型。召回模型负责从海量（千万级）候选文本中快速（毫秒级）筛选出与 Query 相关性较高的 TopK Doc，排序模型会在召回模型筛选出的 TopK Doc 结果基础之上针对每一个 (Query, Doc) Pair 对进行两两匹配计算相关性，排序效果更精准。
+召回模型负责从海量（千万级）候选文本中快速（毫秒级）筛选出与 Query 相关性较高的 TopK Doc，排序模型会在召回模型筛选出的 TopK Doc 结果基础之上针对每一个 (Query, Doc) Pair 对进行两两匹配计算相关性，排序效果更精准。
+
+排序模块有2种选择，第一种基于前沿的预训练模型 ERNIE，训练 Pair-wise 语义匹配模型；第二种是基于RocketQA模型训练的Cross Encoder模形。第一种是Pair-wise的排序算法，基本思路是对样本构建偏序文档对，两两比较，从比较中学习顺序，第二种是Poinet-Wise的算法，只考虑当前Query和每个文档的绝对相关度，并没有考虑其他文档与Query的相关度，但是建模方式比较简单。第一种Pair-wise模型可以说是第二种point-wise模型的改进版本，但对于噪声数据更为敏感，即一个错误的标注会导致多个pair对的错误，用户可以先使用基于Point-wise的Cross Encoder构建一个基础模型，需要进一步优化可以使用Pair-wise的方法优化。
 
 ## 3. 文献检索实践
 
@@ -78,13 +86,13 @@
 
 **语义索引**：由于我们既有无监督数据，又有有监督数据，所以结合 SimCSE 和 In-batch Negatives 方案，并采取 Domain-adaptive Pretraining 优化模型效果。
 
-首先是利用 ERNIE 1.0 模型进行 Domain-adaptive Pretraining，在得到的预训练模型基础上，进行无监督的 SimCSE 训练，最后利用 In-batch Negatives 方法进行微调，得到最终的语义索引模型，把建库的文本放入模型中抽取特征向量，然后把抽取后的向量放到语义索引引擎 milvus 中，利用 milvus 就可以很方便得实现召回了。
+首先是利用 ERNIE模型进行 Domain-adaptive Pretraining，在得到的预训练模型基础上，进行无监督的 SimCSE 训练，最后利用 In-batch Negatives 方法进行微调，得到最终的语义索引模型，把建库的文本放入模型中抽取特征向量，然后把抽取后的向量放到语义索引引擎 milvus 中，利用 milvus 就可以很方便得实现召回了。
 
-**排序**：使用 ERNIE-Gram 的单塔结构对召回后的数据精排序。
+**排序**：使用 ERNIE-Gram 的单塔结构/RocketQA的Cross Encoder对召回后的数据精排序。
 
 #### 3.1.2 评估指标
 
-**模型效果指标**  
+**模型效果指标**
 * 在语义索引召回阶段使用的指标是 Recall@K，表示的是预测的前topK（从最后的按得分排序的召回列表中返回前K个结果）结果和语料库中真实的前 K 个相关结果的重叠率，衡量的是检索系统的查全率。
 
 * 在排序阶段使用的指标为AUC，AUC反映的是分类器对样本的排序能力，如果完全随机得对样本分类，那么AUC应该接近0.5。分类器越可能把真正的正样本排在前面，AUC越大，分类性能越好。
@@ -94,7 +102,7 @@
 
 * 建库性能和 ANN 查询性能快
 
-### 3.2 数据说明
+### 3.2 预置数据说明
 
 数据集来源于某文献检索系统，既有大量无监督数据，又有有监督数据。
 
@@ -104,18 +112,19 @@
 
 （3）使用文献的的query, title, keywords，构造带正标签的数据集，不包含负标签样本，基于 In-batch Negatives 策略进行训练；
 
-（4）在排序阶段，使用点击（作为正样本）和展现未点击（作为负样本）数据构造排序阶段的训练集，使用ERNIE-Gram模型进行精排训练。
+（4）在排序阶段，使用点击（作为正样本）和展现未点击（作为负样本）数据构造排序阶段的训练集，进行精排训练。
 
 |  阶段 |模型 |   训练集 | 评估集（用于评估模型效果） | 召回库 |测试集 |
 | ------------ | ------------ |------------ | ------------ | ------------ | ------------ |
 |  召回 |  Domain-adaptive Pretraining  |  2kw | - | - | - |
 |  召回 |  无监督预训练 - SimCSE |  798w  | 20000 |  300000| 1000 |
 |  召回 |  有监督训练 - In-batch Negatives | 3998  | 20000 |300000  | 1000 |
-|  排序 |  有监督训练 - ERNIE-Gram单塔 Pairwise| 1973538   | 57811 | - | 1000 |
+|  排序 |  有监督训练 - ERNIE-Gram单塔 Pairwise/RocketQA Cross Encoder| 1973538   | 57811 | - | 1000 |
 
 我们将除 Domain-adaptive Pretraining 之外的其他数据集全部开源，下载地址：
 
 - [literature_search_data](https://bj.bcebos.com/v1/paddlenlp/data/literature_search_data.zip)
+- [literature_search_rank](https://paddlenlp.bj.bcebos.com/applications/literature_search_rank.zip)
 
 ```
 ├── milvus # milvus建库数据集
@@ -126,13 +135,147 @@
     ├── dev.csv  # 召回阶段验证集，用于评估召回模型的效果，SimCSE 和 In-batch Negative 共用
     ├── corpus.csv # 构建召回库的数据（模拟实际业务线上的语料库，实际语料库远大于这里的规模），用于评估召回阶段模型效果，SimCSE 和 In-batch Negative 共用
     ├── test.csv # 召回阶段测试数据，预测文本之间的相似度，SimCSE 和 In-batch Negative 共用
+├── data # RocketQA排序数据集
+    ├── test.csv   # 测试集
+    ├── dev_pairwise.csv    # 验证集
+    └── train.csv  # 训练集
 ├── sort # 排序阶段数据集
     ├── train_pairwise.csv  # 排序训练集
     ├── dev_pairwise.csv    # 排序验证集
     └── test_pairwise.csv   # 排序测试集
 ```
 
-### 3.3 运行环境和安装说明
+
+### 3.3 数据格式
+
+1. 对于无监督SimCSE的训练方法，格式参考`train_unsupervised.csv`,即一行条文本即可，无需任何标注。对于召回模型训练需要规定格式的本地数据集，需要准备训练集文件`train.csv`，验证集`dev.csv`，召回集文件`corpus.csv`。
+
+
+训练数据集`train.csv`的格式如下：
+
+```
+query1 \t 用户点击的title1
+query2 \t 用户点击的title2
+```
+训练集合`train.csv`的文件样例：
+```
+从《唐律疏义》看唐代封爵贵族的法律特权  从《唐律疏义》看唐代封爵贵族的法律特权《唐律疏义》,封爵贵族,法律特权
+宁夏社区图书馆服务体系布局现状分析      宁夏社区图书馆服务体系布局现状分析社区图书馆,社区图书馆服务,社区图书馆服务体系
+人口老龄化对京津冀经济  京津冀人口老龄化对区域经济增长的影响京津冀,人口老龄化,区域经济增长,固定效应模型
+英语广告中的模糊语      模糊语在英语广告中的应用及其功能模糊语,英语广告,表现形式,语用功能
+甘氨酸二肽的合成        甘氨酸二肽合成中缩合剂的选择甘氨酸,缩合剂,二肽
+......
+```
+
+验证集`dev.csv`的格式如下：
+
+```
+query1 \t 用户点击的title1
+query2 \t 用户点击的title2
+```
+
+验证集合`train.csv`的文件样例：
+```
+试论我国海岸带经济开发的问题与前景      试论我国海岸带经济开发的问题与前景海岸带,经济开发,问题,前景
+外语阅读焦虑与英语成绩及性别的关系      外语阅读焦虑与英语成绩及性别的关系外语阅读焦虑,外语课堂焦虑,英语成绩,性别
+加油站风险分级管控      加油站工作危害风险分级研究加油站,工作危害分析(JHA),风险分级管控
+```
+召回集合`corpus.csv`主要作用是检验测试集合的句子对能否被正确召回，它的构造主要是提取验证集的第二列的句子，然后加入很多无关的句子，用来检验模型能够正确的从这些文本中找出测试集合对应的第二列的句子，格式如下：
+
+```
+2002-2017年我国法定传染病发病率和死亡率时间变化趋势传染病,发病率,死亡率,病死率
+陕西省贫困地区城乡青春期少女生长发育调查青春期,生长发育,贫困地区
+五丈岩水库溢洪道加固工程中的新材料应用碳纤维布,粘钢加固技术,超细水泥,灌浆技术
+......
+```
+
+2. 对于排序模型的训练，排序模型目前提供了2种，第一种是Pairwise训练的方式，第二种是RocketQA的排序模型，对于第一种排序模型，需要准备训练集`train_pairwise.csv`,验证集`dev_pairwise.csv`两个文件,除此之外还可以准备测试集文件`test.csv`或者`test_pairwise.csv`。
+
+训练数据集`train_pairwise.csv`的格式如下：
+
+```
+query1 \t 用户点击的title1 \t 用户未点击的title2
+query2 \t 用户点击的title3 \t 用户未点击的title4
+```
+
+训练数据集`train_pairwise.csv`的示例如下：
+
+```
+英语委婉语引起的跨文化交际障碍  英语委婉语引起的跨文化交际障碍及其翻译策略研究英语委婉语,跨文化交际障碍,翻译策略        委婉语在英语和汉语中的文化差异委婉语,文化,跨文化交际
+范迪慧 嘉兴市中医院     滋阴疏肝汤联合八穴隔姜灸治疗肾虚肝郁型卵巢功能低下的临床疗效滋阴疏肝汤,八穴隔姜灸,肾虚肝郁型卵巢功能低下,性脉甾类激素,妊娠      温针灸、中药薰蒸在半月板损伤术后康复中的疗效分析膝损伤,半月板,胫骨,中医康复,温针疗法,薰洗
+......
+```
+
+验证数据集`dev_pairwise.csv`的格式如下：
+
+```
+query1 \t title1 \t label
+query2 \t title2 \t label
+```
+验证数据集`dev_pairwise.csv`的示例如下：
+
+```
+作者单位:南州中学       浅谈初中教学管理如何体现人文关怀初中教育,教学管理,人文关怀      1
+作者单位:南州中学       高中美术课堂教学中藏区本土民间艺术的融入路径藏区,传统民间艺术,美术课堂  0
+作者单位:南州中学       列宁关于资产阶级民主革命向 社会主义革命过渡的理论列宁,直接过渡,间接过渡,资产阶级民主革命,社会主义革命   0
+DAA髋关节置换   DAA前侧入路和后外侧入路髋关节置换疗效对比髋关节置换术;直接前侧入路;后外侧入路;髋关节功能;疼痛;并发症    1
+DAA髋关节置换   DAA全髋关节置换术治疗髋关节病变对患者髋关节运动功能的影响直接前侧入路全髋关节置换术,髋关节病变,髋关节运动功能   0
+DAA髋关节置换   护患沟通技巧在急诊输液护理中的应用分析急诊科,输液护理,护理沟通技巧,应用 0
+.......
+```
+训练数据集`test_pairwise.csv`的格式如下，其中这个score得分是召回算出来的相似度或者距离，仅供参考，可以忽略：
+
+```
+query1 \t title1 \t score
+query2 \t title2 \t score
+```
+训练数据集`test_pairwise.csv`的示例如下：
+
+```
+中西方语言与文化的差异  中西方文化差异以及语言体现中西方文化,差异,语言体现      0.43203747272491455
+中西方语言与文化的差异  论中西方文化差异在非言语交际中的体现中西方文化,差异,非言语交际  0.4644506871700287
+中西方语言与文化的差异  中西方体态语文化差异跨文化,体态语,非语言交际,差异       0.4917311668395996
+中西方语言与文化的差异  由此便可以发现两种语言以及两种文化的差异。      0.5039259195327759
+.......
+```
+
+对于第二种基于RocketQA的排序模型。
+
+训练数据集`train.csv`,验证集`dev_pairwise.csv`的格式如下：
+
+```
+query1 \t title1 \t label
+query2 \t title2 \t label
+```
+训练数据集`train.csv`,验证集`dev_pairwise.csv`的示例如下：
+
+```
+(小学数学教材比较) 关键词:新加坡        新加坡与中国数学教材的特色比较数学教材,教材比较,问题解决        0
+徐慧新疆肿瘤医院        头颈部非霍奇金淋巴瘤扩散加权成像ADC值与Ki-67表达相关性分析淋巴瘤,非霍奇金,头颈部肿瘤,磁共振成像 1
+抗生素关性腹泻  鼠李糖乳杆菌GG防治消化系统疾病的研究进展鼠李糖乳杆菌,腹泻,功能性胃肠病,肝脏疾病,幽门螺杆菌      0
+德州市图书馆    图书馆智慧化建设与融合创新服务研究图书馆;智慧化;阅读服务;融合创新       1
+维生素c 综述    维生素C防治2型糖尿病研究进展维生素C;2型糖尿病;氧化应激;自由基;抗氧化剂  0
+.......
+```
+
+训练数据集`test.csv`的格式如下，其中这个score得分是召回算出来的相似度或者距离，仅供参考，可以忽略：
+
+```
+query1 \t title1 \t score
+query2 \t title2 \t score
+```
+训练数据集`test.csv`的示例如下：
+
+```
+加强科研项目管理有效促进医学科研工作    科研项目管理策略科研项目,项目管理,实施,必要性,策略      0.32163668
+加强科研项目管理有效促进医学科研工作    关于推进我院科研发展进程的相关问题研究医院科研,主体,环境,信息化 0.32922596
+加强科研项目管理有效促进医学科研工作    深圳科技计划对高校科研项目资助现状分析与思考基础研究,高校,科技计划,科技创新     0.36869502
+加强科研项目管理有效促进医学科研工作    普通高校科研管理模式的优化与创新普通高校,科研,科研管理  0.3688045
+.......
+```
+
+
+### 3.4 运行环境和安装说明
 
 
 （1）运行环境
@@ -143,7 +286,7 @@ a. 软件环境：
 
 
 - python >= 3.6
-- paddlenlp >= 2.2.1  
+- paddlenlp >= 2.2.1
 - paddlepaddle-gpu >=2.2
 - CUDA Version: 10.2
 - NVIDIA Driver Version: 440.64.00
@@ -163,30 +306,78 @@ c. 依赖安装:
 pip install -r requirements.txt
 ```
 
-## 4. 动手实践——搭建自己的检索系统
+## 4. Neural Search 快速体验实践
+
+PaddleNLP已经基于ERNIE 1.0训练了一个基线模型，如果想快速搭建Neural Search的完整系统。
+
+### 4.1. 召回
+
+- 召回向量抽取服务的搭建请参考：[In-batch Negatives](./recall/in_batch_negative/)， 只需要下载基于ERNIE 1.0的预训练模型，导出成Paddle Serving的格式，然后启动Pipeline Server服务即可
+
+- 召回向量检索服务的搭建请参考：[Milvus](./recall/milvus/)， 需要搭建Milvus并且插入检索数据的向量
+
+【注意】如果使用Neural Search训练好的模型，由于该模型是基于ERNIE 1.0训练的，所以需要把 `model_name_or_path`指定为`ernie 1.0`，向量抽取结果才能正常。
+
+
+### 4.2. 排序
+
+排序服务的搭建请参考 [ernie_matching](./ranking/ernie_matching/)，只需要下载基于ERNIE Gram的预训练模型，导出成Paddle Serving的格式，最后需要启动 Pipeline Serving服务
+
+【注意】如果使用Neural Search训练好的模型，由于该模型是基于ERNIE Gram训练的，所以需要把 `model_name_or_path`指定为`ernie-gram-zh`，向量抽取结果才能正常。
+
+### 4.3. 系统运行
+
+以上召回和排序模型都经过Paddle Serving服务化以后，就可以直接使用下面的命令运行体验：
+
+```
+python3 run_system.py
+```
+输出的结果为：
+
+```
+PipelineClient::predict pack_data time:1656991375.5521955
+PipelineClient::predict before time:1656991375.5529568
+Extract feature time to cost :0.0161135196685791 seconds
+Search milvus time cost is 0.8139839172363281 seconds
+PipelineClient::predict pack_data time:1656991376.3981335
+PipelineClient::predict before time:1656991376.3983877
+time to cost :0.05616641044616699 seconds
+```
+会输出2个文件 `recall_result.csv` 是召回检索的结果，`rank_result.csv` 是排序的结果。csv的示例输出下。
+
+召回的结果：
+
+```
+中西方语言与文化的差异,港台文化对内地中小学生的负面影响,0.055068351328372955
+中西方语言与文化的差异,外来文化在越南的传播与融合,0.05621318891644478
+中西方语言与文化的差异,临终关怀中的“仪式”,0.05705389380455017
+中西方语言与文化的差异,历史的真实与艺术加工,0.05745899677276611
+......
+```
+
+排序的结果：
+
+```
+中西方语言与文化的差异,论中西方教育差异,0.870943009853363
+中西方语言与文化的差异,浅析中西方问候语的差异,0.8468159437179565
+中西方语言与文化的差异,文化认同及其根源,0.8288694620132446
+中西方语言与文化的差异,从历史文化角度分析中西方学校教育的差异,0.8209370970726013
+中西方语言与文化的差异,中西医思维方式的差异,0.8150948882102966
+中西方语言与文化的差异,浅析中韩餐桌文化差异,0.7751647233963013
+......
+```
+
+
+
+## 5. 从头开始搭建自己的检索系统
 
 这里展示了能够从头至尾跑通的完整代码，您使用自己的业务数据，照着跑，能搭建出一个给定 Query，返回 topK 相关文档的小型检索系统。您可以参照我们给出的效果和性能数据来检查自己的运行过程是否正确。
 
-### 4.1 召回阶段
+### 5.1 召回阶段
 
 **召回模型训练**
 
-这里采用 Domain-adaptive Pretraining + SimCSE + In-batch Negatives 方案：
-
-第一步：无监督训练 Domain-adaptive Pretraining
-
-训练用时 16hour55min，可参考：[ERNIE 1.0](./recall/domain_adaptive_pretraining/)  
-
-第二步：无监督训练 SimCSE
-
-训练用时 16hour53min，可参考：[SimCSE](./recall/simcse/)  
-
-第三步：有监督训练
-
-几分钟内训练完成，可参考 [In-batch Negatives](./recall/in_batch_negative/)  
-
-
-此外，我们进行了多组实践，用来对比说明召回阶段各方案的效果：
+我们进行了多组实践，用来对比说明召回阶段各方案的效果：
 
 |  模型 |  Recall@1 | Recall@5 |Recall@10 |Recall@20 |Recall@50 |策略简要说明|
 | ------------ | ------------ | ------------ |--------- |--------- |--------- |--------- |
@@ -197,11 +388,27 @@ pip install -r requirements.txt
 |  Domain-adaptive Pretraining + SimCSE |  51.031 | 66.648| 71.338 | 75.676 |80.144| ERNIE 预训练，SimCSE 无监督训练|
 |  Domain-adaptive Pretraining + SimCSE + In-batch Negatives|  **58.248** | **75.099**| **79.813**| **83.801**|**87.733**| ERNIE 预训练，SimCSE 无监督训训练，In-batch Negatives 有监督训练|
 
-从上述表格可以看出，首先利用ERNIE 1.0 做 Domain-adaptive Pretraining ，然后把训练好的模型加载到 SimCSE 上进行无监督训练，最后利用 In-batch Negatives 在有监督数据上进行训练能够获得最佳的性能。[模型下载](https://paddlenlp.bj.bcebos.com/models/inbatch_model_best.zip)，模型的使用方式参考[In-batch Negatives](./recall/in_batch_negative/) 。
+从上述表格可以看出，首先利用 ERNIE 3.0 做 Domain-adaptive Pretraining ，然后把训练好的模型加载到 SimCSE 上进行无监督训练，最后利用 In-batch Negatives 在有监督数据上进行训练能够获得最佳的性能。[模型下载](https://paddlenlp.bj.bcebos.com/models/inbatch_model_best.zip)，模型的使用方式参考[In-batch Negatives](./recall/in_batch_negative/) 。
+
+
+这里采用 Domain-adaptive Pretraining + SimCSE + In-batch Negatives 方案：
+
+第一步：无监督训练 Domain-adaptive Pretraining
+
+训练用时 16hour55min，可参考：[ERNIE 1.0](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/model_zoo/ernie-1.0)
+
+第二步：无监督训练 SimCSE
+
+训练用时 16hour53min，可参考：[SimCSE](./recall/simcse/)
+
+第三步：有监督训练
+
+几分钟内训练完成，可参考 [In-batch Negatives](./recall/in_batch_negative/)
+
 
 **召回系统搭建**
 
-召回系统使用索引引擎 Milvus，可参考 [milvus_system](./recall/milvus/)。  
+召回系统使用索引引擎 Milvus，可参考 [milvus_system](./recall/milvus/)。
 我们展示一下系统的效果，输入的文本如下：
 
 ```
@@ -233,18 +440,19 @@ pip install -r requirements.txt
 ```
 
 
-### 4.2 排序阶段
+### 5.2 排序阶段
 
-排序阶段使用的模型是 ERNIE-Gram，用时 20h，可参考：
+排序阶段有2种方案，第一种是[ernie_matching](./ranking/ernie_matching/)使用的模型是 ERNIE-3.0-Medium-zh，用时 20h；第二种是基于RocketQA的排序模型[cross_encoder](./ranking/cross_encoder/)，训练用时也是20h左右。
 
-[ernie_matching](./ranking/ernie_matching/)
 
 排序阶段的效果评估：
 
 |  模型 |  AUC |
 | ------------ | ------------ |
 |  Baseline: In-batch Negatives |  0.582 |
-|  ERNIE-Gram |  0.801 |
+| pairwise ERNIE-Gram  |0.801 |
+|  CrossEncoder：rocketqa-base-cross-encoder  |**0.835** |
+
 
 同样输入文本：
 

applications/neural_search/ranking/cross_encoder/README.md

@@ -0,0 +1,399 @@
+
+ **目录**
+
+* [背景介绍](#背景介绍)
+* [CrossEncoder](#CrossEncoder)
+    * [1. 技术方案和评估指标](#技术方案)
+    * [2. 环境依赖](#环境依赖)
+    * [3. 代码结构](#代码结构)
+    * [4. 数据准备](#数据准备)
+    * [5. 模型训练](#模型训练)
+    * [6. 评估](#开始评估)
+    * [7. 预测](#预测)
+    * [8. 部署](#部署)
+
+<a name="背景介绍"></a>
+
+# 背景介绍
+
+基于RocketQA的CrossEncoder训练的单塔模型，该模型用于搜索的排序阶段，对召回的结果进行重新排序的作用。
+
+
+<a name="CrossEncoder"></a>
+
+# CrossEncoder
+
+<a name="技术方案"></a>
+
+## 1. 技术方案和评估指标
+
+### 技术方案
+
+加载基于ERNIE 3.0训练过的RocketQA单塔CrossEncoder模型。
+
+
+### 评估指标
+
+（1）采用 AUC 指标来评估排序模型的排序效果。
+
+**效果评估**
+
+|  训练方式 |  模型  | AUC |
+| ------------ | ------------ |------------ |
+| pairwise|  ERNIE-Gram  |0.801 |
+|  CrossEncoder |  rocketqa-base-cross-encoder  |**0.835** |
+
+<a name="环境依赖"></a>
+
+## 2. 环境依赖和安装说明
+
+**环境依赖**
+
+* python >= 3.7
+* paddlepaddle >= 2.3.7
+* paddlenlp >= 2.3
+* pandas >= 0.25.1
+* scipy >= 1.3.1
+
+<a name="代码结构"></a>
+
+## 3. 代码结构
+
+以下是本项目主要代码结构及说明：
+
+```
+ernie_matching/
+├── deply # 部署
+    ├── cpp
+        ├── rpc_client.py # RPC 客户端的bash脚本
+        ├── http_client.py # http 客户端的bash文件
+        └── start_server.sh # 启动C++服务的脚本
+    └── python
+        ├── deploy.sh # 预测部署bash脚本
+        ├── config_nlp.yml # Pipeline 的配置文件
+        ├── web_service.py # Pipeline 服务端的脚本
+        ├── rpc_client.py # Pipeline RPC客户端的脚本
+        └── predict.py # python 预测部署示例
+|—— scripts
+    ├── export_model.sh # 动态图参数导出静态图参数的bash文件
+    ├── export_to_serving.sh # 导出 Paddle Serving 模型格式的bash文件
+    ├── train_ce.sh # 匹配模型训练的bash文件
+    ├── evaluate_ce.sh # 评估验证文件bash脚本
+    ├── predict_ce.sh # 匹配模型预测脚本的bash文件
+├── export_model.py # 动态图参数导出静态图参数脚本
+├── export_to_serving.py # 导出 Paddle Serving 模型格式的脚本
+├── data.py #  训练样本的转换逻辑
+├── train_ce.py # 模型训练脚本
+├── evaluate.py # 评估验证文件
+├── predict.py # Pair-wise 模型预测脚本，输出文本对是相似度
+
+```
+
+<a name="数据准备"></a>
+
+## 4. 数据准备
+
+### 数据集说明
+
+样例数据如下:
+```
+(小学数学教材比较) 关键词:新加坡        新加坡与中国数学教材的特色比较数学教材,教材比较,问题解决        0
+徐慧新疆肿瘤医院        头颈部非霍奇金淋巴瘤扩散加权成像ADC值与Ki-67表达相关性分析淋巴瘤,非霍奇金,头颈部肿瘤,磁共振成像 1
+抗生素关性腹泻  鼠李糖乳杆菌GG防治消化系统疾病的研究进展鼠李糖乳杆菌,腹泻,功能性胃肠病,肝脏疾病,幽门螺杆菌      0
+德州市图书馆    图书馆智慧化建设与融合创新服务研究图书馆;智慧化;阅读服务;融合创新       1
+维生素c 综述    维生素C防治2型糖尿病研究进展维生素C;2型糖尿病;氧化应激;自由基;抗氧化剂  0
+(白藜芦醇) 关键词:2型糖尿病     2型糖尿病大鼠心肌缺血再灌注损伤转录因子E2相关因子2/血红素氧合酶1信号通路的表达及白藜芦醇的干预研究糖尿病,2型,心肌缺血,再灌注损伤,白藜芦醇       1
+融资偏好        创新型企业产业风险、融资偏好与融资选择融资偏好;产业风险;融资选择        1
+星载激光雷达    星载激光雷达望远镜主镜超轻量化结构设计超轻量化;拓扑优化;集成优化;RMS;有限元仿真 1
+```
+
+
+### 数据集下载
+
+
+- [literature_search_rank](https://paddlenlp.bj.bcebos.com/applications/literature_search_rank.zip)
+
+```
+├── data # 排序数据集
+    ├── test.csv   # 测试集
+    ├── dev_pairwise.csv    # 验证集
+    └── train.csv  # 训练集
+```
+
+<a name="模型训练"></a>
+
+## 5. 模型训练
+
+**排序模型下载链接：**
+
+
+|Model|训练参数配置|硬件|MD5|
+| ------------ | ------------ | ------------ |-----------|
+|[ERNIE-Gram-Sort](https://bj.bcebos.com/v1/paddlenlp/models/ernie_gram_sort.zip)|<div style="width: 150pt">epoch:3 lr:5E-5 bs:64 max_len:64 </div>|<div style="width: 100pt">4卡 v100-16g</div>|d24ece68b7c3626ce6a24baa58dd297d|
+
+
+### 训练环境说明
+
+
+- NVIDIA Driver Version: 440.64.00
+- Ubuntu 16.04.6 LTS (Docker)
+- Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz
+
+
+### 单机单卡训练/单机多卡训练
+
+这里采用单机多卡方式进行训练，通过如下命令，指定 GPU 0,1,2,3 卡。如果采用单机单卡训练，只需要把`--gpu`参数设置成单卡的卡号即可
+
+训练的命令如下：
+
+```
+unset CUDA_VISIBLE_DEVICES
+python -u -m paddle.distributed.launch --gpus "0,1,2,3" --log_dir="logs" train_ce.py \
+        --device gpu \
+        --train_set data/train.csv \
+        --test_file data/dev_pairwise.csv \
+        --save_dir ./checkpoints \
+        --model_name_or_path rocketqa-base-cross-encoder \
+        --batch_size 32 \
+        --save_steps 10000 \
+        --max_seq_len 384 \
+        --learning_rate 1E-5 \
+        --weight_decay  0.01 \
+        --warmup_proportion 0.0 \
+        --logging_steps 10 \
+        --seed 1 \
+        --epochs 3 \
+        --eval_step 1000
+```
+也可以运行bash脚本：
+
+```
+sh scripts/train_ce.sh
+```
+
+<a name="评估"></a>
+
+## 6. 评估
+
+
+```
+python evaluate.py --model_name_or_path rocketqa-base-cross-encoder \
+                   --init_from_ckpt checkpoints/model_80000/model_state.pdparams \
+                   --test_file data/dev_pairwise.csv
+```
+也可以运行bash脚本：
+
+```
+sh scripts/evaluate_ce.sh
+```
+
+
+成功运行后会输出下面的指标：
+
+```
+eval_dev auc:0.829
+```
+
+<a name="预测"></a>
+
+## 7. 预测
+
+### 准备预测数据
+
+待预测数据为 tab 分隔的 tsv 文件，每一行为 1 个文本 Pair，和文本pair的语义索引相似度，(该相似度由召回模型算出，仅供参考)，部分示例如下:
+
+```
+中西方语言与文化的差异  第二语言习得的一大障碍就是文化差异。    0.5160342454910278
+中西方语言与文化的差异  跨文化视角下中国文化对外传播路径琐谈跨文化,中国文化,传播,翻译   0.5145505666732788
+中西方语言与文化的差异  从中西方民族文化心理的差异看英汉翻译语言,文化,民族文化心理,思维方式,翻译        0.5141439437866211
+中西方语言与文化的差异  中英文化差异对翻译的影响中英文化,差异,翻译的影响        0.5138794183731079
+中西方语言与文化的差异  浅谈文化与语言习得文化,语言,文化与语言的关系,文化与语言习得意识,跨文化交际      0.5131710171699524
+```
+
+
+
+### 开始预测
+
+以上述 demo 数据为例，运行如下命令基于我们开源的rocketqa模型开始计算文本 Pair 的语义相似度:
+
+```shell
+unset CUDA_VISIBLE_DEVICES
+python predict.py \
+                --device 'gpu' \
+                --params_path checkpoints/model_80000/model_state.pdparams \
+                --model_name_or_path rocketqa-base-cross-encoder \
+                --test_set data/test.csv \
+                --topk 10 \
+                --batch_size 128 \
+                --max_seq_length 384
+```
+也可以直接执行下面的命令：
+
+```
+sh scripts/predict_ce.sh
+```
+得到下面的输出，分别是query，title和对应的预测概率：
+
+```
+{'text_a': '加强科研项目管理有效促进医学科研工作', 'text_b': '高校\\十四五\\规划中学科建设要处理好五对关系\\十四五\\规划,学科建设,科技创新,人才培养', 'pred_prob': 0.7076062}
+{'text_a': '加强科研项目管理有效促进医学科研工作', 'text_b': '校企科研合作项目管理模式创新校企科研合作项目,管理模式,问题,创新', 'pred_prob': 0.64633846}
+{'text_a': '加强科研项目管理有效促进医学科研工作', 'text_b': '科研项目管理策略科研项目,项目管理,实施,必要性,策略', 'pred_prob': 0.63166416}
+{'text_a': '加强科研项目管理有效促进医学科研工作', 'text_b': '高校科研项目经费管理流程优化研究——以z大学为例高校,科研项目经费\\全流程\\管理,流程优化', 'pred_prob': 0.60351866}
+{'text_a': '加强科研项目管理有效促进医学科研工作', 'text_b': '关于推进我院科研发展进程的相关问题研究医院科研,主体,环境,信息化', 'pred_prob': 0.5688347}
+{'text_a': '加强科研项目管理有效促进医学科研工作', 'text_b': '医学临床科研选题原则和方法医学临床,科学研究,选题', 'pred_prob': 0.55190295}
+```
+
+<a name="部署"></a>
+
+## 8. 部署
+
+### 动转静导出
+
+首先把动态图模型转换为静态图：
+
+```
+python export_model.py \
+                       --params_path checkpoints/model_80000/model_state.pdparams \
+                       --model_name_or_path rocketqa-base-cross-encoder \
+                       --output_path=./output
+```
+也可以运行下面的bash脚本：
+
+```
+sh scripts/export_model.sh
+```
+
+### Paddle Inference
+
+使用PaddleInference
+
+```
+python deploy/python/predict.py --model_dir ./output \
+                                --input_file data/test.csv \
+                                --model_name_or_path rocketqa-base-cross-encoder
+```
+也可以运行下面的bash脚本：
+
+```
+sh deploy/python/deploy.sh
+```
+得到下面的输出，输出的是样本的query，title以及对应的概率：
+
+```
+Data: {'query': '加强科研项目管理有效促进医学科研工作', 'title': '科研项目管理策略科研项目,项目管理,实施,必要性,策略'}   prob: 0.5479063987731934
+Data: {'query': '加强科研项目管理有效促进医学科研工作', 'title': '关于推进我院科研发展进程的相关问题研究医院科研,主体,环境,信息化'}      prob: 0.5151925086975098
+Data: {'query': '加强科研项目管理有效促进医学科研工作', 'title': '深圳科技计划对高校科研项目资助现状分析与思考基础研究,高校,科技计划,科技创新'}          prob: 0.42983829975128174
+Data: {'query': '加强科研项目管理有效促进医学科研工作', 'title': '普通高校科研管理模式的优化与创新普通高校,科研,科研管理'}       prob: 0.465454638004303
+```
+
+### Paddle Serving部署
+
+Paddle Serving 的详细文档请参考 [Pipeline_Design](https://github.com/PaddlePaddle/Serving/blob/v0.7.0/doc/Python_Pipeline/Pipeline_Design_CN.md)和[Serving_Design](https://github.com/PaddlePaddle/Serving/blob/v0.7.0/doc/Serving_Design_CN.md),首先把静态图模型转换成Serving的格式：
+
+```
+python export_to_serving.py \
+    --dirname "output" \
+    --model_filename "inference.pdmodel" \
+    --params_filename "inference.pdiparams" \
+    --server_path "serving_server" \
+    --client_path "serving_client" \
+    --fetch_alias_names "predict"
+
+```
+
+参数含义说明
+* `dirname`: 需要转换的模型文件存储路径，Program 结构文件和参数文件均保存在此目录。
+* `model_filename`： 存储需要转换的模型 Inference Program 结构的文件名称。如果设置为 None ，则使用 `__model__` 作为默认的文件名
+* `params_filename`: 存储需要转换的模型所有参数的文件名称。当且仅当所有模型参数被保>存在一个单独的二进制文件中，它才需要被指定。如果模型参数是存储在各自分离的文件中，设置它的值为 None
+* `server_path`: 转换后的模型文件和配置文件的存储路径。默认值为 serving_server
+* `client_path`: 转换后的客户端配置文件存储路径。默认值为 serving_client
+* `fetch_alias_names`: 模型输出的别名设置，比如输入的 input_ids 等，都可以指定成其他名字，默认不指定
+* `feed_alias_names`: 模型输入的别名设置，比如输出 pooled_out 等，都可以重新指定成其他模型，默认不指定
+
+也可以运行下面的 bash 脚本：
+```
+sh scripts/export_to_serving.sh
+```
+Paddle Serving的部署有两种方式，第一种方式是Pipeline的方式，第二种是C++的方式，下面分别介绍这两种方式的用法：
+
+#### Pipeline方式
+
+修改对应预训练模型的`Tokenizer`：
+
+```
+self.tokenizer = AutoTokenizer.from_pretrained('rocketqa-base-cross-encoder')
+```
+
+启动 Pipeline Server:
+
+```
+python web_service.py
+```
+
+启动客户端调用 Server。
+
+首先修改rpc_client.py中需要预测的样本：
+
+```
+list_data = [{"query":"加强科研项目管理有效促进医学科研工作","title":"科研项目管理策略科研项目,项目管理,实施,必要性,策略"}]`
+```
+然后运行：
+```
+python rpc_client.py
+```
+模型的输出为：
+
+```
+PipelineClient::predict pack_data time:1662354188.422532
+PipelineClient::predict before time:1662354188.423034
+time to cost :0.016808509826660156 seconds
+(1,)
+[0.5479064]
+```
+可以看到客户端发送了1条文本，这条文本的相似的概率值。
+
+#### C++的方式
+
+启动C++的Serving：
+
+```
+python -m paddle_serving_server.serve --model serving_server --port 8600 --gpu_id 0 --thread 5 --ir_optim True
+```
+也可以使用脚本：
+
+```
+sh deploy/cpp/start_server.sh
+```
+Client 可以使用 http 或者 rpc 两种方式，rpc 的方式为：
+
+```
+python deploy/cpp/rpc_client.py
+```
+运行的输出为：
+
+```
+I0905 05:38:28.876770 28507 general_model.cpp:490] [client]logid=0,client_cost=158.124ms,server_cost=156.385ms.
+time to cost :0.15848731994628906 seconds
+[0.54790646]
+```
+可以看到服务端返回了相似度结果
+
+或者使用 http 的客户端访问模式：
+
+```
+python deploy/cpp/http_client.py
+```
+运行的输出为：
+```
+time to cost :0.13054680824279785 seconds
+0.5479064707850817
+```
+可以看到服务端返回了相似度结果
+
+
+## Reference
+
+[1] Xiao, Dongling, Yu-Kun Li, Han Zhang, Yu Sun, Hao Tian, Hua Wu, and Haifeng Wang. “ERNIE-Gram: Pre-Training with Explicitly N-Gram Masked Language Modeling for Natural Language Understanding.” ArXiv:2010.12148 [Cs].
+
+[2] Yingqi Qu, Yuchen Ding, Jing Liu, Kai Liu, Ruiyang Ren, Wayne Xin Zhao, Daxiang Dong, Hua Wu, Haifeng Wang:
+RocketQA: An Optimized Training Approach to Dense Passage Retrieval for Open-Domain Question Answering. NAACL-HLT 2021: 5835-5847

applications/neural_search/ranking/cross_encoder/data.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import numpy as np
+import paddle
 
 
 def convert_example(example,
@@ -51,8 +52,9 @@ def convert_example(example,
     else:
         text = example["text"]
         text_pair = None
-    encoded_inputs = tokenizer(
-        text=text, text_pair=text_pair, max_seq_len=max_seq_length)
+    encoded_inputs = tokenizer(text=text,
+                               text_pair=text_pair,
+                               max_seq_len=max_seq_length)
     input_ids = encoded_inputs["input_ids"]
     token_type_ids = encoded_inputs["token_type_ids"]
 
@@ -60,3 +62,55 @@ def convert_example(example,
         return input_ids, token_type_ids
     label = np.array([example["label"]], dtype="int64")
     return input_ids, token_type_ids, label
+
+
+def read_text_pair(data_path):
+    """Reads data."""
+    with open(data_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            data = line.rstrip().split("\t")
+            if len(data) != 3:
+                continue
+            yield {'text_a': data[0], 'text_b': data[1]}
+
+
+def read_data(data_path):
+    """Reads data."""
+    with open(data_path, 'r', encoding='utf-8', errors='ignore') as f:
+        for i, line in enumerate(f):
+            # Skip column name
+            if (i == 0):
+                continue
+            data = line.rstrip('\n').split("\t")
+            if (len(data) != 3):
+                print(data)
+                continue
+            query = data[0]
+            title = data[1]
+            label = data[-1]
+            # breakpoint()
+            yield {'text_a': query, 'text_b': title, 'label': int(label)}
+
+
+def create_dataloader(dataset,
+                      mode='train',
+                      batch_size=1,
+                      batchify_fn=None,
+                      trans_fn=None):
+    if trans_fn:
+        dataset = dataset.map(trans_fn)
+
+    shuffle = True if mode == 'train' else False
+    if mode == 'train':
+        batch_sampler = paddle.io.DistributedBatchSampler(dataset,
+                                                          batch_size=batch_size,
+                                                          shuffle=shuffle)
+    else:
+        batch_sampler = paddle.io.BatchSampler(dataset,
+                                               batch_size=batch_size,
+                                               shuffle=shuffle)
+
+    return paddle.io.DataLoader(dataset=dataset,
+                                batch_sampler=batch_sampler,
+                                collate_fn=batchify_fn,
+                                return_list=True)

applications/neural_search/ranking/cross_encoder/deploy/cpp/http_client.py

@@ -0,0 +1,78 @@
+# coding:utf-8
+# pylint: disable=doc-string-missing
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import time
+import numpy as np
+import requests
+import json
+
+from paddle_serving_client.httpclient import HttpClient
+from scipy.special import expit
+from paddlenlp.transformers import AutoTokenizer
+import paddlenlp as ppnlp
+
+
+def convert_example(example, tokenizer, max_seq_length=512):
+
+    query, title = example["query"], example["title"]
+    encoded_inputs = tokenizer(text=query,
+                               text_pair=title,
+                               max_seq_len=max_seq_length)
+    input_ids = encoded_inputs["input_ids"]
+    token_type_ids = encoded_inputs["token_type_ids"]
+
+    return input_ids, token_type_ids
+
+
+# 启动python客户端
+endpoint_list = ['127.0.0.1:8600']
+client = HttpClient()
+client.load_client_config('serving_client')
+client.connect(endpoint_list)
+feed_names = client.feed_names_
+fetch_names = client.fetch_names_
+
+# 创建tokenizer
+tokenizer = AutoTokenizer.from_pretrained('rocketqa-base-cross-encoder')
+max_seq_len = 64
+
+# 数据预处理
+list_data = [{
+    "query": "加强科研项目管理有效促进医学科研工作",
+    "title": "科研项目管理策略科研项目,项目管理,实施,必要性,策略"
+}]
+
+input_ids, token_type_ids = [], []
+for example in list_data:
+    input_id, token_type_id = convert_example(example,
+                                              tokenizer,
+                                              max_seq_length=max_seq_len)
+    input_ids.append(input_id)
+    token_type_ids.append(token_type_id)
+
+feed_dict = {}
+feed_dict['input_ids'] = np.array(input_ids)
+feed_dict['token_type_ids'] = np.array(token_type_ids)
+# batch设置为True表示的是批量预测
+b_start = time.time()
+result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)
+b_end = time.time()
+print(result)
+print("time to cost :{} seconds".format(b_end - b_start))
+score = result.outputs[0].tensor[0].float_data
+sim_score = expit(np.array(score))[1]
+print(sim_score)

applications/neural_search/ranking/cross_encoder/deploy/cpp/rpc_client.py

@@ -0,0 +1,76 @@
+# coding:utf-8
+# pylint: disable=doc-string-missing
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import time
+import numpy as np
+
+from paddle_serving_client import Client
+from scipy.special import expit
+from paddlenlp.transformers import AutoTokenizer
+import paddlenlp as ppnlp
+
+
+def convert_example(example, tokenizer, max_seq_length=512):
+
+    query, title = example["query"], example["title"]
+    encoded_inputs = tokenizer(text=query,
+                               text_pair=title,
+                               max_seq_len=max_seq_length)
+
+    input_ids = encoded_inputs["input_ids"]
+    token_type_ids = encoded_inputs["token_type_ids"]
+
+    return input_ids, token_type_ids
+
+
+# 启动python客户端
+endpoint_list = ['127.0.0.1:8600']
+client = Client()
+client.load_client_config('serving_client')
+client.connect(endpoint_list)
+feed_names = client.feed_names_
+fetch_names = client.fetch_names_
+
+# 创建tokenizer
+tokenizer = AutoTokenizer.from_pretrained('rocketqa-base-cross-encoder')
+max_seq_len = 64
+
+# 数据预处理
+list_data = [{
+    "query": "加强科研项目管理有效促进医学科研工作",
+    "title": "科研项目管理策略科研项目,项目管理,实施,必要性,策略"
+}]
+
+input_ids, token_type_ids = [], []
+for example in list_data:
+    input_id, token_type_id = convert_example(example,
+                                              tokenizer,
+                                              max_seq_length=max_seq_len)
+    input_ids.append(input_id)
+    token_type_ids.append(token_type_id)
+
+feed_dict = {}
+feed_dict['input_ids'] = np.array(input_ids)
+feed_dict['token_type_ids'] = np.array(token_type_ids)
+# batch设置为True表示的是批量预测
+b_start = time.time()
+result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)
+# breakpoint()
+sim_score = expit(result['predict'])[:, 1]
+b_end = time.time()
+print("time to cost :{} seconds".format(b_end - b_start))
+print(sim_score)

applications/neural_search/ranking/cross_encoder/deploy/cpp/start_server.sh

@@ -0,0 +1 @@
+python -m paddle_serving_server.serve --model serving_server --port 8600 --gpu_id 0 --thread 5 --ir_optim True

applications/neural_search/ranking/cross_encoder/deploy/python/config_nlp.yml

@@ -0,0 +1,34 @@
+# worker_num, 最大并发数。当build_dag_each_worker=True时, 框架会创建worker_num个进程，每个进程内构建grpcSever和DAG
+# 当build_dag_each_worker=False时，框架会设置主线程grpc线程池的max_workers=worker_num
+worker_num: 20
+# build_dag_each_worker, False，框架在进程内创建一条DAG；True，框架会每个进程内创建多个独立的DAG
+build_dag_each_worker: false
+
+dag:
+  # op资源类型, True, 为线程模型；False，为进程模型
+  is_thread_op: False
+  # 使用性能分析, True，生成Timeline性能数据，对性能有一定影响；False为不使用
+  tracer:
+    interval_s: 10
+# http端口, rpc_port和http_port不允许同时为空。当rpc_port可用且http_port为空时，不自动生成http_port
+http_port: 8088
+# rpc端口, rpc_port和http_port不允许同时为空。当rpc_port为空且http_port不为空时，会自动将rpc_port设置为http_port+1
+rpc_port: 8089
+op:
+  ernie:
+    # 并发数，is_thread_op=True时，为线程并发；否则为进程并发
+    concurrency: 1
+    # 当op配置没有server_endpoints时，从local_service_conf读取本地服务配置
+    local_service_conf:
+      # client类型，包括brpc, grpc和local_predictor.local_predictor不启动Serving服务，进程内预测
+      client_type: local_predictor
+      # ir_optim
+      ir_optim: True
+      # device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu
+      device_type: 1
+      # 计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡
+      devices: "0"
+      # Fetch结果列表，以client_config中fetch_var的alias_name为准, 如果没有设置则全部返回
+      fetch_list: ['predict']
+      # 模型路径
+      model_config: ../../serving_server/

applications/neural_search/ranking/cross_encoder/deploy/python/deploy.sh

@@ -0,0 +1,3 @@
+python deploy/python/predict.py --model_dir ./output \
+                                --input_file data/test.csv \
+                                --model_name_or_path rocketqa-base-cross-encoder

applications/neural_search/ranking/cross_encoder/deploy/python/predict.py

@@ -14,21 +14,30 @@
 
 import argparse
 import os
+import sys
 
-import numpy as np
-import paddle
-import paddlenlp as ppnlp
 from scipy.special import softmax
+from scipy.special import expit
+import numpy as np
+import pandas as pd
+
+import paddle
 from paddle import inference
+import paddle.nn.functional as F
 from paddlenlp.data import Stack, Tuple, Pad
 from paddlenlp.datasets import load_dataset
 from paddlenlp.utils.log import logger
+from paddlenlp.transformers import AutoTokenizer
+
+sys.path.append('.')
 
 # yapf: disable
 parser = argparse.ArgumentParser()
 parser.add_argument("--model_dir", type=str, required=True,
     help="The directory to static model.")
 
+parser.add_argument("--input_file", type=str, required=True,
+    help="The test set file.")
 parser.add_argument("--max_seq_length", default=128, type=int,
     help="The maximum total input sequence length after tokenization. Sequences "
     "longer than this will be truncated, sequences shorter will be padded.")
@@ -51,16 +60,29 @@ parser.add_argument("--benchmark", type=eval, default=False,
     help="To log some information about environment and running.")
 parser.add_argument("--save_log_path", type=str, default="./log_output/",
     help="The file path to save log.")
+parser.add_argument('--model_name_or_path', default="rocketqa-base-cross-encoder", help="The pretrained model used for training")
+
 args = parser.parse_args()
 # yapf: enable
 
 
+def read_text_pair(data_path):
+    """Reads data."""
+    with open(data_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            data = line.rstrip().split("\t")
+            if len(data) != 3:
+                continue
+            yield {'query': data[0], 'title': data[1]}
+
+
 def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
 
     query, title = example["query"], example["title"]
 
-    encoded_inputs = tokenizer(
-        text=query, text_pair=title, max_seq_len=max_seq_length)
+    encoded_inputs = tokenizer(text=query,
+                               text_pair=title,
+                               max_seq_len=max_seq_length)
 
     input_ids = encoded_inputs["input_ids"]
     token_type_ids = encoded_inputs["token_type_ids"]
@@ -73,6 +95,7 @@ def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
 
 
 class Predictor(object):
+
     def __init__(self,
                  model_dir,
                  device="gpu",
@@ -105,10 +128,9 @@ class Predictor(object):
             precision_mode = precision_map[precision]
 
             if args.use_tensorrt:
-                config.enable_tensorrt_engine(
-                    max_batch_size=batch_size,
-                    min_subgraph_size=30,
-                    precision_mode=precision_mode)
+                config.enable_tensorrt_engine(max_batch_size=batch_size,
+                                              min_subgraph_size=30,
+                                              precision_mode=precision_mode)
         elif device == "cpu":
             # set CPU configs accordingly,
             # such as enable_mkldnn, set_cpu_math_library_num_threads
@@ -135,7 +157,7 @@ class Predictor(object):
             import auto_log
             pid = os.getpid()
             self.autolog = auto_log.AutoLogger(
-                model_name="ernie-tiny",
+                model_name=args.model_name_or_path,
                 model_precision=precision,
                 batch_size=self.batch_size,
                 data_shape="dynamic",
@@ -150,7 +172,7 @@ class Predictor(object):
                 warmup=0,
                 logger=logger)
 
-    def predict(self, data, tokenizer, label_map):
+    def predict(self, data, tokenizer):
         """
         Predicts the data labels.
 
@@ -176,8 +198,9 @@ class Predictor(object):
             examples.append((input_ids, segment_ids))
 
         batchify_fn = lambda samples, fn=Tuple(
-            Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input
-            Pad(axis=0, pad_val=tokenizer.pad_token_id),  # segment
+            Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"),  # input
+            Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
+                ),  # segment
         ): fn(samples)
 
         if args.benchmark:
@@ -187,19 +210,15 @@ class Predictor(object):
         self.input_handles[0].copy_from_cpu(input_ids)
         self.input_handles[1].copy_from_cpu(segment_ids)
         self.predictor.run()
-        probs = self.output_handle.copy_to_cpu()
+        sim_score = self.output_handle.copy_to_cpu()
         if args.benchmark:
             self.autolog.times.stamp()
-
-        #probs = softmax(logits, axis=1)
-        idx = np.argmax(probs, axis=1)
-        idx = idx.tolist()
-        labels = [label_map[i] for i in idx]
+        sim_score = expit(sim_score)[:, 1]
 
         if args.benchmark:
             self.autolog.times.end(stamp=True)
 
-        return labels
+        return sim_score
 
 
 if __name__ == "__main__":
@@ -208,10 +227,11 @@ if __name__ == "__main__":
                           args.batch_size, args.use_tensorrt, args.precision,
                           args.cpu_threads, args.enable_mkldnn)
 
-    tokenizer = ppnlp.transformers.ErnieGramTokenizer.from_pretrained(
-        'ernie-gram-zh')
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
 
-    test_ds = load_dataset("lcqmc", splits=["test"])
+    test_ds = load_dataset(read_text_pair,
+                           data_path=args.input_file,
+                           lazy=False)
 
     data = [{'query': d['query'], 'title': d['title']} for d in test_ds]
 
@@ -219,12 +239,11 @@ if __name__ == "__main__":
         data[idx:idx + args.batch_size]
         for idx in range(0, len(data), args.batch_size)
     ]
-    label_map = {0: 'dissimilar', 1: 'similar'}
 
     results = []
     for batch_data in batches:
-        results.extend(predictor.predict(batch_data, tokenizer, label_map))
+        results.extend(predictor.predict(batch_data, tokenizer))
     for idx, text in enumerate(data):
-        print('Data: {} \t Label: {}'.format(text, results[idx]))
+        print('Data: {} \t prob: {}'.format(text, results[idx]))
     if args.benchmark:
         predictor.autolog.report()

applications/neural_search/ranking/cross_encoder/deploy/python/rpc_client.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import time
+import numpy as np
+
+from paddle_serving_server.pipeline import PipelineClient
+
+client = PipelineClient()
+client.connect(['127.0.0.1:8089'])
+
+list_data = [{
+    "query": "加强科研项目管理有效促进医学科研工作",
+    "title": "科研项目管理策略科研项目,项目管理,实施,必要性,策略"
+}]
+feed = {}
+for i, item in enumerate(list_data):
+    feed[str(i)] = str(item)
+
+print(feed)
+start_time = time.time()
+ret = client.predict(feed_dict=feed)
+end_time = time.time()
+print("time to cost :{} seconds".format(end_time - start_time))
+result = np.array(eval(ret.value[0]))
+print(result.shape)
+print(result)

applications/neural_search/ranking/cross_encoder/deploy/python/web_service.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import sys
+import json
+
+from scipy.special import expit
+from paddle_serving_server.web_service import WebService, Op
+
+
+def convert_example(example, tokenizer, max_seq_length=512):
+
+    query, title = example["query"], example["title"]
+    encoded_inputs = tokenizer(text=query,
+                               text_pair=title,
+                               max_seq_len=max_seq_length)
+
+    input_ids = encoded_inputs["input_ids"]
+    token_type_ids = encoded_inputs["token_type_ids"]
+
+    return input_ids, token_type_ids
+
+
+class ErnieOp(Op):
+
+    def init_op(self):
+        from paddlenlp.transformers import AutoTokenizer
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            'rocketqa-base-cross-encoder')
+
+    def preprocess(self, input_dicts, data_id, log_id):
+        from paddlenlp.data import Stack, Tuple, Pad
+
+        (_, input_dict), = input_dicts.items()
+        print("input dict", input_dict)
+        batch_size = len(input_dict.keys())
+        examples = []
+        for i in range(batch_size):
+            example = json.loads(input_dict[str(i)].replace("\'", "\""))
+            input_ids, segment_ids = convert_example(example, self.tokenizer)
+            examples.append((input_ids, segment_ids))
+        batchify_fn = lambda samples, fn=Tuple(
+            Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"
+                ),  # input
+            Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"
+                ),  # segment
+        ): fn(samples)
+        input_ids, segment_ids = batchify_fn(examples)
+        feed_dict = {}
+        feed_dict['input_ids'] = input_ids
+        feed_dict['token_type_ids'] = segment_ids
+        return feed_dict, False, None, ""
+
+    def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
+        new_dict = {}
+        sim_score = expit(fetch_dict["predict"])[:, 1]
+        new_dict["predict"] = str(sim_score)
+        return new_dict, None, ""
+
+
+class ErnieService(WebService):
+
+    def get_pipeline_response(self, read_op):
+        ernie_op = ErnieOp(name="ernie", input_ops=[read_op])
+        return ernie_op
+
+
+ernie_service = ErnieService(name="ernie")
+ernie_service.prepare_pipeline_config("config_nlp.yml")
+ernie_service.run_service()

applications/neural_search/ranking/cross_encoder/evaluate.py

@@ -0,0 +1,125 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import partial
+import argparse
+import os
+import random
+import time
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+import paddle
+import paddle.nn.functional as F
+from paddlenlp.data import Stack, Tuple, Pad
+from paddlenlp.datasets import load_dataset
+from paddlenlp.transformers import LinearDecayWithWarmup, AutoModelForSequenceClassification, AutoTokenizer
+
+from data import create_dataloader
+from data import convert_example, read_data
+
+# yapf: disable
+parser = argparse.ArgumentParser()
+parser.add_argument("--test_file", type=str, required=True, help="The full path of test file")
+parser.add_argument("--max_seq_length", default=128, type=int, help="The maximum total input sequence length after tokenization. "
+    "Sequences longer than this will be truncated, sequences shorter will be padded.")
+parser.add_argument("--batch_size", default=32, type=int, help="Batch size per GPU/CPU for training.")
+parser.add_argument('--model_name_or_path', default="rocketqa-base-cross-encoder", help="The pretrained model used for training")
+parser.add_argument("--init_from_ckpt", type=str, default=None, help="The path of checkpoint to be loaded.")
+parser.add_argument("--seed", type=int, default=1000, help="Random seed for initialization.")
+parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to train model, defaults to gpu.")
+args = parser.parse_args()
+# yapf: enable
+
+
+def set_seed(seed):
+    """sets random seed"""
+    random.seed(seed)
+    np.random.seed(seed)
+    paddle.seed(seed)
+
+
+@paddle.no_grad()
+def evaluate(model, metric, data_loader, phase="dev"):
+    """
+    Given a dataset, it evals model and computes the metric.
+
+    Args:
+        model(obj:`paddle.nn.Layer`): A model to classify texts.
+        data_loader(obj:`paddle.io.DataLoader`): The dataset loader which generates batches.
+        metric(obj:`paddle.metric.Metric`): The evaluation metric.
+    """
+    model.eval()
+    metric.reset()
+
+    for idx, batch in enumerate(data_loader):
+        input_ids, token_type_ids, labels = batch
+
+        pos_probs = model(input_ids=input_ids, token_type_ids=token_type_ids)
+
+        sim_score = F.softmax(pos_probs)
+        metric.update(preds=sim_score.numpy(), labels=labels)
+
+    print("eval_{} auc:{:.3}".format(phase, metric.accumulate()))
+    metric.reset()
+    model.train()
+
+
+def main():
+    paddle.set_device(args.device)
+    rank = paddle.distributed.get_rank()
+    if paddle.distributed.get_world_size() > 1:
+        paddle.distributed.init_parallel_env()
+
+    set_seed(args.seed)
+
+    dev_ds = load_dataset(read_data, data_path=args.test_file, lazy=False)
+
+    model = AutoModelForSequenceClassification.from_pretrained(
+        args.model_name_or_path, num_classes=2)
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
+
+    trans_func_eval = partial(convert_example,
+                              tokenizer=tokenizer,
+                              max_seq_length=args.max_seq_length,
+                              is_pair=True)
+
+    batchify_fn_eval = lambda samples, fn=Tuple(
+        Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
+            ),  # pair_input
+        Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
+            ),  # pair_segment
+        Stack(dtype="int64")  # label
+    ): [data for data in fn(samples)]
+
+    dev_data_loader = create_dataloader(dev_ds,
+                                        mode='dev',
+                                        batch_size=args.batch_size,
+                                        batchify_fn=batchify_fn_eval,
+                                        trans_fn=trans_func_eval)
+
+    if args.init_from_ckpt and os.path.isfile(args.init_from_ckpt):
+        state_dict = paddle.load(args.init_from_ckpt)
+        model.set_dict(state_dict)
+    else:
+        raise ValueError(
+            "Please set --params_path with correct pretrained model file")
+
+    metric = paddle.metric.Auc()
+    evaluate(model, metric, dev_data_loader, "dev")
+
+
+if __name__ == "__main__":
+    main()

applications/neural_search/ranking/cross_encoder/export_model.py

@@ -14,41 +14,43 @@
 
 import argparse
 import os
-from functools import partial
 
 import numpy as np
 import paddle
 import paddle.nn.functional as F
-import paddlenlp as ppnlp
 from paddlenlp.data import Stack, Tuple, Pad
+from paddlenlp.transformers import AutoModelForSequenceClassification, AutoTokenizer
 
 # yapf: disable
 parser = argparse.ArgumentParser()
 parser.add_argument("--params_path", type=str, required=True, default='./checkpoint/model_900/model_state.pdparams', help="The path to model parameters to be loaded.")
 parser.add_argument("--output_path", type=str, default='./output', help="The path of model parameter in static graph to be saved.")
+parser.add_argument('--model_name_or_path', default="rocketqa-base-cross-encoder", help="The pretrained model used for training")
 args = parser.parse_args()
 # yapf: enable
 
 if __name__ == "__main__":
-    # The number of labels should be in accordance with the training dataset.
-    label_map = {0: 'negative', 1: 'positive'}
-    model = ppnlp.transformers.ErnieForSequenceClassification.from_pretrained(
-        "ernie-tiny", num_classes=len(label_map))
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
+    model = AutoModelForSequenceClassification.from_pretrained(
+        args.model_name_or_path, num_classes=2)
 
     if args.params_path and os.path.isfile(args.params_path):
         state_dict = paddle.load(args.params_path)
         model.set_dict(state_dict)
         print("Loaded parameters from %s" % args.params_path)
+    else:
+        raise ValueError(
+            "Please set --params_path with correct pretrained model file")
     model.eval()
 
     # Convert to static graph with specific input description
     model = paddle.jit.to_static(
         model,
         input_spec=[
-            paddle.static.InputSpec(
-                shape=[None, None], dtype="int64"),  # input_ids
-            paddle.static.InputSpec(
-                shape=[None, None], dtype="int64")  # segment_ids
+            paddle.static.InputSpec(shape=[None, None],
+                                    dtype="int64"),  # input_ids
+            paddle.static.InputSpec(shape=[None, None],
+                                    dtype="int64")  # segment_ids
         ])
     # Save in static graph model.
     save_path = os.path.join(args.output_path, "inference")

applications/neural_search/ranking/cross_encoder/export_to_serving.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import paddle_serving_client.io as serving_io
+# yapf: disable
+parser = argparse.ArgumentParser()
+parser.add_argument("--dirname", type=str, required=True,
+                    default='./output', help="Path of saved model files. Program file and parameter files are saved in this directory.")
+parser.add_argument("--model_filename", type=str, required=True,
+                    default='inference.pdmodel', help="The name of file to load the inference program. If it is None, the default filename __model__ will be used.")
+parser.add_argument("--params_filename", type=str, required=True,
+                    default='inference.pdiparams', help="The name of file to load all parameters. It is only used for the case that all parameters were saved in a single binary file. If parameters were saved in separate files, set it as None. Default: None.")
+parser.add_argument("--server_path", type=str, default='./serving_server',
+                    help="The path of server parameter in static graph to be saved.")
+parser.add_argument("--client_path", type=str, default='./serving_client',
+                    help="The path of client parameter in static graph to be saved.")
+parser.add_argument("--feed_alias_names", type=str, default=None,
+                    help='set alias names for feed vars, split by comma \',\', you should run --show_proto to check the number of feed vars')
+parser.add_argument("--fetch_alias_names", type=str, default=None,
+                    help='set alias names for feed vars, split by comma \',\', you should run --show_proto to check the number of fetch vars')
+parser.add_argument("--show_proto", type=bool, default=False,
+                    help='If yes, you can preview the proto and then determine your feed var alias name and fetch var alias name.')
+# yapf: enable
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    serving_io.inference_model_to_serving(
+        dirname=args.dirname,
+        serving_server=args.server_path,
+        serving_client=args.client_path,
+        model_filename=args.model_filename,
+        params_filename=args.params_filename,
+        show_proto=args.show_proto,
+        feed_alias_names=args.feed_alias_names,
+        fetch_alias_names=args.fetch_alias_names)

applications/neural_search/ranking/cross_encoder/predict.py

@@ -0,0 +1,99 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+from functools import partial
+
+import paddle
+import paddle.nn.functional as F
+from paddlenlp.datasets import load_dataset
+from paddlenlp.data import Stack, Tuple, Pad
+from tqdm import tqdm
+from paddlenlp.transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
+
+from data import convert_example, read_text_pair, create_dataloader
+
+# yapf: disable
+parser = argparse.ArgumentParser()
+parser.add_argument("--params_path", type=str, required=True, default="checkpoints/model_900/model_state.pdparams", help="The path to model parameters to be loaded.")
+parser.add_argument("--max_seq_length", type=int, default=128, help="The maximum total input sequence length after tokenization. "
+    "Sequences longer than this will be truncated, sequences shorter will be padded.")
+parser.add_argument("--batch_size", type=int, default=32, help="Batch size per GPU/CPU for training.")
+parser.add_argument("--test_set", type=str, required=True, help="The full path of test_set.")
+parser.add_argument("--topk", type=int, default=10, help="The Topk texts.")
+parser.add_argument('--device', choices=['cpu', 'gpu', 'xpu', 'npu'], default="gpu", help="Select which device to train model, defaults to gpu.")
+parser.add_argument('--model_name_or_path', default="rocketqa-base-cross-encoder", help="The pretrained model used for training")
+args = parser.parse_args()
+# yapf: enable
+
+
+@paddle.no_grad()
+def predict(model, data_loader):
+    results = []
+    model.eval()
+    with paddle.no_grad():
+        for batch in data_loader:
+            input_ids, token_type_ids = batch
+            logits = model(input_ids, token_type_ids)
+            probs = F.softmax(logits)
+            probs = probs.numpy()
+            results.extend(probs[:, 1])
+    return results
+
+
+if __name__ == "__main__":
+    paddle.set_device(args.device)
+
+    test_ds = load_dataset(read_text_pair, data_path=args.test_set, lazy=False)
+    model = AutoModelForSequenceClassification.from_pretrained(
+        args.model_name_or_path, num_classes=2)
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
+
+    trans_func = partial(convert_example,
+                         tokenizer=tokenizer,
+                         max_seq_length=args.max_seq_length,
+                         is_test=True,
+                         is_pair=True)
+
+    batchify_fn = lambda samples, fn=Tuple(
+        Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"),  # input
+        Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
+            ),  # segment
+    ): [data for data in fn(samples)]
+
+    test_data_loader = create_dataloader(test_ds,
+                                         mode='predict',
+                                         batch_size=args.batch_size,
+                                         batchify_fn=batchify_fn,
+                                         trans_fn=trans_func)
+
+    if args.params_path and os.path.isfile(args.params_path):
+        state_dict = paddle.load(args.params_path)
+        model.set_dict(state_dict)
+        print("Loaded parameters from %s" % args.params_path)
+    else:
+        raise ValueError(
+            "Please set --params_path with correct pretrained model file")
+    results = predict(model, test_data_loader)
+    test_ds = load_dataset(read_text_pair, data_path=args.test_set, lazy=False)
+    text_pairs = []
+    for idx, prob in enumerate(results):
+        text_pair = test_ds[idx]
+        text_pair["pred_prob"] = prob
+        text_pairs.append(text_pair)
+    text_pairs = sorted(text_pairs, key=lambda x: x['pred_prob'],
+                        reverse=True)[:args.topk]
+    for item in text_pairs:
+        print(item)

applications/neural_search/ranking/cross_encoder/scripts/evaluate_ce.sh

@@ -0,0 +1,3 @@
+python evaluate.py --model_name_or_path rocketqa-base-cross-encoder \
+                   --init_from_ckpt checkpoints/model_80000/model_state.pdparams \
+                   --test_file data/dev_pairwise.csv

applications/neural_search/ranking/cross_encoder/scripts/export_model.sh

@@ -0,0 +1,4 @@
+python export_model.py \
+                       --params_path checkpoints/model_80000/model_state.pdparams \
+                       --model_name_or_path rocketqa-base-cross-encoder \
+                       --output_path=./output

applications/neural_search/ranking/cross_encoder/scripts/export_to_serving.sh

@@ -0,0 +1,7 @@
+python export_to_serving.py \
+    --dirname "output" \
+    --model_filename "inference.pdmodel" \
+    --params_filename "inference.pdiparams" \
+    --server_path "serving_server" \
+    --client_path "serving_client" \
+    --fetch_alias_names "predict"

applications/neural_search/ranking/cross_encoder/scripts/predict_ce.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+unset CUDA_VISIBLE_DEVICES
+export CUDA_VISIBLE_DEVICES=0
+python predict.py \
+                --device 'gpu' \
+                --params_path checkpoints/model_80000/model_state.pdparams \
+                --model_name_or_path rocketqa-base-cross-encoder \
+                --test_set data/test.csv \
+                --topk 10 \
+                --batch_size 128 \
+                --max_seq_length 384

applications/neural_search/ranking/cross_encoder/scripts/train_ce.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+unset CUDA_VISIBLE_DEVICES
+python -u -m paddle.distributed.launch --gpus "0,1,2,3" --log_dir="logs" train_ce.py \
+        --device gpu \
+        --train_set data/train.csv \
+        --test_file data/dev_pairwise.csv \
+        --save_dir ./checkpoints \
+        --model_name_or_path rocketqa-base-cross-encoder \
+        --batch_size 32 \
+        --save_steps 10000 \
+        --max_seq_len 384 \
+        --learning_rate 1E-5 \
+        --weight_decay  0.01 \
+        --warmup_proportion 0.0 \
+        --logging_steps 10 \
+        --seed 1 \
+        --epochs 3 \
+        --eval_step 1000

applications/neural_search/ranking/cross_encoder/train_ce.py

@@ -24,15 +24,17 @@ import paddle
 import paddle.nn.functional as F
 from paddlenlp.data import Stack, Tuple, Pad
 from paddlenlp.datasets import load_dataset
-from paddlenlp.transformers import AutoModelForSequenceClassification, AutoTokenizer
-from paddlenlp.transformers import LinearDecayWithWarmup
-
-from utils import convert_example
+from paddlenlp.transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
+from paddlenlp.transformers import PolyDecayWithWarmup
+from paddle.distributed.fleet.utils.hybrid_parallel_util import fused_allreduce_gradients
+from data import convert_example, read_data, create_dataloader
 
 # yapf: disable
 parser = argparse.ArgumentParser()
 parser.add_argument("--save_dir", default='./checkpoint', type=str, help="The output directory where the model checkpoints will be written.")
-parser.add_argument("--dataset", choices=["chnsenticorp", "xnli_cn"], default="chnsenticorp", type=str, help="Dataset for classfication tasks.")
+parser.add_argument("--train_set", type=str, required=True, help="The full path of train_set_file.")
+parser.add_argument("--test_file", type=str, required=True, help="The full path of test file")
+
 parser.add_argument("--max_seq_length", default=128, type=int, help="The maximum total input sequence length after tokenization. "
     "Sequences longer than this will be truncated, sequences shorter will be padded.")
 parser.add_argument("--batch_size", default=32, type=int, help="Batch size per GPU/CPU for training.")
@@ -48,10 +50,39 @@ parser.add_argument("--seed", type=int, default=1000, help="random seed for init
 parser.add_argument('--device', choices=['cpu', 'gpu', 'xpu', 'npu'], default="gpu", help="Select which device to train model, defaults to gpu.")
 parser.add_argument("--use_amp", type=distutils.util.strtobool, default=False, help="Enable mixed precision training.")
 parser.add_argument("--scale_loss", type=float, default=2**15, help="The value of scale_loss for fp16.")
+parser.add_argument('--model_name_or_path', default="rocketqa-base-cross-encoder", help="The pretrained model used for training")
+parser.add_argument("--eval_step", default=200, type=int, help="Step interval for evaluation.")
 args = parser.parse_args()
 # yapf: enable
 
 
+@paddle.no_grad()
+def evaluate(model, metric, data_loader, phase="dev"):
+    """
+    Given a dataset, it evals model and computes the metric.
+
+    Args:
+        model(obj:`paddle.nn.Layer`): A model to classify texts.
+        data_loader(obj:`paddle.io.DataLoader`): The dataset loader which generates batches.
+        metric(obj:`paddle.metric.Metric`): The evaluation metric.
+    """
+    model.eval()
+    metric.reset()
+
+    for idx, batch in enumerate(data_loader):
+        input_ids, token_type_ids, labels = batch
+
+        pos_probs = model(input_ids=input_ids, token_type_ids=token_type_ids)
+
+        sim_score = F.softmax(pos_probs)
+
+        metric.update(preds=sim_score.numpy(), labels=labels)
+
+    print("eval_{} auc:{:.3}".format(phase, metric.accumulate()))
+    metric.reset()
+    model.train()
+
+
 def set_seed(seed):
     """sets random seed"""
     random.seed(seed)
@@ -59,103 +90,60 @@ def set_seed(seed):
     paddle.seed(seed)
 
 
-@paddle.no_grad()
-def evaluate(model, criterion, metric, data_loader):
-    """
-    Given a dataset, it evals model and computes the metric.
-
-    Args:
-        model(obj:`paddle.nn.Layer`): A model to classify texts.
-        data_loader(obj:`paddle.io.DataLoader`): The dataset loader which generates batches.
-        criterion(obj:`paddle.nn.Layer`): It can compute the loss.
-        metric(obj:`paddle.metric.Metric`): The evaluation metric.
-    """
-    model.eval()
-    metric.reset()
-    losses = []
-    for batch in data_loader:
-        input_ids, token_type_ids, labels = batch
-        logits = model(input_ids, token_type_ids)
-        loss = criterion(logits, labels)
-        losses.append(loss.numpy())
-        correct = metric.compute(logits, labels)
-        metric.update(correct)
-    accu = metric.accumulate()
-    print("eval loss: %.5f, accuracy: %.5f" % (np.mean(losses), accu))
-    model.train()
-    metric.reset()
-
-
-def create_dataloader(dataset,
-                      mode='train',
-                      batch_size=1,
-                      batchify_fn=None,
-                      trans_fn=None):
-    if trans_fn:
-        dataset = dataset.map(trans_fn)
-
-    shuffle = True if mode == 'train' else False
-    if mode == 'train':
-        batch_sampler = paddle.io.DistributedBatchSampler(
-            dataset, batch_size=batch_size, shuffle=shuffle)
-    else:
-        batch_sampler = paddle.io.BatchSampler(
-            dataset, batch_size=batch_size, shuffle=shuffle)
-
-    return paddle.io.DataLoader(
-        dataset=dataset,
-        batch_sampler=batch_sampler,
-        collate_fn=batchify_fn,
-        return_list=True)
-
-
 def do_train():
     paddle.set_device(args.device)
     rank = paddle.distributed.get_rank()
     if paddle.distributed.get_world_size() > 1:
         paddle.distributed.init_parallel_env()
-
+    dev_count = paddle.distributed.get_world_size()
     set_seed(args.seed)
 
-    train_ds, dev_ds, test_ds = load_dataset(
-        args.dataset, splits=["train", "dev", "test"])
+    train_ds = load_dataset(read_data, data_path=args.train_set, lazy=False)
+    dev_ds = load_dataset(read_data, data_path=args.test_file, lazy=False)
 
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
     model = AutoModelForSequenceClassification.from_pretrained(
-        'ernie-1.0', num_classes=len(train_ds.label_list))
-    tokenizer = AutoTokenizer.from_pretrained('ernie-1.0')
+        args.model_name_or_path, num_classes=2)
+
+    trans_func = partial(convert_example,
+                         tokenizer=tokenizer,
+                         max_seq_length=args.max_seq_length,
+                         is_pair=True)
 
-    trans_func = partial(
-        convert_example,
-        tokenizer=tokenizer,
-        max_seq_length=args.max_seq_length,
-        is_pair=args.dataset == "xnli_cn")
     batchify_fn = lambda samples, fn=Tuple(
-        Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input
-        Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # segment
+        Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"),  # input
+        Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
+            ),  # segment
         Stack(dtype="int64")  # label
     ): [data for data in fn(samples)]
-    train_data_loader = create_dataloader(
-        train_ds,
-        mode='train',
-        batch_size=args.batch_size,
-        batchify_fn=batchify_fn,
-        trans_fn=trans_func)
-    dev_data_loader = create_dataloader(
-        dev_ds,
-        mode='dev',
-        batch_size=args.batch_size,
-        batchify_fn=batchify_fn,
-        trans_fn=trans_func)
+
+    train_data_loader = create_dataloader(train_ds,
+                                          mode='train',
+                                          batch_size=args.batch_size,
+                                          batchify_fn=batchify_fn,
+                                          trans_fn=trans_func)
+    dev_data_loader = create_dataloader(dev_ds,
+                                        mode='dev',
+                                        batch_size=args.batch_size,
+                                        batchify_fn=batchify_fn,
+                                        trans_fn=trans_func)
 
     if args.init_from_ckpt and os.path.isfile(args.init_from_ckpt):
         state_dict = paddle.load(args.init_from_ckpt)
         model.set_dict(state_dict)
+
     model = paddle.DataParallel(model)
 
-    num_training_steps = len(train_data_loader) * args.epochs
+    num_training_examples = len(train_ds)
+    # 4卡 gpu
+    max_train_steps = args.epochs * num_training_examples // args.batch_size // dev_count
 
-    lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps,
-                                         args.warmup_proportion)
+    warmup_steps = int(max_train_steps * args.warmup_proportion)
+
+    print("Device count: %d" % dev_count)
+    print("Num train examples: %d" % num_training_examples)
+    print("Max train steps: %d" % max_train_steps)
+    print("Num warmup steps: %d" % warmup_steps)
 
     # Generate parameter names needed to perform weight decay.
     # All bias and LayerNorm parameters are excluded.
@@ -164,13 +152,14 @@ def do_train():
         if not any(nd in n for nd in ["bias", "norm"])
     ]
     optimizer = paddle.optimizer.AdamW(
-        learning_rate=lr_scheduler,
+        learning_rate=args.learning_rate,
         parameters=model.parameters(),
         weight_decay=args.weight_decay,
-        apply_decay_param_fun=lambda x: x in decay_params)
+        apply_decay_param_fun=lambda x: x in decay_params,
+        grad_clip=paddle.nn.ClipGradByGlobalNorm(1.0))
 
     criterion = paddle.nn.loss.CrossEntropyLoss()
-    metric = paddle.metric.Accuracy()
+    metric = paddle.metric.Auc()
     if args.use_amp:
         scaler = paddle.amp.GradScaler(init_loss_scaling=args.scale_loss)
     global_step = 0
@@ -178,23 +167,13 @@ def do_train():
     for epoch in range(1, args.epochs + 1):
         for step, batch in enumerate(train_data_loader, start=1):
             input_ids, token_type_ids, labels = batch
-            with paddle.amp.auto_cast(
-                    args.use_amp,
-                    custom_white_list=["layer_norm", "softmax", "gelu"], ):
-                logits = model(input_ids, token_type_ids)
-                loss = criterion(logits, labels)
+            logits = model(input_ids, token_type_ids)
+            loss = criterion(logits, labels)
             probs = F.softmax(logits, axis=1)
-            correct = metric.compute(probs, labels)
-            metric.update(correct)
-            acc = metric.accumulate()
+            acc = paddle.metric.accuracy(input=probs, label=labels)
+            loss.backward()
 
-            if args.use_amp:
-                scaler.scale(loss).backward()
-                scaler.minimize(optimizer, loss)
-            else:
-                loss.backward()
-                optimizer.step()
-            lr_scheduler.step()
+            optimizer.step()
             optimizer.clear_grad()
 
             global_step += 1
@@ -205,19 +184,23 @@ def do_train():
                     % (global_step, epoch, step, loss, acc,
                        args.logging_steps / time_diff))
                 tic_train = time.time()
-
-            if global_step % args.valid_steps == 0 and rank == 0:
-                evaluate(model, criterion, metric, dev_data_loader)
-                tic_train = time.time()
-
+            if global_step % args.eval_step == 0 and rank == 0:
+                evaluate(model, metric, dev_data_loader, "dev")
             if global_step % args.save_steps == 0 and rank == 0:
                 save_dir = os.path.join(args.save_dir, "model_%d" % global_step)
-                if not os.path.exists(save_dir):
-                    os.makedirs(save_dir)
-                model._layers.save_pretrained(save_dir)
+                model_to_save = model._layers if isinstance(
+                    model, paddle.DataParallel) else model
+                model_to_save.save_pretrained(save_dir)
                 tokenizer.save_pretrained(save_dir)
                 tic_train = time.time()
 
+    # save final checkpoint
+    save_dir = os.path.join(args.save_dir, "model_%d" % global_step)
+    model_to_save = model._layers if isinstance(model,
+                                                paddle.DataParallel) else model
+    model_to_save.save_pretrained(save_dir)
+    tokenizer.save_pretrained(save_dir)
+
 
 if __name__ == "__main__":
     do_train()

applications/neural_search/ranking/ernie_matching/README.md

@@ -4,7 +4,7 @@
 * [背景介绍](#背景介绍)
 * [ERNIE-Gram](#ERNIE-Gram)
     * [1. 技术方案和评估指标](#技术方案)
-    * [2. 环境依赖](#环境依赖)  
+    * [2. 环境依赖](#环境依赖)
     * [3. 代码结构](#代码结构)
     * [4. 数据准备](#数据准备)
     * [5. 模型训练](#模型训练)
@@ -63,15 +63,24 @@
 ```
 ernie_matching/
 ├── deply # 部署
+    ├── cpp
+        ├── rpc_client.py # RPC 客户端的bash脚本
+        ├── http_client.py # http 客户端的bash文件
+        └── start_server.sh # 启动C++服务的脚本
     └── python
         ├── deploy.sh # 预测部署bash脚本
+        ├── config_nlp.yml # Pipeline 的配置文件
+        ├── web_service.py # Pipeline 服务端的脚本
+        ├── rpc_client.py # Pipeline RPC客户端的脚本
         └── predict.py # python 预测部署示例
 |—— scripts
     ├── export_model.sh # 动态图参数导出静态图参数的bash文件
+    ├── export_to_serving.sh # 导出 Paddle Serving 模型格式的bash文件
     ├── train_pairwise.sh # Pair-wise 单塔匹配模型训练的bash文件
     ├── evaluate.sh # 评估验证文件bash脚本
     ├── predict_pairwise.sh # Pair-wise 单塔匹配模型预测脚本的bash文件
 ├── export_model.py # 动态图参数导出静态图参数脚本
+├── export_to_serving.py # 导出 Paddle Serving 模型格式的脚本
 ├── model.py #  Pair-wise 匹配模型组网
 ├── data.py #  Pair-wise 训练样本的转换逻辑 、Pair-wise 生成随机负例的逻辑
 ├── train_pairwise.py # Pair-wise 单塔匹配模型训练脚本
@@ -143,7 +152,7 @@ ernie_matching/
 训练的命令如下：
 
 ```
-python -u -m paddle.distributed.launch --gpus "0,2,3,4" train_pairwise.py \
+python -u -m paddle.distributed.launch --gpus "0,1,2,3" train_pairwise.py \
         --device gpu \
         --save_dir ./checkpoints \
         --batch_size 32 \
@@ -241,7 +250,9 @@ sh scripts/predict_pairwise.sh
 首先把动态图模型转换为静态图：
 
 ```
-python export_model.py --params_path checkpoints/model_30000/model_state.pdparams --output_path=./output
+python export_model.py --params_path checkpoints/model_30000/model_state.pdparams \
+                       --output_path=./output \
+                       --model_name_or_path ernie-3.0-medium-zh
 ```
 也可以运行下面的bash脚本：
 
@@ -251,21 +262,17 @@ sh scripts/export_model.sh
 
 ### Paddle Inference
 
-修改预测文件路径：
+使用PaddleInference：
 
 ```
-input_file='../../sort/test_pairwise.csv'
-```
-
-然后使用PaddleInference
-
-```
-python predict.py --model_dir=../../output
+python deploy/python/predict.py --model_dir ./output \
+                                --input_file sort/test_pairwise.csv \
+                                --model_name_or_path ernie-3.0-medium-zh
 ```
 也可以运行下面的bash脚本：
 
 ```
-sh deploy.sh
+sh deploy/python/deploy.sh
 ```
 得到下面的输出，输出的是样本的query，title以及对应的概率：
 
@@ -277,6 +284,126 @@ Data: {'query': '中西方语言与文化的差异', 'title': '中英文化差
 Data: {'query': '中西方语言与文化的差异', 'title': '浅谈文化与语言习得文化,语言,文化与语言的关系,文化与语言习得意识,跨文化交际'}     prob: [0.8944413]
 ```
 
+### Paddle Serving部署
+
+Paddle Serving 的详细文档请参考 [Pipeline_Design](https://github.com/PaddlePaddle/Serving/blob/v0.7.0/doc/Python_Pipeline/Pipeline_Design_CN.md)和[Serving_Design](https://github.com/PaddlePaddle/Serving/blob/v0.7.0/doc/Serving_Design_CN.md),首先把静态图模型转换成Serving的格式：
+
+```
+python export_to_serving.py \
+    --dirname "output" \
+    --model_filename "inference.predict.pdmodel" \
+    --params_filename "inference.predict.pdiparams" \
+    --server_path "serving_server" \
+    --client_path "serving_client" \
+    --fetch_alias_names "predict"
+
+```
+
+参数含义说明
+* `dirname`: 需要转换的模型文件存储路径，Program 结构文件和参数文件均保存在此目录。
+* `model_filename`： 存储需要转换的模型 Inference Program 结构的文件名称。如果设置为 None ，则使用 `__model__` 作为默认的文件名
+* `params_filename`: 存储需要转换的模型所有参数的文件名称。当且仅当所有模型参数被保>存在一个单独的二进制文件中，它才需要被指定。如果模型参数是存储在各自分离的文件中，设置它的值为 None
+* `server_path`: 转换后的模型文件和配置文件的存储路径。默认值为 serving_server
+* `client_path`: 转换后的客户端配置文件存储路径。默认值为 serving_client
+* `fetch_alias_names`: 模型输出的别名设置，比如输入的 input_ids 等，都可以指定成其他名字，默认不指定
+* `feed_alias_names`: 模型输入的别名设置，比如输出 pooled_out 等，都可以重新指定成其他模型，默认不指定
+
+也可以运行下面的 bash 脚本：
+```
+sh scripts/export_to_serving.sh
+```
+Paddle Serving的部署有两种方式，第一种方式是Pipeline的方式，第二种是C++的方式，下面分别介绍这两种方式的用法：
+
+#### Pipeline方式
+
+修改`Tokenizer`
+
+```
+self.tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
+```
+
+启动 Pipeline Server:
+
+```
+python web_service.py
+```
+
+启动客户端调用 Server。
+
+首先修改rpc_client.py中需要预测的样本：
+
+```
+list_data = [{"query":"中西方语言与文化的差异","title":"第二语言习得的一大障碍就是文化差异。"}]`
+```
+然后运行：
+```
+python rpc_client.py
+```
+模型的输出为：
+
+```
+PipelineClient::predict pack_data time:1656912047.5986433
+PipelineClient::predict before time:1656912047.599081
+time to cost :0.012039899826049805 seconds
+(1, 1)
+[[0.85112208]]
+```
+可以看到客户端发送了1条文本，这条文本的相似的概率值。
+
+#### C++的方式
+
+启动C++的Serving：
+
+```
+python -m paddle_serving_server.serve --model serving_server --port 8600 --gpu_id 0 --thread 5 --ir_optim True
+```
+也可以使用脚本：
+
+```
+sh deploy/cpp/start_server.sh
+```
+Client 可以使用 http 或者 rpc 两种方式，rpc 的方式为：
+
+```
+python deploy/cpp/rpc_client.py
+```
+运行的输出为：
+
+```
+I0704 05:19:00.443437  1987 general_model.cpp:490] [client]logid=0,client_cost=8.477ms,server_cost=6.458ms.
+time to cost :0.008707761764526367 seconds
+{'predict': array([[0.8511221]], dtype=float32)}
+```
+可以看到服务端返回了相似度结果
+
+或者使用 http 的客户端访问模式：
+
+```
+python deploy/cpp/http_client.py
+```
+运行的输出为：
+```
+time to cost :0.006819009780883789 seconds
+[0.8511220812797546]
+```
+可以看到服务端返回了相似度结果
+
+也可以使用curl方式发送Http请求：
+
+```
+curl -XPOST http://0.0.0.0:8600/GeneralModelService/inference -d  ' {"tensor":[{"int64_data":[    1,    12,   213,    58,   405,   545,    54,    68,    73,
+            5,   859,   712,     2,   131,   177,   405,   545,   489,
+          116,     5,     7,    19,   843,  1767,   113,    10,    68,
+           73,   859,   712, 12043,     2],"elem_type":0,"name":"input_ids","alias_name":"input_ids","shape":[1,32]},
+    {"int64_data":[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+        1, 1, 1, 1, 1, 1, 1, 1, 1, 1],"elem_type":0,"name":"token_type_ids","alias_name":"token_type_ids","shape":[1,32]}
+        ],
+"fetch_var_names":["sigmoid_2.tmp_0"],
+"log_id":0
+}'
+```
+
+
 ## Reference
 
 [1] Xiao, Dongling, Yu-Kun Li, Han Zhang, Yu Sun, Hao Tian, Hua Wu, and Haifeng Wang. “ERNIE-Gram: Pre-Training with Explicitly N-Gram Masked Language Modeling for Natural Language Understanding.” ArXiv:2010.12148 [Cs].

applications/neural_search/ranking/ernie_matching/data.py

@@ -28,17 +28,18 @@ def create_dataloader(dataset,
 
     shuffle = True if mode == 'train' else False
     if mode == 'train':
-        batch_sampler = paddle.io.DistributedBatchSampler(
-            dataset, batch_size=batch_size, shuffle=shuffle)
+        batch_sampler = paddle.io.DistributedBatchSampler(dataset,
+                                                          batch_size=batch_size,
+                                                          shuffle=shuffle)
     else:
-        batch_sampler = paddle.io.BatchSampler(
-            dataset, batch_size=batch_size, shuffle=shuffle)
+        batch_sampler = paddle.io.BatchSampler(dataset,
+                                               batch_size=batch_size,
+                                               shuffle=shuffle)
 
-    return paddle.io.DataLoader(
-        dataset=dataset,
-        batch_sampler=batch_sampler,
-        collate_fn=batchify_fn,
-        return_list=True)
+    return paddle.io.DataLoader(dataset=dataset,
+                                batch_sampler=batch_sampler,
+                                collate_fn=batchify_fn,
+                                return_list=True)
 
 
 def read_text_pair(data_path):
@@ -58,8 +59,9 @@ def convert_pointwise_example(example,
 
     query, title = example["query"], example["title"]
 
-    encoded_inputs = tokenizer(
-        text=query, text_pair=title, max_seq_len=max_seq_length)
+    encoded_inputs = tokenizer(text=query,
+                               text_pair=title,
+                               max_seq_len=max_seq_length)
 
     input_ids = encoded_inputs["input_ids"]
     token_type_ids = encoded_inputs["token_type_ids"]
@@ -80,10 +82,12 @@ def convert_pairwise_example(example,
         query, pos_title, neg_title = example["query"], example[
             "title"], example["neg_title"]
 
-        pos_inputs = tokenizer(
-            text=query, text_pair=pos_title, max_seq_len=max_seq_length)
-        neg_inputs = tokenizer(
-            text=query, text_pair=neg_title, max_seq_len=max_seq_length)
+        pos_inputs = tokenizer(text=query,
+                               text_pair=pos_title,
+                               max_seq_len=max_seq_length)
+        neg_inputs = tokenizer(text=query,
+                               text_pair=neg_title,
+                               max_seq_len=max_seq_length)
 
         pos_input_ids = pos_inputs["input_ids"]
         pos_token_type_ids = pos_inputs["token_type_ids"]
@@ -96,8 +100,9 @@ def convert_pairwise_example(example,
     else:
         query, title = example["query"], example["title"]
 
-        inputs = tokenizer(
-            text=query, text_pair=title, max_seq_len=max_seq_length)
+        inputs = tokenizer(text=query,
+                           text_pair=title,
+                           max_seq_len=max_seq_length)
 
         input_ids = inputs["input_ids"]
         token_type_ids = inputs["token_type_ids"]

applications/neural_search/ranking/ernie_matching/deploy/cpp/http_client.py

@@ -0,0 +1,71 @@
+# coding:utf-8
+# pylint: disable=doc-string-missing
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import time
+import numpy as np
+import requests
+import json
+
+from paddle_serving_client.httpclient import HttpClient
+import paddlenlp as ppnlp
+
+
+def convert_example(example, tokenizer, max_seq_length=512):
+
+    query, title = example["query"], example["title"]
+    encoded_inputs = tokenizer(text=query,
+                               text_pair=title,
+                               max_seq_len=max_seq_length)
+    input_ids = encoded_inputs["input_ids"]
+    token_type_ids = encoded_inputs["token_type_ids"]
+
+    return input_ids, token_type_ids
+
+
+# 启动python客户端
+endpoint_list = ['127.0.0.1:8600']
+client = HttpClient()
+client.load_client_config('serving_client')
+client.connect(endpoint_list)
+feed_names = client.feed_names_
+fetch_names = client.fetch_names_
+
+# 创建tokenizer
+tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-gram-zh')
+max_seq_len = 64
+
+# 数据预处理
+list_data = [{"query": "中西方语言与文化的差异", "title": "第二语言习得的一大障碍就是文化差异。"}]
+
+input_ids, token_type_ids = [], []
+for example in list_data:
+    input_id, token_type_id = convert_example(example,
+                                              tokenizer,
+                                              max_seq_length=max_seq_len)
+    input_ids.append(input_id)
+    token_type_ids.append(token_type_id)
+
+feed_dict = {}
+feed_dict['input_ids'] = np.array(input_ids)
+feed_dict['token_type_ids'] = np.array(token_type_ids)
+# batch设置为True表示的是批量预测
+b_start = time.time()
+result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)
+b_end = time.time()
+print(result)
+print("time to cost :{} seconds".format(b_end - b_start))
+print(result.outputs[0].tensor[0].float_data)

applications/neural_search/ranking/ernie_matching/deploy/cpp/rpc_client.py

@@ -0,0 +1,69 @@
+# coding:utf-8
+# pylint: disable=doc-string-missing
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import time
+import numpy as np
+
+from paddle_serving_client import Client
+import paddlenlp as ppnlp
+
+
+def convert_example(example, tokenizer, max_seq_length=512):
+
+    query, title = example["query"], example["title"]
+    encoded_inputs = tokenizer(text=query,
+                               text_pair=title,
+                               max_seq_len=max_seq_length)
+
+    input_ids = encoded_inputs["input_ids"]
+    token_type_ids = encoded_inputs["token_type_ids"]
+
+    return input_ids, token_type_ids
+
+
+# 启动python客户端
+endpoint_list = ['127.0.0.1:8600']
+client = Client()
+client.load_client_config('serving_client')
+client.connect(endpoint_list)
+feed_names = client.feed_names_
+fetch_names = client.fetch_names_
+
+# 创建tokenizer
+tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-gram-zh')
+max_seq_len = 64
+
+# 数据预处理
+list_data = [{"query": "中西方语言与文化的差异", "title": "第二语言习得的一大障碍就是文化差异。"}]
+
+input_ids, token_type_ids = [], []
+for example in list_data:
+    input_id, token_type_id = convert_example(example,
+                                              tokenizer,
+                                              max_seq_length=max_seq_len)
+    input_ids.append(input_id)
+    token_type_ids.append(token_type_id)
+
+feed_dict = {}
+feed_dict['input_ids'] = np.array(input_ids)
+feed_dict['token_type_ids'] = np.array(token_type_ids)
+# batch设置为True表示的是批量预测
+b_start = time.time()
+result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)
+b_end = time.time()
+print("time to cost :{} seconds".format(b_end - b_start))
+print(result)

applications/neural_search/ranking/ernie_matching/deploy/cpp/start_server.sh

@@ -0,0 +1 @@
+python -m paddle_serving_server.serve --model serving_server --port 8600 --gpu_id 0 --thread 5 --ir_optim True

applications/neural_search/ranking/ernie_matching/deploy/python/config_nlp.yml

@@ -0,0 +1,34 @@
+# worker_num, 最大并发数。当build_dag_each_worker=True时, 框架会创建worker_num个进程，每个进程内构建grpcSever和DAG
+# 当build_dag_each_worker=False时，框架会设置主线程grpc线程池的max_workers=worker_num
+worker_num: 20
+# build_dag_each_worker, False，框架在进程内创建一条DAG；True，框架会每个进程内创建多个独立的DAG
+build_dag_each_worker: false
+
+dag:
+  # op资源类型, True, 为线程模型；False，为进程模型
+  is_thread_op: False
+  # 使用性能分析, True，生成Timeline性能数据，对性能有一定影响；False为不使用
+  tracer:
+    interval_s: 10
+# http端口, rpc_port和http_port不允许同时为空。当rpc_port可用且http_port为空时，不自动生成http_port
+http_port: 8088
+# rpc端口, rpc_port和http_port不允许同时为空。当rpc_port为空且http_port不为空时，会自动将rpc_port设置为http_port+1
+rpc_port: 8089
+op:
+  ernie:
+    # 并发数，is_thread_op=True时，为线程并发；否则为进程并发
+    concurrency: 1
+    # 当op配置没有server_endpoints时，从local_service_conf读取本地服务配置
+    local_service_conf:
+      # client类型，包括brpc, grpc和local_predictor.local_predictor不启动Serving服务，进程内预测
+      client_type: local_predictor
+      # ir_optim
+      ir_optim: True
+      # device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu
+      device_type: 1
+      # 计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡
+      devices: "0"
+      # Fetch结果列表，以client_config中fetch_var的alias_name为准, 如果没有设置则全部返回
+      fetch_list: ['predict']
+      # 模型路径
+      model_config: ../../serving_server/

applications/neural_search/ranking/ernie_matching/deploy/python/deploy.sh

@@ -1 +1,3 @@
-python deploy/python/predict.py --model_dir ./output
+python deploy/python/predict.py --model_dir ./output \
+                                --input_file sort/test_pairwise.csv \
+                                --model_name_or_path ernie-3.0-medium-zh

applications/neural_search/ranking/ernie_matching/deploy/python/predict.py

@@ -14,19 +14,21 @@
 
 import argparse
 import os
+import sys
 
-import numpy as np
-import pandas as pd
-import paddle
-import paddlenlp as ppnlp
 from scipy.special import softmax
 from scipy.special import expit
+import numpy as np
+import pandas as pd
+
+import paddle
 from paddle import inference
+import paddle.nn.functional as F
+
 from paddlenlp.data import Stack, Tuple, Pad
 from paddlenlp.datasets import load_dataset
 from paddlenlp.utils.log import logger
-import paddle.nn.functional as F
-import sys
+from paddlenlp.transformers import AutoTokenizer, AutoModel
 
 sys.path.append('.')
 
@@ -42,7 +44,8 @@ parser.add_argument("--batch_size", default=32, type=int,
     help="Batch size per GPU/CPU for training.")
 parser.add_argument('--device', choices=['cpu', 'gpu', 'xpu'], default="gpu",
     help="Select which device to train model, defaults to gpu.")
-
+parser.add_argument("--input_file", type=str, required=True,
+    help="The test set file.")
 parser.add_argument('--use_tensorrt', default=False, type=eval, choices=[True, False],
     help='Enable to use tensorrt to speed up.')
 parser.add_argument("--precision", default="fp32", type=str, choices=["fp32", "fp16", "int8"],
@@ -57,6 +60,7 @@ parser.add_argument("--benchmark", type=eval, default=False,
     help="To log some information about environment and running.")
 parser.add_argument("--save_log_path", type=str, default="./log_output/",
     help="The file path to save log.")
+parser.add_argument('--model_name_or_path', default="ernie-3.0-medium-zh", help="The pretrained model used for training")
 args = parser.parse_args()
 # yapf: enable
 
@@ -75,8 +79,9 @@ def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
 
     query, title = example["query"], example["title"]
 
-    encoded_inputs = tokenizer(
-        text=query, text_pair=title, max_seq_len=max_seq_length)
+    encoded_inputs = tokenizer(text=query,
+                               text_pair=title,
+                               max_seq_len=max_seq_length)
 
     input_ids = encoded_inputs["input_ids"]
     token_type_ids = encoded_inputs["token_type_ids"]
@@ -89,6 +94,7 @@ def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
 
 
 class Predictor(object):
+
     def __init__(self,
                  model_dir,
                  device="gpu",
@@ -101,8 +107,8 @@ class Predictor(object):
         self.max_seq_length = max_seq_length
         self.batch_size = batch_size
 
-        model_file = model_dir + "/inference.get_pooled_embedding.pdmodel"
-        params_file = model_dir + "/inference.get_pooled_embedding.pdiparams"
+        model_file = model_dir + "/inference.predict.pdmodel"
+        params_file = model_dir + "/inference.predict.pdiparams"
         if not os.path.exists(model_file):
             raise ValueError("not find model file path {}".format(model_file))
         if not os.path.exists(params_file):
@@ -121,10 +127,9 @@ class Predictor(object):
             precision_mode = precision_map[precision]
 
             if args.use_tensorrt:
-                config.enable_tensorrt_engine(
-                    max_batch_size=batch_size,
-                    min_subgraph_size=30,
-                    precision_mode=precision_mode)
+                config.enable_tensorrt_engine(max_batch_size=batch_size,
+                                              min_subgraph_size=30,
+                                              precision_mode=precision_mode)
         elif device == "cpu":
             # set CPU configs accordingly,
             # such as enable_mkldnn, set_cpu_math_library_num_threads
@@ -150,21 +155,22 @@ class Predictor(object):
         if args.benchmark:
             import auto_log
             pid = os.getpid()
-            self.autolog = auto_log.AutoLogger(
-                model_name="ernie-tiny",
-                model_precision=precision,
-                batch_size=self.batch_size,
-                data_shape="dynamic",
-                save_path=args.save_log_path,
-                inference_config=config,
-                pids=pid,
-                process_name=None,
-                gpu_ids=0,
-                time_keys=[
-                    'preprocess_time', 'inference_time', 'postprocess_time'
-                ],
-                warmup=0,
-                logger=logger)
+            self.autolog = auto_log.AutoLogger(model_name="ernie-tiny",
+                                               model_precision=precision,
+                                               batch_size=self.batch_size,
+                                               data_shape="dynamic",
+                                               save_path=args.save_log_path,
+                                               inference_config=config,
+                                               pids=pid,
+                                               process_name=None,
+                                               gpu_ids=0,
+                                               time_keys=[
+                                                   'preprocess_time',
+                                                   'inference_time',
+                                                   'postprocess_time'
+                                               ],
+                                               warmup=0,
+                                               logger=logger)
 
     def predict(self, data, tokenizer):
         """
@@ -192,8 +198,9 @@ class Predictor(object):
             examples.append((input_ids, segment_ids))
 
         batchify_fn = lambda samples, fn=Tuple(
-            Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input
-            Pad(axis=0, pad_val=tokenizer.pad_token_id),  # segment
+            Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"),  # input
+            Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
+                ),  # segment
         ): fn(samples)
 
         if args.benchmark:
@@ -220,13 +227,11 @@ if __name__ == "__main__":
                           args.batch_size, args.use_tensorrt, args.precision,
                           args.cpu_threads, args.enable_mkldnn)
 
-    tokenizer = ppnlp.transformers.ErnieGramTokenizer.from_pretrained(
-        'ernie-gram-zh')
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
 
-    # test_ds = load_dataset("lcqmc", splits=["test"])
-    input_file = 'sort/test_pairwise.csv'
-
-    test_ds = load_dataset(read_text_pair, data_path=input_file, lazy=False)
+    test_ds = load_dataset(read_text_pair,
+                           data_path=args.input_file,
+                           lazy=False)
 
     data = [{'query': d['query'], 'title': d['title']} for d in test_ds]
 

applications/neural_search/ranking/ernie_matching/deploy/python/rpc_client.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import time
+import numpy as np
+
+from paddle_serving_server.pipeline import PipelineClient
+
+client = PipelineClient()
+client.connect(['127.0.0.1:8089'])
+
+list_data = [{"query": "中西方语言与文化的差异", "title": "第二语言习得的一大障碍就是文化差异。"}]
+feed = {}
+for i, item in enumerate(list_data):
+    feed[str(i)] = str(item)
+
+print(feed)
+start_time = time.time()
+ret = client.predict(feed_dict=feed)
+end_time = time.time()
+print("time to cost :{} seconds".format(end_time - start_time))
+result = np.array(eval(ret.value[0]))
+print(result.shape)
+print(result)

applications/neural_search/ranking/ernie_matching/deploy/python/web_service.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import sys
+import json
+
+from paddle_serving_server.web_service import WebService, Op
+
+
+def convert_example(example, tokenizer, max_seq_length=512):
+
+    query, title = example["query"], example["title"]
+    encoded_inputs = tokenizer(text=query,
+                               text_pair=title,
+                               max_seq_len=max_seq_length)
+
+    input_ids = encoded_inputs["input_ids"]
+    token_type_ids = encoded_inputs["token_type_ids"]
+
+    return input_ids, token_type_ids
+
+
+class ErnieOp(Op):
+
+    def init_op(self):
+        from paddlenlp.transformers import AutoTokenizer
+        self.tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
+
+    def preprocess(self, input_dicts, data_id, log_id):
+        from paddlenlp.data import Stack, Tuple, Pad
+
+        (_, input_dict), = input_dicts.items()
+        print("input dict", input_dict)
+        batch_size = len(input_dict.keys())
+        examples = []
+        for i in range(batch_size):
+            example = json.loads(input_dict[str(i)].replace("\'", "\""))
+            input_ids, segment_ids = convert_example(example, self.tokenizer)
+            examples.append((input_ids, segment_ids))
+        batchify_fn = lambda samples, fn=Tuple(
+            Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"
+                ),  # input
+            Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"
+                ),  # segment
+        ): fn(samples)
+        input_ids, segment_ids = batchify_fn(examples)
+        feed_dict = {}
+        feed_dict['input_ids'] = input_ids
+        feed_dict['token_type_ids'] = segment_ids
+        return feed_dict, False, None, ""
+
+    def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
+        new_dict = {}
+        new_dict["predict"] = str(fetch_dict["predict"].tolist())
+        return new_dict, None, ""
+
+
+class ErnieService(WebService):
+
+    def get_pipeline_response(self, read_op):
+        ernie_op = ErnieOp(name="ernie", input_ops=[read_op])
+        return ernie_op
+
+
+ernie_service = ErnieService(name="ernie")
+ernie_service.prepare_pipeline_config("config_nlp.yml")
+ernie_service.run_service()

applications/neural_search/ranking/ernie_matching/evaluate.py

@@ -22,10 +22,9 @@ import numpy as np
 import paddle
 import paddle.nn.functional as F
 
-import paddlenlp as ppnlp
 from paddlenlp.data import Stack, Tuple, Pad
 from paddlenlp.datasets import load_dataset
-from paddlenlp.transformers import LinearDecayWithWarmup
+from paddlenlp.transformers import LinearDecayWithWarmup, AutoModel, AutoTokenizer
 
 from data import create_dataloader, gen_pair
 from data import convert_pairwise_example as convert_example
@@ -41,12 +40,7 @@ parser.add_argument("--test_file", type=str, required=True, help="The full path
 parser.add_argument("--max_seq_length", default=128, type=int, help="The maximum total input sequence length after tokenization. "
     "Sequences longer than this will be truncated, sequences shorter will be padded.")
 parser.add_argument("--batch_size", default=32, type=int, help="Batch size per GPU/CPU for training.")
-parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
-parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
-parser.add_argument("--epochs", default=3, type=int, help="Total number of training epochs to perform.")
-parser.add_argument("--eval_step", default=200, type=int, help="Step interval for evaluation.")
-parser.add_argument('--save_step', default=10000, type=int, help="Step interval for saving checkpoint.")
-parser.add_argument("--warmup_proportion", default=0.0, type=float, help="Linear warmup proption over the training process.")
+parser.add_argument('--model_name_or_path', default="ernie-3.0-medium-zh", help="The pretrained model used for training")
 parser.add_argument("--init_from_ckpt", type=str, default=None, help="The path of checkpoint to be loaded.")
 parser.add_argument("--seed", type=int, default=1000, help="Random seed for initialization.")
 parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to train model, defaults to gpu.")
@@ -77,8 +71,8 @@ def evaluate(model, metric, data_loader, phase="dev"):
     for idx, batch in enumerate(data_loader):
         input_ids, token_type_ids, labels = batch
 
-        pos_probs = model.predict(
-            input_ids=input_ids, token_type_ids=token_type_ids)
+        pos_probs = model.predict(input_ids=input_ids,
+                                  token_type_ids=token_type_ids)
 
         neg_probs = 1.0 - pos_probs
 
@@ -109,7 +103,7 @@ def read_test(src_path, is_predict=False):
         yield {'query': query, 'title': title, 'label': label}
 
 
-def do_train():
+def main():
     paddle.set_device(args.device)
     rank = paddle.distributed.get_rank()
     if paddle.distributed.get_world_size() > 1:
@@ -120,29 +114,27 @@ def do_train():
     dev_ds = load_dataset(read_test, src_path=args.test_file, lazy=False)
     print(dev_ds[0])
 
-    pretrained_model = ppnlp.transformers.ErnieGramModel.from_pretrained(
-        'ernie-gram-zh')
-    tokenizer = ppnlp.transformers.ErnieGramTokenizer.from_pretrained(
-        'ernie-gram-zh')
+    pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
 
-    trans_func_eval = partial(
-        convert_example,
-        tokenizer=tokenizer,
-        max_seq_length=args.max_seq_length,
-        phase="eval")
+    trans_func_eval = partial(convert_example,
+                              tokenizer=tokenizer,
+                              max_seq_length=args.max_seq_length,
+                              phase="eval")
 
     batchify_fn_eval = lambda samples, fn=Tuple(
-        Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"),  # pair_input
-        Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"),  # pair_segment
+        Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"
+            ),  # pair_input
+        Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"
+            ),  # pair_segment
         Stack(dtype="int64")  # label
     ): [data for data in fn(samples)]
 
-    dev_data_loader = create_dataloader(
-        dev_ds,
-        mode='dev',
-        batch_size=args.batch_size,
-        batchify_fn=batchify_fn_eval,
-        trans_fn=trans_func_eval)
+    dev_data_loader = create_dataloader(dev_ds,
+                                        mode='dev',
+                                        batch_size=args.batch_size,
+                                        batchify_fn=batchify_fn_eval,
+                                        trans_fn=trans_func_eval)
 
     model = PairwiseMatching(pretrained_model, margin=args.margin)
 
@@ -155,4 +147,4 @@ def do_train():
 
 
 if __name__ == "__main__":
-    do_train()
+    main()

applications/neural_search/ranking/ernie_matching/export_model.py

@@ -14,49 +14,44 @@
 
 import argparse
 import os
-from functools import partial
 
 import numpy as np
 import paddle
 import paddle.nn.functional as F
-import paddlenlp as ppnlp
 from paddlenlp.data import Stack, Tuple, Pad
-
+from paddlenlp.transformers import AutoModel, AutoTokenizer
 from model import PairwiseMatching
 
 # yapf: disable
 parser = argparse.ArgumentParser()
 parser.add_argument("--params_path", type=str, required=True, default='./checkpoint/model_900/model_state.pdparams', help="The path to model parameters to be loaded.")
 parser.add_argument("--output_path", type=str, default='./output', help="The path of model parameter in static graph to be saved.")
+parser.add_argument('--model_name_or_path', default="ernie-3.0-medium-zh", help="The pretrained model used for training")
 args = parser.parse_args()
 # yapf: enable
 
 if __name__ == "__main__":
-    # If you want to use ernie1.0 model, plesace uncomment the following code
-    # tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
-    # pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained("ernie-1.0")
-
-    pretrained_model = ppnlp.transformers.ErnieGramModel.from_pretrained(
-        'ernie-gram-zh')
-    tokenizer = ppnlp.transformers.ErnieGramTokenizer.from_pretrained(
-        'ernie-gram-zh')
+    pretrained_model = AutoModel.from_pretrained(args.model_name_or_path)
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
     model = PairwiseMatching(pretrained_model)
 
     if args.params_path and os.path.isfile(args.params_path):
         state_dict = paddle.load(args.params_path)
         model.set_dict(state_dict)
         print("Loaded parameters from %s" % args.params_path)
-
+    else:
+        raise ValueError(
+            "Please set --params_path with correct pretrained model file")
     model.eval()
 
     # Convert to static graph with specific input description
     model = paddle.jit.to_static(
         model,
         input_spec=[
-            paddle.static.InputSpec(
-                shape=[None, None], dtype="int64"),  # input_ids
-            paddle.static.InputSpec(
-                shape=[None, None], dtype="int64")  # segment_ids
+            paddle.static.InputSpec(shape=[None, None],
+                                    dtype="int64"),  # input_ids
+            paddle.static.InputSpec(shape=[None, None],
+                                    dtype="int64")  # segment_ids
         ])
     # Save in static graph model.
     save_path = os.path.join(args.output_path, "inference")