modify ChnSentiCorp description

.copyright.hook

@@ -1,134 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import argparse
-import io
-import re
-import sys
-import os
-import datetime
-
-COPYRIGHT = '''Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.'''
-
-def _generate_copyright(comment_mark):
-    copyright=COPYRIGHT.split(os.linesep)
-    header = copyright[0].rstrip()
-
-    p = re.search('(\d{4})', header).group(0)
-    now = datetime.datetime.now()
-
-    header = header.replace(p,str(now.year))
-
-    ans=[comment_mark + " " + header + os.linesep]
-    for idx, line in enumerate(copyright[1:]):
-        ans.append(comment_mark + " " + line.rstrip() + os.linesep)
-
-    return ans
-
-def _get_comment_mark(path):
-    lang_type=re.compile(r"\.(py|sh)$")
-    if lang_type.search(path) is not None:
-        return "#"
-
-    lang_type=re.compile(r"\.(h|c|hpp|cc|cpp|cu|go|cuh|proto)$")
-    if lang_type.search(path) is not None:
-        return "//"
-
-    return None
-
-
-RE_ENCODE = re.compile(r"^[ \t\v]*#.*?coding[:=]", re.IGNORECASE)
-RE_COPYRIGHT = re.compile(r".*Copyright( \(c\))* \d{4}", re.IGNORECASE)
-RE_SHEBANG = re.compile(r"^[ \t\v]*#[ \t]?\!")
-
-def _check_copyright(path):
-    head=[]
-    try:
-        with open(path, encoding="utf-8") as f:
-            head = [next(f) for x in range(4)]
-    except StopIteration:
-        pass
-
-    for idx, line in enumerate(head):
-        if RE_COPYRIGHT.search(line) is not None:
-            return True
-
-    return False
-
-def generate_copyright(path, comment_mark):
-    original_contents = io.open(path, encoding="utf-8").readlines()
-    head = original_contents[0:4]
-
-    insert_line_no=0
-    for i, line in enumerate(head):
-        if RE_ENCODE.search(line) or RE_SHEBANG.search(line):
-            insert_line_no=i+1
-
-    copyright = _generate_copyright(comment_mark)
-    if insert_line_no == 0:
-        new_contents = copyright
-        if len(original_contents) > 0 and len(original_contents[0].strip()) != 0:
-            new_contents.append(os.linesep)
-        new_contents.extend(original_contents)
-    else:
-        new_contents=original_contents[0:insert_line_no]
-        new_contents.append(os.linesep)
-        new_contents.extend(copyright)
-        if len(original_contents) > insert_line_no and len(original_contents[insert_line_no].strip()) != 0:
-            new_contents.append(os.linesep)
-        new_contents.extend(original_contents[insert_line_no:])
-    new_contents="".join(new_contents)
-
-    with io.open(path, 'w') as output_file:
-        output_file.write(new_contents)
-
-
-
-def main(argv=None):
-    parser = argparse.ArgumentParser(
-        description='Checker for copyright declaration.')
-    parser.add_argument('filenames', nargs='*', help='Filenames to check')
-    args = parser.parse_args(argv)
-
-    retv = 0
-    for path in args.filenames:
-        comment_mark = _get_comment_mark(path)
-        if comment_mark is None:
-            print("warning:Unsupported file", path, file=sys.stderr)
-            continue
-
-        if _check_copyright(path):
-            continue
-
-        generate_copyright(path, comment_mark)
-
-
-if __name__ == '__main__':
-    exit(main())

.flake8

@@ -1,7 +0,0 @@
-[flake8]
-ignore = E203, E402, E501, E731, E741, W503, W605, E722
-max-line-length = 119
-
-# E402: module level import not at top of file
-per-file-ignores =
-    __init__.py:F401,F403,E402

.github/CODE_OF_CONDUCT.md

@@ -1,89 +0,0 @@
-**简体中文**🀄 | [English🌎](./CODE_OF_CONDUCT_en.md)
-
-# 贡献者公约
-
-## 我们的承诺
-
-身为社区成员、贡献者和领袖，我们承诺使社区参与者不受骚扰，无论其年龄、体型、可见或不可见的缺陷、族裔、性征、性别认同和表达、经验水平、教育程度、社会与经济地位、国籍、相貌、种族、种姓、肤色、宗教信仰、性倾向或性取向如何。
-
-我们承诺以有助于建立开放、友善、多样化、包容、健康社区的方式行事和互动。
-
-## 我们的准则
-
-有助于为我们的社区创造积极环境的行为例子包括但不限于：
-
-* 表现出对他人的同情和善意
-* 尊重不同的主张、观点和感受
-* 提出和大方接受建设性意见
-* 承担责任并向受我们错误影响的人道歉
-* 注重社区共同诉求，而非个人得失
-
-不当行为例子包括：
-
-* 使用情色化的语言或图像，及性引诱或挑逗
-* 嘲弄、侮辱或诋毁性评论，以及人身或政治攻击
-* 公开或私下的骚扰行为
-* 未经他人明确许可，公布他人的私人信息，如物理或电子邮件地址
-* 其他有理由认定为违反职业操守的不当行为
-
-## 责任和权力
-
-社区领袖有责任解释和落实我们所认可的行为准则，并妥善公正地对他们认为不当、威胁、冒犯或有害的任何行为采取纠正措施。
-
-社区领导有权力和责任删除、编辑或拒绝或拒绝与本行为准则不相符的评论（comment）、提交（commits）、代码、维基（wiki）编辑、议题（issues）或其他贡献，并在适当时机知采取措施的理由。
-
-## 适用范围
-
-本行为准则适用于所有社区场合，也适用于在公共场所代表社区时的个人。
-
-代表社区的情形包括使用官方电子邮件地址、通过官方社交媒体帐户发帖或在线上或线下活动中担任指定代表。
-
-## 监督
-
-辱骂、骚扰或其他不可接受的行为可通过 paddlenlp@baidu.com 向负责监督的社区领袖报告。
-所有投诉都将得到及时和公平的审查和调查。
-
-所有社区领袖都有义务尊重任何事件报告者的隐私和安全。
-
-## 处理方针
-
-社区领袖将遵循下列社区处理方针来明确他们所认定违反本行为准则的行为的处理方式：
-
-### 1. 纠正
-
-**社区影响**：使用不恰当的语言或其他在社区中被认定为不符合职业道德或不受欢迎的行为。
-
-**处理意见**：由社区领袖发出非公开的书面警告，明确说明违规行为的性质，并解释举止如何不妥。或将要求公开道歉。
-
-### 2. 警告
-
-**社区影响**：单个或一系列违规行为。
-
-**处理意见**：警告并对连续性行为进行处理。在指定时间内，不得与相关人员互动，包括主动与行为准则执行者互动。这包括避免在社区场所和外部渠道中的互动。违反这些条款可能会导致临时或永久封禁。
-
-### 3. 临时封禁
-
-**社区影响**: 严重违反社区准则，包括持续的不当行为。
-
-**处理意见**: 在指定时间内，暂时禁止与社区进行任何形式的互动或公开交流。在此期间，不得与相关人员进行公开或私下互动，包括主动与行为准则执行者互动。违反这些条款可能会导致永久封禁。
-
-### 4. 永久封禁
-
-**社区影响**：行为模式表现出违反社区准则，包括持续的不当行为、骚扰个人或攻击或贬低某个类别的个体。
-
-**处理意见**：永久禁止在社区内进行任何形式的公开互动。
-
-## 参见
-
-本行为准则改编自 [Contributor Covenant][homepage] 2.1 版, 参见 [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]。
-
-社区处理方针灵感来源于 [Mozilla's code of conduct enforcement ladder][Mozilla CoC]。
-
-有关本行为准则的常见问题的答案，参见 [https://www.contributor-covenant.org/faq][FAQ]。
-其他语言翻译参见 [https://www.contributor-covenant.org/translations][translations]。
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations

.github/CODE_OF_CONDUCT_en.md

@@ -1,134 +0,0 @@
-[简体中文🀄](./CODE_OF_CONDUCT.md) |  **English**🌎
-
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at
-paddlenlp@baidu.com.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations

.github/CONTRIBUTING_en.md

@@ -1,154 +0,0 @@
-[简体中文🀄](../CONTRIBUTING.md) |  **English**🌎
-
-# Contributing to PaddleNLP
-
-We highly welcome and value your contributions to `PaddleNLP`. The first step to start your contribution is to sign the [PaddlePaddle Contributor License Agreement](https://cla-assistant.io/PaddlePaddle/PaddleNLP).
-
-This document explains our workflow and work style:
-
-## Finding out what to work on Workflow
-
-## Development Workflow
-
-PaddleNLP uses the [Git branching model](http://nvie.com/posts/a-successful-git-branching-model/).  The following steps guide usual contributions.
-
-#### 1. Fork
-
-   Our development community has been growing fastly; it doesn't make sense for everyone to write into the official repo.  So, please file Pull Requests from your fork.  To make a fork,  just head over to the GitHub page and click the ["Fork" button](https://help.github.com/articles/fork-a-repo/).
-
-#### 2. Clone
-
-   To make a copy of your fork to your local computers, please run
-
-   ```bash
-   git clone https://github.com/<your-github-account>/PaddleNLP
-   cd PaddleNLP
-   ```
-
-#### 3. Create the local feature branch
-
-   For daily works like adding a new feature or fixing a bug, please open your feature branch before coding:
-
-   ```bash
-   git checkout -b my-cool-feature
-   ```
-
-#### 4. Set up the development environment
-
-   Before you start coding, you need to setup the development environment. We highly recommend doing all your development in a virtual environment such as
-   [venv](https://docs.python.org/3/library/venv.html) or [conda](https://docs.conda.io/en/latest/). After you setup and activated your virtual environment,
-   run the following command:
-
-   ```bash
-   make install
-   ```
-
-   This will setup all the dependencies of `PaddleNLP` as well as the [`pre-commit`](http://pre-commit.com/) tool.
-
-   If you are working on the `examples` or `applications` module and require importing from `PaddleNLP`, make sure you install `PaddleNLP` in editable mode.
-   If `PaddleNLP` is already installed in the virtual environment, remove it with `pip uninstall paddlenlp` before reinstalling it in editable mode with
-   `pip install -e .`
-
-#### 5. Develop
-
-   As you develop your new exciting feature, keep in mind that it should be covered by unit tests. All of our unit tests can be found under the `tests` directory.
-   You can either modify existing unit test to cover the new feature, or create a new test from scratch.
-   As you finish up the your code, you should make sure the test suite passes. You can run the tests impacted by your changes like this:
-
-   ```bash
-   pytest tests/<test_to_run>.py
-   ```
-
-#### 6. Commit
-
-   We utilizes [`pre-commit`](http://pre-commit.com/) (with [black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/) and
-   [flake8](https://flake8.pycqa.org/en/latest/) under the hood) to check the style of code and documentation in every commit. When you run run `git commit`, you will see
-   something like the following:
-
-   ```
-    ➜  (my-virtual-env) git commit -m "commiting my cool feature"
-    black....................................................................Passed
-    isort....................................................................Passed
-    flake8...................................................................Passed
-    check for merge conflicts................................................Passed
-    check for broken symlinks............................(no files to check)Skipped
-    detect private key.......................................................Passed
-    fix end of files.....................................(no files to check)Skipped
-    trim trailing whitespace.............................(no files to check)Skipped
-    CRLF end-lines checker...............................(no files to check)Skipped
-    CRLF end-lines remover...............................(no files to check)Skipped
-    No-tabs checker......................................(no files to check)Skipped
-    Tabs remover.........................................(no files to check)Skipped
-    copyright_checker........................................................Passed
-   ```
-
-   But most of the time things don't go so smoothly. When your code or documentation doesn't meet the standard, the `pre-commit` check will fail.
-   ```
-    ➜  (my-virtual-env) git commit -m "commiting my cool feature"
-    black....................................................................Passed
-    isort....................................................................Failed
-    - hook id: isort
-    - files were modified by this hook
-
-    Fixing examples/information_extraction/waybill_ie/run_ernie_crf.py
-
-    flake8...................................................................Passed
-    check for merge conflicts................................................Passed
-    check for broken symlinks............................(no files to check)Skipped
-    detect private key.......................................................Passed
-    fix end of files.....................................(no files to check)Skipped
-    trim trailing whitespace.............................(no files to check)Skipped
-    CRLF end-lines checker...............................(no files to check)Skipped
-    CRLF end-lines remover...............................(no files to check)Skipped
-    No-tabs checker......................................(no files to check)Skipped
-    Tabs remover.........................................(no files to check)Skipped
-    copyright_checker........................................................Passed
-   ```
-
-   But **don't panic**!
-   Our tooling will fix most of the style errors automatically. Some errors will need to be addressed manually. Fortunately, the error messages are straight forward and
-   the errors are usually simple to fix. After addressing the errors, you can run `git add <files>` and `git commit` again, which will trigger `pre-commit` again.
-   Once the `pre-commit` checks pass, you are ready to push the code.
-
-   [Google][http://google.com/] or [StackOverflow](https://stackoverflow.com/) are great tools to help you understand the code style errors.
-   Don't worry if you still can't figure it out. You can commit with `git commit -m "style error" --no-verify` and we are happy to help you once you create a Pull Request.
-
-#### 7. Keep pulling
-
-   An experienced Git user pulls from the official repo often -- daily or even hourly, so they notice conflicts with others work early, and it's easier to resolve smaller conflicts.
-
-   ```bash
-   git remote add upstream https://github.com/PaddlePaddle/PaddleNLP
-   git pull upstream develop
-   ```
-
-#### 8. Push and file a pull request
-
-   You can "push" your local work into your forked repo:
-
-   ```bash
-   git push origin my-cool-stuff
-   ```
-
-   The push allows you to create a pull request, requesting owners of this [official repo](https://github.com/PaddlePaddle/PaddleNLP) to pull your change into the official one.
-
-   To create a pull request, please follow [these steps](https://help.github.com/articles/creating-a-pull-request/).
-
-#### 9. Delete local and remote branches
-
-   To keep your local workspace and your fork clean, you might want to remove merged branches:
-
-   ```bash
-   git push origin my-cool-stuff
-   git checkout develop
-   git pull upstream develop
-   git branch -d my-cool-stuff
-   ```
-
-## Code Review
-
--  Please feel free to ping your reviewers by @-mentioning the in the Pull Request.  Please do this after your pull request passes the CI.
-
-- Please answer reviewers' every comment.  If you are to follow the comment, please write "Done"; Otherwise, please start a discussion under the comment.
-
-- If you don't want your reviewers to get overwhelmed by email notifications, you might reply their comments by [in a batch](https://help.github.com/articles/reviewing-proposed-changes-in-a-pull-request/).

.github/ISSUE_TEMPLATE/----.md

@@ -0,0 +1,15 @@
+---
+name: 问题反馈
+about: PaddleNLP问题反馈
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+欢迎您反馈PaddleNLP使用问题，非常感谢您对PaddleNLP的贡献！
+在留下您的问题时，辛苦您同步提供如下信息：
+- 版本、环境信息
+1）PaddleNLP和PaddlePaddle版本：请提供您的PaddleNLP和PaddlePaddle版本号，例如PaddleNLP 2.0.4，PaddlePaddle2.1.1
+2）系统环境：请您描述系统类型，例如Linux/Windows/MacOS/，python版本
+- 复现信息：如为报错，请给出复现环境、复现步骤

.github/ISSUE_TEMPLATE/ask-question.yml

@@ -1,23 +0,0 @@
-name: 🐛 Ask Question
-description: 请描述您使用PaddleNLP时遇到的问题
-title: "[Question]: "
-labels: 
-  - question
-body:
-- type: markdown
-  attributes:
-    value: >
-      #### 你可以在这里提出一个使用/咨询问题，提问之前请确保：
-      
-      - 1）已经百度/谷歌搜索过你的问题，但是没有找到解答；
-      
-      - 2）已经在官网查询过[API文档](https://www.paddlepaddle.org.cn/documentation/docs/zh/api/index_cn.html)与[FAQ](https://www.paddlepaddle.org.cn/documentation/docs/zh/faq/index_cn.html)，但是没有找到解答；
-      
-      - 3）已经在[历史issue](https://github.com/PaddlePaddle/Paddle/issues)中搜索过，没有找到同类issue或issue未被解答。
-      
-- type: textarea
-  id: question
-  attributes:
-    label: 请提出你的问题
-  validations:
-    required: true

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,45 +0,0 @@
-name: 🐛 Bug Report
-description: PaddleNLP问题反馈
-title: "[Bug]: "
-labels: bug
-body: 
-  - type: textarea
-    id: environment
-    attributes:
-      label: 软件环境
-      description: |
-        请使用以下命令给出您本地Paddle相关包信息
-          ```sh
-          pip list | grep paddle
-          
-          ```
-      value: |
-          - paddlepaddle:
-          - paddlepaddle-gpu: 
-          - paddlenlp: 
-      render: Markdown
-    validations:
-      required: true
-  - type: checkboxes
-    id: dumplicated-problem
-    attributes:
-      label: 重复问题
-      description: 是否已在issues中搜索相关问题
-      options:
-      - label: I have searched the existing issues
-        required: true
-  - type: textarea
-    id: descripton
-    attributes:
-      label: 错误描述
-      description: 给出错误详细描述，以便能够更好的追踪相关问题
-      render: Markdown
-    validations:
-      required: true
-  - type: textarea
-    id: mvp-code
-    attributes:
-      label: 稳定复现步骤 & 代码
-      description: 请给出稳定复现该问题的步骤 & 代码，以便相关人员能够快速定位到具体问题。
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/docs-report.yml

@@ -1,32 +0,0 @@
-name: 🐛 Docs Report
-description: PaddleNLP文档反馈
-title: "[Docs]: "
-labels: 
-  - documentation
-
-body: 
-  - type: textarea
-    id: environment
-    attributes:
-      label: 软件环境
-      description: |
-        请使用以下命令给出您本地Paddle相关包信息
-          ```sh
-          pip list | grep paddle
-          
-          ```
-      value: |
-          - paddlepaddle:
-          - paddlepaddle-gpu: 
-          - paddlenlp: 
-      render: Markdown
-    validations:
-      required: true
-  - type: textarea
-    id: description
-    attributes:
-      label: 详细描述
-      description: 请详细描述您想要反馈的具体问题
-      render: Markdown
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,30 +0,0 @@
-name: "\U0001F680 Feature request"
-description: 请详细描述您所需功能
-labels: [ "feature" ]
-body:
-  - type: textarea
-    id: feature-request
-    validations:
-      required: true
-    attributes:
-      label: Feature request
-      description: |
-        对特性提案的清晰而简明的描述。如果论文和代码存在，请提供链接。 
-
-  - type: textarea
-    id: motivation
-    validations:
-      required: true
-    attributes:
-      label: Motivation
-      description: |
-        请概述这项建议的动机。您的特性要求与问题有关吗?
-        
-  - type: textarea
-    id: contribution
-    validations:
-      required: true
-    attributes:
-      label: Your contribution
-      description: |
-        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md)

.github/ISSUE_TEMPLATE/new-model.yaml

@@ -1,28 +0,0 @@
-name: "\U0001F31F 添加新模型"
-description: 请为新模型提交一份说明
-labels: [ "New model" ]
-
-body:
-  - type: textarea
-    id: description-request
-    validations:
-      required: true
-    attributes:
-      label: 简要描述
-      description: |
-        请简要描述模型的类型、解决的问题等。
-
-  - type: checkboxes
-    id: information-tasks
-    attributes:
-      label: 是否已开源
-      options:
-        - label: 已开源
-        - label: 未开源
-
-  - type: textarea
-    id: additional-info
-    attributes:
-      label: 模型详细信息
-      description: |
-        请给出新模型相关信息，如论文地址、现存代码地址等。

.github/ISSUE_TEMPLATE/others.yml

@@ -1,23 +0,0 @@
-name: 🧩 其他 Others
-description: 提出其他问题。
-labels: [others]
-
-body:
-- type: markdown
-  attributes:
-    value: >
-      #### 你可以在这里提出任何前面几类模板不适用的问题，包括但不限于：优化性建议、框架使用体验反馈、版本兼容性问题、报错信息不清楚等。
-
-- type: textarea
-  id: others
-  attributes:
-    label: 问题描述
-  validations:
-    required: true
-    
-- type: markdown
-  attributes:
-    value: >
-      感谢你的贡献 🎉！
-
-

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,18 +1,4 @@
 <!-- Demo: https://github.com/PaddlePaddle/PaddleNLP/pull/26 -->
-#### Before submitting
-
-- [ ] Lint code. If there are lint issues, please format the code first.
-
-```shell
-# Install and register `pre-commit` in the project folder
-pip install pre-commit && pre-commit install
-
-# Process previous code files separately
-pre-commit run --file XXXX.py
-```
-
-- [ ] Add test cases into `tests` folder. If there are codecov issues, please add tests cases first.
-
 ### PR types
 <!-- One of [ New features | Bug fixes | Function optimization | Performance optimization | Breaking changes | Others ] -->
 

.github/actions/check-bypass/action.yml

@@ -1,41 +0,0 @@
-name: "Check bypass"
-description: "A custom action to encapsulate PFCCLab/ci-bypass"
-inputs:
-  github-token:
-    description: "GitHub token"
-    required: true
-  workflow-name:
-    description: "Workflow name"
-    required: true
-outputs:
-  can-skip:
-    description: "Whether the workflow can be skipped."
-    value: ${{ steps.check-bypass.outputs.can-skip }}
-
-runs:
-  using: "composite"
-  steps:
-    - id: check-bypass
-      name: Check Bypass
-      env:
-        CI_TEAM_MEMBERS: '["tianshuo78520a", "swgu98", "risemeup1", "XieYunshen","luotao1","From00"]'
-      uses: PFCCLab/ci-bypass@v1
-      with:
-        github-token: ${{ inputs.github-token }}
-        non-pull-request-event-strategy: 'never-skipped'
-        type: 'composite'
-        composite-rule: |
-          {
-            "any": [
-              {
-                "type": "labeled",
-                "label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
-                "username": ${{ env.CI_TEAM_MEMBERS }}
-              },
-              {
-                "type": "commented",
-                "comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
-                "username": ${{ env.CI_TEAM_MEMBERS }}
-              }
-            ]
-          }

.github/actions/rerun-workflow/action.yml

@@ -1,30 +0,0 @@
-name: 'Rerun Workflow'
-description: 'Re-run GitHub Actions workflow for a given Pull Request'
-inputs:
-  GITHUB_TOKEN:
-    description: 'GitHub token with repo scope'
-    required: true
-  OWNER:
-    description: 'Repository owner'
-    required: true
-  REPO:
-    description: 'Repository name'
-    required: true
-  PR_ID:
-    description: 'Pull Request ID'
-    required: true
-  JOB_NAME:
-    description: 'Job name to rerun'
-    required: true
-
-runs:
-  using: 'composite'
-  steps:
-    - run: bash ./.github/actions/rerun-workflow/rerun.sh
-      shell: bash
-      env:
-        GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }}
-        OWNER: ${{ inputs.OWNER }}
-        REPO: ${{ inputs.REPO }}
-        PR_ID: ${{ inputs.PR_ID }}
-        JOB_NAME: ${{ inputs.JOB_NAME }}

.github/actions/rerun-workflow/rerun.sh

@@ -1,77 +0,0 @@
-# Copyright (c) 2025 PaddleNLP Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-COMMIT_SHA=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
-  "https://api.github.com/repos/$OWNER/$REPO/pulls/$PR_ID" | jq -r '.head.sha')
-
-echo "Commit SHA: $COMMIT_SHA"
-
-response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
-  "https://api.github.com/repos/$OWNER/$REPO/actions/runs?head_sha=$COMMIT_SHA&per_page=100")
-
-echo "Response: $response"
-
-run_ids=$(echo "$response" | jq -r '.workflow_runs[].id')
-
-if [ -n "$run_ids" ]; then
-  echo "Found run_ids for commit $COMMIT_SHA: $run_ids"
-
-  for run_id in $run_ids; do
-    if [ "$JOB_NAME" = "all-failed" ]; then
-      echo "Rerunning all failed jobs for run_id: $run_id"
-
-      rerun_response=$(curl -X POST -s -w "%{http_code}" -o /dev/null \
-        -H "Accept: application/vnd.github.v3+json" \
-        -H "Authorization: Bearer $GITHUB_TOKEN" \
-        "https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/rerun-failed-jobs")
-      if [ "$rerun_response" -eq 201 ]; then
-        echo "Successfully requested rerun for all blocked jobs in run_id: $run_id"
-      else
-        echo "Failed to request rerun for run_id: $run_id with status code $rerun_response"
-      fi
-
-    else
-      jobs_response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
-      "https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/jobs")
-
-      echo "Jobs Response for run_id $run_id: $jobs_response"
-
-      # if [[ "$JOB_NAME" == *"bypass"* ]]; then
-        block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
-        '.jobs[] | select(.name == $job_name) | .id')
-      # else
-      #   block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
-      #   '.jobs[] | select(.name == $job_name and .conclusion != "success") | .id')
-      # fi
-
-      if [ -n "$block_jobs" ]; then
-        echo "Found block jobs for run_id $run_id: $block_jobs"
-
-        for job_id in $block_jobs; do
-          echo "Rerunning job_id: $job_id"
-          curl -X POST -H "Accept: application/vnd.github.v3+json" \
-            -H "Authorization: token $GITHUB_TOKEN" \
-            "https://api.github.com/repos/$OWNER/$REPO/actions/jobs/$job_id/rerun"
-        done
-      else
-        echo "No block jobs found for run_id $run_id with name $JOB_NAME."
-      fi
-    fi
-  done
-else
-  echo "No matching workflow runs found for commit $COMMIT_SHA."
-  exit 1
-fi

.github/codecov.yml

@@ -1,13 +0,0 @@
-codecov:
-  notify:
-    wait_for_ci: false
-
-coverage:
-  status:
-    project:
-      default: 
-        target: 58% # overall project Coverage 
-        threshold: 1% # Allow the coverage to drop by 1%, and posting a success status.
-    patch:
-      default: 
-        target: 80% # lines adjusted  Coverage < 80%  CI will fail

.github/workflows/PaddleFormers-PR-Check.yml

@@ -1,47 +0,0 @@
-name: PaddleFormers PR Check
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-
-env:
-  BRANCH: ${{ github.base_ref }}
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  token: ${{ vars.ACTION_GITHUB_TOKEN }}
-
-jobs:
-  check-approvers:
-    name: Check approval
-    runs-on: 
-      group: APPROVAL
-    steps:
-      - name: Cleanup
-        run: |
-          rm -rf * .[^.]*
-      - name: Update paddle
-        run: |
-          wget -q --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleNLP/PaddleNLP-develop.tar.gz --no-check-certificate
-          tar zxf PaddleNLP-develop.tar.gz --strip-components=1 >/dev/null
-          rm -rf PaddleNLP-develop.tar.gz >/dev/null
-          git fetch origin pull/${PR_ID}/head
-          git checkout -b origin_pr FETCH_HEAD
-          git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-          git fetch upstream
-          git checkout -b test_pr upstream/${BRANCH}
-          git merge --no-edit origin_pr
-          git log --pretty=oneline -10
-
-      - name: Check bypass
-        id: check-bypass
-        uses: ./.github/actions/check-bypass
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          workflow-name: approval
-        
-      - name: Display Required Approvers
-        if: steps.check-bypass.outputs.can-skip != 'true'
-        run: |
-          cd scripts/ci_approval
-          bash run_ci_approval_cherry-pick.sh

.github/workflows/approval.yml

@@ -1,47 +0,0 @@
-name: Approval
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-
-env:
-  BRANCH: ${{ github.base_ref }}
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  token: ${{ vars.ACTION_GITHUB_TOKEN }}
-
-jobs:
-  check-approvers:
-    name: Check approval
-    runs-on: 
-      group: APPROVAL
-    steps:
-      - name: Cleanup
-        run: |
-          rm -rf * .[^.]*
-      - name: Update paddle
-        run: |
-          wget -q --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleNLP/PaddleNLP-develop.tar.gz --no-check-certificate
-          tar zxf PaddleNLP-develop.tar.gz --strip-components=1 >/dev/null
-          rm -rf PaddleNLP-develop.tar.gz >/dev/null
-          git fetch origin pull/${PR_ID}/head
-          git checkout -b origin_pr FETCH_HEAD
-          git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-          git fetch upstream
-          git checkout -b test_pr upstream/${BRANCH}
-          git merge --no-edit origin_pr
-          git log --pretty=oneline -10
-
-      - name: Check bypass
-        id: check-bypass
-        uses: ./.github/actions/check-bypass
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          workflow-name: approval
-        
-      - name: Display Required Approvers
-        if: steps.check-bypass.outputs.can-skip != 'true'
-        run: |
-          cd scripts/ci_approval
-          bash -x run_ci_approval.sh

.github/workflows/check-bypass.yml

@@ -1,52 +0,0 @@
-on:
-  workflow_call:
-    inputs:
-      workflow-name:
-        required: true
-        type: string
-    secrets:
-      github-token:
-        required: true
-    outputs:
-      can-skip:
-        description: "Whether the workflow can be skipped."
-        value: ${{ jobs.check-bypass.outputs.can-skip }}
-
-jobs:
-  check-bypass:
-    name: Check bypass
-    runs-on:
-      group: APPROVAL
-    permissions:
-      contents: read
-    env:
-      CI_TEAM_MEMBERS: '["tianshuo78520a", "swgu98", "risemeup1" , "XieYunshen","luotao1","From00"]'
-    outputs:
-      can-skip: ${{ steps.check-bypass.outputs.can-skip }}
-    steps:
-      - name: Cleanup
-        run: |
-          rm -rf * .[^.]*
-
-      - id: check-bypass
-        name: Check Bypass
-        uses: PFCCLab/ci-bypass@v1
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          non-pull-request-event-strategy: 'never-skipped'
-          type: 'composite'
-          composite-rule: |
-            {
-              "any": [
-                {
-                  "type": "labeled",
-                  "label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
-                  "username": ${{ env.CI_TEAM_MEMBERS }}
-                },
-                {
-                  "type": "commented",
-                  "comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
-                  "username": ${{ env.CI_TEAM_MEMBERS }}
-                }
-              ]
-            }

.github/workflows/lint.yml

@@ -1,113 +0,0 @@
-name: Codestyle Check
-
-on: [push, pull_request]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-env:
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  BRANCH: ${{ github.event.pull_request.base.ref }}
-  TASK: PaddleNLP-CI-Lint-${{ github.event.pull_request.number }}
-
-jobs:
-  check-bypass:
-    name: Check bypass
-    uses: ./.github/workflows/check-bypass.yml
-    with:
-      workflow-name: 'lint'
-    secrets:
-      github-token: ${{ secrets.GITHUB_TOKEN }}
-
-  Lint:
-    name: Lint
-    needs: check-bypass
-    if: ${{ needs.check-bypass.outputs.can-skip != 'true' }}
-    runs-on: [self-hosted, ernie-cpu]
-    steps:
-      - name: Run Container
-        env:
-          work_dir: ${{ github.workspace }}
-          python_version: "3.10"
-        run: |
-          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
-          echo "container_name=${container_name}" >> ${{ github.env }}
-          docker_image="iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu20.04-gcc12.2-cuda12.3-cudnn9.0-nccl2.20.3.1-openmpi4.1.5-latest"
-          docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
-            -v $work_dir/../../..:$work_dir/../../.. \
-            -v $work_dir:/workspace \
-            -v /home/.cache/pip:/home/.cache/pip \
-            -e BRANCH \
-            -e PR_ID \
-            -e COMMIT_ID \
-            -e work_dir \
-            -e no_proxy \
-            -e python_version \
-            -w /workspace ${docker_image}
-
-      - name: Download Code
-        env:
-          work_dir: ${{ github.workspace }}
-        run: |
-          docker exec -t ${container_name} /bin/bash -c '
-          rm -rf * .[^.]*
-          echo "Downloading PaddleNLP.tar"
-          wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
-          echo "Extracting PaddleNLP.tar"
-          tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
-          source $work_dir/../../../proxy
-          cd PaddleNLP
-          git config --global user.name "PaddleCI"
-          git config --global user.email "paddle_ci@example.com"
-          git pull
-          git submodule update --init --recursive --force
-          if [ -n "${PR_ID}" ]; then
-            git fetch origin pull/${PR_ID}/head
-            git checkout -b PR_${PR_ID} FETCH_HEAD
-            git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-            git fetch upstream ${BRANCH}
-            git merge ${BRANCH} --no-edit
-            git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
-          else
-            echo "Not in a pull_request event. Skipping PR-specific operations."
-          fi
-          git log --pretty=oneline -10
-
-          if ! git show-ref --quiet refs/heads/develop; then \
-              echo "local develop branch is missing, creating local develop branch that tracks remote develop branch"
-              git fetch origin develop
-              git branch develop --track origin/develop
-          else
-            echo "local develop branch exist, skipping"
-          fi
-          '
-          
-      - name: Setup Environment
-        run: |
-          docker exec -t $container_name /bin/bash -c '
-          unlink /usr/local/bin/python
-          ln -sf $(which python${python_version}) /usr/local/bin/python
-          set -e
-          python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-          python -m pip config set global.cache-dir "/home/.cache/pip"
-          source $work_dir/../../../proxy
-          python -m pip install --upgrade pip
-          cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
-          make install
-          '
-
-      - name: Test
-        run: |
-          docker exec -t $container_name /bin/bash -c '
-          set -e
-          source $work_dir/../../../proxy
-          cd /workspace/PaddleNLP
-          make lint
-          '
-      
-      - name: Terminate And Delete the Container
-        if: always()
-        run: |
-          docker rm -f $container_name 2>/dev/null || true

.github/workflows/llm.yml

@@ -1,207 +0,0 @@
-# Disabled LLM CI workflow - manually enabled when needed
-name: LLM CI (DISABLED)
-
-#on:
-#  pull_request:
-#    types: [opened, synchronize, reopened]
-#    branches: [develop]
-#    schedule:
-#    - cron: "2 0 * * *"
-#  workflow_call:
-#    inputs:
-#      run_downstream:
-#        required: true
-#        type: string
-#      image_name:
-#        required: true
-#        type: string
-
-#concurrency:
-#  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-#  cancel-in-progress: true
-
-#env:
-#  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  TASK: paddlenlp-CI-${{ github.event.pull_request.number }}-llm
-  ci_scripts: /workspace/PaddleNLP/scripts/regression
-  BRANCH: ${{ github.event.pull_request.base.ref }}
-  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
-  CI_name: llm-ci
-  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
-  HF_ENDPOINT: https://hf-mirror.com
-  STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
-  PPNLP_HOME: /ssd1/paddlenlp
-  HF_DATASETS_CACHE: /ssd1/paddlenlp/huggingface/datasets
-  TRANSFORMERS_CACHE: /ssd1/paddlenlp/huggingface
-  CCACHE_DIR: /home/data/gzcfs/.ccache/gpubox
-  RUN_DOWNSTREAM: ${{ inputs.run_downstream }}
-
-#defaults:
-#  run:
-#    shell: bash
-
-#jobs:
-#  llm-ci:
-#    name: llm-ci (DISABLED)
-    runs-on: [self-hosted, ernie-8gpu]
-    steps:
-      - name: Determine Image Name
-        env:
-          IMAGE_NAME: ${{ inputs.image_name }}
-        run: |
-          if [[ -n "${IMAGE_NAME}" ]]; then
-            echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
-          else
-            echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-paddlenlp-latest" >> "$GITHUB_ENV"
-          fi
-
-      - name: Run Container
-        env:
-          work_dir: ${{ github.workspace }}
-          CACHE_DIR: /home/data/cfs/.cache
-          FLAGS_dynamic_static_unified_comm: "True"
-          python_version: "3.10"
-          paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
-        run: |
-          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
-          echo "container_name=${container_name}" >> "$GITHUB_ENV"
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
-              -v $work_dir/../../..:$work_dir/../../.. \
-              -v $work_dir:/workspace \
-              -v /home/.cache/pip:/home/.cache/pip \
-              -v /ssd1/paddlenlp:/ssd1/paddlenlp \
-              -v /home/data/gzcfs/.ccache/gpubox:/home/data/gzcfs/.ccache/gpubox \
-              -e BRANCH \
-              -e AGILE_COMPILE_BRANCH \
-              -e PR_ID \
-              -e COMMIT_ID \
-              -e work_dir \
-              -e ci_scripts \
-              -e no_proxy \
-              -e CI_name \
-              -e paddle_whl \
-              -e HF_ENDPOINT \
-              -e STUDIO_GIT_HOST \
-              -e PPNLP_HOME \
-              -e HF_DATASETS_CACHE \
-              -e TRANSFORMERS_CACHE \
-              -e CACHE_DIR \
-              -e FLAGS_dynamic_static_unified_comm \
-              -e python_version \
-              -w /workspace --runtime=nvidia $IMAGE_NAME
-          fi
-
-      - name: Download Code
-        env:
-          work_dir: ${{ github.workspace }}
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping.."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            rm -rf * .[^.]*
-            echo "Downloading PaddleNLP.tar.gz"
-            wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
-            echo "Extracting PaddleNLP.tar.gz"
-            tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
-            source $work_dir/../../../proxy
-            cd PaddleNLP
-            git config --global user.name "PaddleCI"
-            git config --global user.email "paddle_ci@example.com"
-            git pull
-            git submodule update --init --recursive --force
-            if [ -n "${PR_ID}" ]; then
-              git fetch origin pull/${PR_ID}/head
-              git checkout -b PR_${PR_ID} FETCH_HEAD
-              git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-              git fetch upstream ${BRANCH}
-              git merge ${BRANCH} --no-edit
-              git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
-            else
-              echo "Not in a pull_request event. Skipping PR-specific operations."
-            fi
-            git log --pretty=oneline -10
-            '
-          fi
-
-      - name: Skip For Bug
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            cd /workspace/PaddleNLP
-            git revert f2477c07272d04244cd3287d1f21c70482a4a85f --no-edit  # 套件PR#10413引入bug-待修复
-            git revert d74c950e15a35b7a100d1688c89318195cc83bca --no-edit  # 框架升级后配合修改，当前因框架bug固定commit适配
-            '
-          fi
-      
-      - name: Test
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            ldconfig
-            unlink /usr/bin/python3
-            ln -sf $(which python${python_version}) /usr/bin/python3
-            pip config set global.cache-dir "/home/.cache/pip"
-            set -e
-            source $work_dir/../../../proxy
-            cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
-            echo "Paddle PR#73283  PR#74484 import bug, set paddle_commit=8ae742 to skip "
-            export paddle_whl=https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/8ae7423e99b2ea96e410968a0ebb3f1795e37205/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
-            timeout 2h bash scripts/regression/run_ci.sh python${python_version} ${paddle_whl}
-            '
-          fi
-
-      - name: Upload Allure-reports & Logs
-        if: always()
-        env:
-          home_path: ${{ github.workspace }}/../../..
-          bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
-          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            unset http_proxy && unset https_proxy
-            if [ ! -f "${{ env.bos_file }}" ]; then
-              wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
-              mkdir ${{ env.home_path }}/bos
-              tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
-            fi
-            if [ ! -f "${{ env.allure_file }}" ]; then
-              wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
-              unzip -q ${{ env.home_path }}/allure-2.19.0.zip -d ${{ env.home_path }}/
-            fi
-            if [[ "${{ env.RUN_DOWNSTREAM }}" == "" && -n "${PR_ID}" ]]; then
-              bos_prefix="${PR_ID}/${COMMIT_ID}"
-            elif [[ "${{ env.RUN_DOWNSTREAM }}" == "true" && -n "${PR_ID}" ]]; then
-              bos_prefix="${PR_ID}/${COMMIT_ID}/test_build"
-            else
-              bos_prefix="schedule/$(date +%Y%m%d)"
-            fi
-            cd /workspace/PaddleNLP/model_logs
-            for FILE in /workspace/PaddleNLP/model_logs/*; do
-              file=$(basename "$FILE")
-              python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleNLP/llm/${bos_prefix}/logs
-              echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/llm/${bos_prefix}/logs/$file"
-            done
-            cd /workspace/PaddleNLP/
-            ${{ env.allure_file }} generate result -o report
-            tar -czf products.tar.gz report model_logs
-            python ${{ env.bos_file }} products.tar.gz paddle-github-action/PR/PaddleNLP/llm/${bos_prefix}/logs
-            echo "products: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/llm/${bos_prefix}/logs/products.tar.gz"
-            '
-          fi
-
-      - name: Terminate And Delete the Container
-        if: always()
-        run: |
-          docker rm -f ${{ env.container_name }} 2>/dev/null || true

.github/workflows/pipelines.yml

@@ -1,30 +0,0 @@
-name: Pipelines-Test
-
-on:
-  push:
-    paths:
-      - 'slm/pipelines/*'
-  pull_request:
-    paths:
-      - 'slm/pipelines/*'
-
-
-jobs:
-  Pipelines-Test:
-    name: Pipelines-Test
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-          cache: 'pip' # caching pip dependencies
-      - name: Install dependencies
-        working-directory: ./slm/pipelines
-        run: |
-          python -m pip install --upgrade pip
-          make install
-          pip install -r tests/requirements.txt
-      - name: run the command
-        working-directory: ./slm/pipelines
-        run: make test

.github/workflows/release.yml

@@ -1,58 +0,0 @@
-name: Release
-
-on: 
-  workflow_dispatch:
-    inputs:
-      package:
-        description: 'Package to be deployed'
-        required: true
-        default: 'paddlenlp'
-        type: choice
-        options:
-        - paddlenlp
-        - paddle-pipelines
-        - ppdiffusers
-
-jobs:
-  Release:
-    name: Release
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
-        with:
-          python-version: '3.10'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Should Deploy
-        id: should-deploy
-        env: 
-          PACKAGE: ${{ inputs.package }}
-        run: python scripts/should_deploy.py  --name $PACKAGE >> $GITHUB_OUTPUT
-
-      - name: Check Branch
-        id: check-branch
-        run: |
-          if [[ ${{ github.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-              echo "match=true" >> $GITHUB_OUTPUT
-          fi  # See: https://stackoverflow.com/a/58869470/1123955
-
-      - name: Is A Publish Branch
-        if: steps.check-branch.outputs.match == 'true' && steps.should-deploy.outputs.should_deploy == 'true'
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.paddlenlp }}
-          PACKAGE: ${{ inputs.package }}
-          PADDLENLP_STABLE_VERSION: "true"
-        run: |
-          make deploy-$PACKAGE
-
-      - name: Should Not Deploy To Pypi Server
-        if: steps.should-deploy.outputs.should_deploy != 'true'
-        run: echo 'should not deploy pypi server'
-
-      - name: Is Not A Publish Branch
-        if: steps.check-branch.outputs.match != 'true'
-        run: echo 'Not A Publish Branch'

.github/workflows/rerun.yml

@@ -1,73 +0,0 @@
-name: Re-run
-
-on:
-  issue_comment:
-    types: [created]
-
-jobs:
-  re-run:
-    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/re-run')  && github.event.comment.user.login == github.event.issue.user.login }}
-    runs-on: [self-hosted, ernie-cpu]
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: Rerun all failed jobs
-        if: ${{ contains(github.event.comment.body, 'all-failed') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'all-failed'
-
-      - name: Rerun LLM
-        if: ${{ contains(github.event.comment.body, 'LLM') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'LLM CI / llm-ci'
-
-      - name: Unittest GPU
-        if: ${{ contains(github.event.comment.body, 'Unittest GPU') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'Unittest GPU CI / unittest-gpu-ci'
-
-      - name: Rerun Unittest CPU
-        if: ${{ contains(github.event.comment.body, 'Unittest CPU') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'Unittest CPU CI / unit-cpu-ci'
-
-      - name: Rerun Distribute (V100)
-        if: ${{ contains(github.event.comment.body, 'Distribute (V100)') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'Distribute CI (V100)/ distribute-v100-ci'
-
-      - name: Rerun Codestyle Check
-        if: ${{ contains(github.event.comment.body, 'Codestyle Check') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'Codestyle Check / Lint'

.github/workflows/stale.yml

@@ -1,35 +0,0 @@
-name: Stale
-
-on:
-  # Allow manual run via GitHub web or CLI
-  workflow_dispatch:
-  schedule:
-    # Run daily at midnight UTC
-    - cron: 0 0 * * *
-
-permissions:
-  issues: write
-  pull-requests: write
-
-jobs:
-  stale:
-    runs-on: ubuntu-24.04
-
-    permissions:
-      issues: write
-      pull-requests: write
-
-    steps:
-      - uses: actions/stale@v6.0.1
-        with:
-          days-before-issue-stale: 60
-          days-before-issue-close: 14
-          stale-issue-label: "stale"
-          stale-issue-message: "This issue is stale because it has been open for 60 days with no activity. 当前issue 60天内无活动，被标记为stale。"
-          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale. 当前issue 被标记为stale已有14天，即将关闭。"
-          exempt-issue-labels: 'triage,keep'
-          days-before-pr-stale: 60
-          days-before-pr-close: -1
-          stale-pr-label: "stale"
-          stale-pr-message: "This Pull Request is stale because it has been open for 60 days with no activity. 当前Pull Request 60天内无活动，被标记为stale。"
-          operations-per-run: 400

.github/workflows/unittest-cpu.yml

@@ -1,120 +0,0 @@
-# Disabled unittest CPU workflow - manually enabled when needed
-name: Unittest CPU CI (DISABLED)
-
-#on: [pull_request]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-env:
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  BRANCH: ${{ github.event.pull_request.base.ref }}
-  TASK: PaddleNLP-CI-${{ github.event.pull_request.number }}-unittest-cpu
-  HF_ENDPOINT: https://hf-mirror.com
-  STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
-  PPNLP_HOME: /home/disk1/cache
-  HF_DATASETS_CACHE: /home/disk1/cache/huggingface/datasets
-  TRANSFORMERS_CACHE: /home/disk1/cache/huggingface
-
-#jobs:
-#  unittest-cpu-ci:
-#    name: unittest-cpu-ci (DISABLED)
-    runs-on: [self-hosted, ernie-cpu]
-    permissions:
-      pull-requests: write
-      contents: read
-      id-token: write
-    steps:
-      - name: Run Container
-        env:
-          work_dir: ${{ github.workspace }}
-          python_version: "3.10"
-        run: |
-          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
-          echo "container_name=${container_name}" >> ${{ github.env }}
-          docker_image="iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu20.04-gcc12.2-cuda12.3-cudnn9.0-nccl2.20.3.1-openmpi4.1.5-latest"
-          docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
-            -v $work_dir/../../..:$work_dir/../../.. \
-            -v $work_dir:/workspace \
-            -v /home/.cache/pip:/home/.cache/pip \
-            -v /home/disk1/cache:/home/disk1/cache \
-            -e BRANCH \
-            -e PR_ID \
-            -e COMMIT_ID \
-            -e work_dir \
-            -e no_proxy \
-            -e python_version \
-            -e HF_ENDPOINT \
-            -e STUDIO_GIT_HOST \
-            -e PPNLP_HOME \
-            -e HF_DATASETS_CACHE \
-            -e TRANSFORMERS_CACHE \
-            -w /workspace ${docker_image}
-
-      - name: Download Code
-        env:
-          work_dir: ${{ github.workspace }}
-        run: |
-          docker exec -t ${container_name} /bin/bash -c '
-          rm -rf * .[^.]*
-          echo "Downloading PaddleNLP.tar"
-          wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
-          echo "Extracting PaddleNLP.tar"
-          tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
-          source $work_dir/../../../proxy
-          cd PaddleNLP
-          git config --global user.name "PaddleCI"
-          git config --global user.email "paddle_ci@example.com"
-          git pull
-          git submodule update --init --recursive --force
-          if [ -n "${PR_ID}" ]; then
-            git fetch origin pull/${PR_ID}/head
-            git checkout -b PR_${PR_ID} FETCH_HEAD
-            git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-            git fetch upstream ${BRANCH}
-            git merge ${BRANCH} --no-edit
-            git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
-          else
-            echo "Not in a pull_request event. Skipping PR-specific operations."
-          fi
-          git log --pretty=oneline -10
-          '
-
-      - name: Setup Environment
-        run: |
-          docker exec -t $container_name /bin/bash -c '
-          unlink /usr/bin/python3
-          ln -sf $(which python${python_version}) /usr/bin/python3
-          set -e
-          python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-          python -m pip config set global.cache-dir "/home/.cache/pip"
-          source $work_dir/../../../proxy
-          python -m pip install --upgrade pip
-          cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
-          pip install -r tests/requirements.txt
-          make install
-          '
-
-      - name: Test
-        run: |
-          docker exec -t $container_name /bin/bash -c '
-          source $work_dir/../../../proxy
-          cd /workspace/PaddleNLP
-          set -e
-          make test
-          '
-
-      # - name: Upload Coverage To Codecov
-      #   uses: codecov/codecov-action@v3
-      #   env:
-      #     CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-      #   with:
-      #     token: ${{ secrets.CODECOV_TOKEN }}
-      #     files: ./PaddleNLP/coverage.xml
-      
-      - name: Terminate And Delete the Container
-        if: always()
-        run: |
-          docker rm -f $container_name 2>/dev/null || true

.github/workflows/unittest-gpu.yml

@@ -1,204 +0,0 @@
-# Disabled unittest GPU workflow - manually enabled when needed
-name: Unittest GPU CI (DISABLED)
-
-#on:
-#  pull_request:
-#    types: [opened, synchronize, reopened]
-#    branches: [develop]
-#  schedule:
-#    - cron: "3 0 * * *"
-#  workflow_call:
-#    inputs:
-#      run_downstream:
-#        required: true
-#        type: string
-#      image_name:
-#        required: true
-#        type: string
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-env:
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  TASK: paddlenlp-CI-${{ github.event.pull_request.number }}-unit-gpu
-  ci_scripts: /workspace/PaddleNLP/scripts/unit_test
-  BRANCH: ${{ github.event.pull_request.base.ref }}
-  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
-  CI_name: unittest-gpu-ci
-  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
-  HF_ENDPOINT: https://hf-mirror.com
-  STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
-  PPNLP_HOME: /ssd1/paddlenlp
-  HF_DATASETS_CACHE: /ssd1/paddlenlp/huggingface/datasets
-  TRANSFORMERS_CACHE: /ssd1/paddlenlp/huggingface
-  CCACHE_DIR: /home/data/gzcfs/.ccache/gpubox
-  RUN_DOWNSTREAM: ${{ inputs.run_downstream }}
-
-defaults:
-  run:
-    shell: bash
-
-#jobs:
-#  unittest-gpu-ci:
-#    name: unittest-gpu-ci (DISABLED)
-    runs-on: [self-hosted, ernie-8gpu]
-    steps:
-      - name: Determine Image Name
-        env:
-          IMAGE_NAME: ${{ inputs.image_name }}
-        run: |
-          if [[ -n "${IMAGE_NAME}" ]]; then
-            echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
-          else
-            echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-paddlenlp-latest" >> "$GITHUB_ENV"
-          fi
-      
-      - name: Run Container
-        env:
-          work_dir: ${{ github.workspace }}
-          CACHE_DIR: /home/data/cfs/.cache
-          FLAGS_dynamic_static_unified_comm: "True"
-          python_version: "3.10"
-          paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
-        run: |
-          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
-          echo "container_name=${container_name}" >> "$GITHUB_ENV"
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
-              -v $work_dir/../../..:$work_dir/../../.. \
-              -v $work_dir:/workspace \
-              -v /home/.cache/pip:/home/.cache/pip \
-              -v /ssd1/paddlenlp:/ssd1/paddlenlp \
-              -v /home/data/gzcfs/.ccache/gpubox:/home/data/gzcfs/.ccache/gpubox \
-              -e BRANCH \
-              -e AGILE_COMPILE_BRANCH \
-              -e PR_ID \
-              -e COMMIT_ID \
-              -e work_dir \
-              -e ci_scripts \
-              -e no_proxy \
-              -e CI_name \
-              -e paddle_whl \
-              -e HF_ENDPOINT \
-              -e STUDIO_GIT_HOST \
-              -e PPNLP_HOME \
-              -e HF_DATASETS_CACHE \
-              -e TRANSFORMERS_CACHE \
-              -e CACHE_DIR \
-              -e FLAGS_dynamic_static_unified_comm \
-              -e python_version \
-              -w /workspace --runtime=nvidia $IMAGE_NAME
-          fi
-          
-      - name: Download Code
-        env:
-          work_dir: ${{ github.workspace }}
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping.."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            rm -rf * .[^.]*
-            echo "Downloading PaddleNLP.tar.gz"
-            wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
-            echo "Extracting PaddleNLP.tar.gz"
-            tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
-            source $work_dir/../../../proxy
-            cd PaddleNLP
-            git config --global user.name "PaddleCI"
-            git config --global user.email "paddle_ci@example.com"
-            git pull
-            git submodule update --init --recursive --force
-            if [ -n "${PR_ID}" ]; then
-              git fetch origin pull/${PR_ID}/head
-              git checkout -b PR_${PR_ID} FETCH_HEAD
-              git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-              git fetch upstream ${BRANCH}
-              git merge ${BRANCH} --no-edit
-              git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
-            else
-              echo "Not in a pull_request event. Skipping PR-specific operations."
-            fi
-            git log --pretty=oneline -10
-            '
-          fi
-      
-      - name: Skip For Bug
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            cd /workspace/PaddleNLP
-            echo "no skip for bug"
-            '
-          fi
-      
-      - name: Test
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            ldconfig
-            unlink /usr/bin/python3
-            ln -sf $(which python${python_version}) /usr/bin/python3
-            pip config set global.cache-dir "/home/.cache/pip"
-            source $work_dir/../../../proxy
-            set -e
-            cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
-            timeout 50m bash scripts/unit_test/ci_unit.sh ${paddle_whl}
-            '
-          fi
-
-      - name: Upload Allure-reports & Logs
-        if: always()
-        env:
-          home_path: ${{ github.workspace }}/../../..
-          bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
-          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            unset http_proxy && unset https_proxy
-            if [ ! -f "${{ env.bos_file }}" ]; then
-              wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
-              mkdir ${{ env.home_path }}/bos
-              tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
-            fi
-            if [ ! -f "${{ env.allure_file }}" ]; then
-              wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
-              unzip -q ${{ env.home_path }}/allure-2.19.0.zip -d ${{ env.home_path }}/
-            fi
-            if [[ "${{ env.RUN_DOWNSTREAM }}" == "" && -n "${PR_ID}" ]]; then
-              bos_prefix="${PR_ID}/${COMMIT_ID}"
-            elif [[ "${{ env.RUN_DOWNSTREAM }}" == "true" && -n "${PR_ID}" ]]; then
-              bos_prefix="${PR_ID}/${COMMIT_ID}/test_build"
-            else
-              bos_prefix="schedule/$(date +%Y%m%d)"
-            fi
-            cd /workspace/PaddleNLP/unittest_logs
-            for FILE in /workspace/PaddleNLP/unittest_logs/*; do
-              file=$(basename "$FILE")
-              python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs
-              echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs/$file"
-            done
-            cd /workspace/PaddleNLP/
-            ${{ env.allure_file }} generate result -o report
-            tar -czf products.tar.gz report unittest_logs
-            python ${{ env.bos_file }} products.tar.gz paddle-github-action/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs
-            echo "report: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs/products.tar.gz"
-            '
-          fi
-
-      - name: Terminate And Delete the Container
-        if: always()
-        run: |
-          docker rm -f $container_name 2>/dev/null || true

.gitignore

@@ -123,21 +123,3 @@ FETCH_HEAD
 
 # vscode
 .vscode
-./ppdiffusers/ppdiffusers/version.py
-
-# third party
-csrc/third_party/
-dataset/
-output/
-
-# gen codes
-autogen/
-
-# cutlass kernel
-!csrc/gpu/cutlass_kernels/gemm/collective/builders
-
-
-#fp8
-ops/csrc/fp8/deep_gemm/include/cutlass
-ops/csrc/fp8/deep_gemm/include/cute
-.ccls-cache

.gitmodules

@@ -1,6 +0,0 @@
-[submodule "csrc/third_party/cutlass"]
-	path = csrc/third_party/cutlass
-	url = https://github.com/NVIDIA/cutlass.git
-[submodule "csrc/third_party/nlohmann_json"]
-	path = csrc/third_party/nlohmann_json
-	url = https://github.com/nlohmann/json.git

.pre-commit-config.yaml

@@ -1,20 +1,9 @@
-exclude: 'slm/model_zoo/gpt-3;csrc/third_party'
 repos:
-# For Python files
--   repo: https://github.com/psf/black.git
-    rev: 22.8.0
+-   repo: https://github.com/google/yapf
+    rev: v0.32.0
     hooks:
-    -   id: black
-        files: \.(py|pyi)$
-        additional_dependencies: [toml]
--   repo: https://github.com/PyCQA/isort
-    rev: 5.11.5
-    hooks:
-    -   id: isort
--   repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
-    hooks:
-    -   id: flake8
+    -   id: yapf
+        files: \.py$
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.1.0
     hooks:
@@ -37,37 +26,3 @@ repos:
         files: \.md$
     -   id: remove-tabs
         files: \.md$
--   repo: local
-    hooks:
-    -   id: copyright_checker
-        name: copyright_checker
-        entry: python .copyright.hook
-        language: system
-        files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|xpu|kps|py|sh)$
-# For Markdown files
--   repo: local
-    hooks:
-    -   id: add-spaces-between-chinese-and-english
-        name: Add spaces between Chinese and English characters
-        entry: python scripts/codestyle/check_spaces.py
-        language: python
-        files: \.(md|markdown)$
-        pass_filenames: true
-# For dead links
--   repo: local
-    hooks:
-    -   id: check-dead-links
-        name: Check dead links
-        entry: python scripts/codestyle/check_dead_links.py
-        language: python
-        files: \.(md|markdown|rst)$
-        pass_filenames: true
-
--   repo: local
-    hooks:
-    -   id: clang-format
-        name: clang-format
-        description: Format files with ClangFormat.
-        entry: bash ./tools/codestyle/clang_format.sh -i
-        language: system
-        files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$

.readthedocs.yaml

@@ -4,10 +4,6 @@
 
 # Required
 version: 2
-build:
-  os: "ubuntu-20.04"
-  tools:
-    python: "3.10"
 
 submodules:
   include: all
@@ -15,7 +11,7 @@ submodules:
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
-   configuration: docs/zh/conf.py
+   configuration: docs/conf.py
 
 # Optionally build your docs in additional formats such as PDF
 #formats:
@@ -23,5 +19,10 @@ sphinx:
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
+  version: 3.8
   install:
     - requirements: docs/requirements.txt
+    - requirements: requirements.txt
+    - method: setuptools
+      path: .
+  system_packages: true

.style.yapf

@@ -0,0 +1,3 @@
+[style]
+based_on_style = pep8
+column_limit = 80

CONTRIBUTING.md

@@ -1,158 +0,0 @@
-**简体中文**🀄 | [English🌎](.github/CONTRIBUTING_en.md)
-
-# Contributing to PaddleNLP
-
-我们非常欢迎并希望您对`PaddleNLP`做出开源贡献。在您开始提交您的贡献之前，请先行签署[PaddlePaddle 贡献者许可协议](https://cla-assistant.io/PaddlePaddle/PaddleNLP)。
-本文接下来将介绍我们的开发与贡献流程：
-
-## 贡献方式
-
-我们欢迎不同的向`PaddleNLP`做出贡献的方式，例如：
-
-- 修复已知的 Issue
-- 提交新的 Issue，例如提出功能需求或者 bug 报告
-- 实现新的模型结构
-
-如果您不知道从哪里开始，请查看 Issues 板块中的`Good First Issue`标签。它为您提供一个对初学者友好的已知 Issue 列表，可以降低贡献的门槛，帮助您开始为开源做出贡献。您只需在您想处理的 Issue 中告知我们您想负责此 Issue 即可。
-
-## 开发流程
-
-PaddleNLP 使用 [Git 分支模型](http://nvie.com/posts/a-successful-git-branching-model/)。对于常见的开源贡献，我们有以下的贡献流程：
-
-### 1. Fork
-
-   因为 PaddleNLP 的开发社区一直在发展，如果每位贡献者都直接向官方 Repo 提交 commit 将会难以管理。因此，请从您的分支中提交 Pull Requests。建议您通过 GitHub 的[“Fork”按钮](https://help.github.com/articles/fork-a-repo/)来创建您的 Fork 分支。
-
-### 2. Clone
-
-   请运行一下命令将您的分支 clone 到本地
-
-   ```bash
-   git clone https://github.com/<your-github-account>/PaddleNLP
-   cd PaddleNLP
-   ```
-
-### 3. 创建本地开发分支
-
-   对于添加新功能或修复错误等日常工作，请在开发前创建您的本地开发分支：
-
-   ```bash
-   git checkout -b my-cool-feature
-   ```
-
-### 4. 配置开发环境
-
-   在开始编码之前，您需要设置开发环境。我们强烈建议您在虚拟环境中进行所有开发，例如[venv](https://docs.python.org/3/library/venv.html)或[conda](https://docs.conda.io/en/latest/)。
-   请您设置并激活虚拟环境后，运行以下命令：
-
-   ```bash
-   make install
-   ```
-
-   这将设置 `PaddleNLP` 的所有依赖以及 [`pre-commit`](http://pre-commit.com/) 工具。
-
-   如果您需要开发 `examples` 或 `applications` 模块并加载 `PaddleNLP`，请确保以可编辑模式（`-e`）安装 `PaddleNLP`。
-   如果在虚拟环境中已经安装 `PaddleNLP` ，请使用 `pip uninstall paddlenlp` 将其删除，然后以可编辑模式重新安装它
-   `pip install -e .`
-
-### 5. 开发
-
-   当您开发时，请确保您新增的代码会被单元测试所覆盖。我们所有的单元测试都可以在 `tests` 目录下找到。
-   您可以修改现有单元测试以覆盖新功能，也可以从头开始创建新测试。
-   当您完成代码时，您应该确保相关的单元测试可以通过。您可以像这样运行受更改影响的测试：
-
-   ```bash
-   pytest tests/<test_to_run>.py
-   ```
-
-### 6. Commit
-
-   我们使用 [`pre-commit`](http://pre-commit.com/)工具（包括[black](https://black.readthedocs.io/en/stable/)、[isort](https:/ /pycqa.github.io/isort/) 和
-   [flake8](https://flake8.pycqa.org/en/latest/)）来检查每次提交中的代码和文档的风格。当你运行 `git commit` 时，你会看到
-   类似于以下内容：
-
-   ```text
-    ➜  (my-virtual-env) git commit -m "committing my cool feature"
-    black....................................................................Passed
-    isort....................................................................Passed
-    flake8...................................................................Passed
-    check for merge conflicts................................................Passed
-    check for broken symlinks............................(no files to check)Skipped
-    detect private key.......................................................Passed
-    fix end of files.....................................(no files to check)Skipped
-    trim trailing whitespace.............................(no files to check)Skipped
-    CRLF end-lines checker...............................(no files to check)Skipped
-    CRLF end-lines remover...............................(no files to check)Skipped
-    No-tabs checker......................................(no files to check)Skipped
-    Tabs remover.........................................(no files to check)Skipped
-    copyright_checker........................................................Passed
-   ```
-
-   但大多数时候事情并没有那么顺利。当您的代码或文档不符合标准时，`pre-commit` 检查将失败。
-
-   ```text
-    ➜  (my-virtual-env) git commit -m "committing my cool feature"
-    black....................................................................Passed
-    isort....................................................................Failed
-    - hook id: isort
-    - files were modified by this hook
-
-    Fixing examples/information_extraction/waybill_ie/run_ernie_crf.py
-
-    flake8...................................................................Passed
-    check for merge conflicts................................................Passed
-    check for broken symlinks............................(no files to check)Skipped
-    detect private key.......................................................Passed
-    fix end of files.....................................(no files to check)Skipped
-    trim trailing whitespace.............................(no files to check)Skipped
-    CRLF end-lines checker...............................(no files to check)Skipped
-    CRLF end-lines remover...............................(no files to check)Skipped
-    No-tabs checker......................................(no files to check)Skipped
-    Tabs remover.........................................(no files to check)Skipped
-    copyright_checker........................................................Passed
-   ```
-
-   我们的工具将自动修复大部分样式错误，但是有些错误需要手动解决。幸运的是，错误信息一般通俗易懂，很容易修复。
-   解决错误后，您可以再次运行 `git add <files>` 和 `git commit`，这将再次触发 pre-commit 。
-   一旦 pre-commit 检查通过，您就可以推送代码了。
-
-   [Google](https://google.com/) 或 [StackOverflow](https://stackoverflow.com/) 是帮助您了解代码风格错误的好工具。
-   如果您仍然无法弄清楚，请不要担心。您可以使用 `git commit -m "style error" --no-verify` 提交，我们很乐意在您创建 Pull Request 后帮助您。
-
-### 7. git pull 与代码冲突
-
-   有经验的 Git 用户经常从官方 Repo 中 git pull。因为这样子他们会及早注意到与其他人的代码冲突，并且让代码冲突更容易解决
-
-   ```bash
-   git remote add upstream https://github.com/PaddlePaddle/PaddleNLP
-   git pull upstream develop
-   ```
-
-### 8. git push 与提交 Pull Request
-
-   您可以将您的本地开发分支中的工作 push 到您的 fork 的分支中：
-
-   ```bash
-   git push origin my-cool-stuff
-   ```
-
-   git push 之后，您可以提交 Pull Request，请求[官方 repo](https://github.com/PaddlePaddle/PaddleNLP) 采纳您的开发工作。请您依照[这些步骤](https://help.github.com/articles/creating-a-pull-request/)创建 Pull Request。
-
-### 9. 删除已经合入的本地和远程分支
-
-   为了保持您本地的工作区和 fork 分支的干净整洁，建议您在 Pull Request 合入之后删除本地的残余分支：
-
-   ```bash
-   git push origin my-cool-stuff
-   git checkout develop
-   git pull upstream develop
-   git branch -d my-cool-stuff
-   ```
-
-## 代码 Review
-
-- 在您的 Pull Request 能够顺利通过本地测试以及 CI 的情况下，您可以在 Pull Request 中 @ 相关的 Reviewer，提醒他们尽快对您的 Pull Request 进行 Review。
-
-- 请处理 Reviewer 的每一条评论。如果您已按照评论修改，请回复“完成”；否则，可以在评论下展开讨论。
-
-- 如果您不希望您的 Reviewer 被电子邮件通知淹没，您可以[批量回复](https://help.github.com/articles/reviewing-proposed-changes-in-a-pull-request/)。

Makefile

@@ -1,79 +0,0 @@
-# Makefile for PaddleNLP
-#
-# 	GitHb: https://github.com/PaddlePaddle/PaddleNLP
-# 	Author: Paddle Team https://github.com/PaddlePaddle
-#
-
-.PHONY: all
-all : lint test
-check_dirs := applications examples model_zoo paddlenlp pipelines ppdiffusers scripts tests 
-# # # # # # # # # # # # # # # Format Block # # # # # # # # # # # # # # # 
-
-format:
-	pre-commit run isort
-	pre-commit run black
-
-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
-# # # # # # # # # # # # # # # Lint Block # # # # # # # # # # # # # # # 
-
-.PHONY: lint
-lint:
-	$(eval modified_py_files := $(shell python scripts/get_modified_files.py $(check_dirs)))
-	@if test -n "$(modified_py_files)"; then \
-		echo ${modified_py_files}; \
-		pre-commit run --files ${modified_py_files}; \
-	else \
-		echo "No library .py files were modified"; \
-	fi	
-
-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
-# # # # # # # # # # # # # # # Test Block # # # # # # # # # # # # # # # 
-
-.PHONY: test
-test: unit-test
-
-unit-test:
-	PYTHONPATH=$(shell pwd) pytest -v \
-		-n auto \
-		--retries 1 --retry-delay 1 \
-		--durations 20 \
-		--cov paddlenlp \
-		--cov-report xml:coverage.xml
-
-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
-.PHONY: install
-install:
-	pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/
-	pip install -r requirements-dev.txt
-	pip install -r requirements.txt
-	pip install -r paddlenlp/experimental/autonlp/requirements.txt
-	pre-commit install
-
-
-.PHONY: deploy-ppdiffusers
-deploy-ppdiffusers:
-	cd ppdiffusers && make install && make
-
-.PHONY: deploy-paddle-pipelines
-deploy-paddle-pipelines:
-	cd pipelines && make install && make
-
-.PHONY: deploy-paddlenlp
-deploy-paddlenlp:
-	# install related package
-	make install
-	# build
-	python3 setup.py sdist bdist_wheel
-	# upload
-	twine upload --skip-existing dist/*
-
-.PHONY: regression-all
-release: 
-	bash ./scripts/regression/run_release.sh 0 0,1 all
-
-.PHONY: regression-key
-key: 
-	bash ./scripts/regression/run_release.sh 0 0,1 p0

README.md

@@ -1,298 +0,0 @@
-**简体中文**🀄 | [English🌎](./README_en.md)
-
-<p align="center">
-  <img src="https://user-images.githubusercontent.com/1371212/175816733-8ec25eb0-9af3-4380-9218-27c154518258.png" align="middle"  width="500" />
-</p>
-
-------------------------------------------------------------------------------------------
-
-<p align="center">
-    <a href="https://paddlenlp.readthedocs.io/en/latest/?badge=latest"><img src="https://readthedocs.org/projects/paddlenlp/badge/?version=latest">
-    <a href="https://github.com/PaddlePaddle/PaddleNLP/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/PaddleNLP?color=ffa"></a>
-    <a href=""><img src="https://img.shields.io/badge/python-3.7+-aff.svg"></a>
-    <a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
-    <a href="https://github.com/PaddlePaddle/PaddleNLP/graphs/contributors"><img src="https://img.shields.io/github/contributors/PaddlePaddle/PaddleNLP?color=9ea"></a>
-    <a href="https://github.com/PaddlePaddle/PaddleNLP/commits"><img src="https://img.shields.io/github/commit-activity/m/PaddlePaddle/PaddleNLP?color=3af"></a>
-    <a href="https://pypi.org/project/paddlenlp/"><img src="https://img.shields.io/pypi/dm/paddlenlp?color=9cf"></a>
-    <a href="https://github.com/PaddlePaddle/PaddleNLP/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/PaddleNLP?color=9cc"></a>
-    <a href="https://github.com/PaddlePaddle/PaddleNLP/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/PaddleNLP?color=ccf"></a>
-    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
-</p>
-
-<h4 align="center">
-  <a href=#特性> 特性 </a> |
-  <a href=#模型支持> 模型支持 </a> |
-  <a href=#安装> 安装 </a> |
-  <a href=#快速开始> 快速开始 </a> |
-  <a href=#社区交流> 社区交流 </a>
-</h4>
-
-**PaddleNLP**是一款基于飞桨深度学习框架的大语言模型(LLM)开发套件，支持在多种硬件上进行高效的大模型训练、无损压缩以及高性能推理。PaddleNLP 具备**简单易用**和**性能极致**的特点，致力于助力开发者实现高效的大模型产业级应用。
-
-<a href="https://trendshift.io/repositories/2246" target="_blank"><img src="https://trendshift.io/api/badge/repositories/2246" alt="PaddlePaddle%2FPaddleNLP | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
-
-## News 📢
-
-* **2025.04.29 PaddleNLP 现已支持 Qwen3 系列模型**: Qwen3 系列模型支持持两种思考模式，预训练约 36 万亿个 token、119 种语言和方言。包括六个 Dense 模型, Qwen3-32B、Qwen3-14B、Qwen3-8B、Qwen3-4B、Qwen3-1.7B 和 Qwen3-0.6B。两个 MoE 模型的权重：Qwen3-235B-A22B，Qwen3-30B-A3B。
-
-* **2025.03.12 [PaddleNLP v3.0 Beta4](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta4)**：全面支持 DeepSeek V3/R1/R1-Distill, 及 QwQ-32B 等热门思考模型。**DeepSeek V3/R1完整版支持 FP8、INT8、4-bit 量化推理，MTP 投机解码**。单机 FP8推理输出超**1000 tokens/s**; 4-bit 推理输出超**2100 tokens/s**! 发布新版推理部署镜像，热门模型[一键部署](https://paddlenlp.readthedocs.io/zh/latest/llm/server/docs/general_model_inference.html)。推理部署[使用文档](https://paddlenlp.readthedocs.io/zh/latest/llm/docs/predict/index.html)全面更新，体验全面提升！自研下一代通用信息抽取模型 PP-UIE [全新发布](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/application/information_extraction)，支持8K 长度信息抽取。新增大模型 Embedding 训练，支持 INF-CL 超大 batch size 训练。新增[MergeKit](https://paddlenlp.readthedocs.io/zh/latest/llm/docs/mergekit.html)模型融合工具，缓解对齐代价。低资源训练全面优化，16G 小显存可以流畅训练。
-
-
-* **2025.02.10 PaddleNLP 现已支持 DeepSeek-R1系列模型，[在线使用](https://aistudio.baidu.com/projectdetail/8775758)**：依托全新的 PaddleNLP 3.0套件，DeepSeek-R1系列模型现已全面支持。凭借数据并行、数据分组切分并行、模型并行、流水线并行以及专家并行等一系列先进的分布式训练能力，结合 Paddle 框架独有的列稀疏注意力掩码表示技术——FlashMask 方法，DeepSeek-R1系列模型在训练过程中显著降低了显存消耗，同时取得了卓越的训练性能提升。
-
-<details><summary> <b>点击展开</b> </summary><div>
-
-* **2025.03.17 《DeepSeek-R1满血版单机部署实测》** 🔥🔥🔥 飞桨框架3.0大模型推理部署全面升级，支持多款主流大模型，DeepSeek-R1满血版实现单机部署，吞吐提升一倍！欢迎广大用户开箱体验～现已开启有奖活动：完成 DeepSeek-R1-MTP 单机部署任务、提交高质量测评 blog，即可实时赢取奖金！💰💰💰
-报名[地址](https://www.wjx.top/vm/OlzzmbG.aspx#)， 活动详情：https://github.com/PaddlePaddle/PaddleNLP/issues/10166 ， 参考文档：https://github.com/PaddlePaddle/PaddleNLP/issues/10157 。
-
-* **2025.03.06 PaddleNLP 现已支持 Qwen/QwQ-32B 模型**: 其模型参数仅有 32B，但其数学推理、编程能力和通用能力可与具备 671B 参数（其中 37B 被激活）的 DeepSeek-R1 媲美。借助 PaddleNLP 3.0套件，现可实现多种并行策略[微调训练](./llm/README.md)、[高性能推理、低比特量化](./llm/docs/predict/qwen.md)和[服务化部署](./llm/server/README.md)。
-
-* **2025.02.20 🔥🔥《PP-UIE 信息抽取智能引擎全新升级》** 强化零样本学习能力，支持极少甚至零标注数据实现高效冷启动与迁移学习，显著降低数据标注成本；具备处理长文本能力，支持 8192 个 Token 长度文档信息抽取，实现跨段落识别关键信息，形成完整理解；提供完整可定制化的训练和推理全流程，训练效率相较于 LLama-Factory 实现了1.8倍的提升。
-2月26日（周三）19：00为您深度解析全新 PP-UIE 技术方案及在部署方面的功能、优势与技巧。报名链接：https://www.wjx.top/vm/mBKC6pb.aspx?udsid=606418
-
-* **2024.12.16 [PaddleNLP v3.0 Beta3](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta3)**：大模型功能全新升级，新增了 Llama-3.2、DeepSeekV2模型，升级了 TokenizerFast，快速分词，重构了 SFTTrainer，一键开启 SFT 训练。此外，PaddleNLP 还支持了优化器状态的卸载和重载功能，实现了精细化的重新计算，训练性能提升7%。在 Unified Checkpoint 方面，进一步优化了异步保存逻辑，新增 Checkpoint 压缩功能，可节省78.5%存储空间。
-最后，在大模型推理方面，升级 Append Attention，支持了 FP8量化，支持投机解码。
-
-* **2024.12.13 📚《飞桨大模型套件 Unified Checkpoint 技术》**，加速模型存储95%，节省空间78%。支持全分布式策略调整自适应转换，提升模型训练的灵活性与可扩展性。训练-压缩-推理统一存储协议，无需手动转换提升全流程体验。Checkpoint 无损压缩结合异步保存，实现秒级存储并降低模型存储成本。适用于智能制造、指挥交通、医疗健康、金融服务等产业实际场景。12月24日（周二）19：00直播为您详细解读该技术如何优化大模型训练流程。报名链接：https://www.wjx.top/vm/huZkHn9.aspx?udsid=787976
-
-* **2024.11.28 📚《FlashRAG-Paddle | 基于 PaddleNLP 的高效开发与评测 RAG 框架》**，为文本更快更好构建准确嵌入表示、加速推理生成速度。PaddleNLP 支持超大 Batch 嵌入表示学习与多硬件高性能推理，涵盖 INT8/INT4量化技术及多种高效注意力机制优化与 TensorCore 深度优化。内置全环节算子融合技术，使得 FlashRAG 推理性能相比 transformers 动态图提升70%以上，结合检索增强知识输出结果更加准确，带来敏捷高效的使用体验。直播时间：12月3日（周二）19：00。报名链接：https://www.wjx.top/vm/eaBa1vA.aspx?udsid=682361
-
-* **2024.08.08 📚《飞桨产业级大语言模型开发利器 PaddleNLP 3.0 重磅发布》**，训压推全流程贯通，主流模型全覆盖。大模型自动并行，千亿模型训推全流程开箱即用。提供产业级高性能精调与对齐解决方案，压缩推理领先，多硬件适配。覆盖产业级智能助手、内容创作、知识问答、关键信息抽取等应用场景。直播时间：8月22日（周四）19：00。报名链接：https://www.wjx.top/vm/Y2f7FFY.aspx?udsid=143844
-
-* **2024.06.27 [PaddleNLP v3.0 Beta](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta0)**：拥抱大模型，体验全升级。统一大模型套件，实现国产计算芯片全流程接入；全面支持飞桨4D 并行配置、高效精调策略、高效对齐算法、高性能推理等大模型产业级应用流程；自研极致收敛的 RsLoRA+算法、自动扩缩容存储机制 Unified Checkpoint 和通用化支持的 FastFFN、FusedQKV 助力大模型训推；主流模型持续支持更新，提供高效解决方案。
-
-* **2024.04.24 [PaddleNLP v2.8](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.8.0)**：自研极致收敛的 RsLoRA+算法，大幅提升 PEFT 训练收敛速度以及训练效果；引入高性能生成加速到 RLHF PPO 算法，打破 PPO 训练中生成速度瓶颈，PPO 训练性能大幅领先。通用化支持 FastFFN、FusedQKV 等多个大模型训练性能优化方式，大模型训练更快、更稳定。
-</div></details>
-
-## 特性
-
-### <a href=#多硬件训推一体> 🔧 多硬件训推一体 </a>
-
-支持英伟达 GPU、昆仑 XPU、昇腾 NPU、燧原 GCU 和海光 DCU 等多个硬件的大模型和自然语言理解模型训练和推理，套件接口支持硬件快速切换，大幅降低硬件切换研发成本。
-当前支持的自然语言理解模型：[多硬件自然语言理解模型列表](./docs/zh/model_zoo/model_list_multy_device.md)
-
-### <a href=#高效易用的预训练> 🚀 高效易用的预训练 </a>
-
-支持纯数据并行策略、分组参数切片的数据并行策略、张量模型并行策略和流水线模型并行策略的4D 高性能训练，Trainer 支持分布式策略配置化，降低复杂分布式组合带来的使用成本；
-[Unified Checkpoint 大模型存储工具](./llm/docs/unified_checkpoint.md)可以使得训练断点支持机器资源动态扩缩容恢复。此外，异步保存，模型存储可加速95%，Checkpoint 压缩，可节省78.5%存储空间。
-
-### <a href=#高效精调> 🤗 高效精调 </a>
-
-精调算法深度结合零填充数据流和 [FlashMask](./llm/docs/flashmask.md) 高性能算子，降低训练无效数据填充和计算，大幅提升精调训练吞吐。
-
-### <a href=#无损压缩和高性能推理> 🎛️ 无损压缩和高性能推理 </a>
-
-大模型套件高性能推理模块内置动态插入和全环节算子融合策略，极大加快并行推理速度。底层实现细节封装化，实现开箱即用的高性能并行推理能力。
-
-## 文档
-更多详细文档, 请访问 [PaddleNLP Documentation](https://paddlenlp.readthedocs.io/).
-
-------------------------------------------------------------------------------------------
-
-## 模型支持
-
-* 模型参数已支持 LLaMA 系列、Baichuan 系列、Bloom 系列、ChatGLM 系列、Gemma 系列、Mistral 系列、OPT 系列和 Qwen 系列，详细列表👉【LLM】模型参数支持列表如下：
-
-|                                                模型系列                                                 | 模型名称                                                                                                                                                                                                                                                                                                                                                                                      |
-|:-------------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [PP-UIE](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/application/information_extraction) | paddlenlp/PP-UIE-0.5B, paddlenlp/PP-UIE-1.5B, paddlenlp/PP-UIE-7B, paddlenlp/PP-UIE-14B                                                                                                                                                                                                                                                                                                       |
-|            [LLaMA](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)             | facebook/llama-7b, facebook/llama-13b, facebook/llama-30b, facebook/llama-65b                                                                                                                                                                                                                                                                                                                 |
-|            [Llama2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)            | meta-llama/Llama-2-7b, meta-llama/Llama-2-7b-chat, meta-llama/Llama-2-13b, meta-llama/Llama-2-13b-chat, meta-llama/Llama-2-70b, meta-llama/Llama-2-70b-chat                                                                                                                                                                                                                                   |
-|            [Llama3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)            | meta-llama/Meta-Llama-3-8B, meta-llama/Meta-Llama-3-8B-Instruct, meta-llama/Meta-Llama-3-70B, meta-llama/Meta-Llama-3-70B-Instruct                                                                                                                                                                                                                                                            |
-|           [Llama3.1](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)           | meta-llama/Meta-Llama-3.1-8B, meta-llama/Meta-Llama-3.1-8B-Instruct, meta-llama/Meta-Llama-3.1-70B, meta-llama/Meta-Llama-3.1-70B-Instruct, meta-llama/Meta-Llama-3.1-405B, meta-llama/Meta-Llama-3.1-405B-Instruct, meta-llama/Llama-Guard-3-8B                                                                                                                                              |
-|           [Llama3.2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)           | meta-llama/Llama-3.2-1B, meta-llama/Llama-3.2-1B-Instruct, meta-llama/Llama-3.2-3B, meta-llama/Llama-3.2-3B-Instruct, meta-llama/Llama-Guard-3-1B                                                                                                                                                                                                                                             |
-|           [Llama3.3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)           | meta-llama/Llama-3.3-70B-Instruct                                                                                                                                                                                                                                                                                                                                                             |
-|         [Baichuan](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/baichuan)          | baichuan-inc/Baichuan-7B, baichuan-inc/Baichuan-13B-Base, baichuan-inc/Baichuan-13B-Chat                                                                                                                                                                                                                                                                                                      |
-|         [Baichuan2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/baichuan)         | baichuan-inc/Baichuan2-7B-Base, baichuan-inc/Baichuan2-7B-Chat, baichuan-inc/Baichuan2-13B-Base, baichuan-inc/Baichuan2-13B-Chat                                                                                                                                                                                                                                                              |
-|            [Bloom](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/bloom)             | bigscience/bloom-560m, bigscience/bloom-560m-bf16, bigscience/bloom-1b1, bigscience/bloom-3b, bigscience/bloom-7b1, bigscience/bloomz-560m, bigscience/bloomz-1b1, bigscience/bloomz-3b, bigscience/bloomz-7b1-mt, bigscience/bloomz-7b1-p3, bigscience/bloomz-7b1, bellegroup/belle-7b-2m                                                                                                    |
-|          [ChatGLM](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/chatglm/)          | THUDM/chatglm-6b, THUDM/chatglm-6b-v1.1                                                                                                                                                                                                                                                                                                                                                       |
-|         [ChatGLM2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/chatglm2)          | THUDM/chatglm2-6b                                                                                                                                                                                                                                                                                                                                                                             |
-|         [ChatGLM3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/chatglm2)          | THUDM/chatglm3-6b                                                                                                                                                                                                                                                                                                                                                                             |
-|       [DeepSeekV2](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/config/deepseek-v2)       | deepseek-ai/DeepSeek-V2, deepseek-ai/DeepSeek-V2-Chat, deepseek-ai/DeepSeek-V2-Lite, deepseek-ai/DeepSeek-V2-Lite-Chat, deepseek-ai/DeepSeek-Coder-V2-Base, deepseek-ai/DeepSeek-Coder-V2-Instruct, deepseek-ai/DeepSeek-Coder-V2-Lite-Base, deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct                                                                                                      |
-|       [DeepSeekV3](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/config/deepseek-v2)       | deepseek-ai/DeepSeek-V3, deepseek-ai/DeepSeek-V3-Base                                                                                                                                                                                                                                                                                                                                         |
-|      [DeepSeek-R1](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/config/deepseek-v2)       | deepseek-ai/DeepSeek-R1, deepseek-ai/DeepSeek-R1-Zero, deepseek-ai/DeepSeek-R1-Distill-Llama-70B, deepseek-ai/DeepSeek-R1-Distill-Llama-8B, deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, deepseek-ai/DeepSeek-R1-Distill-Qwen-7B                                                                            |
-|            [Gemma](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/gemma)             | google/gemma-7b, google/gemma-7b-it, google/gemma-2b, google/gemma-2b-it                                                                                                                                                                                                                                                                                                                      |
-|          [Mistral](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/mistral)           | mistralai/Mistral-7B-Instruct-v0.3, mistralai/Mistral-7B-v0.1                                                                                                                                                                                                                                                                                                                                 |
-|          [Mixtral](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/mixtral)           | mistralai/Mixtral-8x7B-Instruct-v0.1                                                                                                                                                                                                                                                                                                                                                          |
-|              [OPT](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/opt)               | facebook/opt-125m, facebook/opt-350m, facebook/opt-1.3b, facebook/opt-2.7b, facebook/opt-6.7b, facebook/opt-13b, facebook/opt-30b, facebook/opt-66b, facebook/opt-iml-1.3b, opt-iml-max-1.3b                                                                                                                                                                                                  |
-|             [Qwen](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)             | qwen/qwen-7b, qwen/qwen-7b-chat, qwen/qwen-14b, qwen/qwen-14b-chat, qwen/qwen-72b, qwen/qwen-72b-chat,                                                                                                                                                                                                                                                                                        |
-|           [Qwen1.5](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)            | Qwen/Qwen1.5-0.5B, Qwen/Qwen1.5-0.5B-Chat, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-1.8B-Chat, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-4B-Chat, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-7B-Chat, Qwen/Qwen1.5-14B, Qwen/Qwen1.5-14B-Chat, Qwen/Qwen1.5-32B, Qwen/Qwen1.5-32B-Chat, Qwen/Qwen1.5-72B, Qwen/Qwen1.5-72B-Chat, Qwen/Qwen1.5-110B, Qwen/Qwen1.5-110B-Chat, Qwen/Qwen1.5-MoE-A2.7B, Qwen/Qwen1.5-MoE-A2.7B-Chat |
-|            [Qwen2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)             | Qwen/Qwen2-0.5B, Qwen/Qwen2-0.5B-Instruct, Qwen/Qwen2-1.5B, Qwen/Qwen2-1.5B-Instruct, Qwen/Qwen2-7B, Qwen/Qwen2-7B-Instruct, Qwen/Qwen2-72B, Qwen/Qwen2-72B-Instruct, Qwen/Qwen2-57B-A14B, Qwen/Qwen2-57B-A14B-Instruct                                                                                                                                                                       |
-|          [Qwen2-Math](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)          | Qwen/Qwen2-Math-1.5B, Qwen/Qwen2-Math-1.5B-Instruct, Qwen/Qwen2-Math-7B, Qwen/Qwen2-Math-7B-Instruct, Qwen/Qwen2-Math-72B, Qwen/Qwen2-Math-72B-Instruct, Qwen/Qwen2-Math-RM-72B                                                                                                                                                                                                               |
-|           [Qwen2.5](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)            | Qwen/Qwen2.5-0.5B, Qwen/Qwen2.5-0.5B-Instruct, Qwen/Qwen2.5-1.5B, Qwen/Qwen2.5-1.5B-Instruct, Qwen/Qwen2.5-3B, Qwen/Qwen2.5-3B-Instruct, Qwen/Qwen2.5-7B, Qwen/Qwen2.5-7B-Instruct, Qwen/Qwen2.5-7B-Instruct-1M, Qwen/Qwen2.5-14B, Qwen/Qwen2.5-14B-Instruct, Qwen/Qwen2.5-14B-Instruct-1M, Qwen/Qwen2.5-32B, Qwen/Qwen2.5-32B-Instruct, Qwen/Qwen2.5-72B, Qwen/Qwen2.5-72B-Instruct          |
-|         [Qwen2.5-Math](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)         | Qwen/Qwen2.5-Math-1.5B, Qwen/Qwen2.5-Math-1.5B-Instruct, Qwen/Qwen2.5-Math-7B, Qwen/Qwen2.5-Math-7B-Instruct, Qwen/Qwen2.5-Math-72B, Qwen/Qwen2.5-Math-72B-Instruct, Qwen/Qwen2.5-Math-RM-72B                                                                                                                                                                                                 |
-|        [Qwen2.5-Coder](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)         | Qwen/Qwen2.5-Coder-1.5B, Qwen/Qwen2.5-Coder-1.5B-Instruct, Qwen/Qwen2.5-Coder-7B, Qwen/Qwen2.5-Coder-7B-Instruct                                                                                                                                                                                                                                                                              |
-|           [Qwen3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)              | Qwen/Qwen3-0.6B, Qwen/Qwen3-1.7B, Qwen/Qwen3-4B, Qwen/Qwen3-8B, Qwen/Qwen3-14B, Qwen/Qwen3-32B, Qwen/Qwen3-30B-A3B, Qwen/Qwen3-235B-A22B, Qwen/Qwen3-0.6B-Base, Qwen/Qwen3-1.7B-Base, Qwen/Qwen3-4B-Base, Qwen/Qwen3-8B-Base, Qwen/Qwen3-14B-Base, Qwen/Qwen3-30B-A3B-Base                                                                                                                    |
-|             [QwQ](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)              | Qwen/QwQ-32B, Qwen/QwQ-32B-Preview                                                                                                                                                                                                                                                                                                                                                            |
-|            [Yuan2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/yuan/)             | IEITYuan/Yuan2-2B, IEITYuan/Yuan2-51B, IEITYuan/Yuan2-102B                                                                                                                                                                                                                                                                                                                                    |
-
-* 4D 并行和算子优化已支持 LLaMA 系列、Baichuan 系列、Bloom 系列、ChatGLM 系列、Gemma 系列、Mistral 系列、OPT 系列和 Qwen 系列，【LLM】模型4D 并行和算子支持列表如下：
-
-| 模型名称/并行能力支持 | 数据并行 | 张量模型并行 |          | 参数分片并行 |        |        | 流水线并行 |
-|:---------------------:|:--------:|:------------:|:--------:|:------------:|:------:|:------:|:----------:|
-|                       |          |   基础能力   | 序列并行 |    stage1    | stage2 | stage3 |            |
-|         Llama         |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|         Qwen          |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|        Qwen1.5        |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|         Qwen2         |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|     Mixtral(moe)      |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     🚧     |
-|        Mistral        |    ✅     |      ✅       |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|       Baichuan        |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|       Baichuan2       |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|        ChatGLM        |    ✅     |      ✅       |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|       ChatGLM2        |    ✅     |      🚧      |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|       ChatGLM3        |    ✅     |      🚧      |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|         Bloom         |    ✅     |      ✅       |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|      GPT-2/GPT-3      |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|          OPT          |    ✅     |      ✅       |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|         Gemma         |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|         Yuan2         |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     🚧     |
-
-* 大模型预训练、精调（包含 SFT、PEFT 技术）、对齐、量化已支持 LLaMA 系列、Baichuan 系列、Bloom 系列、ChatGLM 系列、Mistral 系列、OPT 系列和 Qwen 系列，【LLM】模型预训练、精调、对齐、量化支持列表如下：
-
-
-| Model                                      | Pretrain | SFT | LoRA | FlashMask | Prefix Tuning | DPO/SimPO/ORPO/KTO | RLHF | Mergekit | Quantization |
-|--------------------------------------------|:--------:|:---:|:----:|:---------:|:-------------:|:------------------:|:----:|:--------:|:------------:|
-| [Llama](./llm/config/llama)                |    ✅     |  ✅  |  ✅   |     ✅     |       ✅       |         ✅          |  ✅   |    ✅     |      ✅       |
-| [Qwen](./llm/config/qwen)                  |    ✅     |  ✅  |  ✅   |     ✅     |       ✅       |         ✅          |  🚧  |    ✅     |      🚧      |
-| [Mixtral](./llm/config/mixtral)            |    ✅     |  ✅  |  ✅   |    🚧     |      🚧       |         ✅          |  🚧  |    ✅     |      🚧      |
-| [Mistral](./llm/config/mistral)            |    ✅     |  ✅  |  ✅   |    🚧     |       ✅       |         ✅          |  🚧  |    ✅     |      🚧      |
-| [Baichuan/Baichuan2](./llm/config/llama)   |    ✅     |  ✅  |  ✅   |     ✅     |       ✅       |         ✅          |  🚧  |    ✅     |      ✅       |
-| [ChatGLM-6B](./llm/config/chatglm)         |    ✅     |  ✅  |  ✅   |    🚧     |       ✅       |         🚧         |  🚧  |    ✅     |      ✅       |
-| [ChatGLM2/ChatGLM3](./llm/config/chatglm2) |    ✅     |  ✅  |  ✅   |    🚧     |       ✅       |         ✅          |  🚧  |    ✅     |      ✅       |
-| [Bloom](./llm/config/bloom)                |    ✅     |  ✅  |  ✅   |    🚧     |       ✅       |         🚧         |  🚧  |    ✅     |      ✅       |
-| [GPT-3](./llm/config/gpt-3)                |    ✅     |  ✅  |  🚧  |    🚧     |      🚧       |         🚧         |  🚧  |    ✅     |      🚧      |
-| [OPT](./llm/config/opt)                    |    ✅     |  ✅  |  ✅   |    🚧     |      🚧       |         🚧         |  🚧  |    ✅     |      🚧      |
-| [Gemma](./llm/config/gemma)                |    ✅     |  ✅  |  ✅   |    🚧     |      🚧       |         ✅          |  🚧  |    ✅     |      🚧      |
-| [Yuan](./llm/config/yuan)                  |    ✅     |  ✅  |  ✅   |    🚧     |      🚧       |         ✅          |  🚧  |    ✅     |      🚧      |
-* [大模型推理](./llm/docs/predict/inference.md)已支持 LLaMA 系列、Qwen 系列、DeepSeek 系列、Mistral 系列、ChatGLM 系列、Bloom 系列和 Baichuan 系列，支持 Weight Only INT8及 INT4推理，支持 WAC（权重、激活、Cache KV）进行 INT8、FP8量化的推理，【LLM】模型推理支持列表如下：
-
-|           模型名称/量化类型支持            | FP16/BF16 | WINT8 | WINT4 | INT8-A8W8 | FP8-A8W8 | INT8-A8W8C8 |
-|:------------------------------------------:|:---------:|:-----:|:-----:|:---------:|:--------:|:-----------:|
-|    [LLaMA](./llm/docs/predict/llama.md)    |     ✅     |   ✅   |   ✅   |     ✅     |    ✅     |      ✅      |
-|     [Qwen](./llm/docs/predict/qwen.md)     |     ✅     |   ✅   |   ✅   |     ✅     |    ✅     |      ✅      |
-| [DeepSeek](./llm/docs/predict/deepseek.md) |     ✅     |   ✅   |   ✅   |    🚧     |    ✅     |     🚧      |
-|   [Qwen-Moe](./llm/docs/predict/qwen.md)   |     ✅     |   ✅   |   ✅   |    🚧     |    🚧    |     🚧      |
-|  [Mixtral](./llm/docs/predict/mixtral.md)  |     ✅     |   ✅   |   ✅   |    🚧     |    🚧    |     🚧      |
-|                  ChatGLM                   |     ✅     |   ✅   |   ✅   |    🚧     |    🚧    |     🚧      |
-|                   Bloom                    |     ✅     |   ✅   |   ✅   |    🚧     |    🚧    |     🚧      |
-|                  BaiChuan                  |     ✅     |   ✅   |   ✅   |     ✅     |    ✅     |     🚧      |
-
-## 安装
-
-### 环境依赖
-
-* python >= 3.8
-* paddlepaddle >= 3.0.0rc1
-
-如果您尚未安装 PaddlePaddle，请参考 [飞桨官网](https://www.paddlepaddle.org.cn/) 进行安装。
-
-### pip 安装
-
-```shell
-pip install --upgrade paddlenlp==3.0.0b4
-```
-
-或者可通过以下命令安装最新 develop 分支代码：
-
-```shell
-pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html
-```
-
-更多关于 PaddlePaddle 和 PaddleNLP 安装的详细教程请查看[Installation](./docs/zh/get_started/installation.rst)。
-
-------------------------------------------------------------------------------------------
-
-## 快速开始
-
-### 大模型文本生成
-
-PaddleNLP 提供了方便易用的 Auto API，能够快速的加载模型和 Tokenizer。这里以使用 `Qwen/Qwen2-0.5B` 模型做文本生成为例：
-
-```python
-from paddlenlp.transformers import AutoTokenizer, AutoModelForCausalLM
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B")
-# if using CPU, please change float16 to float32
-model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B", dtype="float16")
-input_features = tokenizer("你好！请自我介绍一下。", return_tensors="pd")
-outputs = model.generate(**input_features, max_new_tokens=128)
-print(tokenizer.batch_decode(outputs[0], skip_special_tokens=True))
-# ['我是一个AI语言模型，我可以回答各种问题，包括但不限于：天气、新闻、历史、文化、科学、教育、娱乐等。请问您有什么需要了解的吗？']
-```
-
-### 大模型预训练
-
-```shell
-git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # 如已clone或下载PaddleNLP可跳过
-mkdir -p llm/data && cd llm/data
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
-cd .. # change folder to PaddleNLP/llm
-# 如需使用use_fused_rms_norm=true，需要前往slm/model_zoo/gpt-3/external_ops安装fused_ln
-python -u run_pretrain.py ./config/qwen/pretrain_argument_0p5b.json
-```
-
-### 大模型 SFT 精调
-
-```shell
-git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # 如已clone或下载PaddleNLP可跳过
-mkdir -p llm/data && cd llm/data
-wget https://bj.bcebos.com/paddlenlp/datasets/examples/AdvertiseGen.tar.gz && tar -zxvf AdvertiseGen.tar.gz
-cd .. # change folder to PaddleNLP/llm
-python -u run_finetune.py ./config/qwen/sft_argument_0p5b.json
-```
-
-更多大模型全流程步骤，请参考[飞桨大模型套件](./llm)介绍。
-另外我们还提供了快速微调方式, 无需 clone 源代码：
-
-```python
-from paddlenlp.trl import SFTConfig, SFTTrainer
-from datasets import load_dataset
-
-dataset = load_dataset("ZHUI/alpaca_demo", split="train")
-
-training_args = SFTConfig(output_dir="Qwen/Qwen2.5-0.5B-SFT", device="gpu")
-trainer = SFTTrainer(
-    args=training_args,
-    model="Qwen/Qwen2.5-0.5B-Instruct",
-    train_dataset=dataset,
-)
-trainer.train()
-```
-
-更多 PaddleNLP 内容可参考：
-
-* [精选模型库](./slm/model_zoo)，包含优质预训练模型的端到端全流程使用。
-* [多场景示例](./slm/examples)，了解如何使用 PaddleNLP 解决 NLP 多种技术问题，包含基础技术、系统应用与拓展应用。
-* [交互式教程](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)，在🆓免费算力平台 AI Studio 上快速学习 PaddleNLP。
-
-------------------------------------------------------------------------------------------
-
-## 社区交流
-
-* 微信扫描二维码并填写问卷，即可加入交流群与众多社区开发者以及官方团队深度交流.
-
-<div align="center">
-    <img src="https://github.com/user-attachments/assets/3a58cc9f-69c7-4ccb-b6f5-73e966b8051a" width="150" height="150" />
-</div>
-
-## Citation
-
-如果 PaddleNLP 对您的研究有帮助，欢迎引用
-
-```bibtex
-@misc{=paddlenlp,
-    title={PaddleNLP: An Easy-to-use and High Performance NLP Library},
-    author={PaddleNLP Contributors},
-    howpublished = {\url{https://github.com/PaddlePaddle/PaddleNLP}},
-    year={2021}
-}
-```
-
-## Acknowledge
-
-我们借鉴了 Hugging Face 的[Transformers](https://github.com/huggingface/transformers)🤗关于预训练模型使用的优秀设计，在此对 Hugging Face 作者及其开源社区表示感谢。
-
-## License
-
-PaddleNLP 遵循[Apache-2.0开源协议](./LICENSE)。

README.md

@@ -0,0 +1 @@
+README_cn.md

README_cn.md

@@ -0,0 +1,345 @@
+**简体中文**🀄 | [English🌎](./README_en.md)
+
+<p align="center">
+  <img src="https://user-images.githubusercontent.com/1371212/175816733-8ec25eb0-9af3-4380-9218-27c154518258.png" align="middle"  width="500" />
+</p>
+
+------------------------------------------------------------------------------------------
+
+<p align="center">
+    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/PaddleNLP?color=ffa"></a>
+    <a href=""><img src="https://img.shields.io/badge/python-3.6.2+-aff.svg"></a>
+    <a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/graphs/contributors"><img src="https://img.shields.io/github/contributors/PaddlePaddle/PaddleNLP?color=9ea"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/commits"><img src="https://img.shields.io/github/commit-activity/m/PaddlePaddle/PaddleNLP?color=3af"></a>
+    <a href="https://pypi.org/project/paddlenlp/"><img src="https://img.shields.io/pypi/dm/paddlenlp?color=9cf"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/PaddleNLP?color=9cc"></a>
+    <a href="https://github.com/PaddlePaddle/PaddleNLP/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/PaddleNLP?color=ccf"></a>
+</p>
+
+
+<h4 align="center">
+  <a href=#特性> 特性 </a> |
+  <a href=#安装> 安装 </a> |
+  <a href=#快速开始> 快速开始 </a> |
+  <a href=#api文档> API文档 </a> |
+  <a href=#社区交流> 社区交流 </a>
+</h4>
+
+**PaddleNLP**是一款**简单易用**且**功能强大**的自然语言处理开发库。聚合业界**优质预训练模型**并提供**开箱即用**的开发体验，覆盖NLP多场景的模型库搭配**产业实践范例**可满足开发者**灵活定制**的需求。
+
+## News 📢
+* 🍭 2022.6.29 **PaddleNLP v2.3.4**发布！[**ERNIE Tiny**](./model_zoo/ernie-3.0) 全系列中文预训练小模型发布，快速提升预训练模型部署效率，通用信息抽取技术[**UIE Tiny**](./model_zoo/uie) 系列模型全新升级，支持速度更快效果更好的UIE小模型。
+* 🔥 2022.5.16 [**PaddleNLP v2.3**](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.3.0)全新发布！🎉
+  * 💎 发布通用信息抽取技术[**UIE**](./model_zoo/uie)，单模型支持实体识别、关系和事件抽取、情感分析等多种开放域信息抽取任务，不限领域和抽取目标，支持**一键抽取**与全流程**小样本**高效定制开发。
+  * 😊 发布文心大模型[**ERNIE 3.0**](./model_zoo/ernie-3.0)轻量级模型，在[CLUE](https://www.cluebenchmarks.com/)上实现同规模结构效果最佳，并提供**🗜️无损压缩**和**⚙️全场景部署**方案。
+  * 🏥 发布中文医疗领域预训练模型[**ERNIE-Health**](./model_zoo/ernie-health)，[CBLUE](https://github.com/CBLUEbenchmark/CBLUE)中文医疗信息处理评测冠军模型。
+  * 💬 发布大规模百亿开放域对话预训练模型[**PLATO-XL**](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/model_zoo/plato-xl) ，配合⚡**FasterGeneration**⚡快速实现高性能GPU并行推理加速。
+
+* 📬 2021.12.12 **PaddleNLP v2.2**发布！新增开箱即用的NLP能力[Taskflow](./docs/model_zoo/taskflow.md)！配套语义检索、智能问答、评论观点抽取[产业案例](./applications)，快速搭建端到端NLP系统！配套视频课程[直通车](https://aistudio.baidu.com/aistudio/course/introduce/24902)！
+
+## 特性
+
+#### <a href=#开箱即用的nlp工具集> 📦 开箱即用的NLP工具集 </a>
+
+#### <a href=#丰富完备的中文模型库> 🤗 丰富完备的中文模型库 </a>
+
+#### <a href=#产业级端到端系统范例> 🎛️ 产业级端到端系统范例 </a>
+
+#### <a href=#高性能分布式训练与推理> 🚀 高性能分布式训练与推理 </a>
+
+
+### 开箱即用的NLP工具集
+
+Taskflow提供丰富的**📦开箱即用**的产业级NLP预置模型，覆盖自然语言理解与生成两大场景，提供**💪产业级的效果**与**⚡️极致的推理性能**。
+
+![taskflow1](https://user-images.githubusercontent.com/11793384/159693816-fda35221-9751-43bb-b05c-7fc77571dd76.gif)
+
+更多使用方法可参考[Taskflow文档](./docs/model_zoo/taskflow.md)。
+
+### 丰富完备的中文模型库
+
+#### 🀄 业界最全的中文预训练模型
+
+精选 45+ 个网络结构和 500+ 个预训练模型参数，涵盖业界最全的中文预训练模型：既包括文心NLP大模型的ERNIE、PLATO等，也覆盖BERT、GPT、RoBERTa、T5等主流结构。通过`AutoModel` API一键⚡**高速下载**⚡。
+
+```python
+from paddlenlp.transformers import *
+
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+bert = AutoModel.from_pretrained('bert-wwm-chinese')
+albert = AutoModel.from_pretrained('albert-chinese-tiny')
+roberta = AutoModel.from_pretrained('roberta-wwm-ext')
+electra = AutoModel.from_pretrained('chinese-electra-small')
+gpt = AutoModelForPretraining.from_pretrained('gpt-cpm-large-cn')
+```
+
+针对预训练模型计算瓶颈，可以使用API一键使用文心ERNIE-Tiny全系列轻量化模型，降低预训练模型部署难度。
+
+```python
+# 6L768H
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+# 6L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-mini-zh')
+# 4L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-micro-zh')
+# 4L312H
+ernie = AutoModel.from_pretrained('ernie-3.0-nano-zh')
+```
+
+对预训练模型应用范式如语义表示、文本分类、句对匹配、序列标注、问答等，提供统一的API体验。
+
+```python
+import paddle
+from paddlenlp.transformers import *
+
+tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
+text = tokenizer('自然语言处理')
+
+# 语义表示
+model = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+sequence_output, pooled_output = model(input_ids=paddle.to_tensor([text['input_ids']]))
+# 文本分类 & 句对匹配
+model = AutoModelForSequenceClassification.from_pretrained('ernie-3.0-medium-zh')
+# 序列标注
+model = AutoModelForTokenClassification.from_pretrained('ernie-3.0-medium-zh')
+# 问答
+model = AutoModelForQuestionAnswering.from_pretrained('ernie-3.0-medium-zh')
+```
+
+#### 💯 全场景覆盖的应用示例
+
+覆盖从学术到产业的NLP应用示例，涵盖NLP基础技术、NLP系统应用以及拓展应用。全面基于飞桨核心框架2.0全新API体系开发，为开发者提供飞桨文本领域的最佳实践。
+
+精选预训练模型示例可参考[Model Zoo](./model_zoo)，更多场景示例文档可参考[examples目录](./examples)。更有免费算力支持的[AI Studio](https://aistudio.baidu.com)平台的[Notbook交互式教程](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)提供实践。
+
+<details><summary> PaddleNLP预训练模型适用任务汇总（<b>点击展开详情</b>）</summary><div>
+
+| Model              | Sequence Classification | Token Classification | Question Answering | Text Generation | Multiple Choice |
+| :----------------- | ----------------------- | -------------------- | ------------------ | --------------- | --------------- |
+| ALBERT             | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BART               | ✅                       | ✅                    | ✅                  | ✅               | ❌               |
+| BERT               | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BigBird            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BlenderBot         | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| ChineseBERT        | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ConvBERT           | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| CTRL               | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| DistilBERT         | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ELECTRA            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| ERNIE              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| ERNIE-CTM          | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| ERNIE-Doc          | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ERNIE-GEN          | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| ERNIE-Gram         | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ERNIE-M            | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| FNet               | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| Funnel-Transformer | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| GPT                | ✅                       | ✅                    | ❌                  | ✅               | ❌               |
+| LayoutLM           | ✅                       | ✅                    | ❌                  | ❌               | ❌               |
+| LayoutLMv2         | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| LayoutXLM          | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| LUKE               | ❌                       | ✅                    | ✅                  | ❌               | ❌               |
+| mBART              | ✅                       | ❌                    | ✅                  | ❌               | ✅               |
+| MegatronBERT       | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| MobileBERT         | ✅                       | ❌                    | ✅                  | ❌               | ❌               |
+| MPNet              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| NEZHA              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| PP-MiniLM          | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| ProphetNet         | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| Reformer           | ✅                       | ❌                    | ✅                  | ❌               | ❌               |
+| RemBERT            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| RoBERTa            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| RoFormer           | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| SKEP               | ✅                       | ✅                    | ❌                  | ❌               | ❌               |
+| SqueezeBERT        | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| T5                 | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| TinyBERT           | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| UnifiedTransformer | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| XLNet              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+
+</div></details>
+
+可参考[Transformer 文档](/docs/model_zoo/index.rst) 查看目前支持的预训练模型结构、参数和详细用法。
+
+### 产业级端到端系统范例
+
+PaddleNLP针对信息抽取、语义检索、智能问答、情感分析等高频NLP场景，提供了端到端系统范例，打通*数据标注*-*模型训练*-*模型调优*-*预测部署*全流程，持续降低NLP技术产业落地门槛。更多详细的系统级产业范例使用说明请参考[Applications](./applications)。
+
+#### 🔍 语义检索系统
+
+针对无监督数据、有监督数据等多种数据情况，结合SimCSE、In-batch Negatives、ERNIE-Gram单塔模型等，推出前沿的语义检索方案，包含召回、排序环节，打通训练、调优、高效向量检索引擎建库和查询全流程。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514909-8817d79a-72c4-4be1-8080-93d1f682bb46.gif" width="400">
+</div>
+
+
+更多使用说明请参考[语义检索系统](./applications/neural_search)。
+
+#### ❓ 智能问答系统
+
+基于[🚀RocketQA](https://github.com/PaddlePaddle/RocketQA)技术的检索式问答系统，支持FAQ问答、说明书问答等多种业务场景。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514868-1babe981-c675-4f89-9168-dd0a3eede315.gif" width="400">
+</div>
+
+
+更多使用说明请参考[智能问答系统](./applications/question_answering)与[文档智能问答](./applications/doc_vqa)
+
+#### 💌 评论观点抽取与情感分析
+
+基于情感知识增强预训练模型SKEP，针对产品评论进行评价维度和观点抽取，以及细粒度的情感分析。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407260-b7f92800-861c-4207-98f3-2291e0102bbe.png" width="400">
+</div>
+
+更多使用说明请参考[情感分析](./applications/sentiment_analysis)。
+
+#### 🎙️ 智能语音指令解析
+
+集成了[PaddleSpeech](https://github.com/PaddlePaddle/PaddleSpeech)和[百度开放平台](https://ai.baidu.com/)的的语音识别和[UIE](./model_zoo/uie)通用信息抽取等技术，打造智能一体化的语音指令解析系统范例，该方案可应用于智能语音填单、智能语音交互、智能语音检索等场景，提高人机交互效率。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/16698950/168589100-a6c6f346-97bb-47b2-ac26-8d50e71fddc5.png" width="400">
+</div>
+
+更多使用说明请参考[智能语音指令解析](./applications/speech_cmd_analysis)。
+
+### 高性能分布式训练与推理
+
+#### ⚡ FasterTokenizer：高性能文本处理库
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407921-b4395b1d-44bd-41a0-8c58-923ba2b703ef.png" width="400">
+</div>
+
+```python
+AutoTokenizer.from_pretrained("ernie-3.0-medium-zh", use_faster=True)
+```
+
+为了实现更极致的模型部署性能，安装FastTokenizers后只需在`AutoTokenizer` API上打开 `use_faster=True`选项，即可调用C++实现的高性能分词算子，轻松获得超Python百余倍的文本处理加速，更多使用说明可参考[FasterTokenizer文档](./faster_tokenizer)。
+
+#### ⚡️ FasterGeneration：高性能生成加速库
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407831-914dced0-3a5a-40b8-8a65-ec82bf13e53c.gif" width="400">
+</div>
+
+```python
+model = GPTLMHeadModel.from_pretrained('gpt-cpm-large-cn')
+...
+outputs, _ = model.generate(
+    input_ids=inputs_ids, max_length=10, decode_strategy='greedy_search',
+    use_faster=True)
+```
+
+简单地在`generate()`API上打开`use_faster=True`选项，轻松在Transformer、GPT、BART、PLATO、UniLM等生成式预训练模型上获得5倍以上GPU加速，更多使用说明可参考[FasterGeneration文档](./faster_generation)。
+
+#### 🚀 Fleet：飞桨4D混合并行分布式训练技术
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168515134-513f13e0-9902-40ef-98fa-528271dcccda.png" width="300">
+</div>
+
+
+更多关于千亿级AI模型的分布式训练使用说明可参考[GPT-3](./examples/language_model/gpt-3)。
+
+## 安装
+
+### 环境依赖
+
+- python >= 3.6
+- paddlepaddle >= 2.2
+
+### pip安装
+
+```shell
+pip install --upgrade paddlenlp
+```
+
+更多关于PaddlePaddle和PaddleNLP安装的详细教程请查看[Installation](./docs/get_started/installation.rst)。
+
+## 快速开始
+
+这里以信息抽取-命名实体识别任务，UIE模型为例，来说明如何快速使用PaddleNLP:
+
+### 一键预测
+
+PaddleNLP提供[一键预测功能](./docs/model_zoo/taskflow.md)，无需训练，直接输入数据即可开放域抽取结果：
+
+```python
+>>> from pprint import pprint
+>>> from paddlenlp import Taskflow
+
+>>> schema = ['时间', '选手', '赛事名称'] # Define the schema for entity extraction
+>>> ie = Taskflow('information_extraction', schema=schema)
+>>> pprint(ie("2月8日上午北京冬奥会自由式滑雪女子大跳台决赛中中国选手谷爱凌以188.25分获得金牌！"))
+[{'时间': [{'end': 6,
+          'probability': 0.9857378532924486,
+          'start': 0,
+          'text': '2月8日上午'}],
+  '赛事名称': [{'end': 23,
+            'probability': 0.8503089953268272,
+            'start': 6,
+            'text': '北京冬奥会自由式滑雪女子大跳台决赛'}],
+  '选手': [{'end': 31,
+          'probability': 0.8981548639781138,
+          'start': 28,
+          'text': '谷爱凌'}]}]
+```
+
+### 小样本学习
+
+如果对一键预测效果不满意，也可以使用少量数据进行模型精调，进一步提升特定场景的效果，详见[UIE小样本定制训练](./model_zoo/uie/)。
+
+更多PaddleNLP内容可参考：
+- [精选模型库](./model_zoo)，包含优质预训练模型的端到端全流程使用。
+- [多场景示例](./examples)，了解如何使用PaddleNLP解决NLP多种技术问题，包含基础技术、系统应用与拓展应用。
+- [交互式教程](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)，在🆓免费算力平台AI Studio上快速学习PaddleNLP。
+
+
+## API文档
+
+PaddleNLP提供全流程的文本领域API，可大幅提升NLP任务建模的效率：
+
+- 支持[千言](https://www.luge.ai)等丰富中文数据集加载的[Dataset API](https://paddlenlp.readthedocs.io/zh/latest/data_prepare/dataset_list.html)。
+- 提供🤗Hugging Face Style的API，支持 **500+** 优质预训练模型加载的[Transformers API](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/index.html)。
+- 提供30+多语言词向量的[Embedding API](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/embeddings.html)
+
+更多使用方法请参考[API文档](https://paddlenlp.readthedocs.io/zh/latest/)。
+
+## 社区交流
+
+- 微信扫描二维码并填写问卷之后，加入交流群领取福利
+  - 获取5月18-19日每晚20:30《产业级通用信息抽取技术UIE+ERNIE轻量级模型》直播课链接。
+  - 10G重磅NLP学习大礼包！
+
+  <div align="center">
+  <img src="https://user-images.githubusercontent.com/11793384/168411900-d9f3d777-99ab-4b5c-8cdc-ef747a48b864.jpg" width="150" height="150" />
+  </div>
+
+
+## Citation
+
+如果PaddleNLP对您的研究有帮助，欢迎引用
+
+```
+@misc{=paddlenlp,
+    title={PaddleNLP: An Easy-to-use and High Performance NLP Library},
+    author={PaddleNLP Contributors},
+    howpublished = {\url{https://github.com/PaddlePaddle/PaddleNLP}},
+    year={2021}
+}
+```
+
+## Acknowledge
+
+我们借鉴了Hugging Face的[Transformers](https://github.com/huggingface/transformers)🤗关于预训练模型使用的优秀设计，在此对Hugging Face作者及其开源社区表示感谢。
+
+## License
+
+PaddleNLP遵循[Apache-2.0开源协议](./LICENSE)。

README_en.md

@@ -1,4 +1,4 @@
-[简体中文🀄](./README.md) | **English🌎**
+[简体中文🀄](./README_cn.md) |  **English**🌎
 
 <p align="center">
   <img src="https://user-images.githubusercontent.com/1371212/175816733-8ec25eb0-9af3-4380-9218-27c154518258.png" align="middle"  width="500" />
@@ -7,133 +7,311 @@
 ------------------------------------------------------------------------------------------
 
 <p align="center">
-    <a href="https://paddlenlp.readthedocs.io/en/latest/?badge=latest"><img src="https://readthedocs.org/projects/paddlenlp/badge/?version=latest">
+    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/PaddleNLP?color=ffa"></a>
-    <a href=""><img src="https://img.shields.io/badge/python-3.7+-aff.svg"></a>
+    <a href=""><img src="https://img.shields.io/badge/python-3.6.2+-aff.svg"></a>
     <a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/graphs/contributors"><img src="https://img.shields.io/github/contributors/PaddlePaddle/PaddleNLP?color=9ea"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/commits"><img src="https://img.shields.io/github/commit-activity/m/PaddlePaddle/PaddleNLP?color=3af"></a>
     <a href="https://pypi.org/project/paddlenlp/"><img src="https://img.shields.io/pypi/dm/paddlenlp?color=9cf"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/PaddleNLP?color=9cc"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/PaddleNLP?color=ccf"></a>
-    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
 </p>
 
 <h4 align="center">
-    <a href=#Features> Features </a> |
-    <a href=#Support-Models> Supported Models </a> |
-    <a href=#Installation> Installation </a> |
-    <a href=#Quick-start> Quick Start </a> |
-    <a href=#community> Community </a>
+  <a href=#features> Features </a> |
+  <a href=#installation> Installation </a> |
+  <a href=#quick-start> Quick Start </a> |
+  <a href=#api-reference> API Reference </a> |
+  <a href=#community> Community </a>
 </h4>
 
-**PaddleNLP** is a Large Language Model (LLM) development suite based on the PaddlePaddle deep learning framework, supporting efficient large model training, lossless compression, and high-performance inference on various hardware devices. With its **simplicity** and **ultimate performance**, PaddleNLP is dedicated to helping developers achieve efficient industrial applications of large models.
+**PaddleNLP** is an *easy-to-use* and *powerful* NLP library with **Awesome** pre-trained model zoo, supporting wide-range of NLP tasks from research to industrial applications.
 
 ## News 📢
 
-* **2024.06.27 [PaddleNLP v3.0 Beta](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta0)**：Embrace large models and experience a complete upgrade. With a unified large model suite, we achieve full-process access to domestically produced computing chips. We fully support industrial-level application processes for large models, such as PaddlePaddle's 4D parallel configuration, efficient fine-tuning strategies, efficient alignment algorithms, and high-performance reasoning. Our developed RsLoRA+ algorithm, full checkpoint storage mechanism Unified Checkpoint, and generalized support for FastFNN and FusedQKV all contribute to the training and inference of large models. We continuously support updates to mainstream models for providing efficient solutions.
+* 🍭 2022.6.29 **PaddleNLP v2.3.4** Released! Whole series of Chinese pretrained models [**ERNIE Tiny**](./model_zoo/ernie-3.0) are released to quickly improve deployment efficiency. We also provides smaller and faster models [**UIE Tiny**](./model_zoo/uie) for universal information extraction.
 
-* **2024.04.24 [PaddleNLP v2.8](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.8.0)**：Our self-developed RsLoRA+ algorithm with extreme convergence significantly improves the convergence speed and training effectiveness of PEFT training. By introducing high-performance generation acceleration into the RLHF PPO algorithm, we have broken through the generation speed bottleneck in PPO training, achieving a significant lead in PPO training performance. We generally support multiple large model training performance optimization methods such as FastFFN and FusedQKV, making large model training faster and more stable.
+* 🔥 2022.5.16 PaddleNLP [v2.3](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.3.0) Released!🎉
+  * 💎 Release [**UIE** (Universal Information Extraction)](./model_zoo/uie) technique, single model supports multiple **open-domain** IE tasks. Super easy to use and finetune with few examples via [Taskflow](./docs/model_zoo/taskflow.md).
+  * 😊 Release [**ERNIE 3.0**](./model_zoo/ernie-3.0) light-weight model achieved better results compared to ERNIE 2.0 on [CLUE](https://www.cluebenchmarks.com/), also including **🗜️lossless model compression** and **⚙️end-to-end deployment**.
+  * 🏥 Release [**ERNIE-Health**](./model_zoo/ernie-health), a **SOTA** biomedical pretrained model on [CBLUE](https://github.com/CBLUEbenchmark/CBLUE).
+  * 💬 Release [**PLATO-XL**](./model_zoo/plato-xl) with ⚡**FasterGeneration**⚡, the *11B open-domain SOTA chatbot model* can be deployed on multi-GPU and do parallel inference easily.
 
 ## Features
 
-### <a href=#Integrated training and inference on multiple hardware platforms> 🔧 Integrated training and inference on multiple hardware platforms </a>
+#### <a href=#out-of-box-nlp-toolset> 📦 Out-of-Box NLP Toolset </a>
 
-Our development suit supports large model training and inference on multiple hardware platforms, including NVIDIA GPUs, Kunlun XPUs, Ascend NPUs, Enflame GCUs, and Hygon DCUs. The toolkit's interface allows for quick hardware switching, significantly reducing research and development costs associated with hardware transitions.
+#### <a href=#awesome-chinese-model-zoo> 🤗 Awesome Chinese Model Zoo </a>
 
-### <a href=Efficient and easy-to-use pre-training> 🚀 Efficient and easy-to-use pre-training </a>
+#### <a href=#industrial-end-to-end-system> 🎛️ Industrial End-to-end System </a>
 
-We support 4D high-performance training with data parallelism, sharding parallelism, tensor parallelism, and pipeline parallelism. The Trainer supports configurable distributed strategies, reducing the cost associated with complex distributed combinations. The Unified Checkpoint large model storage format supports dynamic scaling of model parameter distribution during training, thereby reducing the migration cost caused by hardware switching.
+#### <a href=#high-performance-distributed-training-and-inference> 🚀 High Performance Distributed Training and Inference </a>
 
-### <a href=#Efficient fine-tuning> 🤗 Efficient fine-tuning </a>
 
-The fine-tuning algorithms are deeply integrated with zero-padding data streams and high-performance FlashMask operators, reducing invalid data padding and computation during training, and significantly improving the throughput of fine-tuning training.
+### Out-of-Box NLP Toolset
 
-### <a href=#Lossless compression and high-performance inference> 🎛️ Lossless compression and high-performance inference </a>
+Taskflow aims to provide off-the-shelf NLP pre-built task covering NLU and NLG technique, in the meanwhile with extreamly fast infernece satisfying industrial scenario.
 
-The high-performance inference module of the large model toolkit incorporates dynamic insertion and operator fusion strategies throughout the entire process, greatly accelerating parallel inference speed. The underlying implementation details are encapsulated, enabling out-of-the-box high-performance parallel inference capabilities.
+![taskflow1](https://user-images.githubusercontent.com/11793384/159693816-fda35221-9751-43bb-b05c-7fc77571dd76.gif)
 
-## Documentation
-For detailed documentation, visit the [PaddleNLP Documentation](https://paddlenlp.readthedocs.io/).
+For more usage please refer to [Taskflow Docs](./docs/model_zoo/taskflow.md).
 
-------------------------------------------------------------------------------------------
+### Awesome Chinese Model Zoo
 
-## Support Models
+#### 🀄 Comprehensive Chinese Transformer Models
+
+We provide **45+** network architectures and over **500+** pretrained models. Not only includes all the SOTA model like ERNIE, PLATO and SKEP released by Baidu, but also integrates most of the high-quality Chinese pretrained model developed by other organizations. Use `AutoModel` API to **⚡SUPER FAST⚡** download pretrained mdoels of different architecture. We welcome all developers to contribute your Transformer models to PaddleNLP!
+
+```python
+from paddlenlp.transformers import *
+
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+bert = AutoModel.from_pretrained('bert-wwm-chinese')
+albert = AutoModel.from_pretrained('albert-chinese-tiny')
+roberta = AutoModel.from_pretrained('roberta-wwm-ext')
+electra = AutoModel.from_pretrained('chinese-electra-small')
+gpt = AutoModelForPretraining.from_pretrained('gpt-cpm-large-cn')
+```
+
+Due to the computation limitation, you can use the ERNIE-Tiny light models to accelerate the deployment of pretrained models.
+```python
+# 6L768H
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+# 6L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-mini-zh')
+# 4L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-micro-zh')
+# 4L312H
+ernie = AutoModel.from_pretrained('ernie-3.0-nano-zh')
+```
+Unified API experience for NLP task like semantic representation, text classification, sentence matching, sequence labeling, question answering, etc.
+
+```python
+import paddle
+from paddlenlp.transformers import *
+
+tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
+text = tokenizer('natural language processing')
+
+# Semantic Representation
+model = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+sequence_output, pooled_output = model(input_ids=paddle.to_tensor([text['input_ids']]))
+# Text Classificaiton and Matching
+model = AutoModelForSequenceClassification.from_pretrained('ernie-3.0-medium-zh')
+# Sequence Labeling
+model = AutoModelForTokenClassification.from_pretrained('ernie-3.0-medium-zh')
+# Question Answering
+model = AutoModelForQuestionAnswering.from_pretrained('ernie-3.0-medium-zh')
+```
+
+#### Wide-range NLP Task Support
+
+PaddleNLP provides rich examples covering mainstream NLP task to help developers accelerate problem solving. You can find our powerful transformer [Model Zoo](./model_zoo), and wide-range NLP application [exmaples](./examples) with detailed instructions.
+
+Also you can run our interactive [Notebook tutorial](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995) on AI Studio, a powerful platform with **FREE** computing resource.
+
+<details><summary> PaddleNLP Transformer model summary (<b>click to show details</b>) </summary><div>
+
+| Model              | Sequence Classification | Token Classification | Question Answering | Text Generation | Multiple Choice |
+| :----------------- | ----------------------- | -------------------- | ------------------ | --------------- | --------------- |
+| ALBERT             | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BART               | ✅                       | ✅                    | ✅                  | ✅               | ❌               |
+| BERT               | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BigBird            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BlenderBot         | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| ChineseBERT        | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ConvBERT           | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| CTRL               | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| DistilBERT         | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ELECTRA            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| ERNIE              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| ERNIE-CTM          | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| ERNIE-Doc          | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ERNIE-GEN          | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| ERNIE-Gram         | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ERNIE-M            | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| FNet               | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| Funnel-Transformer | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| GPT                | ✅                       | ✅                    | ❌                  | ✅               | ❌               |
+| LayoutLM           | ✅                       | ✅                    | ❌                  | ❌               | ❌               |
+| LayoutLMv2         | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| LayoutXLM          | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| LUKE               | ❌                       | ✅                    | ✅                  | ❌               | ❌               |
+| mBART              | ✅                       | ❌                    | ✅                  | ❌               | ✅               |
+| MegatronBERT       | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| MobileBERT         | ✅                       | ❌                    | ✅                  | ❌               | ❌               |
+| MPNet              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| NEZHA              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| PP-MiniLM          | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| ProphetNet         | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| Reformer           | ✅                       | ❌                    | ✅                  | ❌               | ❌               |
+| RemBERT            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| RoBERTa            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| RoFormer           | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| SKEP               | ✅                       | ✅                    | ❌                  | ❌               | ❌               |
+| SqueezeBERT        | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| T5                 | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| TinyBERT           | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| UnifiedTransformer | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| XLNet              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+
+</div></details>
+
+For more pretrained model usage, please refer to [Transformer API Docs](./docs/model_zoo/index.rst).
+
+### Industrial End-to-end System
+
+We provide high value scenarios including information extraction, semantic retrieval, questionn answering high-value.
+
+For more details industial cases please refer to [Applications](./applications).
+
+
+#### 🔍 Neural Search System
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514909-8817d79a-72c4-4be1-8080-93d1f682bb46.gif" width="400">
+</div>
+
+
+For more details please refer to [Neural Search](./applications/neural_search).
+
+#### ❓ Question Answering System
+
+We provide question answering pipeline which can support FAQ system, Document-level Visual Question answering system based on [🚀RocketQA](https://github.com/PaddlePaddle/RocketQA).
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514868-1babe981-c675-4f89-9168-dd0a3eede315.gif" width="400">
+</div>
+
+
+For more details please refer to [Question Answering](./applications/question_answering) and [Document VQA](./applications/doc_vqa).
+
+
+#### 💌 Opinion Extraction and Sentiment Analysis
+
+We build an opinion extraction system for product review and fine-grained sentiment analysis based on [SKEP](https://arxiv.org/abs/2005.05635) Model.
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407260-b7f92800-861c-4207-98f3-2291e0102bbe.png" width="300">
+</div>
+
+
+For more details please refer to [Sentiment Analysis](./applications/sentiment_analysis).
+
+#### 🎙️ Speech Command Analysis
+
+Integrated ASR Model, Information Extraction, we provide a speech command analysis pipeline that show how to use PaddleNLP and [PaddleSpeech](https://github.com/PaddlePaddle/PaddleSpeech) to solve Speech + NLP real scenarios.
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168412618-04897a47-79c9-4fe7-a054-5dc1f6a1f75c.png" width="500">
+</div>
+
+
+For more details please refer to [Speech Command Analysis](./applications/speech_cmd_analysis).
+
+### High Performance Distributed Training and Inference
+
+#### ⚡ FasterTokenizer: High Performance Text Preprocessing Library
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407921-b4395b1d-44bd-41a0-8c58-923ba2b703ef.png" width="400">
+</div>
+
+```python
+AutoTokenizer.from_pretrained("ernie-3.0-medium-zh", use_faster=True)
+```
+
+Set `use_faster=True` to use C++ Tokenizer kernel to achieve 100x faster on text pre-processing. For more usage please refer to [FasterTokenizer](./faster_tokenizer).
+
+#### ⚡ FasterGeneration: High Perforance Generation Library
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407831-914dced0-3a5a-40b8-8a65-ec82bf13e53c.gif" width="400">
+</div>
+
+```python
+model = GPTLMHeadModel.from_pretrained('gpt-cpm-large-cn')
+...
+outputs, _ = model.generate(
+    input_ids=inputs_ids, max_length=10, decode_strategy='greedy_search',
+    use_faster=True)
+```
+
+Set `use_faster=True` to achieve 5x speedup for Transformer, GPT, BART, PLATO, UniLM text generation. For more usage please refer to [FasterGeneration](./faster_generation).
+
+#### 🚀 Fleet: 4D Hybrid Distributed Training
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168515134-513f13e0-9902-40ef-98fa-528271dcccda.png" width="300">
+</div>
+
+
+For more super large-scale model pre-training details please refer to [GPT-3](./examples/language_model/gpt-3).
 
-Detailed list 👉 [Supported Model List](https://github.com/PaddlePaddle/PaddleNLP/issues/8663)
 
 ## Installation
 
 ### Prerequisites
 
-* python >= 3.8
-* paddlepaddle >= 3.0.0b0
+* python >= 3.6
+* paddlepaddle >= 2.2
 
-### Pip Installation
+More information about PaddlePaddle installation please refer to [PaddlePaddle's Website](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/conda/linux-conda.html).
+
+### Python pip Installation
 
-```shell
-pip install --upgrade paddlenlp==3.0.0b3
 ```
-
-or you can install the latest develop branch code with the following command:
-
-```shell
-pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html
+pip install --upgrade paddlenlp
 ```
 
-More information about PaddlePaddle installation please refer to [PaddlePaddle's Website](https://www.paddlepaddle.org.cn).
-
-------------------------------------------------------------------------------------------
-
 ## Quick Start
 
-### Text generation with large language model
-
-PaddleNLP provides a convenient and easy-to-use Auto API, which can quickly load models and Tokenizers. Here, we use the `Qwen/Qwen2-0.5B` large model as an example for text generation:
+**Taskflow** aims to provide off-the-shelf NLP pre-built task covering NLU and NLG scenario, in the meanwhile with extreamly fast infernece satisfying industrial applications.
 
 ```python
->>> from paddlenlp.transformers import AutoTokenizer, AutoModelForCausalLM
->>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B")
->>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B", dtype="float16")
->>> input_features = tokenizer("你好！请自我介绍一下。", return_tensors="pd")
->>> outputs = model.generate(**input_features, max_length=128)
->>> print(tokenizer.batch_decode(outputs[0], skip_special_tokens=True))
-['我是一个AI语言模型，我可以回答各种问题，包括但不限于：天气、新闻、历史、文化、科学、教育、娱乐等。请问您有什么需要了解的吗？']
+from paddlenlp import Taskflow
+
+# Chinese Word Segmentation
+seg = Taskflow("word_segmentation")
+seg("第十四届全运会在西安举办")
+>>> ['第十四届', '全运会', '在', '西安', '举办']
+
+# POS Tagging
+tag = Taskflow("pos_tagging")
+tag("第十四届全运会在西安举办")
+>>> [('第十四届', 'm'), ('全运会', 'nz'), ('在', 'p'), ('西安', 'LOC'), ('举办', 'v')]
+
+# Named Entity Recognition
+ner = Taskflow("ner")
+ner("《孤女》是2010年九州出版社出版的小说，作者是余兼羽")
+>>> [('《', 'w'), ('孤女', '作品类_实体'), ('》', 'w'), ('是', '肯定词'), ('2010年', '时间类'), ('九州出版社', '组织机构类'), ('出版', '场景事件'), ('的', '助词'), ('小说', '作品类_概念'), ('，', 'w'), ('作者', '人物类_概念'), ('是', '肯定词'), ('余兼羽', '人物类_实体')]
+
+# Dependency Parsing
+ddp = Taskflow("dependency_parsing")
+ddp("9月9日上午纳达尔在亚瑟·阿什球场击败俄罗斯球员梅德韦杰夫")
+>>> [{'word': ['9月9日', '上午', '纳达尔', '在', '亚瑟·阿什球场', '击败', '俄罗斯', '球员', '梅德韦杰夫'], 'head': [2, 6, 6, 5, 6, 0, 8, 9, 6], 'deprel': ['ATT', 'ADV', 'SBV', 'MT', 'ADV', 'HED', 'ATT', 'ATT', 'VOB']}]
+
+# Sentiment Analysis
+senta = Taskflow("sentiment_analysis")
+senta("这个产品用起来真的很流畅，我非常喜欢")
+>>> [{'text': '这个产品用起来真的很流畅，我非常喜欢', 'label': 'positive', 'score': 0.9938690066337585}]
 ```
 
-### Pre-training for large language model
+## API Reference
 
-```shell
-git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # if cloned or downloaded, can skip this step
-mkdir -p llm/data && cd llm/data
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
-cd .. # change folder to PaddleNLP/llm
-python -u -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" run_pretrain.py ./config/llama/pretrain_argument.json
-```
+- Support [LUGE](https://www.luge.ai/) dataset loading and compatible with Hugging Face [Datasets](https://huggingface.co/datasets). For more details please refer to [Dataset API](https://paddlenlp.readthedocs.io/zh/latest/data_prepare/dataset_list.html).
+- Using Hugging Face style API to load 500+ selected transformer models and download with fast speed. For more information please refer to [Transformers API](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/index.html).
+- One-line of code to load pre-trained word embedding. For more usage please refer to [Embedding API](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/embeddings.html).
 
-### SFT finetuning forlarge language model
-
-```shell
-git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # if cloned or downloaded, can skip this step
-mkdir -p llm/data && cd llm/data
-wget https://bj.bcebos.com/paddlenlp/datasets/examples/AdvertiseGen.tar.gz && tar -zxvf AdvertiseGen.tar.gz
-cd .. # change folder to PaddleNLP/llm
-python -u -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" run_finetune.py ./config/llama/sft_argument.json
-```
-
-For more steps in the entire large model process, please refer to the[Large Model Full-Process Suite](./llm).
-
-For more PaddleNLP content, please refer to:
-
-* [Model Library](./slm/model_zoo)，which includes end-to-end usage of high-quality pre-trained models.
-* [Multi-scenario Examples](./slm/examples)，to understand how to use PaddleNLP to solve various NLP technical problems, including basic techniques, system applications, and extended applications.
-* [Interactive Tutorial](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)，to quickly learn PaddleNLP on the free computing platform AI Studio.
-
-------------------------------------------------------------------------------------------
+Please find all PaddleNLP API Reference from our [readthedocs](https://paddlenlp.readthedocs.io/).
 
 ## Community
 
+### Special Interest Group (SIG)
+
+Welcome to join [PaddleNLP SIG](https://iwenjuan.baidu.com/?code=bkypg8) for contribution, eg. Dataset, Models and Toolkit.
+
 ### Slack
 
 To connect with other users and contributors, welcome to join our [Slack channel](https://paddlenlp.slack.com/).
@@ -142,15 +320,14 @@ To connect with other users and contributors, welcome to join our [Slack channel
 
 Scan the QR code below with your Wechat⬇️. You can access to official technical exchange group. Look forward to your participation.
 
-<div align="center">
-    <img src="https://github.com/user-attachments/assets/3a58cc9f-69c7-4ccb-b6f5-73e966b8051a" width="150" height="150" />
-</div>
+ <div align="center">
+ <img src="https://user-images.githubusercontent.com/11793384/168411900-d9f3d777-99ab-4b5c-8cdc-ef747a48b864.jpg" width="150" height="150" />
+ </div>
 
 ## Citation
 
-If you find PaddleNLP useful in your research, please consider citing
-
-```bibtext
+If you find PaddleNLP useful in your research, please consider cite
+```
 @misc{=paddlenlp,
     title={PaddleNLP: An Easy-to-use and High Performance NLP Library},
     author={PaddleNLP Contributors},
@@ -161,7 +338,7 @@ If you find PaddleNLP useful in your research, please consider citing
 
 ## Acknowledge
 
-We have borrowed from Hugging Face's [Transformers](https://github.com/huggingface/transformers)🤗 excellent design on pretrained models usage, and we would like to express our gratitude to the authors of Hugging Face and its open source community.
+We have borrowed from Hugging Face's [Transformer](https://github.com/huggingface/transformers)🤗 excellent design on pretrained models usage, and we would like to express our gratitude to the authors of Hugging Face and its open source community.
 
 ## License
 

applications/doc_vqa/.gitignore

@@ -0,0 +1,17 @@
+OCR_process/*.json
+*.png
+*.json
+answers/*
+checkpoints/*
+__pycache__/*
+OCR_process/demo_pics/*
+Rerank/log/*
+Rerank/checkpoints/*
+Rerank/data/*
+Rerank/output/*
+Rerank/__pycache__/*
+Extraction/log/*
+Extraction/checkpoints/*
+Extraction/data/*
+Extraction/output/*
+Extraction/__pycache__/*

applications/doc_vqa/Extraction/change_to_mrc.py

@@ -0,0 +1,37 @@
+import sys
+import json
+import numpy as np
+
+
+def get_top1_from_ranker(path):
+    with open(path, "r", encoding="utf-8") as f:
+        scores = [float(line.strip()) for line in f.readlines()]
+        top_id = np.argmax(scores)
+
+    return top_id
+
+
+def get_ocr_result_by_id(path, top_id):
+    with open(path, "r", encoding="utf-8") as f:
+        reses = f.readlines()
+        res = reses[top_id]
+    return json.loads(res)
+
+
+def write_to_file(doc, path):
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(doc, f, ensure_ascii=False)
+        f.write("\n")
+
+
+if __name__ == "__main__":
+    question = sys.argv[1]
+    ranker_result_path = "../Rerank/data/demo.score"
+    ocr_result_path = "../OCR_process/demo_ocr_res.json"
+    save_path = "data/demo_test.json"
+    top_id = get_top1_from_ranker(ranker_result_path)
+    doc = get_ocr_result_by_id(ocr_result_path, top_id)
+    doc["question"] = question
+    doc["img_id"] = str(top_id + 1)
+
+    write_to_file(doc, save_path)

applications/doc_vqa/Extraction/docvqa.py

@@ -0,0 +1,368 @@
+import os
+import cv2
+import sys
+import json
+import copy
+import collections
+import numpy as np
+from tqdm import tqdm
+
+import paddle
+from paddle.io import Dataset
+
+sys.path.insert(0, "../")
+
+
+class DocVQAExample(object):
+
+    def __init__(self,
+                 question,
+                 doc_tokens,
+                 doc_boxes=[],
+                 answer=None,
+                 labels=None,
+                 image=None):
+        self.question = question
+        self.doc_tokens = doc_tokens
+        self.doc_boxes = doc_boxes
+        self.image = image
+        self.answer = answer
+        self.labels = labels
+
+
+class DocVQAFeatures(object):
+    """A single set of features of data."""
+
+    def __init__(self,
+                 example_index,
+                 input_ids,
+                 input_mask,
+                 segment_ids,
+                 boxes=None,
+                 label=None):
+        self.example_index = example_index
+        self.input_ids = input_ids
+        self.input_mask = input_mask
+        self.segment_ids = segment_ids
+        self.boxes = boxes
+        self.label = label
+
+
+class DocVQA(Dataset):
+
+    def __init__(self,
+                 args,
+                 tokenizer,
+                 label2id_map,
+                 max_seq_len=512,
+                 max_query_length=20,
+                 max_doc_length=512,
+                 max_span_num=1):
+        super(DocVQA, self).__init__()
+        self.tokenizer = tokenizer
+        self.label2id_map = label2id_map
+        self.max_seq_len = max_seq_len
+        self.max_query_length = max_query_length
+        self.max_doc_length = max_doc_length
+        self.max_span_num = max_span_num
+        self.sample_list = None
+        self.args = args
+
+        self.docvqa_inputs = self.docvqa_input()
+
+    def check_is_max_context(self, doc_spans, cur_span_index, position):
+        """Check if this is the 'max context' doc span for the token."""
+
+        # Because of the sliding window approach taken to scoring documents, a single
+        # token can appear in multiple documents. E.g.
+        #  Doc: the man went to the store and bought a gallon of milk
+        #  Span A: the man went to the
+        #  Span B: to the store and bought
+        #  Span C: and bought a gallon of
+        #  ...
+        #
+        # Now the word 'bought' will have two scores from spans B and C. We only
+        # want to consider the score with "maximum context", which we define as
+        # the *minimum* of its left and right context (the *sum* of left and
+        # right context will always be the same, of course).
+        #
+        # In the example the maximum context for 'bought' would be span C since
+        # it has 1 left context and 3 right context, while span B has 4 left context
+        # and 0 right context.
+        best_score = None
+        best_span_index = None
+        for (span_index, doc_span) in enumerate(doc_spans):
+            end = doc_span.start + doc_span.length - 1
+            if position < doc_span.start:
+                continue
+            if position > end:
+                continue
+            num_left_context = position - doc_span.start
+            num_right_context = end - position
+            score = min(num_left_context,
+                        num_right_context) + 0.01 * doc_span.length
+            if best_score is None or score > best_score:
+                best_score = score
+            best_span_index = span_index
+
+        return cur_span_index == best_span_index
+
+    def convert_examples_to_features(self, examples, tokenizer, label_map,
+                                     max_seq_length, max_span_num,
+                                     max_doc_length, max_query_length):
+
+        if "[CLS]" in self.tokenizer.get_vocab():
+            start_token = "[CLS]"
+            end_token = "[SEP]"
+        else:
+            start_token = "<s>"
+            end_token = "</s>"
+
+        features = []
+        total = len(examples)
+        for (example_index, example) in enumerate(examples):
+            query_tokens = tokenizer.tokenize(example.question)
+            if len(query_tokens) > max_query_length:
+                query_tokens = query_tokens[0:max_query_length]
+
+            all_doc_tokens = example.doc_tokens
+            all_doc_boxes_tokens = example.doc_boxes
+
+            cls_token_box = [0, 0, 0, 0]
+            sep_token_box = [1000, 1000, 1000, 1000]
+            pad_token_box = [0, 0, 0, 0]
+            ques_token_box = [0, 0, 0, 0]
+
+            # The -3 accounts for [CLS], [SEP] and [SEP]
+            max_tokens_for_doc = max_seq_length - len(query_tokens) - 3
+
+            # We can have documents that are longer than the maximum sequence length.
+            # To deal with this we do a sliding window approach, where we take chunks
+            # of the up to our max length with a stride of `doc_stride`.
+            _DocSpan = collections.namedtuple("DocSpan", ["start", "length"])
+            doc_spans = []
+            start_offset = 0
+            while start_offset < len(all_doc_tokens):
+                length = len(all_doc_tokens) - start_offset
+                if length > max_tokens_for_doc:
+                    length = max_tokens_for_doc
+                doc_spans.append(_DocSpan(start=start_offset, length=length))
+                if start_offset + length == len(all_doc_tokens):
+                    break
+                start_offset += length
+
+            spans_input_ids = []
+            spans_input_mask = []
+            spans_segment_ids = []
+            spans_boxes_tokens = []
+            for (doc_span_index, doc_span) in enumerate(doc_spans):
+                if doc_span_index == max_span_num:
+                    break
+                tokens = []
+                boxes_tokens = []
+                token_is_max_context = {}
+                segment_ids = []
+                tokens.append(start_token)
+                boxes_tokens.append(cls_token_box)
+                segment_ids.append(0)
+                for token in query_tokens:
+                    tokens.append(token)
+                    boxes_tokens.append(ques_token_box)
+                    segment_ids.append(0)
+                tokens.append(end_token)
+                boxes_tokens.append(sep_token_box)
+                segment_ids.append(0)
+                for i in range(doc_span.length):
+                    split_token_index = doc_span.start + i
+                    is_max_context = self.check_is_max_context(
+                        doc_spans, doc_span_index, split_token_index)
+                    token_is_max_context[len(tokens)] = is_max_context
+                    tokens.append(all_doc_tokens[split_token_index])
+                    boxes_tokens.append(all_doc_boxes_tokens[split_token_index])
+                    segment_ids.append(0)
+
+                tokens.append(end_token)
+                boxes_tokens.append(sep_token_box)
+                segment_ids.append(0)
+                input_ids = tokenizer.convert_tokens_to_ids(tokens)
+                # The mask has 1 for real tokens and 0 for padding tokens. Only real
+                # tokens are attended to.
+                input_mask = [1] * len(input_ids)
+                # Zero-pad up to the sequence length.
+                while len(input_ids) < max_seq_length:
+                    input_ids.append(0)
+                    input_mask.append(0)
+                    segment_ids.append(0)
+                    boxes_tokens.append(pad_token_box)
+                assert len(input_ids) == max_seq_length
+                assert len(input_mask) == max_seq_length
+                assert len(segment_ids) == max_seq_length
+                assert len(boxes_tokens) == max_seq_length
+
+                spans_input_ids.append(input_ids)
+                spans_input_mask.append(input_mask)
+                spans_segment_ids.append(segment_ids)
+                spans_boxes_tokens.append(boxes_tokens)
+
+            # Padding
+            # padding spans
+            # max_span_num: max_seg_num
+            # spans_input_ids: the tokens in each segment
+            if len(spans_input_ids) > max_span_num:
+                spans_input_ids = spans_input_ids[0:max_span_num]
+                spans_input_mask = spans_input_mask[0:max_span_num]
+                spans_segment_ids = spans_segment_ids[0:max_span_num]
+                spans_boxes_tokens = spans_boxes_tokens[0:max_span_num]
+            while len(spans_input_ids) < max_span_num:
+                tokens = []
+                boxes_tokens = []
+                segment_ids = []
+                tokens.append(start_token)
+                boxes_tokens.append(cls_token_box)
+                segment_ids.append(0)
+                for token in query_tokens:
+                    tokens.append(token)
+                    boxes_tokens.append(ques_token_box)
+                    segment_ids.append(0)
+                tokens.append(end_token)
+                boxes_tokens.append(sep_token_box)
+                segment_ids.append(0)
+                tokens.append(end_token)
+                boxes_tokens.append(sep_token_box)
+                segment_ids.append(0)
+                input_ids = tokenizer.convert_tokens_to_ids(tokens)
+                input_mask = [1] * len(input_ids)
+                while len(input_ids) < max_seq_length:
+                    input_ids.append(0)
+                    input_mask.append(0)
+                    segment_ids.append(0)
+                    boxes_tokens.append(pad_token_box)
+                spans_input_ids.append(input_ids)
+                spans_input_mask.append(input_mask)
+                spans_segment_ids.append(segment_ids)
+                spans_boxes_tokens.append(boxes_tokens)
+
+            # padding labels
+            labels = example.labels
+            sep_id = tokenizer.convert_tokens_to_ids(end_token)
+            labels = ["O"] * (spans_input_ids[0].index(sep_id) + 1) + labels
+            if len(labels) > 512:
+                labels = labels[:512]
+
+            if len(labels) < 512:
+                labels += ["O"] * (512 - len(labels))
+            assert len(spans_input_ids[0]) == len(labels)
+
+            label_ids = []
+            for lid, l in enumerate(labels):
+                if l not in label_map:
+                    label_ids.append(0)
+                else:
+                    label_ids.append(label_map[l])
+
+            feature = DocVQAFeatures(
+                example_index=example_index,
+                input_ids=spans_input_ids,
+                input_mask=spans_input_mask,
+                segment_ids=spans_segment_ids,
+                boxes=spans_boxes_tokens,
+                label=label_ids,
+            )
+            features.append(feature)
+        return features
+
+    def create_examples(self, data, is_test=False):
+        """Creates examples for the training and dev sets."""
+        examples = []
+        for sample in tqdm(data, total=len(data)):
+            question = sample["question"]
+            doc_tokens = sample["document"]
+            doc_boxes = sample["document_bbox"]
+            labels = sample['labels'] if not is_test else []
+
+            x_min, y_min = min(doc_boxes, key=lambda x: x[0])[0], min(
+                doc_boxes, key=lambda x: x[2])[2]
+            x_max, y_max = max(doc_boxes, key=lambda x: x[1])[1], max(
+                doc_boxes, key=lambda x: x[3])[3]
+            width = x_max - x_min
+            height = y_max - y_min
+
+            if max(width, height) < 1000:
+                scale_x = 1
+                scale_y = 1
+            else:
+                scale_x = 1000 / max(width, height)
+                scale_y = 1000 / max(width, height)
+
+            scaled_doc_boxes = [[
+                round((b[0] - x_min) * scale_x),
+                round((b[2] - y_min) * scale_y),
+                round((b[1] - x_min) * scale_x),
+                round((b[3] - y_min) * scale_y)
+            ] for b in doc_boxes]
+
+            for box, oribox in zip(scaled_doc_boxes, doc_boxes):
+                if box[0] < 0:
+                    print(box, oribox)
+                if box[2] - box[0] < 0:
+                    print(box, oribox)
+                if box[3] - box[1] < 0:
+                    print(box, oribox)
+                for pos in box:
+                    if pos > 1000:
+                        print(width, height, box, oribox)
+
+            example = DocVQAExample(question=question,
+                                    doc_tokens=doc_tokens,
+                                    doc_boxes=scaled_doc_boxes,
+                                    labels=labels)
+            examples.append(example)
+        return examples
+
+    def docvqa_input(self):
+        data = []
+        if self.args.do_train:
+            dataset = self.args.train_file
+        elif self.args.do_test:
+            dataset = self.args.test_file
+        with open(dataset, 'r', encoding='utf8') as f:
+            for index, line in enumerate(f):
+                data.append(json.loads(line.strip()))
+
+            # read the examples from train/test xlm files
+            examples = self.create_examples(data, is_test=self.args.do_test)
+
+        features = self.convert_examples_to_features(
+            examples,
+            self.tokenizer,
+            self.label2id_map,
+            max_seq_length=self.max_seq_len,
+            max_doc_length=self.max_doc_length,
+            max_span_num=self.max_span_num,
+            max_query_length=self.max_query_length)
+
+        all_input_ids = paddle.to_tensor([f.input_ids for f in features],
+                                         dtype="int64")
+        all_input_mask = paddle.to_tensor([f.input_mask for f in features],
+                                          dtype="int64")
+        all_segment_ids = paddle.to_tensor([f.segment_ids for f in features],
+                                           dtype="int64")
+        all_bboxes = paddle.to_tensor([f.boxes for f in features],
+                                      dtype="int64")
+        all_labels = paddle.to_tensor([f.label for f in features],
+                                      dtype="int64")
+        self.sample_list = [
+            np.array(all_input_ids),
+            np.array(all_input_mask),
+            np.array(all_segment_ids),
+            np.array(all_bboxes),
+            np.array(all_labels)
+        ]
+
+    def __getitem__(self, idx):
+        return self.sample_list[0][idx], self.sample_list[1][
+            idx], self.sample_list[2][idx], self.sample_list[3][
+                idx], self.sample_list[4][idx]
+
+    def __len__(self, ):
+        return self.sample_list[0].shape[0]

applications/doc_vqa/Extraction/model.py

@@ -0,0 +1,229 @@
+import numpy as np
+
+import paddle
+import paddle.nn as nn
+import paddle.fluid as fluid
+from paddlenlp.transformers import LayoutXLMPretrainedModel
+
+
+class Crf_decoding(paddle.fluid.dygraph.Layer):
+
+    def __init__(self, param_attr, size=None, is_test=True, dtype='float32'):
+        super(Crf_decoding, self).__init__()
+
+        self._dtype = dtype
+        self._size = size
+        self._is_test = is_test
+        self._param_attr = param_attr
+        self._transition = self.create_parameter(
+            attr=self._param_attr,
+            shape=[self._size + 2, self._size],
+            dtype=self._dtype)
+
+    @property
+    def weight(self):
+        return self._transition
+
+    @weight.setter
+    def weight(self, value):
+        self._transition = value
+
+    def forward(self, input, label=None, length=None):
+
+        viterbi_path = self._helper.create_variable_for_type_inference(
+            dtype=self._dtype)
+        this_inputs = {
+            "Emission": [input],
+            "Transition": self._transition,
+            "Label": label
+        }
+        if length is not None:
+            this_inputs['Length'] = [length]
+        self._helper.append_op(type='crf_decoding',
+                               inputs=this_inputs,
+                               outputs={"ViterbiPath": [viterbi_path]},
+                               attrs={
+                                   "is_test": self._is_test,
+                               })
+        return viterbi_path
+
+
+class Chunk_eval(paddle.fluid.dygraph.Layer):
+
+    def __init__(self,
+                 num_chunk_types,
+                 chunk_scheme,
+                 excluded_chunk_types=None):
+        super(Chunk_eval, self).__init__()
+        self.num_chunk_types = num_chunk_types
+        self.chunk_scheme = chunk_scheme
+        self.excluded_chunk_types = excluded_chunk_types
+
+    def forward(self, input, label, seq_length=None):
+
+        precision = self._helper.create_variable_for_type_inference(
+            dtype="float32")
+        recall = self._helper.create_variable_for_type_inference(
+            dtype="float32")
+        f1_score = self._helper.create_variable_for_type_inference(
+            dtype="float32")
+        num_infer_chunks = self._helper.create_variable_for_type_inference(
+            dtype="int64")
+        num_label_chunks = self._helper.create_variable_for_type_inference(
+            dtype="int64")
+        num_correct_chunks = self._helper.create_variable_for_type_inference(
+            dtype="int64")
+
+        this_input = {"Inference": [input], "Label": [label]}
+        if seq_length is not None:
+            this_input["SeqLength"] = [seq_length]
+
+        self._helper.append_op(type='chunk_eval',
+                               inputs=this_input,
+                               outputs={
+                                   "Precision": [precision],
+                                   "Recall": [recall],
+                                   "F1-Score": [f1_score],
+                                   "NumInferChunks": [num_infer_chunks],
+                                   "NumLabelChunks": [num_label_chunks],
+                                   "NumCorrectChunks": [num_correct_chunks]
+                               },
+                               attrs={
+                                   "num_chunk_types":
+                                   self.num_chunk_types,
+                                   "chunk_scheme":
+                                   self.chunk_scheme,
+                                   "excluded_chunk_types":
+                                   self.excluded_chunk_types or []
+                               })
+        return (precision, recall, f1_score, num_infer_chunks, num_label_chunks,
+                num_correct_chunks)
+
+
+class Linear_chain_crf(paddle.fluid.dygraph.Layer):
+
+    def __init__(self, param_attr, size=None, is_test=False, dtype='float32'):
+        super(Linear_chain_crf, self).__init__()
+
+        self._param_attr = param_attr
+        self._dtype = dtype
+        self._size = size
+        self._is_test = is_test
+        self._transition = self.create_parameter(
+            attr=self._param_attr,
+            shape=[self._size + 2, self._size],
+            dtype=self._dtype)
+
+    @property
+    def weight(self):
+        return self._transition
+
+    @weight.setter
+    def weight(self, value):
+        self._transition = value
+
+    def forward(self, input, label, length=None):
+
+        alpha = self._helper.create_variable_for_type_inference(
+            dtype=self._dtype)
+        emission_exps = self._helper.create_variable_for_type_inference(
+            dtype=self._dtype)
+        transition_exps = self._helper.create_variable_for_type_inference(
+            dtype=self._dtype)
+        log_likelihood = self._helper.create_variable_for_type_inference(
+            dtype=self._dtype)
+        this_inputs = {
+            "Emission": [input],
+            "Transition": self._transition,
+            "Label": [label]
+        }
+        if length is not None:
+            this_inputs['Length'] = [length]
+        self._helper.append_op(type='linear_chain_crf',
+                               inputs=this_inputs,
+                               outputs={
+                                   "Alpha": [alpha],
+                                   "EmissionExps": [emission_exps],
+                                   "TransitionExps": transition_exps,
+                                   "LogLikelihood": log_likelihood
+                               },
+                               attrs={
+                                   "is_test": self._is_test,
+                               })
+        return log_likelihood
+
+
+class LayoutXLMForTokenClassification_with_CRF(LayoutXLMPretrainedModel):
+
+    def __init__(self, layoutxlm, num_classes, dropout=None):
+        super(LayoutXLMForTokenClassification_with_CRF, self).__init__()
+        self.num_classes = num_classes
+        if isinstance(layoutxlm, dict):
+            self.layoutxlm = LayoutXLMModel(**layoutxlm)
+        else:
+            self.layoutxlm = layoutxlm
+        self.dropout = nn.Dropout(dropout if dropout is not None else self.
+                                  layoutxlm.config["hidden_dropout_prob"])
+        self.emission_classifier = nn.Linear(
+            self.layoutxlm.config["hidden_size"], self.num_classes)
+        self.emission_classifier.apply(self.init_weights)
+        self.linear_chain_crf = Linear_chain_crf(
+            size=self.num_classes,
+            param_attr=paddle.fluid.ParamAttr(name='liner_chain_crfw'))
+        self.crf_decoding = Crf_decoding(
+            param_attr=paddle.fluid.ParamAttr(name='crfw_decode'),
+            size=self.num_classes)
+        self.crf_decoding.weight = self.linear_chain_crf.weight
+        self.crfw = fluid.layers.create_parameter(
+            shape=[self.num_classes + 2, self.num_classes],
+            dtype='float32',
+            name='crfw')
+        self.mask_crfw = fluid.layers.create_parameter(
+            shape=[self.num_classes + 2, self.num_classes],
+            dtype='float32',
+            name='mask_matrix')
+
+    def get_input_embeddings(self):
+        return self.layoutxlm.embeddings.word_embeddings
+
+    def forward(self,
+                input_ids=None,
+                bbox=None,
+                attention_mask=None,
+                token_type_ids=None,
+                labels=None,
+                image=None,
+                position_ids=None,
+                head_mask=None,
+                is_train=False):
+
+        input_ids = input_ids.squeeze(axis=1)
+        bbox = bbox.squeeze(axis=1)
+        attention_mask = attention_mask.squeeze(axis=1)
+        token_type_ids = token_type_ids.squeeze(axis=1)
+        outputs = self.layoutxlm(
+            input_ids=input_ids,
+            bbox=bbox,
+            image=image,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+        )
+        seq_length = input_ids.shape[1]
+        # sequence out and image out
+        sequence_logits, image_output = outputs[0][:, :seq_length], outputs[
+            0][:, seq_length:]
+        emission = self.emission_classifier(sequence_logits)
+        length = paddle.sum(attention_mask, axis=1)
+        labels = labels.reshape([-1, seq_length, 1])
+
+        # standard crf loss
+        crf_cost = self.linear_chain_crf(input=emission,
+                                         label=labels,
+                                         length=length)
+        crf_decode = self.crf_decoding(input=emission, length=length)
+        if is_train:
+            return [crf_cost]
+        else:
+            return [crf_cost, crf_decode]

applications/doc_vqa/Extraction/run_docvqa.py

@@ -0,0 +1,450 @@
+import os
+import sys
+import copy
+import json
+import random
+import logging
+import warnings
+import argparse
+import numpy as np
+from collections import OrderedDict, Counter
+
+import paddle
+from paddle.static import InputSpec
+from paddle.jit import to_static
+from paddlenlp.transformers import LayoutXLMModel, LayoutXLMTokenizer
+
+from docvqa import DocVQA
+from model import LayoutXLMForTokenClassification_with_CRF
+
+warnings.filterwarnings("ignore")
+logger = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    # yapf: disable
+    parser.add_argument("--model_name_or_path", default=None, type=str, required=True)
+    parser.add_argument("--do_train", default=False, type=bool, required=False)
+    parser.add_argument("--do_test", default=False, type=bool, required=False)
+    parser.add_argument("--test_file", default=None, type=str, required=False)
+    parser.add_argument("--train_file", default=None, type=str, required=False)
+    parser.add_argument("--output_dir", default=None, type=str, required=True)
+    parser.add_argument("--max_seq_len", default=512, type=int)
+    parser.add_argument("--max_query_length", default=20, type=int)
+    parser.add_argument("--max_doc_length", default=512, type=int)
+    parser.add_argument("--max_span_num", default=1, type=int)
+    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.")
+    parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for eval.")
+    parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
+    parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
+    parser.add_argument("--num_train_epochs", default=3, type=int, help="Total number of training epochs to perform.")
+    parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")
+    parser.add_argument("--eval_steps", type=int, default=10, help="eval every X updates steps.")
+    parser.add_argument("--save_steps", type=int, default=50, help="Save checkpoint every X updates steps.")
+    parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
+    parser.add_argument("--init_checkpoint", type=str, default=None, help="the initialized checkpoint")
+    parser.add_argument("--save_path", type=str, default=None, help="the initialized checkpoint")
+    # yapf: enable
+    args = parser.parse_args()
+    return args
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    paddle.seed(args.seed)
+
+
+def get_label_maps():
+    labels = ["O", "I-ans", "B-ans", "E-ans"]
+    label2id_map = {label: idx for idx, label in enumerate(labels)}
+    id2label_map = {idx: label for idx, label in enumerate(labels)}
+    return label2id_map, id2label_map
+
+
+def main(args):
+    os.makedirs(args.output_dir, exist_ok=True)
+    logging.basicConfig(
+        filename=os.path.join(args.output_dir, "train.log")
+        if paddle.distributed.get_rank() == 0 else None,
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO
+        if paddle.distributed.get_rank() == 0 else logging.WARN,
+    )
+
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    logger.addHandler(ch)
+
+    label2id_map, id2label_map = get_label_maps()
+    pad_token_label_id = paddle.nn.CrossEntropyLoss().ignore_index
+
+    # dist mode
+    if paddle.distributed.get_world_size() > 1:
+        paddle.distributed.init_parallel_env()
+
+    tokenizer = LayoutXLMTokenizer.from_pretrained(args.model_name_or_path)
+
+    if args.do_test:
+        model = LayoutXLMForTokenClassification_with_CRF.from_pretrained(
+            args.init_checkpoint)
+        evaluate(args,
+                 model,
+                 tokenizer,
+                 label2id_map,
+                 id2label_map,
+                 pad_token_label_id,
+                 global_step=0)
+        exit(0)
+
+    if args.init_checkpoint:
+        logger.info('Init checkpoint from {}'.format(args.init_checkpoint))
+        model = LayoutXLMForTokenClassification_with_CRF.from_pretrained(
+            args.init_checkpoint)
+    else:
+        base_model = LayoutXLMModel.from_pretrained(args.model_name_or_path)
+        model = LayoutXLMForTokenClassification_with_CRF(
+            base_model, num_classes=len(label2id_map), dropout=None)
+
+    # dist mode
+    if paddle.distributed.get_world_size() > 1:
+        model = paddle.DataParallel(model)
+
+    train_dataset = DocVQA(args,
+                           tokenizer,
+                           label2id_map,
+                           max_seq_len=args.max_seq_len,
+                           max_query_length=args.max_query_length,
+                           max_doc_length=args.max_doc_length,
+                           max_span_num=args.max_span_num)
+
+    train_sampler = paddle.io.DistributedBatchSampler(
+        train_dataset, batch_size=args.per_gpu_train_batch_size, shuffle=False)
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(
+        1, paddle.distributed.get_world_size())
+
+    train_dataloader = paddle.io.DataLoader(train_dataset,
+                                            batch_sampler=train_sampler,
+                                            num_workers=0,
+                                            use_shared_memory=True,
+                                            collate_fn=None)
+
+    t_total = len(train_dataloader) * args.num_train_epochs
+    # build linear decay with warmup lr sch
+    lr_scheduler = paddle.optimizer.lr.PolynomialDecay(
+        learning_rate=args.learning_rate,
+        decay_steps=t_total,
+        end_lr=0.0,
+        power=1.0)
+    if args.warmup_steps > 0:
+        lr_scheduler = paddle.optimizer.lr.LinearWarmup(
+            lr_scheduler,
+            args.warmup_steps,
+            start_lr=0,
+            end_lr=args.learning_rate)
+
+    optimizer = paddle.optimizer.AdamW(learning_rate=lr_scheduler,
+                                       parameters=model.parameters(),
+                                       epsilon=args.adam_epsilon,
+                                       weight_decay=args.weight_decay)
+
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d",
+                args.per_gpu_train_batch_size)
+    logger.info(
+        "  Total train batch size (w. parallel, distributed) = %d",
+        args.train_batch_size * paddle.distributed.get_world_size(),
+    )
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss = 0.0
+    set_seed(args)
+    best_metrics = None
+    for epoch_id in range(args.num_train_epochs):
+        print('epoch id:{}'.format(epoch_id))
+        for step, batch in enumerate(train_dataloader):
+            model.train()
+            input_ids, input_mask, segment_ids, bboxes, labels = batch
+            if input_ids.shape[0] != args.per_gpu_train_batch_size:
+                continue
+            outputs = model(input_ids=input_ids,
+                            bbox=bboxes,
+                            attention_mask=input_mask,
+                            token_type_ids=segment_ids,
+                            labels=labels,
+                            is_train=True)
+            # model outputs are always tuple in paddlenlp (see doc)
+            loss = outputs[0]
+            loss = loss.mean()
+            if global_step % 50 == 0:
+                logger.info(
+                    "[epoch {}/{}][iter: {}/{}] lr: {:.5f}, train loss: {:.5f}, "
+                    .format(epoch_id, args.num_train_epochs, step,
+                            len(train_dataloader), lr_scheduler.get_lr(),
+                            loss.numpy()[0]))
+
+            loss.backward()
+            tr_loss += loss.item()
+            optimizer.step()
+            lr_scheduler.step()  # Update learning rate schedule
+            optimizer.clear_grad()
+            global_step += 1
+
+            if paddle.distributed.get_rank(
+            ) == 0 and args.save_steps > 0 and global_step % args.save_steps == 0:
+                # Save model checkpoint
+                output_dir = os.path.join(args.output_dir,
+                                          "checkpoint-{}".format(global_step))
+                os.makedirs(output_dir, exist_ok=True)
+                if paddle.distributed.get_rank() == 0:
+                    model.save_pretrained(output_dir)
+                    tokenizer.save_pretrained(output_dir)
+                    paddle.save(args,
+                                os.path.join(output_dir, "training_args.bin"))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+
+def _tokenize_chinese_chars(text):
+    """
+    :param text: input text, unicode string
+    :return:
+        tokenized text, list
+    """
+
+    def _is_chinese_char(cp):
+        """Checks whether CP is the codepoint of a CJK character."""
+        # This defines a "chinese character" as anything in the CJK Unicode block:
+        #     https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+        #
+        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+        # despite its name. The modern Korean Hangul alphabet is a different block,
+        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+        # space-separated words, so they are not treated specially and handled
+        # like the all of the other languages.
+        if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #
+            (cp >= 0x3400 and cp <= 0x4DBF) or  #
+            (cp >= 0x20000 and cp <= 0x2A6DF) or  #
+            (cp >= 0x2A700 and cp <= 0x2B73F) or  #
+            (cp >= 0x2B740 and cp <= 0x2B81F) or  #
+            (cp >= 0x2B820 and cp <= 0x2CEAF) or
+            (cp >= 0xF900 and cp <= 0xFAFF) or  #
+            (cp >= 0x2F800 and cp <= 0x2FA1F)):  #
+            return True
+
+        return False
+
+    output = []
+    buff = ""
+    for char in text:
+        cp = ord(char)
+        if _is_chinese_char(cp) or char == "=":
+            if buff != "":
+                output.append(buff)
+                buff = ""
+            output.append(char)
+        else:
+            buff += char
+
+    if buff != "":
+        output.append(buff)
+
+    return output
+
+
+def fast_f1(text1, text2):
+    common_char = Counter(text1) & Counter(text2)
+    len_seq1 = len(text1)
+    len_seq2 = len(text2)
+    len_common = sum(common_char.values())
+    if len_common == 0:
+        return 0.0
+    precision = 1.0 * len_common / len_seq2
+    recall = 1.0 * len_common / len_seq1
+    return (2.0 * precision * recall) / (precision + recall)
+
+
+def _normalize(in_str):
+    """
+    normalize the input unicode string
+    """
+    in_str = in_str.lower()
+    sp_char = [
+        u':', u'_', u'`', u'，', u'。', u'：', u'？', u'！', u'(', u')', u'“', u'”',
+        u'；', u'’', u'《', u'》', u'……', u'·', u'、', u',', u'「', u'」', u'（', u'）',
+        u'－', u'～', u'『', u'』', '|'
+    ]
+    out_segs = []
+    for char in in_str:
+        if char in sp_char:
+            continue
+        else:
+            out_segs.append(char)
+    return ''.join(out_segs)
+
+
+def calc_f1_score(answer, prediction):
+    f1_scores = []
+    ans_segs = _tokenize_chinese_chars(_normalize(answer))
+    prediction_segs = _tokenize_chinese_chars(_normalize(prediction))
+    f1 = fast_f1(prediction_segs, ans_segs)
+    return f1
+
+
+def decode(tokenizer, res):
+    sep_id = tokenizer._convert_token_to_id("</s>")
+    text_res = []
+    all_f1 = []
+    save_f1 = []
+    for i in range(len(res)):
+        input_ids, label_ids, predict_ids, bbox = res[i]
+        remove_pos = len(' '.join(
+            [str(x) for x in input_ids]).split('2 6 ')[0].strip(' ').split(
+                ' ')) + 2  # remove the question bbox and sep bbox
+        start_pos = input_ids.index(sep_id)
+        query_text = []
+        for idx in range(1, start_pos):
+            input_id = input_ids[idx]
+            query_text.append(tokenizer._convert_id_to_token(int(input_id)))
+
+        #label texts and predict texts
+        text_label, text_predict = [], []
+        label_bbox_index, predict_bbox_index = [], []
+        for idx in range(start_pos + 1, len(input_ids)):
+            input_id, label_id, predict_id = input_ids[idx], label_ids[
+                idx], predict_ids[idx]
+
+            if label_id in [1, 2, 3]:
+                text_label.append(tokenizer._convert_id_to_token(int(input_id)))
+                label_bbox_index.append(idx - remove_pos + 1)
+            if predict_id in [1, 2, 3]:
+                text_predict.append(
+                    tokenizer._convert_id_to_token(int(input_id)))
+                predict_bbox_index.append(idx - remove_pos + 1)
+        text_res.append([
+            ''.join(query_text), ''.join(text_label), ''.join(text_predict),
+            label_bbox_index, predict_bbox_index
+        ])
+
+        f1 = calc_f1_score(''.join(text_label), ''.join(text_predict))
+        save_f1.append(f1)
+
+        if len(''.join(text_label)) > 10:
+            all_f1.append(f1)
+    if len(all_f1) > 0:
+        print("F1: ", sum(all_f1) / len(all_f1))
+
+    assert len(text_res) == len(save_f1)
+    return text_res
+
+
+def evaluate(args,
+             model,
+             tokenizer,
+             label2id_map,
+             id2label_map,
+             pad_token_label_id,
+             prefix="",
+             global_step=0):
+
+    eval_dataset = DocVQA(args,
+                          tokenizer,
+                          label2id_map,
+                          max_seq_len=512,
+                          max_query_length=20,
+                          max_doc_length=512,
+                          max_span_num=1)
+
+    args.eval_batch_size = args.per_gpu_eval_batch_size * max(
+        1, paddle.distributed.get_world_size())
+
+    eval_dataloader = paddle.io.DataLoader(eval_dataset,
+                                           batch_size=args.eval_batch_size,
+                                           num_workers=0,
+                                           use_shared_memory=True,
+                                           collate_fn=None)
+
+    # Eval!
+    logger.info("***** Running evaluation %s *****", prefix)
+    logger.info("  Num examples = %d", len(eval_dataset))
+    logger.info("  Batch size = %d", args.eval_batch_size)
+    model.eval()
+    res = []
+    for idx, batch in enumerate(eval_dataloader):
+        with paddle.no_grad():
+            input_ids, input_mask, segment_ids, bboxes, labels = batch
+
+            if input_ids.shape[0] != args.eval_batch_size:
+                continue
+            outputs = model(input_ids=input_ids,
+                            bbox=bboxes,
+                            attention_mask=input_mask,
+                            token_type_ids=segment_ids,
+                            labels=labels,
+                            is_train=False)
+            labels = labels.numpy()
+            crf_decode = outputs[1].numpy()
+            bboxes = bboxes.squeeze().numpy()
+            input_ids = input_ids.squeeze(axis=1).numpy()
+
+            for index in range(input_ids.shape[0]):
+                res.append([
+                    list(input_ids[index]),
+                    list(labels[index]),
+                    list(crf_decode[index]), bboxes[index]
+                ])
+
+    origin_inputs = []
+    with open(args.test_file, 'r', encoding='utf8') as f:
+        for line in f:
+            line = json.loads(line.strip())
+            origin_inputs.append({
+                'img_name': line['img_name'],
+                'question': line['question'],
+                'bboxes': line['document_bbox'],
+                'img_id': line['img_id']
+            })
+
+    text_res = decode(tokenizer, res)
+
+    with open(args.save_path, 'w', encoding='utf8') as f:
+        for line_res, line_text, line_label in zip(res, text_res,
+                                                   origin_inputs):
+            line_json = {}
+            line_json['img_name'] = line_label['img_name']
+            line_json['img_id'] = line_label['img_id']
+            line_json['question'] = line_label['question']
+            line_json['label_answer'] = line_text[1]
+            line_json['predict_answer'] = line_text[2]
+            all_boxes = line_res[3]
+            label_bbox_index, predict_bbox_index = line_text[3], line_text[4]
+            label_bboxes, predict_bboxes = [], []
+            for i in range(len(line_label['bboxes'])):
+                if i in label_bbox_index:
+                    label_bboxes.append(line_label['bboxes'][i])
+                if i in predict_bbox_index:
+                    predict_bboxes.append(line_label['bboxes'][i])
+            line_json['label_bboxes'] = label_bboxes
+            line_json['predict_bboxes'] = predict_bboxes
+            json.dump(line_json, f, ensure_ascii=False)
+            f.write('\n')
+
+
+def print_arguments(args):
+    """print arguments"""
+    print('-----------  Configuration Arguments -----------')
+    for arg, value in sorted(vars(args).items()):
+        print('%s: %s' % (arg, value))
+    print('------------------------------------------------')
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    print_arguments(args)
+    main(args)

applications/doc_vqa/Extraction/run_test.sh

@@ -0,0 +1,24 @@
+export CUDA_VISIBLE_DEVICES=0
+
+QUESTION=$1
+
+python3 change_to_mrc.py ${QUESTION}
+
+python3  ./run_docvqa.py \
+    --model_name_or_path "layoutxlm-base-uncased" \
+    --max_seq_len 512 \
+    --do_test true \
+	--test_file "data/demo_test.json" \
+	--num_train_epochs 100 \
+    --eval_steps 6000 \
+    --save_steps 6000 \
+    --output_dir "output/" \
+    --save_path "data/decode_res.json" \
+	--init_checkpoint "./checkpoints/layoutxlm/" \
+    --learning_rate 3e-5 \
+    --warmup_steps 12000 \
+    --per_gpu_train_batch_size 4 \
+    --per_gpu_eval_batch_size 1 \
+    --seed 2048
+
+python3 view.py

applications/doc_vqa/Extraction/run_train.sh

@@ -0,0 +1,18 @@
+export CUDA_VISIBLE_DEVICES=0
+
+python3 ./run_docvqa.py \
+    --model_name_or_path "layoutxlm-base-uncased" \
+    --max_seq_len 512 \
+    --train_file "data/train.json" \
+    --init_checkpoint "checkpoints/base_model" \
+	--do_train true \
+    --num_train_epochs 50 \
+    --eval_steps 24000 \
+    --save_steps 40 \
+    --output_dir "output" \
+    --save_path "data/decode_res.json" \
+    --learning_rate 3e-5 \
+    --warmup_steps 40 \
+    --per_gpu_train_batch_size 4 \
+    --per_gpu_eval_batch_size 4 \
+    --seed 2048

applications/doc_vqa/Extraction/view.py

@@ -0,0 +1,53 @@
+import cv2
+import json
+import numpy as np
+
+
+def view_ocr_result(img_path, bboxes, opath):
+    image = cv2.imread(img_path)
+    for char_bbox in bboxes:
+        x_min, x_max, y_min, y_max = char_bbox
+        cv2.rectangle(image, (x_min, y_min), (x_max, y_max), (0, 0, 255), 1)
+    cv2.imwrite(opath, image)
+
+
+def _highlight_bbox(img, bbox):
+    x = bbox[0]
+    w = bbox[1] - x
+    y = bbox[2]
+    h = bbox[3] - y
+    sub_img = img[y:y + h, x:x + w]
+    colored_rect = np.zeros(sub_img.shape, dtype=np.uint8)
+    colored_rect[:, :, 2] = 255
+    colored_rect[:, :, 1] = 255
+    res = cv2.addWeighted(sub_img, 0.5, colored_rect, 0.5, 1.0)
+    img[y:y + h, x:x + w] = res
+
+
+def highlight_ans(source_img_path, output_img_path, ans_bbox):
+    image = cv2.imread(source_img_path)
+    for bbox in ans_bbox:
+        _highlight_bbox(image, bbox)
+    cv2.imwrite(output_img_path, image)
+
+
+def highlight_img(source_img_path, output_img_path):
+    image = cv2.imread(source_img_path)
+    height = image.shape[0]
+    width = image.shape[1]
+    bbox = [0, width - 1, 0, height - 1]
+    _highlight_bbox(image, bbox)
+    cv2.imwrite(output_img_path, image)
+
+
+if __name__ == '__main__':
+    res_path = "./data/decode_res.json"
+    result = {}
+    with open(res_path, "r", encoding="utf-8") as f:
+        line = f.readline()
+        result = json.loads(line.strip())
+
+    img_path = '../OCR_process/demo_pics/demo_{}.png'.format(result["img_id"])
+    img_save_path = "../answer.png"
+    highlight_ans(img_path, img_save_path, result['predict_bboxes'])
+    print("extraction result has been saved to answer.png")

applications/doc_vqa/OCR_process/ocr_process.py

@@ -0,0 +1,273 @@
+import os
+import json
+from paddleocr import PaddleOCR
+import numpy as np
+import re
+import time
+import sys
+from multiprocessing import Process, Queue
+
+from paddlenlp.transformers import LayoutXLMTokenizer
+
+tokenizer = LayoutXLMTokenizer.from_pretrained("layoutxlm-base-uncased")
+
+
+def get_all_chars(tokenizer):
+    all_chr = []
+    for i in range(30000):
+        tok_chr = tokenizer.tokenize(chr(i))
+        tok_chr = [tc.replace("▁", "") for tc in tok_chr]
+        while "" in tok_chr:
+            tok_chr.remove("")
+        tok_chr = ''.join(tok_chr)
+        if len(tok_chr) != 1:
+            all_chr.append(i)
+    return all_chr
+
+
+def merge_bbox(tok_bboxes):
+    min_gx = min([box[0] for box in tok_bboxes])
+    max_gx = max([box[1] for box in tok_bboxes])
+    min_gy = min([box[2] for box in tok_bboxes])
+    max_gy = max([box[3] for box in tok_bboxes])
+    height_g = max_gy - min_gy
+    width_g = max_gx - min_gx
+    height_m = 0
+    width_m = 0
+    for box in tok_bboxes:
+        x_min, x_max, y_min, y_max = box
+        height_m += y_max - y_min
+        width_m += x_max - x_min
+    height_m = height_m / len(tok_bboxes)
+    if (height_g -
+            height_m) < 0.5 * height_m and width_g - width_m < 0.1 * width_m:
+        return False, [min_gx, max_gx, min_gy, max_gy]
+    else:
+        return True, tok_bboxes[0]
+
+
+def xlm_parse(ocr_res, tokenizer):
+
+    doc_tokens, doc_bboxes = [], []
+    all_chr = get_all_chars(tokenizer)
+
+    try:
+        new_tokens, new_token_boxes = [], []
+        for item in ocr_res:
+            new_tokens.extend(item["tokens"])
+            new_token_boxes.extend(item["token_box"])
+
+        # get layoutxlm tokenizer results and get the final results
+        temp_span_text = ''.join(new_tokens)
+        temp_span_bbox = new_token_boxes
+        span_text = ""
+        span_bbox = []
+        # drop blank space
+        for text, bbox in zip(temp_span_text, temp_span_bbox):
+            if text == " ":
+                continue
+            else:
+                span_text += text
+                span_bbox += [bbox]
+
+        # span_tokens starts with "_"
+        span_tokens = tokenizer.tokenize(span_text)
+        span_tokens[0] = span_tokens[0].replace("▁", "")
+        while "" in span_tokens:
+            span_tokens.remove("")
+
+        doc_bboxes = []
+        i = 0
+        for tid, tok in enumerate(span_tokens):
+            tok = tok.replace("▁", "")
+            if tok == "":
+                doc_bboxes.append(span_bbox[i])
+                continue
+            if tok == "<unk>":
+                if tid + 1 == len(span_tokens):
+                    tok_len = 1
+                else:
+                    if span_tokens[tid + 1] == "<unk>":
+                        tok_len = 1
+                    else:
+                        for j in range(i, len(span_text)):
+                            if span_text[j].lower() == span_tokens[tid + 1][0]:
+                                break
+                        tok_len = j - i
+            elif ord(span_text[i]) in all_chr:
+                if tid + 1 == len(span_tokens):
+                    tok_len = 1
+                elif "°" in tok and "C" in span_tokens[tid + 1]:
+                    tok_len = len(tok) - 1
+                    if tok_len == 0:
+                        doc_bboxes.append(span_bbox[i])
+                        continue
+                elif span_text[i] == "ⅱ":
+                    if tok == "ii":
+                        if span_text[i + 1] != "i":
+                            tok_len = len(tok) - 1
+                        else:
+                            tok_len = len(tok)
+                    elif tok == "i":
+                        tok_len = len(tok) - 1
+                        if tok_len == 0:
+                            doc_bboxes.append(span_bbox[i])
+                            continue
+                elif "m" in tok and "2" == span_tokens[tid + 1][0]:
+                    tok_len = len(tok) - 1
+                    if tok_len == 0:
+                        doc_bboxes.append(span_bbox[i])
+                        continue
+                elif ord(span_text[i + 1]) in all_chr:
+                    tok_len = 1
+                else:
+                    for j in range(i, len(span_text)):
+                        if span_text[j].lower() == span_tokens[tid + 1][0]:
+                            break
+                        if span_text[j].lower() == "，" and span_tokens[
+                                tid + 1][0] == ",":
+                            break
+                        if span_text[j].lower() == "；" and span_tokens[
+                                tid + 1][0] == ";":
+                            break
+                        if span_text[j].lower() == "）" and span_tokens[
+                                tid + 1][0] == ")":
+                            break
+                        if span_text[j].lower() == "（" and span_tokens[
+                                tid + 1][0] == "(":
+                            break
+                        if span_text[j].lower() == "￥" and span_tokens[
+                                tid + 1][0] == "¥":
+                            break
+
+                    tok_len = j - i
+
+            else:
+                if "<EFBFBD>" == span_text[i]:
+                    tok_len = len(tok) + 1
+                elif tok == "......" and "…" in span_text[i:i + 6]:
+                    tok_len = len(tok) - 2
+                elif "ⅱ" in span_text[i + len(tok) - 1]:
+                    if tok == "i":
+                        tok_len = 1
+                    else:
+                        tok_len = len(tok) - 1
+                elif "°" in tok and "C" in span_tokens[tid + 1]:
+                    tok_len = len(tok) - 1
+                else:
+                    tok_len = len(tok)
+
+            assert i + tok_len <= len(span_bbox)
+            tok_bboxes = span_bbox[i:i + tok_len]
+            _, merged_bbox = merge_bbox(tok_bboxes)
+
+            doc_bboxes.append(merged_bbox)
+            i = i + tok_len
+    except:
+        print('Error')
+        span_tokens = ['▁'] * 512
+        doc_bboxes = [[0, 0, 0, 0]] * 512
+
+    return span_tokens, doc_bboxes
+
+
+def tokenize_ocr_res(ocr_reses):
+    '''
+    input:
+        ocr_res: the ocr result of the image
+    return:
+        new_reses: {
+            pid: {
+                "text": all text in each ocr_res,
+                "bounding_box": the bounding box of the ocr_res,
+                "tokens": all chars in ocr_res,
+                "token_box: bounding box of each chars in ocr_res
+                }
+        }
+    '''
+    new_reses = []
+    for img_name, ocr_res in ocr_reses:
+        new_res = []
+        for para in ocr_res:
+            text = para["text"]
+            text_box = para["bbox"]
+            x_min, y_min = [int(min(idx)) for idx in zip(*text_box)]
+            x_max, y_max = [int(max(idx)) for idx in zip(*text_box)]
+            text_chars = list(text.lower())
+            char_num = 0
+            for char in text_chars:
+                if re.match("[^\x00-\xff]", char):
+                    char_num += 2
+                else:
+                    char_num += 1
+            width = x_max - x_min
+            shift = x_min
+            new_token_boxes, new_tokens = [], []
+            for char in text_chars:
+                if re.match("[^\x00-\xff]", char):
+                    tok_x_max = shift + width / char_num * 2
+                else:
+                    tok_x_max = shift + width / char_num * 1
+                tok_x_min = shift
+                tok_y_min = y_min
+                tok_y_max = y_max
+
+                shift = tok_x_max
+                new_token_boxes.append(
+                    [round(tok_x_min),
+                     round(tok_x_max), tok_y_min, tok_y_max])
+                new_tokens.append(char)
+            new_res.append({
+                "text": para["text"],
+                "bounding_box": para["bbox"],
+                "tokens": new_tokens,
+                "token_box": new_token_boxes
+            })
+        new_reses.append((img_name, new_res))
+    return new_reses
+
+
+def process_input(ocr_reses, tokenizer, save_ocr_path):
+    ocr_reses = tokenize_ocr_res(ocr_reses)
+
+    examples = []
+    for img_name, ocr_res in ocr_reses:
+        doc_tokens, doc_bboxes = xlm_parse(ocr_res, tokenizer)
+        doc_tokens.insert(0, '▁')
+        doc_bboxes.insert(0, doc_bboxes[0])
+        example = {
+            "img_name": img_name,
+            "document": doc_tokens,
+            "document_bbox": doc_bboxes
+        }
+        examples.append(example)
+
+    with open(save_ocr_path, 'w', encoding='utf8') as f:
+        for example in examples:
+            json.dump(example, f, ensure_ascii=False)
+            f.write('\n')
+
+    print(f"ocr parsing results has been save to: {save_ocr_path}")
+
+
+def ocr_preprocess(img_dir):
+    ocr = PaddleOCR(use_angle_cls=True, lang="ch", use_gpu=True)
+    ocr_reses = []
+    img_names = sorted(os.listdir(img_dir),
+                       key=lambda x: int(x.split("_")[1].split(".")[0]))
+    for img_name in img_names:
+        img_path = os.path.join(img_dir, img_name)
+        parsing_res = ocr.ocr(img_path, cls=True)
+        ocr_res = []
+        for para in parsing_res:
+            ocr_res.append({"text": para[1][0], "bbox": para[0]})
+        ocr_reses.append((img_name, ocr_res))
+
+    return ocr_reses
+
+
+if __name__ == '__main__':
+    img_dir = "./demo_pics"
+    save_path = "./demo_ocr_res.json"
+    ocr_results = ocr_preprocess(img_dir)
+    process_input(ocr_results, tokenizer, save_path)

applications/doc_vqa/README.md

@@ -0,0 +1,131 @@
+# 汽车说明书跨模态智能问答
+
+## 1. 项目说明
+
+**跨模态文档问答** 是跨模态的文档抽取任务，要求文档智能模型在文档中抽取能够回答文档相关问题的答案，需要模型在抽取和理解文档中文本信息的同时，还能充分利用文档的布局、字体、颜色等视觉信息，这比单一模态的信息抽取任务更具挑战性。
+
+这种基于跨模态文档阅读理解技术的智能问答能力，可以深度解析非结构化文档中排版复杂的图文/图表内容，直接定位问题答案。
+
+本项目将基于跨模态文档问答技术实现**汽车说明书问答系统**，该系统能够对用户提出的问题，自动从汽车说明书中寻找答案并进行回答。
+
+如下图所示， 用户提出问题："如何更换前风窗玻璃的刮水片"，跨模态文档问答引擎将从库中寻找相关的文档，然后通过跨模态阅读理解模型抽取出相应的答案，并进行了高亮展示。
+
+<center><img width="883" alt="image" src="https://user-images.githubusercontent.com/35913314/169781111-0734729d-3c7b-400d-8e92-e56548bb7dc5.png"></center>
+
+通过使用汽车说明书问答系统，能够极大地解决传统汽车售后的压力：
+- 用户：用户没有耐心查阅说明书，打客服电话需要等待
+- 售后客服：需要配置大量客服人员，且客服专业知识培训周期长
+- 构建问题库：需要投入大量人力整理常见问题库，并且固定的问题库难以覆盖灵活多变的提问
+
+对于用户来说，汽车说明书问答系统能够支持通过车机助手/APP/小程序为用户提供即问即答的的功能。对于常见问题，用户不再需要查阅说明书，也无需打客服电话，从而缓解了人工客服的压力。
+
+对于客服来讲，汽车说明书问答系统帮助客服人员快速定位答案，高效查阅文档，提高客服的专业水平，同时也能够缩短客服的培训周期。
+
+## 2. 安装说明
+
+#### 环境要求
+
+- paddlepaddle >= 2.2.0
+- paddlenlp >= 2.3.1
+- paddleocr >= 2.5
+
+安装相关问题可参考[PaddlePaddle](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/pip/linux-pip.html)和[PaddleNLP](https://paddlenlp.readthedocs.io/zh/latest/get_started/installation.html)文档。
+
+
+## 3. 整体流程
+
+汽车说明书问答系统针对用户提出的汽车使用相关问题，智能化地在汽车说明书中找出对应答案，并返回给用户。本项目提供的汽车说明书问答系统的使用流程如下图所示。本项目提供的汽车说明书问答系统主要包括 3 个模块：OCR处理模块、排序模块和跨模态阅读理解模块。
+
+在使用汽车说明书问答模型回答问题之前，需要先使用PaddleOCR对离线提供的汽车说明书文档进解析，并将解析结果保存下来，以备后续排序模块使用。
+
+对于用户提问的问题，首先会被传入排序模块，排序模块会针对该问题对解析的文档进行排序打分，其结果将会被传入跨模态阅读理解模块。阅读理解模块将从分数最高的说明书文档中，抽取用户问题的答案，并返回给用户。
+
+<center><img width="864" alt="image" src="https://user-images.githubusercontent.com/35913314/170222662-c438ff2a-a1df-44e5-8a83-f14dc0814b9d.png"></center>
+
+下面将具体介绍各个模块的功能。
+
+## 4. OCR处理模块
+
+本项目提供了包含10张图片的汽车说明书，为方便后续处理，首先需要通过 PaddleOCR 对汽车说明书进行识别，记录汽车说明书上的文字和文字布局信息， 以方便后续使用计算机视觉和自然语言处理方面的技术进行问答任务。
+
+本项目提供的汽车说明书图片可点击[这里](https://paddlenlp.bj.bcebos.com/images/applications/automobile.tar.gz)进行下载，下载后解压放至 `./OCR_process/demo_pics` 目录下，然后通过如下命令，使用 PaddleOCR 对图片进行解析。
+
+```shell
+cd OCR_process/
+python3 ocr_process.py
+cd ..
+```
+
+解析后的结果存放至 `./OCR_process/demo_ocr_res.json` 中。
+
+## 5. 排序模块
+对于用户提出的问题，如果从所有的汽车说明书图片中去寻找答案会比较耗时且耗费资源。因此这里使用了一个基于[RocketQA](https://arxiv.org/pdf/2010.08191.pdf)的排序模块，该模块将根据用户提出的问题对汽车说明书的不同图片进行打分排序，这样便可以获取和问题最相关的图片，并使用跨模态阅读理解模块在该问题上进行抽取答案。
+
+本项目提供了140条汽车说明书相关的训练样本，用于排序模型的训练， 同时也提供了一个基于RocketQA的预先训练好的基线模型 base_model。 本模块可以使用 base_model 在汽车说明书训练样本上进一步微调。
+
+其中，汽车说明书的训练集可点击[这里](https://paddlenlp.bj.bcebos.com/data/automobile_rerank_train.tsv) 进行下载，下载后将其重命名为 `train.tsv` ，存放至 `./Rerank/data/` 目录下。
+
+同时，base_model 是 [Dureader retrieval](https://arxiv.org/abs/2203.10232) 数据集训练的排序模型， 可点击[这里](https://paddlenlp.bj.bcebos.com/models/base_ranker.tar.gz) 进行下载，解压后可获得包含模型的目录 `base_model`，将其放至 `./Rerank/checkpoints` 目录下。
+
+可使用如下代码进行训练：
+
+```shell
+cd Rerank
+bash run_train.sh ./data/train.tsv ./checkpoints/base_model 50 1
+cd ..
+```
+其中，参数依次为训练数据地址，base_model 地址，训练轮次，节点数。
+
+在模型训练完成后，可将模型重命名为 `ranker` 存放至 `./checkpoints/` 目录下，接下来便可以使用如下命令，根据给定的汽车说明书相关问题，对汽车说明书的图片进行打分。代码如下：
+
+```shell
+cd Rerank
+bash run_test.sh 后备箱怎么开
+cd ..
+```
+
+其中，后一项为用户问题，命令执行完成后，分数文件将会保存至 `./Rerank/data/demo.score` 中。
+
+
+## 6. 跨模态阅读理解模块
+本项目首先获取排序模块输出的结果中评分最高的图片，然后将会使用跨模态的语言模型 LayoutXLM 从该图片中去抽取用户提问的答案。在获取答案后，将会对答案在该图片中进行高亮显示并返回用户。
+
+本项目提供了28条汽车说明书相关的训练样本，用于跨模态阅读理解模型的训练， 同时也提供了一个预先训练好的基线模型 base_model。 本模块可以使用 base_model 在汽车说明书训练样本上进一步微调，增强模型对汽车说明书领域的理解。
+
+其中，汽车说明书的阅读理解训练集可点击[这里](https://paddlenlp.bj.bcebos.com/data/automobile_mrc_train.json) 进行下载，下载后将其重命名为 `train.json`，存放至 `./Extraction/data/` 目录下。
+
+同时，base_model 是 [Dureader VIS](https://aclanthology.org/2022.findings-acl.105.pdf) 数据集训练的跨模态阅读理解模型， 可点击[这里](https://paddlenlp.bj.bcebos.com/models/base_mrc.tar.gz) 进行下载，解压后可获得包含模型的目录 `base_model`，将其放至 `./Extraction/checkpoints` 目录下。
+
+可使用如下代码进行训练：
+
+```shell
+cd Extraction
+bash run_train.sh
+cd ..
+```
+
+在模型训练完成后，可将模型重命名为 `layoutxlm` 存放至 `./checkpoints/` 目录下，接下来便可以使用如下命令，根据给定的汽车说明书相关问题，从得分最高的汽车说明书图片中抽取答案。代码如下：
+
+```shell
+cd Extraction
+bash run_test.sh 后备箱怎么开
+cd ..
+```
+
+其中，后一项为用户问题，命令执行完成后，最终结果将会保存至 `./answer.png` 中。
+
+## 7. 全流程预测
+本项目提供了全流程预测的功能，可通过如下命令进行一键式预测：
+
+```shell
+bash run_test.sh 后备箱怎么开
+```
+
+其中，后一项参数为用户问题，最终结果将会保存至 `./answer.png` 中。
+
+**备注**：在运行命令前，请确保已使用第4节介绍的命令对原始汽车说明书图片完成了文档解析。
+
+
+下图展示了用户提问的三个问题："后备箱怎么开"，"钥匙怎么充电" 和 "NFC解锁注意事项"， 可以看到，本项目的汽车说明书问答系统能够精准地找到答案并进行高亮显示。
+
+<center><img src="https://user-images.githubusercontent.com/35913314/169012902-1a42bd14-976f-4da8-b5b5-d8e7352b68df.png"/></center>

applications/doc_vqa/Rerank/change_to_rerank.py

@@ -0,0 +1,19 @@
+import sys
+import json
+
+question = sys.argv[1]
+
+with open('../OCR_process/demo_ocr_res.json', 'r', encoding='utf8') as f:
+    paras = []
+    for line in f:
+        line = json.loads(line.strip())
+        document = line['document']
+        para = []
+        for token in document:
+            token = token.replace('▁', '')
+            para.append(token)
+        paras.append(''.join(para))
+
+with open('./data/demo.tsv', 'w', encoding='utf8') as f:
+    for para in paras:
+        f.write('{}\t\t{}\t0\n'.format(question, para))

applications/doc_vqa/Rerank/config/ernie_base_1.0_CN/ernie_config.json

@@ -0,0 +1,12 @@
+{
+  "attention_probs_dropout_prob": 0.1, 
+  "hidden_act": "relu", 
+  "hidden_dropout_prob": 0.1, 
+  "hidden_size": 768, 
+  "initializer_range": 0.02, 
+  "max_position_embeddings": 513, 
+  "num_attention_heads": 12, 
+  "num_hidden_layers": 12, 
+  "type_vocab_size": 2, 
+  "vocab_size": 18000
+}

applications/doc_vqa/Rerank/run_test.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+export CUDA_VISIBLE_DEVICES=0
+
+QUESTION=$1
+
+if [ ! -d output ]; then
+    mkdir output
+fi
+if [ ! -d log ]; then
+    mkdir log
+fi
+
+python3 change_to_rerank.py ${QUESTION}
+
+python3 -u ./src/train_ce.py \
+                   --use_cuda true \
+                   --verbose true \
+                   --do_train false \
+                   --do_val false \
+                   --do_test true \
+                   --batch_size 128 \
+                   --init_checkpoint "./checkpoints/ranker" \
+                   --test_set "./data/demo.tsv" \
+                   --test_save "data/demo.score" \
+                   --max_seq_len 384 \
+                   --for_cn true \
+                   --vocab_path "config/ernie_base_1.0_CN/vocab.txt" \
+                   --ernie_config_path "config/ernie_base_1.0_CN/ernie_config.json"
+                   1>>log/train.log 2>&1
+

applications/doc_vqa/Rerank/run_train.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+export CUDA_VISIBLE_DEVICES=0
+
+if [ $# != 4 ];then
+    echo "USAGE: sh run_train.sh \$TRAIN_SET \$MODEL_PATH \$epoch \$nodes_count"
+    exit 1
+fi
+
+TRAIN_SET=$1
+MODEL_PATH=$2
+epoch=$3
+node=$4
+
+CHECKPOINT_PATH=output
+if [ ! -d output ]; then
+    mkdir output
+fi
+if [ ! -d log ]; then
+    mkdir log
+fi
+
+lr=1e-5
+batch_size=32
+train_exampls=`cat $TRAIN_SET | wc -l`
+save_steps=$[$train_exampls/$batch_size/$node]
+data_size=$[$save_steps*$batch_size*$node]
+new_save_steps=$[$save_steps*$epoch/2]
+
+python3 -m paddle.distributed.launch \
+    --log_dir log \
+    ./src/train_ce.py \
+                   --use_cuda true \
+                   --verbose true \
+                   --do_train true \
+                   --do_val false \
+                   --do_test false \
+                   --use_mix_precision false \
+                   --train_data_size ${data_size} \
+                   --batch_size ${batch_size} \
+                   --init_pretraining_params ${MODEL_PATH} \
+                   --train_set ${TRAIN_SET} \
+                   --save_steps ${new_save_steps} \
+                   --validation_steps ${new_save_steps} \
+                   --checkpoints ${CHECKPOINT_PATH} \
+                   --weight_decay  0.01 \
+                   --warmup_proportion 0.0 \
+                   --epoch $epoch \
+                   --max_seq_len 384 \
+                   --for_cn true \
+                   --vocab_path config/ernie_base_1.0_CN/vocab.txt \
+                   --ernie_config_path config/ernie_base_1.0_CN/ernie_config.json \
+                   --learning_rate ${lr} \
+                   --skip_steps 10 \
+                   --num_iteration_per_drop_scope 1 \
+                   --num_labels 2 \
+                   --random_seed 1 
+

applications/doc_vqa/Rerank/src/batching.py

@@ -0,0 +1,77 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Mask, padding and batching."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+from six.moves import xrange
+
+
+def pad_batch_data(insts,
+                   pad_idx=0,
+                   return_pos=False,
+                   return_input_mask=False,
+                   return_max_len=False,
+                   return_num_token=False,
+                   return_seq_lens=False):
+    """
+    Pad the instances to the max sequence length in batch, and generate the
+    corresponding position data and attention bias.
+    """
+    return_list = []
+    max_len = max(len(inst) for inst in insts)
+    # Any token included in dict can be used to pad, since the paddings' loss
+    # will be masked out by weights and make no effect on parameter gradients.
+
+    inst_data = np.array(
+        [inst + list([pad_idx] * (max_len - len(inst))) for inst in insts])
+    return_list += [inst_data.astype("int64").reshape([-1, max_len, 1])]
+
+    # position data
+    if return_pos:
+        inst_pos = np.array([
+            list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst))
+            for inst in insts
+        ])
+
+        return_list += [inst_pos.astype("int64").reshape([-1, max_len, 1])]
+
+    if return_input_mask:
+        # This is used to avoid attention on paddings.
+        input_mask_data = np.array(
+            [[1] * len(inst) + [0] * (max_len - len(inst)) for inst in insts])
+        input_mask_data = np.expand_dims(input_mask_data, axis=-1)
+        return_list += [input_mask_data.astype("float32")]
+
+    if return_max_len:
+        return_list += [max_len]
+
+    if return_num_token:
+        num_token = 0
+        for inst in insts:
+            num_token += len(inst)
+        return_list += [num_token]
+
+    if return_seq_lens:
+        seq_lens = np.array([len(inst) for inst in insts])
+        return_list += [seq_lens.astype("int64").reshape([-1, 1])]
+
+    return return_list if len(return_list) > 1 else return_list[0]
+
+
+if __name__ == "__main__":
+    pass

applications/doc_vqa/Rerank/src/cross_encoder.py

@@ -0,0 +1,336 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model for classifier."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import time
+import logging
+import numpy as np
+from six.moves import xrange
+from scipy.stats import pearsonr, spearmanr
+
+import paddle.fluid as fluid
+
+from model.ernie import ErnieModel
+
+log = logging.getLogger(__name__)
+
+
+def create_model(args,
+                 pyreader_name,
+                 ernie_config,
+                 is_prediction=False,
+                 task_name=""):
+    pyreader = fluid.layers.py_reader(capacity=50,
+                                      shapes=[[-1, args.max_seq_len, 1],
+                                              [-1, args.max_seq_len, 1],
+                                              [-1, args.max_seq_len, 1],
+                                              [-1, args.max_seq_len, 1],
+                                              [-1, args.max_seq_len, 1],
+                                              [-1, 1], [-1, 1]],
+                                      dtypes=[
+                                          'int64', 'int64', 'int64', 'int64',
+                                          'float32', 'int64', 'int64'
+                                      ],
+                                      lod_levels=[0, 0, 0, 0, 0, 0, 0],
+                                      name=task_name + "_" + pyreader_name,
+                                      use_double_buffer=True)
+
+    (src_ids, sent_ids, pos_ids, task_ids, input_mask, labels,
+     qids) = fluid.layers.read_file(pyreader)
+
+    def _model(is_noise=False):
+        ernie = ErnieModel(src_ids=src_ids,
+                           position_ids=pos_ids,
+                           sentence_ids=sent_ids,
+                           task_ids=task_ids,
+                           input_mask=input_mask,
+                           config=ernie_config,
+                           is_noise=is_noise)
+
+        cls_feats = ernie.get_pooled_output()
+        if not is_noise:
+            cls_feats = fluid.layers.dropout(
+                x=cls_feats,
+                dropout_prob=0.1,
+                dropout_implementation="upscale_in_train")
+        logits = fluid.layers.fc(
+            input=cls_feats,
+            size=args.num_labels,
+            param_attr=fluid.ParamAttr(
+                name=task_name + "_cls_out_w",
+                initializer=fluid.initializer.TruncatedNormal(scale=0.02)),
+            bias_attr=fluid.ParamAttr(
+                name=task_name + "_cls_out_b",
+                initializer=fluid.initializer.Constant(0.)))
+        """
+        if is_prediction:
+            probs = fluid.layers.softmax(logits)
+            feed_targets_name = [
+                src_ids.name, sent_ids.name, pos_ids.name, input_mask.name
+            ]
+            if ernie_version == "2.0":
+                feed_targets_name += [task_ids.name]
+            return pyreader, probs, feed_targets_name
+        """
+
+        num_seqs = fluid.layers.create_tensor(dtype='int64')
+        ## add focal loss
+        ce_loss, probs = fluid.layers.softmax_with_cross_entropy(
+            logits=logits, label=labels, return_softmax=True)
+        loss = fluid.layers.mean(x=ce_loss)
+        accuracy = fluid.layers.accuracy(input=probs,
+                                         label=labels,
+                                         total=num_seqs)
+        graph_vars = {
+            "loss": loss,
+            "probs": probs,
+            "accuracy": accuracy,
+            "labels": labels,
+            "num_seqs": num_seqs,
+            "qids": qids
+        }
+        return graph_vars
+
+    if not is_prediction:
+        graph_vars = _model(is_noise=True)
+        old_loss = graph_vars["loss"]
+        token_emb = fluid.default_main_program().global_block().var(
+            "word_embedding")
+        # print(token_emb)
+        token_emb.stop_gradient = False
+        token_gradient = fluid.gradients(old_loss, token_emb)[0]
+        token_gradient.stop_gradient = False
+        epsilon = 1e-8
+        norm = (fluid.layers.sqrt(
+            fluid.layers.reduce_sum(fluid.layers.square(token_gradient)) +
+            epsilon))
+        gp = (0.01 * token_gradient) / norm
+        gp.stop_gradient = True
+        fluid.layers.assign(token_emb + gp, token_emb)
+        graph_vars = _model()
+        fluid.layers.assign(token_emb - gp, token_emb)
+    else:
+        graph_vars = _model()
+
+    return pyreader, graph_vars
+
+
+def evaluate_mrr(preds):
+    last_qid = None
+    total_mrr = 0.0
+    qnum = 0.0
+    rank = 0.0
+    correct = False
+    for qid, score, label in preds:
+        if qid != last_qid:
+            rank = 0.0
+            qnum += 1
+            correct = False
+            last_qid = qid
+
+        rank += 1
+        if not correct and label != 0:
+            total_mrr += 1.0 / rank
+            correct = True
+
+    return total_mrr / qnum
+
+
+def evaluate(exe,
+             test_program,
+             test_pyreader,
+             graph_vars,
+             eval_phase,
+             use_multi_gpu_test=False,
+             metric='simple_accuracy'):
+    train_fetch_list = [
+        graph_vars["loss"].name, graph_vars["accuracy"].name,
+        graph_vars["num_seqs"].name
+    ]
+
+    if eval_phase == "train":
+        if "learning_rate" in graph_vars:
+            train_fetch_list.append(graph_vars["learning_rate"].name)
+        outputs = exe.run(fetch_list=train_fetch_list, program=test_program)
+        ret = {"loss": np.mean(outputs[0]), "accuracy": np.mean(outputs[1])}
+        if "learning_rate" in graph_vars:
+            ret["learning_rate"] = float(outputs[3][0])
+        return ret
+
+    test_pyreader.start()
+    total_cost = 0.0
+    total_acc = 0.0
+    total_num_seqs = 0.0
+    total_label_pos_num = 0.0
+    total_pred_pos_num = 0.0
+    total_correct_num = 0.0
+    qids, labels, scores, preds = [], [], [], []
+    time_begin = time.time()
+
+    fetch_list = [
+        graph_vars["loss"].name, graph_vars["accuracy"].name,
+        graph_vars["probs"].name, graph_vars["labels"].name,
+        graph_vars["num_seqs"].name, graph_vars["qids"].name
+    ]
+    while True:
+        try:
+            if use_multi_gpu_test:
+                np_loss, np_acc, np_probs, np_labels, np_num_seqs, np_qids = exe.run(
+                    fetch_list=fetch_list)
+            else:
+                np_loss, np_acc, np_probs, np_labels, np_num_seqs, np_qids = exe.run(
+                    program=test_program, fetch_list=fetch_list)
+            total_cost += np.sum(np_loss * np_num_seqs)
+            total_acc += np.sum(np_acc * np_num_seqs)
+            total_num_seqs += np.sum(np_num_seqs)
+            labels.extend(np_labels.reshape((-1)).tolist())
+            if np_qids is None:
+                np_qids = np.array([])
+            qids.extend(np_qids.reshape(-1).tolist())
+            scores.extend(np_probs[:, 1].reshape(-1).tolist())
+            np_preds = np.argmax(np_probs, axis=1).astype(np.float32)
+            preds.extend(np_preds)
+            total_label_pos_num += np.sum(np_labels)
+            total_pred_pos_num += np.sum(np_preds)
+            total_correct_num += np.sum(np.dot(np_preds, np_labels))
+        except fluid.core.EOFException:
+            test_pyreader.reset()
+            break
+    time_end = time.time()
+    cost = total_cost / total_num_seqs
+    elapsed_time = time_end - time_begin
+
+    evaluate_info = ""
+    if metric == 'acc_and_f1':
+        ret = acc_and_f1(preds, labels)
+        evaluate_info = "[%s evaluation] ave loss: %f, ave_acc: %f, f1: %f, data_num: %d, elapsed time: %f s" \
+            % (eval_phase, cost, ret['acc'], ret['f1'], total_num_seqs, elapsed_time)
+    elif metric == 'matthews_corrcoef':
+        ret = matthews_corrcoef(preds, labels)
+        evaluate_info = "[%s evaluation] ave loss: %f, matthews_corrcoef: %f, data_num: %d, elapsed time: %f s" \
+            % (eval_phase, cost, ret, total_num_seqs, elapsed_time)
+    elif metric == 'pearson_and_spearman':
+        ret = pearson_and_spearman(scores, labels)
+        evaluate_info = "[%s evaluation] ave loss: %f, pearson:%f, spearman:%f, corr:%f, data_num: %d, elapsed time: %f s" \
+            % (eval_phase, cost, ret['pearson'], ret['spearman'], ret['corr'], total_num_seqs, elapsed_time)
+    elif metric == 'simple_accuracy':
+        ret = simple_accuracy(preds, labels)
+        evaluate_info = "[%s evaluation] ave loss: %f, acc:%f, data_num: %d, elapsed time: %f s" \
+            % (eval_phase, cost, ret, total_num_seqs, elapsed_time)
+    elif metric == "acc_and_f1_and_mrr":
+        ret_a = acc_and_f1(preds, labels)
+        preds = sorted(zip(qids, scores, labels),
+                       key=lambda elem: (elem[0], -elem[1]))
+        ret_b = evaluate_mrr(preds)
+        evaluate_info = "[%s evaluation] ave loss: %f, acc: %f, f1: %f, mrr: %f, data_num: %d, elapsed time: %f s" \
+            % (eval_phase, cost, ret_a['acc'], ret_a['f1'], ret_b, total_num_seqs, elapsed_time)
+    else:
+        raise ValueError('unsupported metric {}'.format(metric))
+    return evaluate_info
+
+
+def matthews_corrcoef(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+    tp = np.sum((labels == 1) & (preds == 1))
+    tn = np.sum((labels == 0) & (preds == 0))
+    fp = np.sum((labels == 0) & (preds == 1))
+    fn = np.sum((labels == 1) & (preds == 0))
+
+    mcc = ((tp * tn) - (fp * fn)) / np.sqrt(
+        (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
+    return mcc
+
+
+def f1_score(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+
+    tp = np.sum((labels == 1) & (preds == 1))
+    tn = np.sum((labels == 0) & (preds == 0))
+    fp = np.sum((labels == 0) & (preds == 1))
+    fn = np.sum((labels == 1) & (preds == 0))
+    p = tp / (tp + fp)
+    r = tp / (tp + fn)
+    f1 = (2 * p * r) / (p + r + 1e-8)
+    return f1
+
+
+def pearson_and_spearman(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+
+    pearson_corr = pearsonr(preds, labels)[0]
+    spearman_corr = spearmanr(preds, labels)[0]
+    return {
+        "pearson": pearson_corr,
+        "spearmanr": spearman_corr,
+        "corr": (pearson_corr + spearman_corr) / 2,
+    }
+
+
+def acc_and_f1(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+
+    acc = simple_accuracy(preds, labels)
+    f1 = f1_score(preds, labels)
+    return {
+        "acc": acc,
+        "f1": f1,
+        "acc_and_f1": (acc + f1) / 2,
+    }
+
+
+def simple_accuracy(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+    return (preds == labels).mean()
+
+
+def predict(exe, test_program, test_pyreader, graph_vars, dev_count=1):
+    test_pyreader.start()
+    qids, scores, probs = [], [], []
+    preds = []
+
+    fetch_list = [graph_vars["probs"].name, graph_vars["qids"].name]
+
+    while True:
+        try:
+            if dev_count == 1:
+                np_probs, np_qids = exe.run(program=test_program,
+                                            fetch_list=fetch_list)
+            else:
+                np_probs, np_qids = exe.run(fetch_list=fetch_list)
+
+            if np_qids is None:
+                np_qids = np.array([])
+            qids.extend(np_qids.reshape(-1).tolist())
+            np_preds = np.argmax(np_probs, axis=1).astype(np.float32)
+            preds.extend(np_preds)
+            probs.append(np_probs)
+
+        except fluid.core.EOFException:
+            test_pyreader.reset()
+            break
+
+    probs = np.concatenate(probs, axis=0).reshape([len(preds), -1])
+
+    return qids, preds, probs

applications/doc_vqa/Rerank/src/finetune_args.py

@@ -0,0 +1,117 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import os
+import time
+import argparse
+
+from utils.args import ArgumentGroup
+
+# yapf: disable
+parser = argparse.ArgumentParser(__doc__)
+model_g = ArgumentGroup(parser, "model", "model configuration and paths.")
+model_g.add_arg("ernie_config_path",         str,  None,           "Path to the json file for ernie model config.")
+model_g.add_arg("init_checkpoint",          str,  None,           "Init checkpoint to resume training from.")
+model_g.add_arg("init_pretraining_params",  str,  None,
+                "Init pre-training params which preforms fine-tuning from. If the "
+                 "arg 'init_checkpoint' has been set, this argument wouldn't be valid.")
+model_g.add_arg("checkpoints",              str,  "checkpoints",  "Path to save checkpoints.")
+
+model_g.add_arg("is_classify",    bool, True,  "is_classify")
+model_g.add_arg("is_regression",  bool, False, "is_regression")
+model_g.add_arg("task_id",           int,    0,       "task id")
+
+train_g = ArgumentGroup(parser, "training", "training options.")
+train_g.add_arg("epoch",             int,    3,       "Number of epoches for fine-tuning.")
+train_g.add_arg("learning_rate",     float,  5e-5,    "Learning rate used to train with warmup.")
+train_g.add_arg("lr_scheduler",      str,    "linear_warmup_decay",
+                "scheduler of learning rate.", choices=['linear_warmup_decay', 'noam_decay'])
+train_g.add_arg("weight_decay",      float,  0.01,    "Weight decay rate for L2 regularizer.")
+train_g.add_arg("warmup_proportion", float,  0.1,
+                "Proportion of training steps to perform linear learning rate warmup for.")
+train_g.add_arg("save_steps",        int,    10000,   "The steps interval to save checkpoints.")
+train_g.add_arg("validation_steps",  int,    1000,    "The steps interval to evaluate model performance.")
+train_g.add_arg("use_recompute",          bool,   False,   "Whether to use recompute optimizer for training.")
+train_g.add_arg("use_mix_precision",          bool,   False,   "Whether to use mix-precision optimizer for training.")
+train_g.add_arg("use_cross_batch",          bool,   False,   "Whether to use cross-batch for training.")
+train_g.add_arg("use_lamb",          bool,   False,   "Whether to use LambOptimizer for training.")
+train_g.add_arg("use_dynamic_loss_scaling",    bool,   True,   "Whether to use dynamic loss scaling.")
+
+train_g.add_arg("test_save",            str,    "./checkpoints/test_result",       "test_save")
+train_g.add_arg("metric",               str,    "simple_accuracy",   "metric")
+train_g.add_arg("incr_every_n_steps",          int,    100,   "Increases loss scaling every n consecutive.")
+train_g.add_arg("decr_every_n_nan_or_inf",     int,    2,
+                "Decreases loss scaling every n accumulated steps with nan or inf gradients.")
+train_g.add_arg("incr_ratio",                  float,  2.0,
+                "The multiplier to use when increasing the loss scaling.")
+train_g.add_arg("decr_ratio",                  float,  0.8,
+                "The less-than-one-multiplier to use when decreasing.")
+
+
+
+log_g = ArgumentGroup(parser,     "logging", "logging related.")
+log_g.add_arg("skip_steps",          int,    10,    "The steps interval to print loss.")
+log_g.add_arg("verbose",             bool,   False, "Whether to output verbose log.")
+
+data_g = ArgumentGroup(parser, "data", "Data paths, vocab paths and data processing options")
+data_g.add_arg("tokenizer",           str, "FullTokenizer",
+              "ATTENTION: the INPUT must be splited by Word with blank while using SentencepieceTokenizer or WordsegTokenizer")
+data_g.add_arg("train_set",           str,  None,  "Path to training data.")
+data_g.add_arg("test_set",            str,  None,  "Path to test data.")
+data_g.add_arg("dev_set",             str,  None,  "Path to validation data.")
+data_g.add_arg("vocab_path",          str,  None,  "Vocabulary path.")
+data_g.add_arg("max_seq_len",         int,  512,   "Number of words of the longest seqence.")
+data_g.add_arg("q_max_seq_len",       int,  32,   "Number of words of the longest seqence.")
+data_g.add_arg("p_max_seq_len",       int,  256,   "Number of words of the longest seqence.")
+data_g.add_arg("train_data_size",     int,  0,     "Number of training data's total examples. Set for distribute.")
+data_g.add_arg("batch_size",          int,  32,    "Total examples' number in batch for training. see also --in_tokens.")
+data_g.add_arg("predict_batch_size",  int,  None,    "Total examples' number in batch for predict. see also --in_tokens.")
+data_g.add_arg("in_tokens",           bool, False,
+              "If set, the batch size will be the maximum number of tokens in one batch. "
+              "Otherwise, it will be the maximum number of examples in one batch.")
+data_g.add_arg("do_lower_case",       bool, True,
+               "Whether to lower case the input text. Should be True for uncased models and False for cased models.")
+data_g.add_arg("random_seed",         int,  None,     "Random seed.")
+data_g.add_arg("label_map_config",    str,  None,  "label_map_path.")
+data_g.add_arg("num_labels",          int,  2,     "label number")
+data_g.add_arg("diagnostic",          str,  None,  "GLUE Diagnostic Dataset")
+data_g.add_arg("diagnostic_save",     str,  None,  "GLUE Diagnostic save f")
+data_g.add_arg("max_query_length",          int,   64,    "Max query length.")
+data_g.add_arg("max_answer_length",         int,   100,    "Max answer length.")
+data_g.add_arg("doc_stride",                int,   128,
+               "When splitting up a long document into chunks, how much stride to take between chunks.")
+data_g.add_arg("n_best_size",               int,   20,
+               "The total number of n-best predictions to generate in the nbest_predictions.json output file.")
+data_g.add_arg("chunk_scheme", type=str,  default="IOB", choices=["IO", "IOB", "IOE", "IOBES"], help="chunk scheme")
+
+run_type_g = ArgumentGroup(parser, "run_type", "running type options.")
+run_type_g.add_arg("use_cuda",                     bool,   True,  "If set, use GPU for training.")
+run_type_g.add_arg("is_distributed",    bool,   False,  "If set, then start distributed training.")
+run_type_g.add_arg("use_fast_executor",            bool,   False, "If set, use fast parallel executor (in experiment).")
+run_type_g.add_arg("num_iteration_per_drop_scope", int,    10,    "Iteration intervals to drop scope.")
+run_type_g.add_arg("do_train",                     bool,   True,  "Whether to perform training.")
+run_type_g.add_arg("do_val",                       bool,   True,  "Whether to perform evaluation on dev data set.")
+run_type_g.add_arg("do_test",                      bool,   True,  "Whether to perform evaluation on test data set.")
+run_type_g.add_arg("output_item",                  int,   3,  "Test output format.")
+run_type_g.add_arg("output_file_name",             str,   None,  "Test output file name")
+run_type_g.add_arg("test_data_cnt",             int,  1110000 ,  "total cnt of testset")
+run_type_g.add_arg("use_multi_gpu_test",           bool,   False, "Whether to perform evaluation using multiple gpu cards")
+run_type_g.add_arg("metrics",                      bool,   True,  "Whether to perform evaluation on test data set.")
+run_type_g.add_arg("shuffle",                      bool,   True,  "")
+run_type_g.add_arg("for_cn",                       bool,   False,  "model train for cn or for other langs.")

applications/doc_vqa/Rerank/src/index_search.py

@@ -0,0 +1,71 @@
+import sys
+import time
+import faiss
+import math
+import numpy as np
+
+
+def read_embed(file_name, dim=768, bs=3000):
+    if file_name.endswith('npy'):
+        i = 0
+        emb_np = np.load(file_name)
+        while (i < len(emb_np)):
+            vec_list = emb_np[i:i + bs]
+            i += bs
+            yield vec_list
+    else:
+        vec_list = []
+        with open(file_name) as inp:
+            for line in inp:
+                data = line.strip()
+                vector = [float(item) for item in data.split(' ')]
+                assert len(vector) == dim
+                vec_list.append(vector)
+                if len(vec_list) == bs:
+                    yield vec_list
+                    vec_list = []
+            if vec_list:
+                yield vec_list
+
+
+def load_qid(file_name):
+    qid_list = []
+    with open(file_name) as inp:
+        for line in inp:
+            line = line.strip()
+            qid = line.split('\t')[0]
+            qid_list.append(qid)
+    return qid_list
+
+
+def search(index, emb_file, qid_list, outfile, top_k):
+    q_idx = 0
+    with open(outfile, 'w') as out:
+        for batch_vec in read_embed(emb_file):
+            q_emb_matrix = np.array(batch_vec)
+            res_dist, res_p_id = index.search(q_emb_matrix.astype('float32'),
+                                              top_k)
+            for i in range(len(q_emb_matrix)):
+                qid = qid_list[q_idx]
+                for j in range(top_k):
+                    pid = res_p_id[i][j]
+                    score = res_dist[i][j]
+                    out.write('%s\t%s\t%s\t%s\n' % (qid, pid, j + 1, score))
+                q_idx += 1
+
+
+def main():
+    part = sys.argv[1]
+    topk = int(sys.argv[2])
+    q_text_file = sys.argv[3]
+    outfile = 'output/res.top%s-part%s' % (topk, part)
+
+    qid_list = load_qid(q_text_file)
+
+    engine = faiss.read_index("output/para.index.part%s" % part)
+    emb_file = 'output/query.emb.npy'
+    search(engine, emb_file, qid_list, outfile, topk)
+
+
+if __name__ == "__main__":
+    main()

applications/doc_vqa/Rerank/src/merge.py

@@ -0,0 +1,48 @@
+import sys
+
+shift = int(sys.argv[1])
+top = int(sys.argv[2])
+total_part = int(sys.argv[3])
+
+f_list = []
+for part in range(total_part):
+    f0 = open('output/res.top%s-part%s' % (top, part))
+    f_list.append(f0)
+
+line_list = []
+for part in range(total_part):
+    line = f_list[part].readline()
+    line_list.append(line)
+
+out = open('output/dev.res.top%s' % top, 'w')
+last_q = ''
+ans_list = {}
+while line_list[-1]:
+    cur_list = []
+    for line in line_list:
+        sub = line.strip().split('\t')
+        cur_list.append(sub)
+
+    if last_q == '':
+        last_q = cur_list[0][0]
+    if cur_list[0][0] != last_q:
+        rank = sorted(ans_list.items(), key=lambda a: a[1], reverse=True)
+        for i in range(top):
+            out.write("%s\t%s\t%s\t%s\n" %
+                      (last_q, rank[i][0], i + 1, rank[i][1]))
+        ans_list = {}
+    for i, sub in enumerate(cur_list):
+        ans_list[int(sub[1]) + shift * i] = float(sub[-1])
+    last_q = cur_list[0][0]
+
+    line_list = []
+    for f0 in f_list:
+        line = f0.readline()
+        line_list.append(line)
+
+rank = sorted(ans_list.items(), key=lambda a: a[1], reverse=True)
+for i in range(top):
+    out.write("%s\t%s\t%s\t%s\n" % (last_q, rank[i][0], i + 1, rank[i][1]))
+out.close()
+
+print('output/dev.res.top%s' % top)

applications/doc_vqa/Rerank/src/model/ernie.py

@@ -0,0 +1,284 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Ernie model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import json
+import six
+import logging
+from io import open
+
+import paddle.fluid as fluid
+
+from model.transformer_encoder import encoder, pre_process_layer
+
+log = logging.getLogger(__name__)
+
+
+class ErnieConfig(object):
+
+    def __init__(self, config_path):
+        self._config_dict = self._parse(config_path)
+
+    def _parse(self, config_path):
+        try:
+            with open(config_path, 'r', encoding='utf8') as json_file:
+                config_dict = json.load(json_file)
+        except Exception:
+            raise IOError("Error in parsing Ernie model config file '%s'" %
+                          config_path)
+        else:
+            return config_dict
+
+    def __getitem__(self, key):
+        return self._config_dict.get(key, None)
+
+    def print_config(self):
+        for arg, value in sorted(six.iteritems(self._config_dict)):
+            log.info('%s: %s' % (arg, value))
+        log.info('------------------------------------------------')
+
+
+class ErnieModel(object):
+
+    def __init__(self,
+                 src_ids,
+                 position_ids,
+                 sentence_ids,
+                 task_ids,
+                 input_mask,
+                 config,
+                 weight_sharing=True,
+                 model_name='',
+                 is_noise=False):
+
+        self._emb_size = config['hidden_size']
+        self._n_layer = config['num_hidden_layers']
+        self._n_head = config['num_attention_heads']
+        self._voc_size = config['vocab_size']
+        self._max_position_seq_len = config['max_position_embeddings']
+        if config['sent_type_vocab_size']:
+            self._sent_types = config['sent_type_vocab_size']
+        else:
+            self._sent_types = config['type_vocab_size']
+
+        self._use_task_id = config['use_task_id']
+        if self._use_task_id:
+            self._task_types = config['task_type_vocab_size']
+        self._hidden_act = config['hidden_act']
+        self._prepostprocess_dropout = config['hidden_dropout_prob']
+        self._attention_dropout = config['attention_probs_dropout_prob']
+        if is_noise:
+            self._prepostprocess_dropout = 0
+            self._attention_dropout = 0
+        self._weight_sharing = weight_sharing
+        self.checkpoints = []
+
+        self._word_emb_name = "word_embedding"
+        self._pos_emb_name = "pos_embedding"
+        self._sent_emb_name = "sent_embedding"
+        self._task_emb_name = "task_embedding"
+        self._emb_dtype = "float32"
+
+        # Initialize all weigths by truncated normal initializer, and all biases
+        # will be initialized by constant zero by default.
+        self._param_initializer = fluid.initializer.TruncatedNormal(
+            scale=config['initializer_range'])
+
+        self._build_model(model_name, src_ids, position_ids, sentence_ids,
+                          task_ids, input_mask)
+
+    def _build_model(self, model_name, src_ids, position_ids, sentence_ids,
+                     task_ids, input_mask):
+        # padding id in vocabulary must be set to 0
+        emb_out = fluid.layers.embedding(
+            input=src_ids,
+            size=[self._voc_size, self._emb_size],
+            dtype=self._emb_dtype,
+            param_attr=fluid.ParamAttr(name=model_name + self._word_emb_name,
+                                       initializer=self._param_initializer),
+            is_sparse=False)
+
+        position_emb_out = fluid.layers.embedding(
+            input=position_ids,
+            size=[self._max_position_seq_len, self._emb_size],
+            dtype=self._emb_dtype,
+            param_attr=fluid.ParamAttr(name=model_name + self._pos_emb_name,
+                                       initializer=self._param_initializer))
+
+        sent_emb_out = fluid.layers.embedding(
+            sentence_ids,
+            size=[self._sent_types, self._emb_size],
+            dtype=self._emb_dtype,
+            param_attr=fluid.ParamAttr(name=model_name + self._sent_emb_name,
+                                       initializer=self._param_initializer))
+
+        emb_out = emb_out + position_emb_out
+        emb_out = emb_out + sent_emb_out
+
+        if self._use_task_id:
+            task_emb_out = fluid.layers.embedding(
+                task_ids,
+                size=[self._task_types, self._emb_size],
+                dtype=self._emb_dtype,
+                param_attr=fluid.ParamAttr(name=model_name +
+                                           self._task_emb_name,
+                                           initializer=self._param_initializer))
+
+            emb_out = emb_out + task_emb_out
+
+        emb_out = pre_process_layer(emb_out,
+                                    'nd',
+                                    self._prepostprocess_dropout,
+                                    name=model_name + 'pre_encoder')
+
+        self_attn_mask = fluid.layers.matmul(x=input_mask,
+                                             y=input_mask,
+                                             transpose_y=True)
+
+        self_attn_mask = fluid.layers.scale(x=self_attn_mask,
+                                            scale=10000.0,
+                                            bias=-1.0,
+                                            bias_after_scale=False)
+        n_head_self_attn_mask = fluid.layers.stack(x=[self_attn_mask] *
+                                                   self._n_head,
+                                                   axis=1)
+        n_head_self_attn_mask.stop_gradient = True
+
+        self._enc_out, self.checkpoints = encoder(
+            enc_input=emb_out,
+            attn_bias=n_head_self_attn_mask,
+            n_layer=self._n_layer,
+            n_head=self._n_head,
+            d_key=self._emb_size // self._n_head,
+            d_value=self._emb_size // self._n_head,
+            d_model=self._emb_size,
+            d_inner_hid=self._emb_size * 4,
+            prepostprocess_dropout=self._prepostprocess_dropout,
+            attention_dropout=self._attention_dropout,
+            relu_dropout=0,
+            hidden_act=self._hidden_act,
+            preprocess_cmd="",
+            postprocess_cmd="dan",
+            param_initializer=self._param_initializer,
+            model_name=model_name,
+            name=model_name + 'encoder')
+
+    def get_sequence_output(self):
+        return self._enc_out
+
+    def get_cls_output(self):
+        """Get the first feature of each sequence for classification"""
+        cls_output = fluid.layers.slice(input=self._enc_out,
+                                        axes=[1],
+                                        starts=[0],
+                                        ends=[1])
+        cls_output = fluid.layers.squeeze(cls_output, axes=[1])
+        return cls_output
+
+    def get_pooled_output(self):
+        """Get the first feature of each sequence for classification"""
+        next_sent_feat = fluid.layers.slice(input=self._enc_out,
+                                            axes=[1],
+                                            starts=[0],
+                                            ends=[1])
+        next_sent_feat = fluid.layers.fc(
+            input=next_sent_feat,
+            size=self._emb_size,
+            act="tanh",
+            param_attr=fluid.ParamAttr(name="pooled_fc.w_0",
+                                       initializer=self._param_initializer),
+            bias_attr="pooled_fc.b_0")
+        return next_sent_feat
+
+    def get_lm_output(self, mask_label, mask_pos):
+        """Get the loss & accuracy for pretraining"""
+
+        mask_pos = fluid.layers.cast(x=mask_pos, dtype='int32')
+
+        # extract the first token feature in each sentence
+        self.next_sent_feat = self.get_pooled_output()
+        reshaped_emb_out = fluid.layers.reshape(x=self._enc_out,
+                                                shape=[-1, self._emb_size])
+        # extract masked tokens' feature
+        mask_feat = fluid.layers.gather(input=reshaped_emb_out, index=mask_pos)
+
+        # transform: fc
+        mask_trans_feat = fluid.layers.fc(
+            input=mask_feat,
+            size=self._emb_size,
+            act=self._hidden_act,
+            param_attr=fluid.ParamAttr(name='mask_lm_trans_fc.w_0',
+                                       initializer=self._param_initializer),
+            bias_attr=fluid.ParamAttr(name='mask_lm_trans_fc.b_0'))
+
+        # transform: layer norm
+        mask_trans_feat = fluid.layers.layer_norm(
+            mask_trans_feat,
+            begin_norm_axis=len(mask_trans_feat.shape) - 1,
+            param_attr=fluid.ParamAttr(
+                name='mask_lm_trans_layer_norm_scale',
+                initializer=fluid.initializer.Constant(1.)),
+            bias_attr=fluid.ParamAttr(
+                name='mask_lm_trans_layer_norm_bias',
+                initializer=fluid.initializer.Constant(1.)))
+        # transform: layer norm
+        #mask_trans_feat = pre_process_layer(
+        #    mask_trans_feat, 'n', name='mask_lm_trans')
+
+        mask_lm_out_bias_attr = fluid.ParamAttr(
+            name="mask_lm_out_fc.b_0",
+            initializer=fluid.initializer.Constant(value=0.0))
+        if self._weight_sharing:
+            fc_out = fluid.layers.matmul(
+                x=mask_trans_feat,
+                y=fluid.default_main_program().global_block().var(
+                    self._word_emb_name),
+                transpose_y=True)
+            fc_out += fluid.layers.create_parameter(shape=[self._voc_size],
+                                                    dtype=self._emb_dtype,
+                                                    attr=mask_lm_out_bias_attr,
+                                                    is_bias=True)
+
+        else:
+            fc_out = fluid.layers.fc(input=mask_trans_feat,
+                                     size=self._voc_size,
+                                     param_attr=fluid.ParamAttr(
+                                         name="mask_lm_out_fc.w_0",
+                                         initializer=self._param_initializer),
+                                     bias_attr=mask_lm_out_bias_attr)
+
+        mask_lm_loss = fluid.layers.softmax_with_cross_entropy(logits=fc_out,
+                                                               label=mask_label)
+        mean_mask_lm_loss = fluid.layers.mean(mask_lm_loss)
+
+        return mean_mask_lm_loss
+
+    def get_task_output(self, task, task_labels):
+        task_fc_out = fluid.layers.fc(input=self.next_sent_feat,
+                                      size=task["num_labels"],
+                                      param_attr=fluid.ParamAttr(
+                                          name=task["task_name"] + "_fc.w_0",
+                                          initializer=self._param_initializer),
+                                      bias_attr=task["task_name"] + "_fc.b_0")
+        task_loss, task_softmax = fluid.layers.softmax_with_cross_entropy(
+            logits=task_fc_out, label=task_labels, return_softmax=True)
+        task_acc = fluid.layers.accuracy(input=task_softmax, label=task_labels)
+        mean_task_loss = fluid.layers.mean(task_loss)
+        return mean_task_loss, task_acc

applications/doc_vqa/Rerank/src/model/transformer_encoder.py

@@ -0,0 +1,336 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Transformer encoder."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from functools import partial
+
+import paddle.fluid as fluid
+import paddle.fluid.layers as layers
+
+
+def multi_head_attention(queries,
+                         keys,
+                         values,
+                         attn_bias,
+                         d_key,
+                         d_value,
+                         d_model,
+                         n_head=1,
+                         dropout_rate=0.,
+                         cache=None,
+                         param_initializer=None,
+                         name='multi_head_att'):
+    """
+    Multi-Head Attention. Note that attn_bias is added to the logit before
+    computing softmax activiation to mask certain selected positions so that
+    they will not considered in attention weights.
+    """
+    keys = queries if keys is None else keys
+    values = keys if values is None else values
+
+    if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3):
+        raise ValueError(
+            "Inputs: quries, keys and values should all be 3-D tensors.")
+
+    def __compute_qkv(queries, keys, values, n_head, d_key, d_value):
+        """
+        Add linear projection to queries, keys, and values.
+        """
+        q = layers.fc(input=queries,
+                      size=d_key * n_head,
+                      num_flatten_dims=2,
+                      param_attr=fluid.ParamAttr(name=name + '_query_fc.w_0',
+                                                 initializer=param_initializer),
+                      bias_attr=name + '_query_fc.b_0')
+        k = layers.fc(input=keys,
+                      size=d_key * n_head,
+                      num_flatten_dims=2,
+                      param_attr=fluid.ParamAttr(name=name + '_key_fc.w_0',
+                                                 initializer=param_initializer),
+                      bias_attr=name + '_key_fc.b_0')
+        v = layers.fc(input=values,
+                      size=d_value * n_head,
+                      num_flatten_dims=2,
+                      param_attr=fluid.ParamAttr(name=name + '_value_fc.w_0',
+                                                 initializer=param_initializer),
+                      bias_attr=name + '_value_fc.b_0')
+        return q, k, v
+
+    def __split_heads(x, n_head):
+        """
+        Reshape the last dimension of inpunt tensor x so that it becomes two
+        dimensions and then transpose. Specifically, input a tensor with shape
+        [bs, max_sequence_length, n_head * hidden_dim] then output a tensor
+        with shape [bs, n_head, max_sequence_length, hidden_dim].
+        """
+        hidden_size = x.shape[-1]
+        # The value 0 in shape attr means copying the corresponding dimension
+        # size of the input as the output dimension size.
+        reshaped = layers.reshape(x=x,
+                                  shape=[0, 0, n_head, hidden_size // n_head],
+                                  inplace=True)
+
+        # permuate the dimensions into:
+        # [batch_size, n_head, max_sequence_len, hidden_size_per_head]
+        return layers.transpose(x=reshaped, perm=[0, 2, 1, 3])
+
+    def __combine_heads(x):
+        """
+        Transpose and then reshape the last two dimensions of inpunt tensor x
+        so that it becomes one dimension, which is reverse to __split_heads.
+        """
+        if len(x.shape) == 3: return x
+        if len(x.shape) != 4:
+            raise ValueError("Input(x) should be a 4-D Tensor.")
+
+        trans_x = layers.transpose(x, perm=[0, 2, 1, 3])
+        # The value 0 in shape attr means copying the corresponding dimension
+        # size of the input as the output dimension size.
+        return layers.reshape(x=trans_x,
+                              shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]],
+                              inplace=True)
+
+    def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate):
+        """
+        Scaled Dot-Product Attention
+        """
+        scaled_q = layers.scale(x=q, scale=d_key**-0.5)
+        product = layers.matmul(x=scaled_q, y=k, transpose_y=True)
+        if attn_bias:
+            product += attn_bias
+        weights = layers.softmax(product)
+        if dropout_rate:
+            weights = layers.dropout(weights,
+                                     dropout_prob=dropout_rate,
+                                     dropout_implementation="upscale_in_train",
+                                     is_test=False)
+        out = layers.matmul(weights, v)
+        return out
+
+    q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value)
+
+    if cache is not None:  # use cache and concat time steps
+        # Since the inplace reshape in __split_heads changes the shape of k and
+        # v, which is the cache input for next time step, reshape the cache
+        # input from the previous time step first.
+        k = cache["k"] = layers.concat(
+            [layers.reshape(cache["k"], shape=[0, 0, d_model]), k], axis=1)
+        v = cache["v"] = layers.concat(
+            [layers.reshape(cache["v"], shape=[0, 0, d_model]), v], axis=1)
+
+    q = __split_heads(q, n_head)
+    k = __split_heads(k, n_head)
+    v = __split_heads(v, n_head)
+
+    ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_key,
+                                                  dropout_rate)
+
+    out = __combine_heads(ctx_multiheads)
+
+    # Project back to the model size.
+    proj_out = layers.fc(input=out,
+                         size=d_model,
+                         num_flatten_dims=2,
+                         param_attr=fluid.ParamAttr(
+                             name=name + '_output_fc.w_0',
+                             initializer=param_initializer),
+                         bias_attr=name + '_output_fc.b_0')
+    return proj_out
+
+
+def positionwise_feed_forward(x,
+                              d_inner_hid,
+                              d_hid,
+                              dropout_rate,
+                              hidden_act,
+                              param_initializer=None,
+                              name='ffn'):
+    """
+    Position-wise Feed-Forward Networks.
+    This module consists of two linear transformations with a ReLU activation
+    in between, which is applied to each position separately and identically.
+    """
+    hidden = layers.fc(input=x,
+                       size=d_inner_hid,
+                       num_flatten_dims=2,
+                       act=hidden_act,
+                       param_attr=fluid.ParamAttr(
+                           name=name + '_fc_0.w_0',
+                           initializer=param_initializer),
+                       bias_attr=name + '_fc_0.b_0')
+    if dropout_rate:
+        hidden = layers.dropout(hidden,
+                                dropout_prob=dropout_rate,
+                                dropout_implementation="upscale_in_train",
+                                is_test=False)
+    out = layers.fc(input=hidden,
+                    size=d_hid,
+                    num_flatten_dims=2,
+                    param_attr=fluid.ParamAttr(name=name + '_fc_1.w_0',
+                                               initializer=param_initializer),
+                    bias_attr=name + '_fc_1.b_0')
+    return out
+
+
+def pre_post_process_layer(prev_out,
+                           out,
+                           process_cmd,
+                           dropout_rate=0.,
+                           name=''):
+    """
+    Add residual connection, layer normalization and droput to the out tensor
+    optionally according to the value of process_cmd.
+    This will be used before or after multi-head attention and position-wise
+    feed-forward networks.
+    """
+    for cmd in process_cmd:
+        if cmd == "a":  # add residual connection
+            out = out + prev_out if prev_out else out
+        elif cmd == "n":  # add layer normalization
+            out_dtype = out.dtype
+            if out_dtype == fluid.core.VarDesc.VarType.FP16:
+                out = layers.cast(x=out, dtype="float32")
+            out = layers.layer_norm(
+                out,
+                begin_norm_axis=len(out.shape) - 1,
+                param_attr=fluid.ParamAttr(
+                    name=name + '_layer_norm_scale',
+                    initializer=fluid.initializer.Constant(1.)),
+                bias_attr=fluid.ParamAttr(
+                    name=name + '_layer_norm_bias',
+                    initializer=fluid.initializer.Constant(0.)))
+            if out_dtype == fluid.core.VarDesc.VarType.FP16:
+                out = layers.cast(x=out, dtype="float16")
+        elif cmd == "d":  # add dropout
+            if dropout_rate:
+                out = layers.dropout(out,
+                                     dropout_prob=dropout_rate,
+                                     dropout_implementation="upscale_in_train",
+                                     is_test=False)
+    return out
+
+
+pre_process_layer = partial(pre_post_process_layer, None)
+post_process_layer = pre_post_process_layer
+
+
+def encoder_layer(enc_input,
+                  attn_bias,
+                  n_head,
+                  d_key,
+                  d_value,
+                  d_model,
+                  d_inner_hid,
+                  prepostprocess_dropout,
+                  attention_dropout,
+                  relu_dropout,
+                  hidden_act,
+                  preprocess_cmd="n",
+                  postprocess_cmd="da",
+                  param_initializer=None,
+                  name=''):
+    """The encoder layers that can be stacked to form a deep encoder.
+    This module consits of a multi-head (self) attention followed by
+    position-wise feed-forward networks and both the two components companied
+    with the post_process_layer to add residual connection, layer normalization
+    and droput.
+    """
+    attn_output = multi_head_attention(pre_process_layer(enc_input,
+                                                         preprocess_cmd,
+                                                         prepostprocess_dropout,
+                                                         name=name +
+                                                         '_pre_att'),
+                                       None,
+                                       None,
+                                       attn_bias,
+                                       d_key,
+                                       d_value,
+                                       d_model,
+                                       n_head,
+                                       attention_dropout,
+                                       param_initializer=param_initializer,
+                                       name=name + '_multi_head_att')
+    attn_output = post_process_layer(enc_input,
+                                     attn_output,
+                                     postprocess_cmd,
+                                     prepostprocess_dropout,
+                                     name=name + '_post_att')
+    ffd_output = positionwise_feed_forward(pre_process_layer(
+        attn_output,
+        preprocess_cmd,
+        prepostprocess_dropout,
+        name=name + '_pre_ffn'),
+                                           d_inner_hid,
+                                           d_model,
+                                           relu_dropout,
+                                           hidden_act,
+                                           param_initializer=param_initializer,
+                                           name=name + '_ffn')
+    return post_process_layer(attn_output,
+                              ffd_output,
+                              postprocess_cmd,
+                              prepostprocess_dropout,
+                              name=name + '_post_ffn'), ffd_output
+
+
+def encoder(enc_input,
+            attn_bias,
+            n_layer,
+            n_head,
+            d_key,
+            d_value,
+            d_model,
+            d_inner_hid,
+            prepostprocess_dropout,
+            attention_dropout,
+            relu_dropout,
+            hidden_act,
+            preprocess_cmd="n",
+            postprocess_cmd="da",
+            param_initializer=None,
+            model_name='',
+            name=''):
+    """
+    The encoder is composed of a stack of identical layers returned by calling
+    encoder_layer.
+    """
+    checkpoints = []
+    for i in range(n_layer):
+        enc_output, cp = encoder_layer(enc_input,
+                                       attn_bias,
+                                       n_head,
+                                       d_key,
+                                       d_value,
+                                       d_model,
+                                       d_inner_hid,
+                                       prepostprocess_dropout,
+                                       attention_dropout,
+                                       relu_dropout,
+                                       hidden_act,
+                                       preprocess_cmd,
+                                       postprocess_cmd,
+                                       param_initializer=param_initializer,
+                                       name=name + '_layer_' + str(i))
+        checkpoints.append(cp)
+        enc_input = enc_output
+    enc_output = pre_process_layer(enc_output,
+                                   preprocess_cmd,
+                                   prepostprocess_dropout,
+                                   name=model_name + "post_encoder")
+
+    return enc_output, checkpoints

applications/doc_vqa/Rerank/src/optimization.py

@@ -0,0 +1,137 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Optimization and learning rate scheduling."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import numpy as np
+
+import paddle.fluid as fluid
+from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy
+
+
+def linear_warmup_decay(learning_rate, warmup_steps, num_train_steps):
+    """ Applies linear warmup of learning rate from 0 and decay to 0."""
+    with fluid.default_main_program()._lr_schedule_guard():
+        lr = fluid.layers.tensor.create_global_var(
+            shape=[1],
+            value=0.0,
+            dtype='float32',
+            persistable=True,
+            name="scheduled_learning_rate")
+
+        global_step = fluid.layers.learning_rate_scheduler._decay_step_counter()
+
+        with fluid.layers.control_flow.Switch() as switch:
+            with switch.case(global_step < warmup_steps):
+                warmup_lr = learning_rate * (global_step / warmup_steps)
+                fluid.layers.tensor.assign(warmup_lr, lr)
+            with switch.default():
+                decayed_lr = fluid.layers.learning_rate_scheduler.polynomial_decay(
+                    learning_rate=learning_rate,
+                    decay_steps=num_train_steps,
+                    end_learning_rate=0.0,
+                    power=1.0,
+                    cycle=False)
+                fluid.layers.tensor.assign(decayed_lr, lr)
+
+        return lr
+
+
+def optimization(loss,
+                 warmup_steps,
+                 num_train_steps,
+                 learning_rate,
+                 train_program,
+                 startup_prog,
+                 weight_decay,
+                 scheduler='linear_warmup_decay',
+                 use_dynamic_loss_scaling=False,
+                 incr_every_n_steps=1000,
+                 decr_every_n_nan_or_inf=2,
+                 incr_ratio=2.0,
+                 decr_ratio=0.8,
+                 dist_strategy=None,
+                 use_lamb=False):
+    if warmup_steps > 0:
+        if scheduler == 'noam_decay':
+            scheduled_lr = fluid.layers.learning_rate_scheduler\
+             .noam_decay(1/(warmup_steps *(learning_rate ** 2)),
+                         warmup_steps)
+        elif scheduler == 'linear_warmup_decay':
+            scheduled_lr = linear_warmup_decay(learning_rate, warmup_steps,
+                                               num_train_steps)
+        else:
+            raise ValueError("Unkown learning rate scheduler, should be "
+                             "'noam_decay' or 'linear_warmup_decay'")
+        if use_lamb:
+            optimizer = fluid.optimizer.LambOptimizer(
+                learning_rate=scheduled_lr)
+        else:
+            optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr)
+    else:
+        scheduled_lr = fluid.layers.create_global_var(
+            name=fluid.unique_name.generate("learning_rate"),
+            shape=[1],
+            value=learning_rate,
+            dtype='float32',
+            persistable=True)
+        if use_lamb:
+            optimizer = fluid.optimizer.LambOptimizer(
+                learning_rate=scheduled_lr)
+        else:
+            optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr)
+        optimizer._learning_rate_map[
+            fluid.default_main_program()] = scheduled_lr
+
+    fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByGlobalNorm(
+        clip_norm=1.0))
+
+    def exclude_from_weight_decay(name):
+        if name.find("layer_norm") > -1:
+            return True
+        bias_suffix = ["_bias", "_b", ".b_0"]
+        for suffix in bias_suffix:
+            if name.endswith(suffix):
+                return True
+        return False
+
+    param_list = dict()
+
+    for param in train_program.global_block().all_parameters():
+        param_list[param.name] = param * 1.0
+        param_list[param.name].stop_gradient = True
+
+    if dist_strategy is not None:
+        # use fleet api
+        optimizer = fleet.distributed_optimizer(optimizer,
+                                                strategy=dist_strategy)
+
+    _, param_grads = optimizer.minimize(loss)
+
+    if weight_decay > 0:
+        for param, grad in param_grads:
+            if exclude_from_weight_decay(param.name):
+                continue
+            with param.block.program._optimized_guard(
+                [param, grad]), fluid.framework.name_scope("weight_decay"):
+                updated_param = param - param_list[
+                    param.name] * weight_decay * scheduled_lr
+                fluid.layers.assign(output=param, input=updated_param)
+
+    return scheduled_lr

applications/doc_vqa/Rerank/src/reader_ce.py

@@ -0,0 +1,357 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import sys
+import os
+import json
+import random
+import logging
+import numpy as np
+import six
+from io import open
+from collections import namedtuple
+
+import tokenization
+from batching import pad_batch_data
+
+log = logging.getLogger(__name__)
+
+if six.PY3:
+    import io
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
+
+
+def csv_reader(fd, delimiter='\t', trainer_id=0, trainer_num=1):
+
+    def gen():
+        for i, line in enumerate(fd):
+            if i % trainer_num == trainer_id:
+                slots = line.rstrip('\n').split(delimiter)
+                if len(slots) == 1:
+                    yield slots,
+                else:
+                    yield slots
+
+    return gen()
+
+
+class BaseReader(object):
+
+    def __init__(self,
+                 vocab_path,
+                 label_map_config=None,
+                 max_seq_len=512,
+                 total_num=0,
+                 do_lower_case=True,
+                 in_tokens=False,
+                 is_inference=False,
+                 random_seed=None,
+                 tokenizer="FullTokenizer",
+                 for_cn=True,
+                 task_id=0):
+        self.max_seq_len = max_seq_len
+        self.tokenizer = tokenization.FullTokenizer(vocab_file=vocab_path,
+                                                    do_lower_case=do_lower_case)
+        self.vocab = self.tokenizer.vocab
+        self.pad_id = self.vocab["[PAD]"]
+        self.cls_id = self.vocab["[CLS]"]
+        self.sep_id = self.vocab["[SEP]"]
+        self.in_tokens = in_tokens
+        self.is_inference = is_inference
+        self.for_cn = for_cn
+        self.task_id = task_id
+
+        np.random.seed(random_seed)
+
+        self.current_example = 0
+        self.current_epoch = 0
+        self.num_examples = 0
+        self.total_num = total_num
+
+        if label_map_config:
+            with open(label_map_config, encoding='utf8') as f:
+                self.label_map = json.load(f)
+        else:
+            self.label_map = None
+
+    def get_train_progress(self):
+        """Gets progress for training phase."""
+        return self.current_example, self.current_epoch
+
+    def _read_tsv(self, input_file, quotechar=None):
+        """Reads a tab separated value file."""
+        with open(input_file, 'r', encoding='utf8') as f:
+            reader = csv_reader(f)
+            headers = next(reader)
+            Example = namedtuple('Example', headers)
+
+            examples = []
+            for line in reader:
+                example = Example(*line)
+                examples.append(example)
+            return examples
+
+    def _truncate_seq_pair(self, tokens_a, tokens_b, max_length):
+        """Truncates a sequence pair in place to the maximum length."""
+
+        # This is a simple heuristic which will always truncate the longer sequence
+        # one token at a time. This makes more sense than truncating an equal percent
+        # of tokens from each, since if one sequence is very short then each token
+        # that's truncated likely contains more information than a longer sequence.
+        while True:
+            total_length = len(tokens_a) + len(tokens_b)
+            if total_length <= max_length:
+                break
+            if len(tokens_a) > len(tokens_b):
+                tokens_a.pop()
+            else:
+                tokens_b.pop()
+
+    def _convert_example_to_record(self, example, max_seq_length, tokenizer):
+        """Converts a single `Example` into a single `Record`."""
+
+        query = tokenization.convert_to_unicode(example.query)
+        tokens_a = tokenizer.tokenize(query)
+        tokens_b = None
+
+        title = tokenization.convert_to_unicode(example.title)
+        tokens_b = tokenizer.tokenize(title)
+
+        para = tokenization.convert_to_unicode(example.para)
+        tokens_para = tokenizer.tokenize(para)
+
+        tokens_b.extend(tokens_para)
+
+        self._truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
+
+        # The convention in BERT/ERNIE is:
+        # (a) For sequence pairs:
+        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
+        #  type_ids: 0     0  0    0    0     0       0 0     1  1  1  1   1 1
+        # (b) For single sequences:
+        #  tokens:   [CLS] the dog is hairy . [SEP]
+        #  type_ids: 0     0   0   0  0     0 0
+        #
+        # Where "type_ids" are used to indicate whether this is the first
+        # sequence or the second sequence. The embedding vectors for `type=0` and
+        # `type=1` were learned during pre-training and are added to the wordpiece
+        # embedding vector (and position vector). This is not *strictly* necessary
+        # since the [SEP] token unambiguously separates the sequences, but it makes
+        # it easier for the model to learn the concept of sequences.
+        #
+        # For classification tasks, the first vector (corresponding to [CLS]) is
+        # used as as the "sentence vector". Note that this only makes sense because
+        # the entire model is fine-tuned.
+        tokens = []
+        text_type_ids = []
+        tokens.append("[CLS]")
+        text_type_ids.append(0)
+        for token in tokens_a:
+            tokens.append(token)
+            text_type_ids.append(0)
+        tokens.append("[SEP]")
+        text_type_ids.append(0)
+
+        if tokens_b:
+            for token in tokens_b:
+                tokens.append(token)
+                text_type_ids.append(1)
+            tokens.append("[SEP]")
+            text_type_ids.append(1)
+
+        token_ids = tokenizer.convert_tokens_to_ids(tokens)
+        position_ids = list(range(len(token_ids)))
+
+        if self.is_inference:
+            Record = namedtuple('Record',
+                                ['token_ids', 'text_type_ids', 'position_ids'])
+            record = Record(token_ids=token_ids,
+                            text_type_ids=text_type_ids,
+                            position_ids=position_ids)
+        else:
+            if self.label_map:
+                label_id = self.label_map[example.label]
+            else:
+                label_id = example.label
+
+            Record = namedtuple('Record', [
+                'token_ids', 'text_type_ids', 'position_ids', 'label_id', 'qid'
+            ])
+
+            qid = None
+            if "qid" in example._fields:
+                qid = example.qid
+
+            record = Record(token_ids=token_ids,
+                            text_type_ids=text_type_ids,
+                            position_ids=position_ids,
+                            label_id=label_id,
+                            qid=qid)
+        return record
+
+    def _prepare_batch_data(self, examples, batch_size, phase=None):
+        """generate batch records"""
+        batch_records, max_len = [], 0
+        for index, example in enumerate(examples):
+            if phase == "train":
+                self.current_example = index
+            record = self._convert_example_to_record(example, self.max_seq_len,
+                                                     self.tokenizer)
+            max_len = max(max_len, len(record.token_ids))
+            if self.in_tokens:
+                to_append = (len(batch_records) + 1) * max_len <= batch_size
+            else:
+                to_append = len(batch_records) < batch_size
+            if to_append:
+                batch_records.append(record)
+            else:
+                yield self._pad_batch_records(batch_records)
+                batch_records, max_len = [record], len(record.token_ids)
+
+        if batch_records:
+            yield self._pad_batch_records(batch_records)
+
+    def get_num_examples(self, input_file):
+        #        examples = self._read_tsv(input_file)
+        #        return len(examples)
+        return self.num_examples
+
+    def data_generator(self,
+                       input_file,
+                       batch_size,
+                       epoch,
+                       dev_count=1,
+                       trainer_id=0,
+                       trainer_num=1,
+                       shuffle=True,
+                       phase=None):
+
+        if phase == 'train':
+            #            examples = examples[trainer_id: (len(examples) //trainer_num) * trainer_num : trainer_num]
+            self.num_examples_per_node = self.total_num // trainer_num
+            self.num_examples = self.num_examples_per_node * trainer_num
+            examples = self._read_tsv(input_file,
+                                      trainer_id=trainer_id,
+                                      trainer_num=trainer_num,
+                                      num_examples=self.num_examples_per_node)
+            log.info('apply sharding %d/%d' % (trainer_id, trainer_num))
+        else:
+            examples = self._read_tsv(input_file)
+
+        def wrapper():
+            all_dev_batches = []
+            for epoch_index in range(epoch):
+                if phase == "train":
+                    self.current_example = 0
+                    self.current_epoch = epoch_index
+                if shuffle:
+                    np.random.shuffle(examples)
+
+                for batch_data in self._prepare_batch_data(examples,
+                                                           batch_size,
+                                                           phase=phase):
+                    if len(all_dev_batches) < dev_count:
+                        all_dev_batches.append(batch_data)
+                    if len(all_dev_batches) == dev_count:
+                        for batch in all_dev_batches:
+                            yield batch
+                        all_dev_batches = []
+
+        def f():
+            try:
+                for i in wrapper():
+                    yield i
+            except Exception as e:
+                import traceback
+                traceback.print_exc()
+
+        return f
+
+
+class ClassifyReader(BaseReader):
+
+    def _read_tsv(self,
+                  input_file,
+                  quotechar=None,
+                  trainer_id=0,
+                  trainer_num=1,
+                  num_examples=0):
+        """Reads a tab separated value file."""
+        with open(input_file, 'r', encoding='utf8') as f:
+            reader = csv_reader(f,
+                                trainer_id=trainer_id,
+                                trainer_num=trainer_num)
+            #            headers = next(reader)
+            headers = 'query\ttitle\tpara\tlabel'.split('\t')
+            text_indices = [
+                index for index, h in enumerate(headers) if h != "label"
+            ]
+            Example = namedtuple('Example', headers)
+
+            examples = []
+            for cnt, line in enumerate(reader):
+                if num_examples != 0 and cnt == num_examples:
+                    break
+                for index, text in enumerate(line):
+                    if index in text_indices:
+                        if self.for_cn:
+                            line[index] = text.replace(' ', '')
+                        else:
+                            line[index] = text
+                example = Example(*line)
+                examples.append(example)
+            return examples
+
+    def _pad_batch_records(self, batch_records):
+        batch_token_ids = [record.token_ids for record in batch_records]
+        batch_text_type_ids = [record.text_type_ids for record in batch_records]
+        batch_position_ids = [record.position_ids for record in batch_records]
+
+        if not self.is_inference:
+            batch_labels = [record.label_id for record in batch_records]
+            batch_labels = np.array(batch_labels).astype("int64").reshape(
+                [-1, 1])
+
+            if batch_records[0].qid:
+                batch_qids = [record.qid for record in batch_records]
+                batch_qids = np.array(batch_qids).astype("int64").reshape(
+                    [-1, 1])
+            else:
+                batch_qids = np.array([]).astype("int64").reshape([-1, 1])
+
+        # padding
+        padded_token_ids, input_mask = pad_batch_data(batch_token_ids,
+                                                      pad_idx=self.pad_id,
+                                                      return_input_mask=True)
+        padded_text_type_ids = pad_batch_data(batch_text_type_ids,
+                                              pad_idx=self.pad_id)
+        padded_position_ids = pad_batch_data(batch_position_ids,
+                                             pad_idx=self.pad_id)
+        padded_task_ids = np.ones_like(padded_token_ids,
+                                       dtype="int64") * self.task_id
+
+        return_list = [
+            padded_token_ids, padded_text_type_ids, padded_position_ids,
+            padded_task_ids, input_mask
+        ]
+        if not self.is_inference:
+            return_list += [batch_labels, batch_qids]
+
+        return return_list

applications/doc_vqa/Rerank/src/tokenization.py

@@ -0,0 +1,421 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import six
+import collections
+import unicodedata
+from io import open
+
+
+def convert_to_unicode(text):
+    """Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
+    if six.PY3:
+        if isinstance(text, str):
+            return text
+        elif isinstance(text, bytes):
+            return text.decode("utf-8", "ignore")
+        else:
+            raise ValueError("Unsupported string type: %s" % (type(text)))
+    elif six.PY2:
+        if isinstance(text, str):
+            return text.decode("utf-8", "ignore")
+        elif isinstance(text, unicode):
+            return text
+        else:
+            raise ValueError("Unsupported string type: %s" % (type(text)))
+    else:
+        raise ValueError("Not running on Python2 or Python 3?")
+
+
+def printable_text(text):
+    """Returns text encoded in a way suitable for print or `tf.logging`."""
+
+    # These functions want `str` for both Python2 and Python3, but in one case
+    # it's a Unicode string and in the other it's a byte string.
+    if six.PY3:
+        if isinstance(text, str):
+            return text
+        elif isinstance(text, bytes):
+            return text.decode("utf-8", "ignore")
+        else:
+            raise ValueError("Unsupported string type: %s" % (type(text)))
+    elif six.PY2:
+        if isinstance(text, str):
+            return text
+        elif isinstance(text, unicode):
+            return text.encode("utf-8")
+        else:
+            raise ValueError("Unsupported string type: %s" % (type(text)))
+    else:
+        raise ValueError("Not running on Python2 or Python 3?")
+
+
+def load_vocab(vocab_file):
+    """Loads a vocabulary file into a dictionary."""
+    vocab = collections.OrderedDict()
+    with open(vocab_file, encoding='utf8') as fin:
+        for num, line in enumerate(fin):
+            items = convert_to_unicode(line.strip()).split("\t")
+            if len(items) > 2:
+                break
+            token = items[0]
+            index = items[1] if len(items) == 2 else num
+            token = token.strip()
+            vocab[token] = int(index)
+    return vocab
+
+
+def convert_by_vocab(vocab, items):
+    """Converts a sequence of [tokens|ids] using the vocab."""
+    output = []
+    for item in items:
+        output.append(vocab[item])
+    return output
+
+
+def convert_tokens_to_ids(vocab, tokens):
+    return convert_by_vocab(vocab, tokens)
+
+
+def convert_ids_to_tokens(inv_vocab, ids):
+    return convert_by_vocab(inv_vocab, ids)
+
+
+def whitespace_tokenize(text):
+    """Runs basic whitespace cleaning and splitting on a peice of text."""
+    text = text.strip()
+    if not text:
+        return []
+    tokens = text.split()
+    return tokens
+
+
+class FullTokenizer(object):
+    """Runs end-to-end tokenziation."""
+
+    def __init__(self, vocab_file, do_lower_case=True):
+        self.vocab = load_vocab(vocab_file)
+        self.inv_vocab = {v: k for k, v in self.vocab.items()}
+        self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)
+        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+
+    def tokenize(self, text):
+        split_tokens = []
+        for token in self.basic_tokenizer.tokenize(text):
+            for sub_token in self.wordpiece_tokenizer.tokenize(token):
+                split_tokens.append(sub_token)
+
+        return split_tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return convert_by_vocab(self.vocab, tokens)
+
+    def convert_ids_to_tokens(self, ids):
+        return convert_by_vocab(self.inv_vocab, ids)
+
+
+class CharTokenizer(object):
+    """Runs end-to-end tokenziation."""
+
+    def __init__(self, vocab_file, do_lower_case=True):
+        self.vocab = load_vocab(vocab_file)
+        self.inv_vocab = {v: k for k, v in self.vocab.items()}
+        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+
+    def tokenize(self, text):
+        split_tokens = []
+        for token in text.lower().split(" "):
+            for sub_token in self.wordpiece_tokenizer.tokenize(token):
+                split_tokens.append(sub_token)
+
+        return split_tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return convert_by_vocab(self.vocab, tokens)
+
+    def convert_ids_to_tokens(self, ids):
+        return convert_by_vocab(self.inv_vocab, ids)
+
+
+class BasicTokenizer(object):
+    """Runs basic tokenization (punctuation splitting, lower casing, etc.)."""
+
+    def __init__(self, do_lower_case=True):
+        """Constructs a BasicTokenizer.
+
+        Args:
+            do_lower_case: Whether to lower case the input.
+        """
+        self.do_lower_case = do_lower_case
+
+    def tokenize(self, text):
+        """Tokenizes a piece of text."""
+        text = convert_to_unicode(text)
+        text = self._clean_text(text)
+
+        # This was added on November 1st, 2018 for the multilingual and Chinese
+        # models. This is also applied to the English models now, but it doesn't
+        # matter since the English models were not trained on any Chinese data
+        # and generally don't have any Chinese data in them (there are Chinese
+        # characters in the vocabulary because Wikipedia does have some Chinese
+        # words in the English Wikipedia.).
+        text = self._tokenize_chinese_chars(text)
+
+        orig_tokens = whitespace_tokenize(text)
+        split_tokens = []
+        for token in orig_tokens:
+            if self.do_lower_case:
+                token = token.lower()
+                token = self._run_strip_accents(token)
+            split_tokens.extend(self._run_split_on_punc(token))
+
+        output_tokens = whitespace_tokenize(" ".join(split_tokens))
+        return output_tokens
+
+    def _run_strip_accents(self, text):
+        """Strips accents from a piece of text."""
+        text = unicodedata.normalize("NFD", text)
+        output = []
+        for char in text:
+            cat = unicodedata.category(char)
+            if cat == "Mn":
+                continue
+            output.append(char)
+        return "".join(output)
+
+    def _run_split_on_punc(self, text):
+        """Splits punctuation on a piece of text."""
+        chars = list(text)
+        i = 0
+        start_new_word = True
+        output = []
+        while i < len(chars):
+            char = chars[i]
+            if _is_punctuation(char):
+                output.append([char])
+                start_new_word = True
+            else:
+                if start_new_word:
+                    output.append([])
+                start_new_word = False
+                output[-1].append(char)
+            i += 1
+
+        return ["".join(x) for x in output]
+
+    def _tokenize_chinese_chars(self, text):
+        """Adds whitespace around any CJK character."""
+        output = []
+        for char in text:
+            cp = ord(char)
+            if self._is_chinese_char(cp):
+                output.append(" ")
+                output.append(char)
+                output.append(" ")
+            else:
+                output.append(char)
+        return "".join(output)
+
+    def _is_chinese_char(self, cp):
+        """Checks whether CP is the codepoint of a CJK character."""
+        # This defines a "chinese character" as anything in the CJK Unicode block:
+        #     https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+        #
+        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+        # despite its name. The modern Korean Hangul alphabet is a different block,
+        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+        # space-separated words, so they are not treated specially and handled
+        # like the all of the other languages.
+        if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #
+            (cp >= 0x3400 and cp <= 0x4DBF) or  #
+            (cp >= 0x20000 and cp <= 0x2A6DF) or  #
+            (cp >= 0x2A700 and cp <= 0x2B73F) or  #
+            (cp >= 0x2B740 and cp <= 0x2B81F) or  #
+            (cp >= 0x2B820 and cp <= 0x2CEAF) or
+            (cp >= 0xF900 and cp <= 0xFAFF) or  #
+            (cp >= 0x2F800 and cp <= 0x2FA1F)):  #
+            return True
+
+        return False
+
+    def _clean_text(self, text):
+        """Performs invalid character removal and whitespace cleanup on text."""
+        output = []
+        for char in text:
+            cp = ord(char)
+            if cp == 0 or cp == 0xfffd or _is_control(char):
+                continue
+            if _is_whitespace(char):
+                output.append(" ")
+            else:
+                output.append(char)
+        return "".join(output)
+
+
+class WordpieceTokenizer(object):
+    """Runs WordPiece tokenziation."""
+
+    def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100):
+        self.vocab = vocab
+        self.unk_token = unk_token
+        self.max_input_chars_per_word = max_input_chars_per_word
+
+    def tokenize(self, text):
+        """Tokenizes a piece of text into its word pieces.
+
+        This uses a greedy longest-match-first algorithm to perform tokenization
+        using the given vocabulary.
+
+        For example:
+            input = "unaffable"
+            output = ["un", "##aff", "##able"]
+
+        Args:
+            text: A single token or whitespace separated tokens. This should have
+                already been passed through `BasicTokenizer.
+
+        Returns:
+            A list of wordpiece tokens.
+        """
+
+        text = convert_to_unicode(text)
+
+        output_tokens = []
+        for token in whitespace_tokenize(text):
+            chars = list(token)
+            if len(chars) > self.max_input_chars_per_word:
+                output_tokens.append(self.unk_token)
+                continue
+
+            is_bad = False
+            start = 0
+            sub_tokens = []
+            while start < len(chars):
+                end = len(chars)
+                cur_substr = None
+                while start < end:
+                    substr = "".join(chars[start:end])
+                    if start > 0:
+                        substr = "##" + substr
+                    if substr in self.vocab:
+                        cur_substr = substr
+                        break
+                    end -= 1
+                if cur_substr is None:
+                    is_bad = True
+                    break
+                sub_tokens.append(cur_substr)
+                start = end
+
+            if is_bad:
+                output_tokens.append(self.unk_token)
+            else:
+                output_tokens.extend(sub_tokens)
+        return output_tokens
+
+
+def _is_whitespace(char):
+    """Checks whether `chars` is a whitespace character."""
+    # \t, \n, and \r are technically contorl characters but we treat them
+    # as whitespace since they are generally considered as such.
+    if char == " " or char == "\t" or char == "\n" or char == "\r":
+        return True
+    cat = unicodedata.category(char)
+    if cat == "Zs":
+        return True
+    return False
+
+
+def _is_control(char):
+    """Checks whether `chars` is a control character."""
+    # These are technically control characters but we count them as whitespace
+    # characters.
+    if char == "\t" or char == "\n" or char == "\r":
+        return False
+    cat = unicodedata.category(char)
+    if cat.startswith("C"):
+        return True
+    return False
+
+
+def _is_punctuation(char):
+    """Checks whether `chars` is a punctuation character."""
+    cp = ord(char)
+    # We treat all non-letter/number ASCII as punctuation.
+    # Characters such as "^", "$", and "`" are not in the Unicode
+    # Punctuation class but we treat them as punctuation anyways, for
+    # consistency.
+    if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64)
+            or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):
+        return True
+    cat = unicodedata.category(char)
+    if cat.startswith("P"):
+        return True
+    return False
+
+
+def tokenize_chinese_chars(text):
+    """Adds whitespace around any CJK character."""
+
+    def _is_chinese_char(cp):
+        """Checks whether CP is the codepoint of a CJK character."""
+        # This defines a "chinese character" as anything in the CJK Unicode block:
+        #     https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+        #
+        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+        # despite its name. The modern Korean Hangul alphabet is a different block,
+        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+        # space-separated words, so they are not treated specially and handled
+        # like the all of the other languages.
+        if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #
+            (cp >= 0x3400 and cp <= 0x4DBF) or  #
+            (cp >= 0x20000 and cp <= 0x2A6DF) or  #
+            (cp >= 0x2A700 and cp <= 0x2B73F) or  #
+            (cp >= 0x2B740 and cp <= 0x2B81F) or  #
+            (cp >= 0x2B820 and cp <= 0x2CEAF) or
+            (cp >= 0xF900 and cp <= 0xFAFF) or  #
+            (cp >= 0x2F800 and cp <= 0x2FA1F)):  #
+            return True
+
+        return False
+
+    def _is_whitespace(c):
+        if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
+            return True
+        return False
+
+    output = []
+    buff = ""
+    for char in text:
+        cp = ord(char)
+        if _is_chinese_char(cp) or _is_whitespace(char):
+            if buff != "":
+                output.append(buff)
+                buff = ""
+            output.append(char)
+        else:
+            buff += char
+
+    if buff != "":
+        output.append(buff)
+
+    return output

applications/doc_vqa/Rerank/src/train_ce.py

@@ -0,0 +1,392 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Finetuning on classification tasks."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import os
+import time
+import logging
+import warnings
+import multiprocessing
+
+# NOTE(paddle-dev): All of these flags should be
+# set before `import paddle`. Otherwise, it would
+# not take any effect.
+os.environ['FLAGS_eager_delete_tensor_gb'] = '0'  # enable gc
+
+import paddle
+import paddle.fluid as fluid
+if hasattr(paddle, 'enable_static'):
+    paddle.enable_static()
+from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy
+import paddle.fluid.incubate.fleet.base.role_maker as role_maker
+
+import reader_ce as reader_ce
+from model.ernie import ErnieConfig
+from cross_encoder import create_model, evaluate, predict
+from optimization import optimization
+from utils.args import print_arguments, check_cuda, prepare_logger
+from utils.init import init_pretraining_params, init_checkpoint
+from finetune_args import parser
+
+warnings.filterwarnings("ignore")
+args = parser.parse_args()
+log = logging.getLogger()
+
+
+def main(args):
+    ernie_config = ErnieConfig(args.ernie_config_path)
+    ernie_config.print_config()
+
+    if args.use_cuda:
+        dev_list = fluid.cuda_places()
+        place = dev_list[0]
+        dev_count = len(dev_list)
+    else:
+        place = fluid.CPUPlace()
+        dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
+    exe = fluid.Executor(place)
+
+    reader = reader_ce.ClassifyReader(vocab_path=args.vocab_path,
+                                      label_map_config=args.label_map_config,
+                                      max_seq_len=args.max_seq_len,
+                                      total_num=args.train_data_size,
+                                      do_lower_case=args.do_lower_case,
+                                      in_tokens=args.in_tokens,
+                                      random_seed=args.random_seed,
+                                      tokenizer=args.tokenizer,
+                                      for_cn=args.for_cn,
+                                      task_id=args.task_id)
+
+    if not (args.do_train or args.do_val or args.do_test):
+        raise ValueError("For args `do_train`, `do_val` and `do_test`, at "
+                         "least one of them must be True.")
+
+    if args.do_test:
+        assert args.test_save is not None
+    startup_prog = fluid.Program()
+    if args.random_seed is not None:
+        startup_prog.random_seed = args.random_seed
+
+    if args.predict_batch_size == None:
+        args.predict_batch_size = args.batch_size
+
+    if args.do_train:
+        role = role_maker.PaddleCloudRoleMaker(is_collective=True)
+        fleet.init(role)
+        dev_count = fleet.worker_num()
+
+        train_data_generator = reader.data_generator(
+            input_file=args.train_set,
+            batch_size=args.batch_size,
+            epoch=args.epoch,
+            dev_count=1,
+            trainer_id=fleet.worker_index(),
+            trainer_num=fleet.worker_num(),
+            shuffle=True,
+            phase="train")
+
+        num_train_examples = reader.get_num_examples(args.train_set)
+
+        if args.in_tokens:
+            max_train_steps = args.epoch * num_train_examples // (
+                args.batch_size // args.max_seq_len) // dev_count
+        else:
+            max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count
+
+        warmup_steps = int(max_train_steps * args.warmup_proportion)
+        log.info("Device count: %d" % dev_count)
+        log.info("Num train examples: %d" % num_train_examples)
+        log.info("Max train steps: %d" % max_train_steps)
+        log.info("Num warmup steps: %d" % warmup_steps)
+
+        train_program = fluid.Program()
+
+        # use fleet api
+        exec_strategy = fluid.ExecutionStrategy()
+        if args.use_fast_executor:
+            exec_strategy.use_experimental_executor = True
+        exec_strategy.num_threads = dev_count
+        if args.is_distributed:
+            exec_strategy.num_threads = 3
+
+        exec_strategy.num_iteration_per_drop_scope = args.num_iteration_per_drop_scope
+
+        dist_strategy = DistributedStrategy()
+        dist_strategy.exec_strategy = exec_strategy
+        dist_strategy.nccl_comm_num = 1
+        if args.is_distributed:
+            dist_strategy.nccl_comm_num = 2
+        dist_strategy.use_hierarchical_allreduce = True
+
+        if args.use_mix_precision:
+            dist_strategy.use_amp = True
+
+        with fluid.program_guard(train_program, startup_prog):
+            with fluid.unique_name.guard():
+                train_pyreader, graph_vars = create_model(
+                    args,
+                    pyreader_name='train_reader',
+                    ernie_config=ernie_config)
+                scheduled_lr = optimization(
+                    loss=graph_vars["loss"],
+                    warmup_steps=warmup_steps,
+                    num_train_steps=max_train_steps,
+                    learning_rate=args.learning_rate,
+                    train_program=train_program,
+                    startup_prog=startup_prog,
+                    weight_decay=args.weight_decay,
+                    scheduler=args.lr_scheduler,
+                    use_dynamic_loss_scaling=args.use_dynamic_loss_scaling,
+                    incr_every_n_steps=args.incr_every_n_steps,
+                    decr_every_n_nan_or_inf=args.decr_every_n_nan_or_inf,
+                    incr_ratio=args.incr_ratio,
+                    decr_ratio=args.decr_ratio,
+                    dist_strategy=dist_strategy)
+
+        if args.verbose:
+            if args.in_tokens:
+                lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
+                    program=train_program,
+                    batch_size=args.batch_size // args.max_seq_len)
+            else:
+                lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
+                    program=train_program, batch_size=args.batch_size)
+            log.info("Theoretical memory usage in training: %.3f - %.3f %s" %
+                     (lower_mem, upper_mem, unit))
+
+    if args.do_val or args.do_test:
+        test_prog = fluid.Program()
+        with fluid.program_guard(test_prog, startup_prog):
+            with fluid.unique_name.guard():
+                test_pyreader, graph_vars = create_model(
+                    args,
+                    pyreader_name='test_reader',
+                    ernie_config=ernie_config,
+                    is_prediction=True)
+
+        test_prog = test_prog.clone(for_test=True)
+
+    train_program = fleet.main_program
+
+    exe = fluid.Executor(place)
+    exe.run(startup_prog)
+
+    if args.do_train:
+        if args.init_checkpoint and args.init_pretraining_params:
+            log.warning(
+                "WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
+                "both are set! Only arg 'init_checkpoint' is made valid.")
+        if args.init_checkpoint:
+            init_checkpoint(exe,
+                            args.init_checkpoint,
+                            main_program=startup_prog)
+        elif args.init_pretraining_params:
+            init_pretraining_params(exe,
+                                    args.init_pretraining_params,
+                                    main_program=startup_prog)
+    elif args.do_val or args.do_test:
+        if not args.init_checkpoint:
+            raise ValueError("args 'init_checkpoint' should be set if"
+                             "only doing validation or testing!")
+        init_checkpoint(exe, args.init_checkpoint, main_program=startup_prog)
+
+    if args.do_train:
+        train_exe = exe
+        train_pyreader.decorate_tensor_provider(train_data_generator)
+    else:
+        train_exe = None
+
+    test_exe = exe
+
+    current_epoch = 0
+    steps = 0
+    if args.do_train:
+        train_pyreader.start()
+        if warmup_steps > 0:
+            graph_vars["learning_rate"] = scheduled_lr
+
+        ce_info = []
+        time_begin = time.time()
+        last_epoch = 0
+        while True:
+            try:
+                steps += 1
+
+                if fleet.worker_index() != 0:
+                    train_exe.run(fetch_list=[], program=train_program)
+                    continue
+
+                if steps % args.skip_steps != 0:
+                    train_exe.run(fetch_list=[], program=train_program)
+
+                else:
+                    outputs = evaluate(train_exe,
+                                       train_program,
+                                       train_pyreader,
+                                       graph_vars,
+                                       "train",
+                                       metric=args.metric)
+
+                    if args.verbose:
+                        verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size(
+                        )
+                        verbose += "learning rate: %f" % (
+                            outputs["learning_rate"]
+                            if warmup_steps > 0 else args.learning_rate)
+                        log.info(verbose)
+
+                    current_example, current_epoch = reader.get_train_progress()
+                    time_end = time.time()
+                    used_time = time_end - time_begin
+
+                    log.info(
+                        "epoch: %d, progress: %d/%d, step: %d, ave loss: %f, "
+                        "ave acc: %f, speed: %f steps/s" %
+                        (current_epoch, current_example * dev_count,
+                         num_train_examples, steps, outputs["loss"],
+                         outputs["accuracy"], args.skip_steps / used_time))
+                    ce_info.append(
+                        [outputs["loss"], outputs["accuracy"], used_time])
+
+                    time_begin = time.time()
+
+                if steps % args.save_steps == 0:
+                    save_path = os.path.join(args.checkpoints,
+                                             "step_" + str(steps))
+                    fluid.io.save_persistables(exe, save_path,
+                                               fleet._origin_program)
+
+                if steps % args.validation_steps == 0:
+                    # evaluate dev set
+                    if args.do_val:
+                        evaluate_wrapper(args, reader, exe, test_prog,
+                                         test_pyreader, graph_vars,
+                                         current_epoch, steps)
+
+                    if args.do_test:
+                        predict_wrapper(args, reader, exe, test_prog,
+                                        test_pyreader, graph_vars,
+                                        current_epoch, steps)
+
+                if last_epoch != current_epoch:
+                    last_epoch = current_epoch
+
+            except fluid.core.EOFException:
+                save_path = os.path.join(args.checkpoints, "step_" + str(steps))
+                fluid.io.save_persistables(exe, save_path,
+                                           fleet._origin_program)
+                train_pyreader.reset()
+                break
+
+    # final eval on dev set
+    if args.do_val:
+        evaluate_wrapper(args, reader, exe, test_prog, test_pyreader,
+                         graph_vars, current_epoch, steps)
+
+    # final eval on test set
+    if args.do_test:
+        predict_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars)
+
+    # final eval on dianostic, hack for glue-ax
+    if args.diagnostic:
+        test_pyreader.decorate_tensor_provider(
+            reader.data_generator(args.diagnostic,
+                                  batch_size=args.batch_size,
+                                  epoch=1,
+                                  dev_count=1,
+                                  shuffle=False))
+
+        log.info("Final diagnostic")
+        qids, preds, probs = predict(test_exe, test_prog, test_pyreader,
+                                     graph_vars)
+        assert len(qids) == len(preds), '{} v.s. {}'.format(
+            len(qids), len(preds))
+        with open(args.diagnostic_save, 'w') as f:
+            for id, s, p in zip(qids, preds, probs):
+                f.write('{}\t{}\t{}\n'.format(id, s, p))
+
+        log.info("Done final diagnostic, saving to {}".format(
+            args.diagnostic_save))
+
+
+def evaluate_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars,
+                     epoch, steps):
+    # evaluate dev set
+    for ds in args.dev_set.split(','):
+        test_pyreader.decorate_tensor_provider(
+            reader.data_generator(ds,
+                                  batch_size=args.predict_batch_size,
+                                  epoch=1,
+                                  dev_count=1,
+                                  shuffle=False))
+        log.info("validation result of dataset {}:".format(ds))
+        evaluate_info = evaluate(exe,
+                                 test_prog,
+                                 test_pyreader,
+                                 graph_vars,
+                                 "dev",
+                                 metric=args.metric)
+        log.info(evaluate_info +
+                 ', file: {}, epoch: {}, steps: {}'.format(ds, epoch, steps))
+
+
+def predict_wrapper(args,
+                    reader,
+                    exe,
+                    test_prog,
+                    test_pyreader,
+                    graph_vars,
+                    epoch=None,
+                    steps=None):
+    test_sets = args.test_set.split(',')
+    save_dirs = args.test_save.split(',')
+    assert len(test_sets) == len(save_dirs)
+
+    for test_f, save_f in zip(test_sets, save_dirs):
+        test_pyreader.decorate_tensor_provider(
+            reader.data_generator(test_f,
+                                  batch_size=args.predict_batch_size,
+                                  epoch=1,
+                                  dev_count=1,
+                                  shuffle=False))
+
+        if epoch is not None or steps is not None:
+            save_path = save_f + '.' + str(epoch) + '.' + str(steps)
+        else:
+            save_path = save_f
+        log.info("testing {}, save to {}".format(test_f, save_path))
+        qids, preds, probs = predict(exe, test_prog, test_pyreader, graph_vars)
+
+        save_dir = os.path.dirname(save_path)
+        if not os.path.exists(save_dir):
+            os.makedirs(save_dir)
+        else:
+            log.warning('save dir exsits: %s, will skip saving' % save_dir)
+
+        with open(save_path, 'w') as f:
+            for p in probs:
+                f.write('{}\n'.format(p[1]))
+
+
+if __name__ == '__main__':
+    prepare_logger(log)
+    print_arguments(args)
+    check_cuda(args.use_cuda)
+    main(args)

applications/doc_vqa/Rerank/src/utils/args.py

@@ -0,0 +1,91 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Arguments for configuration."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import six
+import os
+import sys
+import argparse
+import logging
+
+import paddle.fluid as fluid
+
+log = logging.getLogger(__name__)
+
+
+def prepare_logger(logger, debug=False, save_to_file=None):
+    formatter = logging.Formatter(
+        fmt=
+        '[%(levelname)s] %(asctime)s [%(filename)12s:%(lineno)5d]:\t%(message)s'
+    )
+    console_hdl = logging.StreamHandler()
+    console_hdl.setFormatter(formatter)
+    logger.addHandler(console_hdl)
+    if save_to_file is not None and not os.path.exists(save_to_file):
+        file_hdl = logging.FileHandler(save_to_file)
+        file_hdl.setFormatter(formatter)
+        logger.addHandler(file_hdl)
+    logger.setLevel(logging.DEBUG)
+    logger.propagate = False
+
+
+def str2bool(v):
+    # because argparse does not support to parse "true, False" as python
+    # boolean directly
+    return v.lower() in ("true", "t", "1")
+
+
+class ArgumentGroup(object):
+
+    def __init__(self, parser, title, des):
+        self._group = parser.add_argument_group(title=title, description=des)
+
+    def add_arg(self,
+                name,
+                type,
+                default,
+                help,
+                positional_arg=False,
+                **kwargs):
+        prefix = "" if positional_arg else "--"
+        type = str2bool if type == bool else type
+        self._group.add_argument(prefix + name,
+                                 default=default,
+                                 type=type,
+                                 help=help + ' Default: %(default)s.',
+                                 **kwargs)
+
+
+def print_arguments(args):
+    log.info('-----------  Configuration Arguments -----------')
+    for arg, value in sorted(six.iteritems(vars(args))):
+        log.info('%s: %s' % (arg, value))
+    log.info('------------------------------------------------')
+
+
+def check_cuda(use_cuda, err = \
+    "\nYou can not set use_cuda = True in the model because you are using paddlepaddle-cpu.\n \
+    Please: 1. Install paddlepaddle-gpu to run your models on GPU or 2. Set use_cuda = False to run models on CPU.\n"
+                                                                                                                     ):
+    try:
+        if use_cuda == True and fluid.is_compiled_with_cuda() == False:
+            log.error(err)
+            sys.exit(1)
+    except Exception as e:
+        pass

applications/doc_vqa/Rerank/src/utils/init.py

@@ -0,0 +1,73 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import os
+import six
+import ast
+import copy
+import logging
+import numpy as np
+
+import paddle.fluid as fluid
+
+log = logging.getLogger(__name__)
+
+
+def init_checkpoint(exe, init_checkpoint_path, main_program):
+    assert os.path.exists(
+        init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path
+
+    def existed_persitables(var):
+        if not fluid.io.is_persistable(var):
+            return False
+        if not os.path.exists(os.path.join(init_checkpoint_path, var.name)):
+            print("Var not exists: [%s]\t%s" %
+                  (var.name, os.path.join(init_checkpoint_path, var.name)))
+        #else:
+        #    print ("Var exists: [%s]" % (var.name))
+        return os.path.exists(os.path.join(init_checkpoint_path, var.name))
+
+    fluid.io.load_vars(exe,
+                       init_checkpoint_path,
+                       main_program=main_program,
+                       predicate=existed_persitables)
+    log.info("Load model from {}".format(init_checkpoint_path))
+
+
+def init_pretraining_params(exe, pretraining_params_path, main_program):
+    assert os.path.exists(pretraining_params_path
+                          ), "[%s] cann't be found." % pretraining_params_path
+
+    def existed_params(var):
+        if not isinstance(var, fluid.framework.Parameter):
+            return False
+        if not os.path.exists(os.path.join(pretraining_params_path, var.name)):
+            print("Var not exists: [%s]\t%s" %
+                  (var.name, os.path.join(pretraining_params_path, var.name)))
+        #else:
+        #    print ("Var exists: [%s]" % (var.name))
+        return os.path.exists(os.path.join(pretraining_params_path, var.name))
+
+    fluid.io.load_vars(exe,
+                       pretraining_params_path,
+                       main_program=main_program,
+                       predicate=existed_params)
+    log.info(
+        "Load pretraining parameters from {}.".format(pretraining_params_path))

applications/doc_vqa/run_test.sh

@@ -0,0 +1,20 @@
+export CUDA_VISIBLE_DEVICES=0
+
+QUESTION=$1
+
+# Question: NFC咋开门
+
+if [ $# != 1 ];then
+    echo "USAGE: sh script/run_cross_encoder_test.sh \$QUESTION"
+    exit 1
+fi
+
+# compute scores for QUESTION and OCR parsing results  with Rerank module
+cd Rerank
+bash run_test.sh ${QUESTION}
+cd ..
+
+# extraction answer for QUESTION from the top1 of rank
+cd Extraction
+bash run_test.sh ${QUESTION}
+cd ..

applications/experimental/pipelines/README.md

@@ -0,0 +1,26 @@
+## PaddleNLP智能文本产线
+
+本项目基于百度领先NLP技术与飞桨PaddleNLP模型库，为开发者提供低成本搭建端到端NLP应用系统的能力。
+
+## 效果速览
+本项目展示了端到端智能问答系统和端到端语义检索系统的效果速览。
+
+### 智能问答系统
+我们针对中国重点城市的基本信息搭建了百科知识智能问答系统, 您可以根据实际需求基于自己的业务数据快速搭智能问答系统。[点击这里](./examples/question-answering/)快速构建端到端智能问答系统。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514868-1babe981-c675-4f89-9168-dd0a3eede315.gif" width="500">
+</div>
+
+### 语义检索系统
+我们针对[DuReader](https://github.com/baidu/DuReader)数据集搭建了语义检索系统, 您可以根据实际需求基于自己的业务数据快速搭语义检索系统。[点击这里](./examples/semantic-search/)快速构建端到端语义检索系统。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514909-8817d79a-72c4-4be1-8080-93d1f682bb46.gif" width="500">
+</div>
+
+## Acknowledge
+
+我们借鉴了 Deepset.ai [Haystack](https://github.com/deepset-ai/haystack) 优秀的框架设计，在此对[Haystack](https://github.com/deepset-ai/haystack)作者及其开源社区表示感谢。
+
+We learn form the excellent framework design of Deepset.ai [Haystack](https://github.com/deepset-ai/haystack), and we would like to express our thanks to the authors of Haystack and their open source community.

applications/experimental/pipelines/examples/question-answering/README.md

@@ -0,0 +1,130 @@
+# 端到端智能问答系统
+
+## 1. 场景概述
+
+问答系统是信息检索系统的一种高级形式，通过对用户输入的问题进行理解，然后从知识库中寻找答案，并直接反馈给用户。在很多具体场景下我们都会对问答系统有需求。
+
+在日常生活中，用户会经常碰到很多复杂的规章制度、规则条款。例如：火车提前多久可以免费退票？、在北京工作几年可以办理居住证？
+
+在平时工作中，员工也会面对公司多种多样的政策。比如：商业保险理赔需要什么材料？打车报销的具体流程是什么？
+
+这些情况下，传统做法需要仔细阅读政策文件、规章制度、或者咨询相关工作人员才能得到答案，费时费力。现在我们则可以针对这类常见的业务场景快速搭建一套智能问答系统，高效地回答用户的常见问题，提升用户体验的同时，也降低了客服人员的工作负荷及企业成本。
+
+## 2. 产品功能介绍
+
+本项目提供了低成本搭建端到端问答系统的能力。用户只需要处理好自己的业务数据，就可以使用本项目预置的问答系统模型(召回模型、排序模型、阅读理解模型)快速搭建一个针对自己业务数据的问答系统，并可以提供基于[Streamlit](https://streamlit.io/) 的 Web 可视化服务。
+
+### 2.1 系统特色
+
++ 端到端
+    + 提供包括数据建库、模型服务部署、WebUI 可视化一整套端到端问答系统能力
+    + 多源数据支持: 支持对 Txt、Word、PDF、Image 多源数据进行解析、识别并写入 ANN 数据库
++ 效果好
+    + 依托百度领先的NLP技术，包括[ERNIE](https://github.com/PaddlePaddle/ERNIE)语义理解技术与[RocketQA](https://github.com/PaddlePaddle/RocketQA)开放域问答技术
+    + 预置领先的深度学习模型
+
+## 3. 快速开始: 城市百科知识问答系统搭建
+
+### 3.1 运行环境和安装说明
+
+本实验采用了以下的运行环境进行，详细说明如下，用户也可以在自己 GPU 硬件环境进行：
+
+a. 软件环境：
+- python >= 3.6
+- paddlenlp >= 2.2.1
+- paddlepaddle-gpu >=2.2
+- CUDA Version: 10.2
+- NVIDIA Driver Version: 440.64.00
+- Ubuntu 16.04.6 LTS (Docker)
+
+b. 硬件环境：
+
+- NVIDIA Tesla V100 16GB x4卡
+- Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz
+
+c. 依赖安装：
+```bash
+# 1) 安装 pipelines package
+cd ${HOME}/PaddleNLP/applications/experimental/pipelines/
+python setup.py install
+# 2) 安装 RestAPI 相关依赖
+python ./rest_api/setup.py install
+# 3) 安装 Streamlit WebUI 相关依赖
+python ./ui/setup.py install
+```
+### 3.2 数据说明
+问答知识库数据是我们爬取了百度百科上对国内重点城市的百科介绍文档。我们将所有文档中的非结构化文本数据抽取出来， 按照段落切分后作为问答系统知识库的数据，一共包含 365 个城市的百科介绍文档、切分后共 1318 个段落。
+
+### 3.3 一键体验问答系统
+我们预置了搭建城市百科知识问答系统的代码示例，您可以通过如下命令快速体验问答系统的效果。
+
+
+```bash
+# 我们建议在 GPU 环境下运行本示例，运行速度较快
+# 设置 1 个空闲的 GPU 卡，此处假设 0 卡为空闲 GPU
+export CUDA_VISIBLE_DEVICES=0
+python examples/question-answering/dense_qa_example.py --device gpu
+# 如果只有 CPU 机器，可以通过 --device 参数指定 cpu 即可, 运行耗时较长
+unset CUDA_VISIBLE_DEVICES
+python examples/question-answering/dense_qa_example.py --device cpu
+```
+
+### 3.4 构建 Web 可视化问答系统
+
+整个 Web 可视化问答系统主要包含 3 大组件: 1. 基于 ElasticSearch 的 ANN 服务 2. 基于 RestAPI 构建模型服务 3. 基于 Streamlit 构建 WebUI。接下来我们依次搭建这 3 个服务并串联构成可视化的问答系统
+
+#### 3.4.1 启动 ANN 服务
+1. 参考官方文档下载安装 [elasticsearch-8.1.2](https://www.elastic.co/cn/start) 并解压。
+2. 启动 ES 服务
+```bash
+./bin/elasticsearch
+```
+3. 检查确保 ES 服务启动成功
+```bash
+curl http://localhost:9200/_aliases?pretty=true
+```
+备注：ES 服务默认开启端口为 9200
+
+#### 3.4.2 文档数据写入 ANN 索引库
+```
+# 以百科城市数据为例建立 ANN 索引库
+python utils/offline_ann.py
+```
+#### 3.4.3 启动 RestAPI 模型服务
+```bash
+# 指定智能问答系统的Yaml配置文件
+export PIPELINE_YAML_PATH=rest_api/pipeline/dense_qa.yaml
+# 使用端口号 8891 启动模型服务
+python rest_api/application.py 8891
+```
+#### 3.4.4 启动 WebUI
+```bash
+# 配置模型服务地址
+export API_ENDPOINT=http://127.0.0.1:8891
+# 在指定端口 8502 启动 WebUI
+python -m streamlit run ui/webapp_question_answering.py --server.port 8502
+```
+
+到这里您就可以打开浏览器访问 http://127.0.0.1:8502 地址体验城市百科知识问答系统服务了。
+
+## FAQ
+
+#### 语义检索系统可以跑通，但终端输出字符是乱码怎么解决？
+
++ 通过如下命令设置操作系统默认编码为 zh_CN.UTF-8
+```bash
+export LANG=zh_CN.UTF-8
+```
+
+## Reference
+[1]Y. Sun et al., “[ERNIE 3.0: Large-scale Knowledge Enhanced Pre-training for Language Understanding and Generation](https://arxiv.org/pdf/2107.02137.pdf),” arXiv:2107.02137 [cs], Jul. 2021, Accessed: Jan. 17, 2022. [Online]. Available: http://arxiv.org/abs/2107.02137
+
+[2]Y. Qu et al., “[RocketQA: An Optimized Training Approach to Dense Passage Retrieval for Open-Domain Question Answering](https://arxiv.org/abs/2010.08191),” arXiv:2010.08191 [cs], May 2021, Accessed: Aug. 16, 2021. [Online]. Available: http://arxiv.org/abs/2010.08191
+
+[3]H. Tang, H. Li, J. Liu, Y. Hong, H. Wu, and H. Wang, “[DuReader_robust: A Chinese Dataset Towards Evaluating Robustness and Generalization of Machine Reading Comprehension in Real-World Applications](https://arxiv.org/pdf/2004.11142.pdf).” arXiv, Jul. 21, 2021. Accessed: May 15, 2022. [Online]. Available: http://arxiv.org/abs/2004.11142
+
+## Acknowledge
+
+我们借鉴了 Deepset.ai [Haystack](https://github.com/deepset-ai/haystack) 优秀的框架设计，在此对[Haystack](https://github.com/deepset-ai/haystack)作者及其开源社区表示感谢。
+
+We learn form the excellent framework design of Deepset.ai [Haystack](https://github.com/deepset-ai/haystack), and we would like to express our thanks to the authors of Haystack and their open source community.

applications/experimental/pipelines/examples/question-answering/dense_qa_example.py

@@ -0,0 +1,116 @@
+### 城市百科知识智能问答系统
+import argparse
+import logging
+import os
+
+import paddle
+from pipelines.document_stores import FAISSDocumentStore
+from pipelines.utils import convert_files_to_dicts, fetch_archive_from_http, print_answers
+from pipelines.nodes import ErnieReader, ErnieRanker, DensePassageRetriever
+
+# yapf: disable
+parser = argparse.ArgumentParser()
+parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to run dense_qa system, defaults to gpu.")
+parser.add_argument("--index_name", default='faiss_index', type=str, help="The ann index name of FAISS.")
+parser.add_argument("--max_seq_len_query", default=64, type=int, help="The maximum total length of query after tokenization.")
+parser.add_argument("--max_seq_len_passage", default=256, type=int, help="The maximum total length of passage after tokenization.")
+parser.add_argument("--retriever_batch_size", default=16, type=int, help="The batch size of retriever to extract passage embedding for building ANN index.")
+args = parser.parse_args()
+# yapf: enable
+
+
+def dense_qa_pipeline():
+
+    use_gpu = True if args.device == 'gpu' else False
+
+    faiss_document_store = "faiss_document_store.db"
+    if os.path.exists(args.index_name) and os.path.exists(faiss_document_store):
+        # connect to existed FAISS Index
+        document_store = FAISSDocumentStore.load(args.index_name)
+        retriever = DensePassageRetriever(
+            document_store=document_store,
+            query_embedding_model="rocketqa-zh-dureader-query-encoder",
+            passage_embedding_model="rocketqa-zh-dureader-query-encoder",
+            max_seq_len_query=args.max_seq_len_query,
+            max_seq_len_passage=args.max_seq_len_passage,
+            batch_size=args.retriever_batch_size,
+            use_gpu=use_gpu,
+            embed_title=False,
+        )
+    else:
+        doc_dir = "data/baike"
+        city_data = "https://paddlenlp.bj.bcebos.com/applications/baike.zip"
+        fetch_archive_from_http(url=city_data, output_dir=doc_dir)
+        dicts = convert_files_to_dicts(dir_path=doc_dir, split_paragraphs=True)
+
+        if os.path.exists(args.index_name):
+            os.remove(args.index_name)
+        if os.path.exists(faiss_document_store):
+            os.remove(faiss_document_store)
+
+        document_store = FAISSDocumentStore(embedding_dim=768,
+                                            faiss_index_factory_str="Flat")
+        document_store.write_documents(dicts)
+
+        retriever = DensePassageRetriever(
+            document_store=document_store,
+            query_embedding_model="rocketqa-zh-dureader-query-encoder",
+            passage_embedding_model="rocketqa-zh-dureader-query-encoder",
+            max_seq_len_query=args.max_seq_len_query,
+            max_seq_len_passage=args.max_seq_len_passage,
+            batch_size=args.retriever_batch_size,
+            use_gpu=use_gpu,
+            embed_title=False,
+        )
+
+        # update Embedding
+        document_store.update_embeddings(retriever)
+
+        # save index
+        document_store.save(args.index_name)
+
+    ### Ranker
+    ranker = ErnieRanker(
+        model_name_or_path="rocketqa-zh-dureader-cross-encoder",
+        use_gpu=use_gpu)
+
+    reader = ErnieReader(
+        model_name_or_path="ernie-gram-zh-finetuned-dureader-robust",
+        use_gpu=use_gpu,
+        num_processes=1)
+
+    # ### Pipeline
+    from pipelines.pipelines import ExtractiveQAPipeline
+
+    pipe = ExtractiveQAPipeline(reader, ranker, retriever)
+
+    pipeline_params = {
+        "Retriever": {
+            "top_k": 50
+        },
+        "Ranker": {
+            "top_k": 1
+        },
+        "Reader": {
+            "top_k": 1
+        }
+    }
+
+    prediction = pipe.run(query="北京市有多少个行政区？", params=pipeline_params)
+    print_answers(prediction, details="minimum")
+
+    prediction = pipe.run(query="上海常住人口有多少？", params=pipeline_params)
+    print_answers(prediction, details="minimum")
+
+    prediction = pipe.run(query="广州市总面积多大？", params=pipeline_params)
+    print_answers(prediction, details="minimum")
+
+    prediction = pipe.run(query="河北省的省会在哪里？", params=pipeline_params)
+    print_answers(prediction, details="minimum")
+
+    prediction = pipe.run(query="安徽省的简称是什么？", params=pipeline_params)
+    print_answers(prediction, details="minimum")
+
+
+if __name__ == "__main__":
+    dense_qa_pipeline()

applications/experimental/pipelines/examples/semantic-search/README.md

@@ -0,0 +1,124 @@
+# 端到端语义检索系统
+
+## 1. 场景概述
+
+检索系统存在于我们日常使用的很多产品中，比如商品搜索系统、学术文献检索系等等，本方案提供了检索系统完整实现。限定场景是用户通过输入检索词 Query，快速在海量数据中查找相似文档。
+
+所谓语义检索（也称基于向量的检索），是指检索系统不再拘泥于用户 Query 字面本身，而是能精准捕捉到用户 Query 后面的真正意图并以此来搜索，从而更准确地向用户返回最符合的结果。通过使用最先进的语义索引模型找到文本的向量表示，在高维向量空间中对它们进行索引，并度量查询向量与索引文档的相似程度，从而解决了关键词索引带来的缺陷。
+
+例如下面两组文本 Pair，如果基于关键词去计算相似度，两组的相似度是相同的。而从实际语义上看，第一组相似度高于第二组。
+
+```
+车头如何放置车牌    前牌照怎么装
+车头如何放置车牌    后牌照怎么装
+```
+
+语义检索系统的关键就在于，采用语义而非关键词方式进行召回，达到更精准、更广泛得召回相似结果的目的。
+
+## 2. 产品功能介绍
+
+本项目提供了低成本搭建端到端语义检索系统的能力。用户只需要处理好自己的业务数据，就可以使用本项目预置的语义检索系统模型(召回模型、排序模型)快速搭建一个针对自己业务数据的问答系统，并可以提供 Web 化产品服务。
+
+### 2.1 系统特色
+
++ 端到端
+    + 提供包括数据建库、模型服务部署、WebUI 可视化一整套端到端语义检索系统能力
+    + 多源数据支持: 支持对 Txt、Word、PDF、Image 多源数据进行解析、识别并写入 ANN 数据库
++ 效果好
+    + 依托百度领先的NLP技术，包括[ERNIE](https://github.com/PaddlePaddle/ERNIE)语义理解技术与[RocketQA](https://github.com/PaddlePaddle/RocketQA)开放域问答技术
+    + 预置领先的深度学习模型
+
+## 3. 快速开始: 快速搭建语义检索系统
+
+### 3.1 运行环境和安装说明
+
+本实验采用了以下的运行环境进行，详细说明如下，用户也可以在自己 GPU 硬件环境进行：
+
+a. 软件环境：
+- python >= 3.6
+- paddlenlp >= 2.2.1
+- paddlepaddle-gpu >=2.2
+- CUDA Version: 10.2
+- NVIDIA Driver Version: 440.64.00
+- Ubuntu 16.04.6 LTS (Docker)
+
+b. 硬件环境：
+
+- NVIDIA Tesla V100 16GB x4卡
+- Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz
+
+c. 依赖安装：
+```bash
+# 1) 安装 pipelines package
+cd ${HOME}/PaddleNLP/applications/experimental/pipelines/
+python setup.py install
+# 2) 安装 RestAPI 相关依赖
+python ./rest_api/setup.py install
+# 3) 安装 Streamlit WebUI 相关依赖
+python ./ui/setup.py install
+```
+### 3.2 数据说明
+语义检索数据库的数据来自于[DuReader-Robust数据集](https://github.com/baidu/DuReader/tree/master/DuReader-Robust)，共包含 46972 个段落文本。
+
+### 3.3 一键体验语义检索系统
+我们预置了基于[DuReader-Robust数据集](https://github.com/baidu/DuReader/tree/master/DuReader-Robust)搭建语义检索系统的代码示例，您可以通过如下命令快速体验语义检索系统的效果
+```bash
+# 我们建议在 GPU 环境下运行本示例，运行速度较快
+# 设置 1 个空闲的 GPU 卡，此处假设 0 卡为空闲 GPU
+export CUDA_VISIBLE_DEVICES=0
+python examples/semantic-search/semantic_search_example.py --device gpu
+# 如果只有 CPU 机器，可以通过 --device 参数指定 cpu 即可, 运行耗时较长
+unset CUDA_VISIBLE_DEVICES
+python examples/semantic-search/semantic_search_example.py --device cpu
+```
+
+### 3.4 构建 Web 可视化语义检索系统
+
+整个 Web 可视化语义检索系统主要包含 3 大组件: 1. 基于 ElasticSearch 的 ANN 服务 2. 基于 RestAPI 构建模型服务 3. 基于 Streamlit 构建 WebUI，接下来我们依次搭建这 3 个服务并最终形成可视化的语义检索系统。
+
+#### 3.4.1 启动 ANN 服务
+1. 参考官方文档下载安装 [elasticsearch-8.1.2](https://www.elastic.co/cn/start) 并解压。
+2. 启动 ES 服务
+```bash
+./bin/elasticsearch
+```
+3. 检查确保 ES 服务启动成功
+```bash
+curl http://localhost:9200/_aliases?pretty=true```
+```
+备注：ES 服务默认开启端口为 9200
+
+#### 3.4.2 文档数据写入 ANN 索引库
+```
+# 以DuReader-Robust 数据集为例建立 ANN 索引库
+python utils/offline_ann.py
+```
+#### 3.4.3 启动 RestAPI 模型服务
+```bash
+# 指定语义检索系统的Yaml配置文件
+export PIPELINE_YAML_PATH=rest_api/pipeline/semantic_search.yaml
+# 使用端口号 8891 启动模型服务
+python rest_api/application.py 8891
+```
+#### 3.4.4 启动 WebUI
+```bash
+# 配置模型服务地址
+export API_ENDPOINT=http://127.0.0.1:8891
+# 在指定端口 8502 启动 WebUI
+python -m streamlit run ui/webapp_semantic_search.py --server.port 8502
+```
+
+到这里您就可以打开浏览器访问 http://127.0.0.1:8502 地址体验语义检索系统服务了。
+
+## Reference
+[1]Y. Sun et al., “[ERNIE 3.0: Large-scale Knowledge Enhanced Pre-training for Language Understanding and Generation](https://arxiv.org/pdf/2107.02137.pdf),” arXiv:2107.02137 [cs], Jul. 2021, Accessed: Jan. 17, 2022. [Online]. Available: http://arxiv.org/abs/2107.02137
+
+[2]Y. Qu et al., “[RocketQA: An Optimized Training Approach to Dense Passage Retrieval for Open-Domain Question Answering](https://arxiv.org/abs/2010.08191),” arXiv:2010.08191 [cs], May 2021, Accessed: Aug. 16, 2021. [Online]. Available: http://arxiv.org/abs/2010.08191
+
+[3]H. Tang, H. Li, J. Liu, Y. Hong, H. Wu, and H. Wang, “[DuReader_robust: A Chinese Dataset Towards Evaluating Robustness and Generalization of Machine Reading Comprehension in Real-World Applications](https://arxiv.org/pdf/2004.11142.pdf).” arXiv, Jul. 21, 2021. Accessed: May 15, 2022. [Online]. Available: http://arxiv.org/abs/2004.11142
+
+## Acknowledge
+
+我们借鉴了 Deepset.ai [Haystack](https://github.com/deepset-ai/haystack) 优秀的框架设计，在此对[Haystack](https://github.com/deepset-ai/haystack)作者及其开源社区表示感谢。
+
+We learn form the excellent framework design of Deepset.ai [Haystack](https://github.com/deepset-ai/haystack), and we would like to express our thanks to the authors of Haystack and their open source community.

applications/experimental/pipelines/examples/semantic-search/semantic_search_example.py

@@ -0,0 +1,94 @@
+import os
+import argparse
+
+import paddle
+from pipelines.document_stores import FAISSDocumentStore
+from pipelines.nodes import DensePassageRetriever, ErnieRanker
+from pipelines.utils import convert_files_to_dicts, fetch_archive_from_http, print_documents
+
+# yapf: disable
+parser = argparse.ArgumentParser()
+parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to run dense_qa system, defaults to gpu.")
+parser.add_argument("--index_name", default='faiss_index', type=str, help="The ann index name of FAISS.")
+parser.add_argument("--max_seq_len_query", default=64, type=int, help="The maximum total length of query after tokenization.")
+parser.add_argument("--max_seq_len_passage", default=256, type=int, help="The maximum total length of passage after tokenization.")
+parser.add_argument("--retriever_batch_size", default=16, type=int, help="The batch size of retriever to extract passage embedding for building ANN index.")
+args = parser.parse_args()
+# yapf: enable
+
+
+def semantic_search_tutorial():
+
+    use_gpu = True if args.device == 'gpu' else False
+
+    faiss_document_store = "faiss_document_store.db"
+    if os.path.exists(args.index_name) and os.path.exists(faiss_document_store):
+        # connect to existed FAISS Index
+        document_store = FAISSDocumentStore.load(args.index_name)
+        retriever = DensePassageRetriever(
+            document_store=document_store,
+            query_embedding_model="rocketqa-zh-dureader-query-encoder",
+            passage_embedding_model="rocketqa-zh-dureader-query-encoder",
+            max_seq_len_query=args.max_seq_len_query,
+            max_seq_len_passage=args.max_seq_len_passage,
+            batch_size=args.retriever_batch_size,
+            use_gpu=use_gpu,
+            embed_title=False,
+        )
+    else:
+        doc_dir = "data/dureader_robust_processed"
+        dureader_data = "https://paddlenlp.bj.bcebos.com/applications/dureader_robust_processed.zip"
+
+        fetch_archive_from_http(url=dureader_data, output_dir=doc_dir)
+        dicts = convert_files_to_dicts(dir_path=doc_dir, split_paragraphs=True)
+
+        if os.path.exists(args.index_name):
+            os.remove(args.index_name)
+        if os.path.exists(faiss_document_store):
+            os.remove(faiss_document_store)
+
+        document_store = FAISSDocumentStore(embedding_dim=768,
+                                            faiss_index_factory_str="Flat")
+        document_store.write_documents(dicts)
+
+        retriever = DensePassageRetriever(
+            document_store=document_store,
+            query_embedding_model="rocketqa-zh-dureader-query-encoder",
+            passage_embedding_model="rocketqa-zh-dureader-query-encoder",
+            max_seq_len_query=args.max_seq_len_query,
+            max_seq_len_passage=args.max_seq_len_passage,
+            batch_size=args.retriever_batch_size,
+            use_gpu=use_gpu,
+            embed_title=False,
+        )
+
+        # update Embedding
+        document_store.update_embeddings(retriever)
+
+        # save index
+        document_store.save(args.index_name)
+
+    ### Ranker
+    ranker = ErnieRanker(
+        model_name_or_path="rocketqa-zh-dureader-cross-encoder",
+        use_gpu=use_gpu)
+
+    ### Pipeline
+    from pipelines.pipelines import SemanticSearchPipeline
+    pipe = SemanticSearchPipeline(retriever, ranker)
+
+    prediction = pipe.run(query="亚马逊河流的介绍",
+                          params={
+                              "Retriever": {
+                                  "top_k": 50
+                              },
+                              "Ranker": {
+                                  "top_k": 5
+                              }
+                          })
+
+    print_documents(prediction)
+
+
+if __name__ == "__main__":
+    semantic_search_tutorial()

applications/experimental/pipelines/pipelines/__init__.py

@@ -0,0 +1,167 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = '0.1.0a0'  # Maybe dev is better
+
+from typing import Union
+from types import ModuleType
+
+try:
+    from importlib import metadata
+except (ModuleNotFoundError, ImportError):
+    # Python <= 3.7
+    import importlib_metadata as metadata  # type: ignore
+
+# This configuration must be done before any import to apply to all submodules
+import logging
+
+logging.basicConfig(format="%(levelname)s - %(name)s -  %(message)s",
+                    datefmt="%m/%d/%Y %H:%M:%S",
+                    level=logging.WARNING)
+logging.getLogger("pipelines").setLevel(logging.INFO)
+
+from pipelines import utils
+from pipelines import pipelines
+from pipelines.schema import Document, Answer, Label, Span
+from pipelines.nodes import BaseComponent
+from pipelines.pipelines import Pipeline
+
+import pandas as pd
+
+pd.options.display.max_colwidth = 80
+
+# ###########################################
+# Enable old style imports (temporary)
+import sys
+
+logger = logging.getLogger(__name__)
+
+
+# Wrapper emitting a warning on import
+def DeprecatedModule(mod,
+                     deprecated_attributes=None,
+                     is_module_deprecated=True):
+    """
+    Return a wrapped object that warns about deprecated accesses at import
+    """
+
+    class DeprecationWrapper:
+        warned = []
+
+        def __getattr__(self, attr):
+            is_a_deprecated_attr = deprecated_attributes and attr in deprecated_attributes
+            is_a_deprecated_module = is_module_deprecated and attr not in [
+                "__path__", "__spec__", "__name__"
+            ]
+            warning_already_emitted = attr in self.warned
+            attribute_exists = getattr(mod, attr) is not None
+
+            if (is_a_deprecated_attr or is_a_deprecated_module
+                ) and not warning_already_emitted and attribute_exists:
+                logger.warn(
+                    f"Object '{attr}' is imported through a deprecated path. Please check out the docs for the new import path."
+                )
+                self.warned.append(attr)
+            return getattr(mod, attr)
+
+    return DeprecationWrapper()
+
+
+# All modules to be aliased need to be imported here
+
+# This self-import is used to monkey-patch, keep for now
+import pipelines  # pylint: disable=import-self
+from pipelines.nodes import (file_converter, preprocessor, ranker, reader,
+                             retriever)
+
+# Note that we ignore the ImportError here because if the user did not install
+# the correct dependency group for a document store, we don't need to setup
+# import warnings for that, so the import here is useless and should fail silently.
+
+document_stores: Union[ModuleType, None] = None
+try:
+    from pipelines import document_stores
+except ImportError:
+    pass
+
+from pipelines.nodes.file_classifier import FileTypeClassifier
+from pipelines.nodes.other import JoinDocuments, Docs2Answers, JoinAnswers
+from pipelines.utils import preprocessing
+from pipelines.utils import cleaning
+
+# For the alias to work as an importable module (like `from pipelines import reader`),
+# modules need to be set as attributes of their parent model.
+# To make chain imports work (`from pipelines.reader import FARMReader`) the module
+# needs to be also present in sys.modules with its complete import path.
+
+setattr(preprocessor, "utils", DeprecatedModule(preprocessing))
+setattr(preprocessor, "cleaning", DeprecatedModule(cleaning))
+sys.modules["pipelines.preprocessor.utils"] = DeprecatedModule(preprocessing)
+sys.modules["pipelines.preprocessor.cleaning"] = DeprecatedModule(cleaning)
+
+setattr(pipelines, "document_store", DeprecatedModule(document_stores))
+setattr(
+    pipelines, "file_converter",
+    DeprecatedModule(file_converter,
+                     deprecated_attributes=["FileTypeClassifier"]))
+setattr(
+    pipelines,
+    "pipeline",
+    DeprecatedModule(
+        pipelines,
+        deprecated_attributes=[
+            "JoinDocuments",
+            "Docs2Answers",
+        ],
+    ),
+)
+setattr(
+    pipelines, "preprocessor",
+    DeprecatedModule(preprocessor, deprecated_attributes=["utils", "cleaning"]))
+setattr(pipelines, "ranker", DeprecatedModule(ranker))
+setattr(pipelines, "reader", DeprecatedModule(reader))
+setattr(pipelines, "retriever", DeprecatedModule(retriever))
+
+sys.modules["pipelines.document_store"] = DeprecatedModule(document_stores)
+sys.modules["pipelines.file_converter"] = DeprecatedModule(file_converter)
+sys.modules["pipelines.pipeline"] = DeprecatedModule(pipelines)
+sys.modules["pipelines.preprocessor"] = DeprecatedModule(
+    preprocessor, deprecated_attributes=["utils", "cleaning"])
+sys.modules["pipelines.ranker"] = DeprecatedModule(ranker)
+sys.modules["pipelines.reader"] = DeprecatedModule(reader)
+sys.modules["pipelines.retriever"] = DeprecatedModule(retriever)
+
+# To be imported from modules, classes need only to be set as attributes,
+# they don't need to be present in sys.modules too.
+# Adding them to sys.modules would enable `import pipelines.pipelines.JoinDocuments`,
+# which I believe it's a very rare import style.
+setattr(file_converter, "FileTypeClassifier", FileTypeClassifier)
+setattr(pipelines, "Docs2Answers", Docs2Answers)
+
+# This last line is used to throw the deprecation error for imports like `from pipelines import connector`
+deprecated_attributes = [
+    "document_store",
+    "file_converter",
+    "pipeline",
+    "preprocessor",
+    "ranker",
+    "reader",
+    "retriever",
+]
+
+sys.modules["pipelines"] = DeprecatedModule(
+    pipelines,
+    is_module_deprecated=False,
+    deprecated_attributes=deprecated_attributes)

applications/experimental/pipelines/pipelines/data_handler/dataset.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import numbers
+from typing import Iterable, List
+
+import numpy as np
+from paddlenlp.datasets import IterDataset, MapDataset
+
+from pipelines.utils.common_utils import flatten_list
+
+logger = logging.getLogger(__name__)
+
+
+def convert_features_to_dataset(features):
+    """
+    Converts a list of feature dictionaries (one for each sample) into a Paddle Dataset.
+
+    :param features: A list of dictionaries. Each dictionary corresponds to one sample. Its keys are the
+                     names of the type of feature and the keys are the features themselves.
+    :Return: a Paddle dataset and a list of tensor names.
+    """
+    # features can be an empty list in cases where down sampling occurs
+    if len(features) == 0:
+        return None, None
+    tensor_names = list(features[0].keys())
+    all_tensors = []
+    for t_name in tensor_names:
+        try:
+            # Checking whether a non-integer will be silently converted to Paddle.long
+            check = features[0][t_name]
+            if isinstance(check, numbers.Number):
+                base = check
+            # extract a base variable from a nested lists or tuples
+            elif isinstance(check, list):
+                base = list(flatten_list(check))[0]
+            # extract a base variable from numpy arrays
+            else:
+                base = check.ravel()[0]
+            if not np.issubdtype(type(base), np.integer):
+                logger.warning(
+                    f"Problem during conversion to Paddle tensors:\n"
+                    f"A non-integer value for feature '{t_name}' with a value of: "
+                    f"'{base}' will be converted to a Paddle tensor of dtype long."
+                )
+        except:
+            logger.debug(f"Could not determine type for feature '{t_name}'. "
+                         "Converting now to a tensor of default type long.")
+
+        # Convert all remaining python objects to Paddle long tensors
+        cur_tensor = [sample[t_name] for sample in features]
+        all_tensors.append(cur_tensor)
+
+    # Todo(tianxin): When set to IterDataset, throw Exception with paddle.io.BatchSampler
+    # all_tensors: List[List[all_token_ids], List[all_segment_ids]]
+    # list(zip(*all_tensors)): List[([token_ids], [segment_ids]), ([token_ids], [segment_ids])]
+    # For Question Answering: tensor_names: ['input_ids', 'padding_mask', 'segment_ids', 'passage_start_t', 'start_of_word', 'labels', 'id', 'seq_2_start_t', 'span_mask']
+    dataset = MapDataset(list(zip(*all_tensors)))
+    return dataset, tensor_names

applications/experimental/pipelines/pipelines/data_handler/inputs.py

@@ -16,6 +16,7 @@ from typing import List, Union
 
 
 class Question:
+
     def __init__(self, text: str, uid: str = None):
         self.text = text
         self.uid = uid
@@ -26,7 +27,9 @@ class Question:
 
 
 class QAInput:
-    def __init__(self, doc_text: str, questions: Union[List[Question], Question]):
+
+    def __init__(self, doc_text: str, questions: Union[List[Question],
+                                                       Question]):
         self.doc_text = doc_text
         if type(questions) == Question:
             self.questions = [questions]

applications/experimental/pipelines/pipelines/data_handler/predictions.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List, Any, Optional, Tuple, Union, Dict
+
 import logging
 from abc import ABC
-from typing import Any, Dict, List, Optional, Tuple, Union
 
 logger = logging.getLogger(__name__)
 
@@ -98,13 +99,15 @@ class QACandidate:
         self.meta = None
 
     def set_context_window(self, context_window_size: int, clear_text: str):
-        window_str, start_ch, end_ch = self._create_context_window(context_window_size, clear_text)
+        window_str, start_ch, end_ch = self._create_context_window(
+            context_window_size, clear_text)
         self.context_window = window_str
         self.offset_context_window_start = start_ch
         self.offset_context_window_end = end_ch
 
     def set_answer_string(self, token_offsets: List[int], document_text: str):
-        pred_str, self.offset_answer_start, self.offset_answer_end = self._span_to_string(token_offsets, document_text)
+        pred_str, self.offset_answer_start, self.offset_answer_end = self._span_to_string(
+            token_offsets, document_text)
         self.offset_unit = "char"
         self._add_answer(pred_str)
 
@@ -133,7 +136,8 @@ class QACandidate:
                     f"({self.offset_answer_start}, {self.offset_answer_end}) with a span answer. "
                 )
 
-    def _create_context_window(self, context_window_size: int, clear_text: str) -> Tuple[str, int, int]:
+    def _create_context_window(self, context_window_size: int,
+                               clear_text: str) -> Tuple[str, int, int]:
         """
         Extract from the clear_text a window that contains the answer and (usually) some amount of text on either
         side of the answer. Useful for cases where the answer and its surrounding context needs to be
@@ -168,7 +172,8 @@ class QACandidate:
         window_str = clear_text[window_start_ch:window_end_ch]
         return window_str, window_start_ch, window_end_ch
 
-    def _span_to_string(self, token_offsets: List[int], clear_text: str) -> Tuple[str, int, int]:
+    def _span_to_string(self, token_offsets: List[int],
+                        clear_text: str) -> Tuple[str, int, int]:
         """
         Generates a string answer span using self.offset_answer_start and self.offset_answer_end. If the candidate
         is a no answer, an empty string is returned
@@ -230,7 +235,10 @@ class QACandidate:
         self.aggregation_level = "document"
 
     def to_list(self) -> List[Optional[Union[str, int, float]]]:
-        return [self.answer, self.offset_answer_start, self.offset_answer_end, self.score, self.passage_id]
+        return [
+            self.answer, self.offset_answer_start, self.offset_answer_end,
+            self.score, self.passage_id
+        ]
 
 
 class QAPred(Pred):
@@ -276,7 +284,8 @@ class QAPred(Pred):
         self.n_passages = self.prediction[0].n_passages_in_doc
         for qa_candidate in self.prediction:
             qa_candidate.set_answer_string(token_offsets, self.context)
-            qa_candidate.set_context_window(self.context_window_size, self.context)
+            qa_candidate.set_context_window(self.context_window_size,
+                                            self.context)
 
     def to_json(self, squad=False) -> Dict:
         """
@@ -286,16 +295,16 @@ class QAPred(Pred):
         """
         answers = self._answers_to_json(self.id, squad)
         ret = {
-            "task": "qa",
-            "predictions": [
-                {
-                    "question": self.question,
-                    "id": self.id,
-                    "ground_truth": self.ground_truth_answer,
-                    "answers": answers,
-                    "no_ans_gap": self.no_answer_gap,  # Add no_ans_gap to current no_ans_boost for switching top prediction
-                }
-            ],
+            "task":
+            "qa",
+            "predictions": [{
+                "question": self.question,
+                "id": self.id,
+                "ground_truth": self.ground_truth_answer,
+                "answers": answers,
+                "no_ans_gap": self.
+                no_answer_gap,  # Add no_ans_gap to current no_ans_boost for switching top prediction
+            }],
         }
         if squad:
             del ret["predictions"][0]["id"]  # type: ignore
@@ -324,7 +333,8 @@ class QAPred(Pred):
                 "offset_answer_start": qa_candidate.offset_answer_start,
                 "offset_answer_end": qa_candidate.offset_answer_end,
                 "context": qa_candidate.context_window,
-                "offset_context_start": qa_candidate.offset_context_window_start,
+                "offset_context_start":
+                qa_candidate.offset_context_window_start,
                 "offset_context_end": qa_candidate.offset_context_window_end,
                 "document_id": ext_id,
             }

applications/experimental/pipelines/pipelines/data_handler/samples.py

@@ -13,9 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
-from typing import List, Optional, Union
+from typing import Union, Optional, List
 
+import logging
 import numpy as np
 
 logger = logging.getLogger(__name__)
@@ -26,7 +26,11 @@ class Sample:
     the human readable clear_text. Over the course of data preprocessing, this object is populated
     with tokenized and featurized versions of the data."""
 
-    def __init__(self, id: str, clear_text: dict, tokenized: Optional[dict] = None, features: Optional[dict] = None):
+    def __init__(self,
+                 id: str,
+                 clear_text: dict,
+                 tokenized: Optional[dict] = None,
+                 features: Optional[dict] = None):
         """
         :param id: The unique id of the sample
         :param clear_text: A dictionary containing various human readable fields (e.g. text, label).
@@ -41,12 +45,13 @@ class Sample:
     def __str__(self):
 
         if self.clear_text:
-            clear_text_str = "\n \t".join([k + ": " + str(v) for k, v in self.clear_text.items()])
+            clear_text_str = "\n \t".join(
+                [k + ": " + str(v) for k, v in self.clear_text.items()])
             if len(clear_text_str) > 3000:
                 clear_text_str = (
-                    clear_text_str[:3_000] + f"\nTHE REST IS TOO LONG TO DISPLAY. "
-                    f"Remaining chars :{len(clear_text_str)-3_000}"
-                )
+                    clear_text_str[:3_000] +
+                    f"\nTHE REST IS TOO LONG TO DISPLAY. "
+                    f"Remaining chars :{len(clear_text_str)-3_000}")
         else:
             clear_text_str = "None"
 
@@ -55,26 +60,25 @@ class Sample:
                 features = self.features[0]
             else:
                 features = self.features
-            feature_str = "\n \t".join([k + ": " + str(v) for k, v in features.items()])
+            feature_str = "\n \t".join(
+                [k + ": " + str(v) for k, v in features.items()])
         else:
             feature_str = "None"
 
         if self.tokenized:
-            tokenized_str = "\n \t".join([k + ": " + str(v) for k, v in self.tokenized.items()])
+            tokenized_str = "\n \t".join(
+                [k + ": " + str(v) for k, v in self.tokenized.items()])
             if len(tokenized_str) > 3000:
-                tokenized_str = (
-                    tokenized_str[:3_000] + f"\nTHE REST IS TOO LONG TO DISPLAY. "
-                    f"Remaining chars: {len(tokenized_str)-3_000}"
-                )
+                tokenized_str = (tokenized_str[:3_000] +
+                                 f"\nTHE REST IS TOO LONG TO DISPLAY. "
+                                 f"Remaining chars: {len(tokenized_str)-3_000}")
         else:
             tokenized_str = "None"
-        s = (
-            f"ID: {self.id}\n"
-            f"Clear Text: \n \t{clear_text_str}\n"
-            f"Tokenized: \n \t{tokenized_str}\n"
-            f"Features: \n \t{feature_str}\n"
-            "_____________________________________________________"
-        )
+        s = (f"ID: {self.id}\n"
+             f"Clear Text: \n \t{clear_text_str}\n"
+             f"Tokenized: \n \t{tokenized_str}\n"
+             f"Features: \n \t{feature_str}\n"
+             "_____________________________________________________")
         return s
 
 
@@ -115,8 +119,9 @@ def process_answers(answers, doc_offsets, passage_start_c, passage_start_t):
         else:
             answer_start_c = answer["answer_start"]
         answer_end_c = answer_start_c + answer_len_c - 1
-        answer_start_t = offset_to_token_idx_vectorized(doc_offsets, answer_start_c)
-        answer_end_t = offset_to_token_idx_vectorized(doc_offsets, answer_end_c)
+        answer_start_t = offset_to_token_idx_vecorized(doc_offsets,
+                                                       answer_start_c)
+        answer_end_t = offset_to_token_idx_vecorized(doc_offsets, answer_end_c)
 
         # TODO: Perform check that answer can be recovered from document?
         # This section converts start and end so that they are relative to the passage
@@ -126,7 +131,11 @@ def process_answers(answers, doc_offsets, passage_start_c, passage_start_t):
         answer_start_t -= passage_start_t
         answer_end_t -= passage_start_t
 
-        curr_answer_clear = {"text": answer_text, "start_c": answer_start_c, "end_c": answer_end_c}
+        curr_answer_clear = {
+            "text": answer_text,
+            "start_c": answer_start_c,
+            "end_c": answer_end_c
+        }
         curr_answer_tokenized = {
             "start_t": answer_start_t,
             "end_t": answer_end_t,
@@ -185,19 +194,20 @@ def offset_to_token_idx(token_offsets, ch_idx) -> Optional[int]:
     """Returns the idx of the token at the given character idx"""
     n_tokens = len(token_offsets)
     for i in range(n_tokens):
-        if (i + 1 == n_tokens) or (token_offsets[i] <= ch_idx < token_offsets[i + 1]):
+        if (i + 1 == n_tokens) or (token_offsets[i] <= ch_idx <
+                                   token_offsets[i + 1]):
             return i
     return None
 
 
-def offset_to_token_idx_vectorized(token_offsets, ch_idx):
+def offset_to_token_idx_vecorized(token_offsets, ch_idx):
     """Returns the idx of the token at the given character idx"""
     # case ch_idx is at end of tokens
     if ch_idx >= np.max(token_offsets):
         # TODO check "+ 1" (it is needed for making end indices compliant with old offset_to_token_idx() function)
-        # check whether end token is inclusive or exclusive
+        # check whether end token is incluse or exclusive
         idx = np.argmax(token_offsets) + 1
-    # looking for the first occurrence of token_offsets larger than ch_idx and taking one position to the left.
+    # looking for the first occurence of token_offsets larger than ch_idx and taking one position to the left.
     # This is needed to overcome n special_tokens at start of sequence
     # and failsafe matching (the character start might not always coincide with a token offset, e.g. when starting at whitespace)
     else:

applications/experimental/pipelines/pipelines/document_stores/__init__.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import importlib
+from pipelines.utils.import_utils import safe_import
+from pipelines.document_stores.base import BaseDocumentStore, BaseKnowledgeGraph, KeywordDocumentStore
+
+ElasticsearchDocumentStore = safe_import(
+    "pipelines.document_stores.elasticsearch", "ElasticsearchDocumentStore",
+    "elasticsearch")
+OpenDistroElasticsearchDocumentStore = safe_import(
+    "pipelines.document_stores.elasticsearch",
+    "OpenDistroElasticsearchDocumentStore", "elasticsearch")
+OpenSearchDocumentStore = safe_import("pipelines.document_stores.elasticsearch",
+                                      "OpenSearchDocumentStore",
+                                      "elasticsearch")
+
+FAISSDocumentStore = safe_import("pipelines.document_stores.faiss",
+                                 "FAISSDocumentStore", "faiss")
+
+from pipelines.document_stores.utils import (
+    eval_data_from_json,
+    eval_data_from_jsonl,
+    es_index_to_document_store,
+)

applications/experimental/pipelines/pipelines/document_stores/base.py

@@ -0,0 +1,677 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Generator, Optional, Dict, List, Set, Union
+
+import logging
+import collections
+import numpy as np
+from itertools import islice
+from abc import abstractmethod
+from pathlib import Path
+
+try:
+    from typing import Literal
+except ImportError:
+    from typing_extensions import Literal  # type: ignore
+
+from pipelines.schema import Document, Label
+from pipelines.nodes.base import BaseComponent
+from pipelines.errors import DuplicateDocumentError
+from pipelines.nodes.preprocessor import PreProcessor
+from pipelines.document_stores.utils import eval_data_from_json, eval_data_from_jsonl
+
+logger = logging.getLogger(__name__)
+
+try:
+    from numba import njit  # pylint: disable=import-error
+except (ImportError, ModuleNotFoundError):
+    logger.info("Numba not found, replacing njit() with no-op implementation. "
+                "Enable it with 'pip install numba'.")
+
+    def njit(f):
+        return f
+
+
+@njit  # (fastmath=True)
+def expit(x: float) -> float:
+    return 1 / (1 + np.exp(-x))
+
+
+class BaseKnowledgeGraph(BaseComponent):
+    """
+    Base class for implementing Knowledge Graphs.
+    """
+
+    outgoing_edges = 1
+
+    def run(self,
+            sparql_query: str,
+            index: Optional[str] = None,
+            headers: Optional[Dict[str, str]] = None):  # type: ignore
+        result = self.query(sparql_query=sparql_query,
+                            index=index,
+                            headers=headers)
+        output = {"sparql_result": result}
+        return output, "output_1"
+
+    def query(self,
+              sparql_query: str,
+              index: Optional[str] = None,
+              headers: Optional[Dict[str, str]] = None):
+        raise NotImplementedError
+
+
+class BaseDocumentStore(BaseComponent):
+    """
+    Base class for implementing Document Stores.
+    """
+
+    index: Optional[str]
+    label_index: Optional[str]
+    similarity: Optional[str]
+    duplicate_documents_options: tuple = ("skip", "overwrite", "fail")
+    ids_iterator = None
+
+    @abstractmethod
+    def write_documents(
+        self,
+        documents: Union[List[dict], List[Document]],
+        index: Optional[str] = None,
+        batch_size: int = 10_000,
+        duplicate_documents: Optional[str] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Indexes documents for later queries.
+
+        :param documents: a list of Python dictionaries or a list of pipelines Document objects.
+                          For documents as dictionaries, the format is {"text": "<the-actual-text>"}.
+                          Optionally: Include meta data via {"text": "<the-actual-text>",
+                          "meta":{"name": "<some-document-name>, "author": "somebody", ...}}
+                          It can be used for filtering and is accessible in the responses of the Finder.
+        :param index: Optional name of index where the documents shall be written to.
+                      If None, the DocumentStore's default index (self.index) will be used.
+        :param batch_size: Number of documents that are passed to bulk function at a time.
+        :param duplicate_documents: Handle duplicates document based on parameter options.
+                                    Parameter options : ( 'skip','overwrite','fail')
+                                    skip: Ignore the duplicates documents
+                                    overwrite: Update any existing documents with the same ID when adding documents.
+                                    fail: an error is raised if the document ID of the document being added already
+                                    exists.
+        :param headers: Custom HTTP headers to pass to document store client if supported (e.g. {'Authorization': 'Basic YWRtaW46cm9vdA=='} for basic authentication)
+
+        :return: None
+        """
+        pass
+
+    @abstractmethod
+    def get_all_documents(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[str, Union[Dict, List, str, int, float,
+                                          bool]]] = None,
+        return_embedding: Optional[bool] = None,
+        batch_size: int = 10_000,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+        """
+        Get documents from the document store.
+
+        :param index: Name of the index to get the documents from. If None, the
+                      DocumentStore's default index (self.index) will be used.
+        :param filters: Optional filters to narrow down the search space to documents whose metadata fulfill certain
+                        conditions.
+                        Filters are defined as nested dictionaries. The keys of the dictionaries can be a logical
+                        operator (`"$and"`, `"$or"`, `"$not"`), a comparison operator (`"$eq"`, `"$in"`, `"$gt"`,
+                        `"$gte"`, `"$lt"`, `"$lte"`) or a metadata field name.
+                        Logical operator keys take a dictionary of metadata field names and/or logical operators as
+                        value. Metadata field names take a dictionary of comparison operators as value. Comparison
+                        operator keys take a single value or (in case of `"$in"`) a list of values as value.
+                        If no logical operator is provided, `"$and"` is used as default operation. If no comparison
+                        operator is provided, `"$eq"` (or `"$in"` if the comparison value is a list) is used as default
+                        operation.
+
+                            __Example__:
+                            ```python
+                            filters = {
+                                "$and": {
+                                    "type": {"$eq": "article"},
+                                    "date": {"$gte": "2015-01-01", "$lt": "2021-01-01"},
+                                    "rating": {"$gte": 3},
+                                    "$or": {
+                                        "genre": {"$in": ["economy", "politics"]},
+                                        "publisher": {"$eq": "nytimes"}
+                                    }
+                                }
+                            }
+                            ```
+
+        :param return_embedding: Whether to return the document embeddings.
+        :param batch_size: Number of documents that are passed to bulk function at a time.
+        :param headers: Custom HTTP headers to pass to document store client if supported (e.g. {'Authorization': 'Basic YWRtaW46cm9vdA=='} for basic authentication)
+        """
+        pass
+
+    @abstractmethod
+    def get_all_documents_generator(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[str, Union[Dict, List, str, int, float,
+                                          bool]]] = None,
+        return_embedding: Optional[bool] = None,
+        batch_size: int = 10_000,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Generator[Document, None, None]:
+        """
+        Get documents from the document store. Under-the-hood, documents are fetched in batches from the
+        document store and yielded as individual documents. This method can be used to iteratively process
+        a large number of documents without having to load all documents in memory.
+
+        :param index: Name of the index to get the documents from. If None, the
+                      DocumentStore's default index (self.index) will be used.
+        :param filters: Optional filters to narrow down the search space to documents whose metadata fulfill certain
+                        conditions.
+                        Filters are defined as nested dictionaries. The keys of the dictionaries can be a logical
+                        operator (`"$and"`, `"$or"`, `"$not"`), a comparison operator (`"$eq"`, `"$in"`, `"$gt"`,
+                        `"$gte"`, `"$lt"`, `"$lte"`) or a metadata field name.
+                        Logical operator keys take a dictionary of metadata field names and/or logical operators as
+                        value. Metadata field names take a dictionary of comparison operators as value. Comparison
+                        operator keys take a single value or (in case of `"$in"`) a list of values as value.
+                        If no logical operator is provided, `"$and"` is used as default operation. If no comparison
+                        operator is provided, `"$eq"` (or `"$in"` if the comparison value is a list) is used as default
+                        operation.
+
+                        __Example__:
+                        ```python
+                        filters = {
+                            "$and": {
+                                "type": {"$eq": "article"},
+                                "date": {"$gte": "2015-01-01", "$lt": "2021-01-01"},
+                                "rating": {"$gte": 3},
+                                "$or": {
+                                    "genre": {"$in": ["economy", "politics"]},
+                                    "publisher": {"$eq": "nytimes"}
+                                }
+                            }
+                        }
+                        ```
+
+        :param return_embedding: Whether to return the document embeddings.
+        :param batch_size: When working with large number of documents, batching can help reduce memory footprint.
+        :param headers: Custom HTTP headers to pass to document store client if supported (e.g. {'Authorization': 'Basic YWRtaW46cm9vdA=='} for basic authentication)
+        """
+        pass
+
+    def __iter__(self):
+        if not self.ids_iterator:
+            self.ids_iterator = [x.id for x in self.get_all_documents()]
+        return self
+
+    def __next__(self):
+        if len(self.ids_iterator) == 0:
+            raise StopIteration
+        curr_id = self.ids_iterator[0]
+        ret = self.get_document_by_id(curr_id)
+        self.ids_iterator = self.ids_iterator[1:]
+        return ret
+
+    @abstractmethod
+    def get_all_labels(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[str, Union[Dict, List, str, int, float,
+                                          bool]]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Label]:
+        pass
+
+    @abstractmethod
+    def get_document_by_id(
+            self,
+            id: str,
+            index: Optional[str] = None,
+            headers: Optional[Dict[str, str]] = None) -> Optional[Document]:
+        pass
+
+    @abstractmethod
+    def get_document_count(
+        self,
+        filters: Optional[Dict[str, Union[Dict, List, str, int, float,
+                                          bool]]] = None,
+        index: Optional[str] = None,
+        only_documents_without_embedding: bool = False,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> int:
+        pass
+
+    @staticmethod
+    @njit  # (fastmath=True)
+    def normalize_embedding(emb: np.ndarray) -> None:
+        """
+        Performs L2 normalization of embeddings vector inplace. Input can be a single vector (1D array) or a matrix
+        (2D array).
+        """
+        # Might be extended to other normalizations in future
+
+        # Single vec
+        if len(emb.shape) == 1:
+            norm = np.sqrt(emb.dot(emb))  # faster than np.linalg.norm()
+            if norm != 0.0:
+                emb /= norm
+        # 2D matrix
+        else:
+            for vec in emb:
+                vec = np.ascontiguousarray(vec)
+                norm = np.sqrt(vec.dot(vec))
+                if norm != 0.0:
+                    vec /= norm
+
+    def finalize_raw_score(self, raw_score: float,
+                           similarity: Optional[str]) -> float:
+        if similarity == "cosine":
+            return (raw_score + 1) / 2
+        else:
+            return float(expit(raw_score / 100))
+
+    @abstractmethod
+    def query_by_embedding(
+        self,
+        query_emb: np.ndarray,
+        filters: Optional[Dict[str, Union[Dict, List, str, int, float,
+                                          bool]]] = None,
+        top_k: int = 10,
+        index: Optional[str] = None,
+        return_embedding: Optional[bool] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+        pass
+
+    @abstractmethod
+    def get_label_count(self,
+                        index: Optional[str] = None,
+                        headers: Optional[Dict[str, str]] = None) -> int:
+        pass
+
+    @abstractmethod
+    def write_labels(
+        self,
+        labels: Union[List[Label], List[dict]],
+        index: Optional[str] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        pass
+
+    def add_eval_data(
+        self,
+        filename: str,
+        doc_index: str = "eval_document",
+        label_index: str = "label",
+        batch_size: Optional[int] = None,
+        preprocessor: Optional[PreProcessor] = None,
+        max_docs: Union[int, bool] = None,
+        open_domain: bool = False,
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Adds a SQuAD-formatted file to the DocumentStore in order to be able to perform evaluation on it.
+        If a jsonl file and a batch_size is passed to the function, documents are loaded batchwise
+        from disk and also indexed batchwise to the DocumentStore in order to prevent out of memory errors.
+
+        :param filename: Name of the file containing evaluation data (json or jsonl)
+        :param doc_index: Elasticsearch index where evaluation documents should be stored
+        :param label_index: Elasticsearch index where labeled questions should be stored
+        :param batch_size: Optional number of documents that are loaded and processed at a time.
+                           When set to None (default) all documents are processed at once.
+        :param preprocessor: Optional PreProcessor to preprocess evaluation documents.
+                             It can be used for splitting documents into passages (and assigning labels to corresponding passages).
+                             Currently the PreProcessor does not support split_by sentence, cleaning nor split_overlap != 0.
+                             When set to None (default) preprocessing is disabled.
+        :param max_docs: Optional number of documents that will be loaded.
+                         When set to None (default) all available eval documents are used.
+        :param open_domain: Set this to True if your file is an open domain dataset where two different answers to the
+                            same question might be found in different contexts.
+        :param headers: Custom HTTP headers to pass to document store client if supported (e.g. {'Authorization': 'Basic YWRtaW46cm9vdA=='} for basic authentication)
+
+        """
+        # TODO improve support for PreProcessor when adding eval data
+        if preprocessor is not None:
+            assert preprocessor.split_by != "sentence", (
+                f"Split by sentence not supported.\n"
+                f"Please set 'split_by' to either 'word' or 'passage' in the supplied PreProcessor."
+            )
+            assert preprocessor.split_respect_sentence_boundary == False, (
+                f"split_respect_sentence_boundary not supported yet.\n"
+                f"Please set 'split_respect_sentence_boundary' to False in the supplied PreProcessor."
+            )
+            assert preprocessor.split_overlap == 0, (
+                f"Overlapping documents are currently not supported when adding eval data.\n"
+                f"Please set 'split_overlap=0' in the supplied PreProcessor.")
+            assert preprocessor.clean_empty_lines == False, (
+                f"clean_empty_lines currently not supported when adding eval data.\n"
+                f"Please set 'clean_empty_lines=False' in the supplied PreProcessor."
+            )
+            assert preprocessor.clean_whitespace == False, (
+                f"clean_whitespace is currently not supported when adding eval data.\n"
+                f"Please set 'clean_whitespace=False' in the supplied PreProcessor."
+            )
+            assert preprocessor.clean_header_footer == False, (
+                f"clean_header_footer is currently not supported when adding eval data.\n"
+                f"Please set 'clean_header_footer=False' in the supplied PreProcessor."
+            )
+
+        file_path = Path(filename)
+        if file_path.suffix == ".json":
+            if batch_size is None:
+                docs, labels = eval_data_from_json(filename,
+                                                   max_docs=max_docs,
+                                                   preprocessor=preprocessor,
+                                                   open_domain=open_domain)
+                self.write_documents(docs, index=doc_index, headers=headers)
+                self.write_labels(labels, index=label_index, headers=headers)
+            else:
+                jsonl_filename = (file_path.parent /
+                                  (file_path.stem + ".jsonl")).as_posix()
+                logger.info(
+                    f"Adding evaluation data batch-wise is not compatible with json-formatted SQuAD files. "
+                    f"Converting json to jsonl to: {jsonl_filename}")
+                squad_json_to_jsonl(filename, jsonl_filename)
+                self.add_eval_data(jsonl_filename,
+                                   doc_index,
+                                   label_index,
+                                   batch_size,
+                                   open_domain=open_domain,
+                                   headers=headers)
+
+        elif file_path.suffix == ".jsonl":
+            for docs, labels in eval_data_from_jsonl(filename,
+                                                     batch_size,
+                                                     max_docs=max_docs,
+                                                     preprocessor=preprocessor,
+                                                     open_domain=open_domain):
+                if docs:
+                    self.write_documents(docs, index=doc_index, headers=headers)
+                if labels:
+                    self.write_labels(labels,
+                                      index=label_index,
+                                      headers=headers)
+
+        else:
+            logger.error("File needs to be in json or jsonl format.")
+
+    def delete_all_documents(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[str, Union[Dict, List, str, int, float,
+                                          bool]]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        pass
+
+    @abstractmethod
+    def delete_documents(
+        self,
+        index: Optional[str] = None,
+        ids: Optional[List[str]] = None,
+        filters: Optional[Dict[str, Union[Dict, List, str, int, float,
+                                          bool]]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        pass
+
+    @abstractmethod
+    def delete_labels(
+        self,
+        index: Optional[str] = None,
+        ids: Optional[List[str]] = None,
+        filters: Optional[Dict[str, Union[Dict, List, str, int, float,
+                                          bool]]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        pass
+
+    @abstractmethod
+    def _create_document_field_map(self) -> Dict:
+        pass
+
+    def run(self,
+            documents: List[dict],
+            index: Optional[str] = None,
+            headers: Optional[Dict[str, str]] = None,
+            id_hash_keys: Optional[List[str]] = None):  # type: ignore
+        """
+        Run requests of document stores
+
+        Comment: We will gradually introduce the primitives. The doument stores also accept dicts and parse them to documents.
+        In the future, however, only documents themselves will be accepted. Parsing the dictionaries in the run function
+        is therefore only an interim solution until the run function also accepts documents.
+
+        :param documents: A list of dicts that are documents.
+        :param headers: A list of headers.
+        :param index: Optional name of index where the documents shall be written to.
+                      If None, the DocumentStore's default index (self.index) will be used.
+        :param id_hash_keys: List of the fields that the hashes of the ids are generated from.
+        """
+
+        field_map = self._create_document_field_map()
+        doc_objects = [
+            Document.from_dict(d,
+                               field_map=field_map,
+                               id_hash_keys=id_hash_keys) for d in documents
+        ]
+        self.write_documents(documents=doc_objects,
+                             index=index,
+                             headers=headers)
+        return {}, "output_1"
+
+    @abstractmethod
+    def get_documents_by_id(
+        self,
+        ids: List[str],
+        index: Optional[str] = None,
+        batch_size: int = 10_000,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+        pass
+
+    def _drop_duplicate_documents(self,
+                                  documents: List[Document]) -> List[Document]:
+        """
+        Drop duplicates documents based on same hash ID
+
+        :param documents: A list of pipelines Document objects.
+        :return: A list of pipelines Document objects.
+        """
+        _hash_ids: Set = set([])
+        _documents: List[Document] = []
+
+        for document in documents:
+            if document.id in _hash_ids:
+                logger.info(
+                    f"Duplicate Documents: Document with id '{document.id}' already exists in index "
+                    f"'{self.index}'")
+                continue
+            _documents.append(document)
+            _hash_ids.add(document.id)
+
+        return _documents
+
+    def _handle_duplicate_documents(
+        self,
+        documents: List[Document],
+        index: Optional[str] = None,
+        duplicate_documents: Optional[str] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Checks whether any of the passed documents is already existing in the chosen index and returns a list of
+        documents that are not in the index yet.
+
+        :param documents: A list of pipelines Document objects.
+        :param duplicate_documents: Handle duplicates document based on parameter options.
+                                    Parameter options : ( 'skip','overwrite','fail')
+                                    skip (default option): Ignore the duplicates documents
+                                    overwrite: Update any existing documents with the same ID when adding documents.
+                                    fail: an error is raised if the document ID of the document being added already
+                                    exists.
+        :param headers: Custom HTTP headers to pass to document store client if supported (e.g. {'Authorization': 'Basic YWRtaW46cm9vdA=='} for basic authentication)
+        :return: A list of pipelines Document objects.
+        """
+
+        index = index or self.index
+        if duplicate_documents in ("skip", "fail"):
+            documents = self._drop_duplicate_documents(documents)
+            documents_found = self.get_documents_by_id(
+                ids=[doc.id for doc in documents], index=index, headers=headers)
+            ids_exist_in_db: List[str] = [doc.id for doc in documents_found]
+
+            if len(ids_exist_in_db) > 0 and duplicate_documents == "fail":
+                raise DuplicateDocumentError(
+                    f"Document with ids '{', '.join(ids_exist_in_db)} already exists"
+                    f" in index = '{index}'.")
+
+            documents = list(
+                filter(lambda doc: doc.id not in ids_exist_in_db, documents))
+
+        return documents
+
+    def _get_duplicate_labels(
+            self,
+            labels: list,
+            index: str = None,
+            headers: Optional[Dict[str, str]] = None) -> List[Label]:
+        """
+        Return all duplicate labels
+        :param labels: List of Label objects
+        :param index: add an optional index attribute to labels. It can be later used for filtering.
+        :param headers: Custom HTTP headers to pass to document store client if supported (e.g. {'Authorization': 'Basic YWRtaW46cm9vdA=='} for basic authentication)
+        :return: List of labels
+        """
+        index = index or self.label_index
+        new_ids: List[str] = [label.id for label in labels]
+        duplicate_ids: List[str] = []
+
+        for label_id, count in collections.Counter(new_ids).items():
+            if count > 1:
+                duplicate_ids.append(label_id)
+
+        for label in self.get_all_labels(index=index, headers=headers):
+            if label.id in new_ids:
+                duplicate_ids.append(label.id)
+
+        return [label for label in labels if label.id in duplicate_ids]
+
+
+class KeywordDocumentStore(BaseDocumentStore):
+    """
+    Base class for implementing Document Stores that support keyword searches.
+    """
+
+    @abstractmethod
+    def query(
+        self,
+        query: Optional[str],
+        filters: Optional[Dict[str, Union[Dict, List, str, int, float,
+                                          bool]]] = None,
+        top_k: int = 10,
+        custom_query: Optional[str] = None,
+        index: Optional[str] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+        """
+        Scan through documents in DocumentStore and return a small number documents
+        that are most relevant to the query as defined by keyword matching algorithms like BM25.
+
+        :param query: The query
+        :param filters: Optional filters to narrow down the search space to documents whose metadata fulfill certain
+                        conditions.
+                        Filters are defined as nested dictionaries. The keys of the dictionaries can be a logical
+                        operator (`"$and"`, `"$or"`, `"$not"`), a comparison operator (`"$eq"`, `"$in"`, `"$gt"`,
+                        `"$gte"`, `"$lt"`, `"$lte"`) or a metadata field name.
+                        Logical operator keys take a dictionary of metadata field names and/or logical operators as
+                        value. Metadata field names take a dictionary of comparison operators as value. Comparison
+                        operator keys take a single value or (in case of `"$in"`) a list of values as value.
+                        If no logical operator is provided, `"$and"` is used as default operation. If no comparison
+                        operator is provided, `"$eq"` (or `"$in"` if the comparison value is a list) is used as default
+                        operation.
+
+                            __Example__:
+                            ```python
+                            filters = {
+                                "$and": {
+                                    "type": {"$eq": "article"},
+                                    "date": {"$gte": "2015-01-01", "$lt": "2021-01-01"},
+                                    "rating": {"$gte": 3},
+                                    "$or": {
+                                        "genre": {"$in": ["economy", "politics"]},
+                                        "publisher": {"$eq": "nytimes"}
+                                    }
+                                }
+                            }
+                            # or simpler using default operators
+                            filters = {
+                                "type": "article",
+                                "date": {"$gte": "2015-01-01", "$lt": "2021-01-01"},
+                                "rating": {"$gte": 3},
+                                "$or": {
+                                    "genre": ["economy", "politics"],
+                                    "publisher": "nytimes"
+                                }
+                            }
+                            ```
+
+                            To use the same logical operator multiple times on the same level, logical operators take
+                            optionally a list of dictionaries as value.
+
+                            __Example__:
+                            ```python
+                            filters = {
+                                "$or": [
+                                    {
+                                        "$and": {
+                                            "Type": "News Paper",
+                                            "Date": {
+                                                "$lt": "2019-01-01"
+                                            }
+                                        }
+                                    },
+                                    {
+                                        "$and": {
+                                            "Type": "Blog Post",
+                                            "Date": {
+                                                "$gte": "2019-01-01"
+                                            }
+                                        }
+                                    }
+                                ]
+                            }
+                            ```
+
+        :param top_k: How many documents to return per query.
+        :param custom_query: Custom query to be executed.
+        :param index: The name of the index in the DocumentStore from which to retrieve documents
+        :param headers: Custom HTTP headers to pass to document store client if supported (e.g. {'Authorization': 'Basic YWRtaW46cm9vdA=='} for basic authentication)
+        """
+
+
+def get_batches_from_generator(iterable, n):
+    """
+    Batch elements of an iterable into fixed-length chunks or blocks.
+    """
+    it = iter(iterable)
+    x = tuple(islice(it, n))
+    while x:
+        yield x
+        x = tuple(islice(it, n))

applications/experimental/pipelines/pipelines/document_stores/faiss.py

@@ -0,0 +1,754 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pipelines.nodes.retriever import BaseRetriever
+
+import json
+import logging
+from pathlib import Path
+from typing import Union, List, Optional, Dict, Generator
+from tqdm.auto import tqdm
+import warnings
+import numpy as np
+from inspect import Signature, signature
+
+try:
+    import faiss
+    from pipelines.document_stores.sql import (
+        SQLDocumentStore,
+    )  # its deps are optional, but get installed with the `faiss` extra
+except (ImportError, ModuleNotFoundError) as ie:
+    from pipelines.utils.import_utils import _optional_component_not_installed
+
+    _optional_component_not_installed(__name__, "faiss", ie)
+
+from pipelines.schema import Document
+from pipelines.document_stores.base import get_batches_from_generator
+
+logger = logging.getLogger(__name__)
+
+
+class FAISSDocumentStore(SQLDocumentStore):
+    """
+    Document store for very large scale embedding based dense retrievers.
+
+    It implements the FAISS library(https://github.com/facebookresearch/faiss)
+    to perform similarity search on vectors.
+
+    The document text and meta-data (for filtering) are stored using the SQLDocumentStore, while
+    the vector embeddings are indexed in a FAISS Index.
+    """
+
+    def __init__(
+        self,
+        sql_url: str = "sqlite:///faiss_document_store.db",
+        vector_dim: int = None,
+        embedding_dim: int = 768,
+        faiss_index_factory_str: str = "Flat",
+        faiss_index: "Optional[faiss.swigfaiss.Index]" = None,
+        return_embedding: bool = False,
+        index_name: str = "document",
+        similarity: str = "dot_product",
+        embedding_field: str = "embedding",
+        progress_bar: bool = True,
+        duplicate_documents: str = "overwrite",
+        faiss_index_path: Union[str, Path] = None,
+        faiss_config_path: Union[str, Path] = None,
+        isolation_level: str = None,
+        **kwargs,
+    ):
+        """
+        :param sql_url: SQL connection URL for database. It defaults to local file based SQLite DB. For large scale
+                        deployment, Postgres is recommended.
+        :param vector_dim: Deprecated. Use embedding_dim instead.
+        :param embedding_dim: The embedding vector size. Default: 768.
+        :param faiss_index_factory_str: Create a new FAISS index of the specified type.
+                                        The type is determined from the given string following the conventions
+                                        of the original FAISS index factory.
+                                        Recommended options:
+                                        - "Flat" (default): Best accuracy (= exact). Becomes slow and RAM intense for > 1 Mio docs.
+                                        - "HNSW": Graph-based heuristic. If not further specified,
+                                                  we use the following config:
+                                                  HNSW64, efConstruction=80 and efSearch=20
+                                        - "IVFx,Flat": Inverted Index. Replace x with the number of centroids aka nlist.
+                                                          Rule of thumb: nlist = 10 * sqrt (num_docs) is a good starting point.
+                                        For more details see:
+                                        - Overview of indices https://github.com/facebookresearch/faiss/wiki/Faiss-indexes
+                                        - Guideline for choosing an index https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index
+                                        - FAISS Index factory https://github.com/facebookresearch/faiss/wiki/The-index-factory
+                                        Benchmarks: XXX
+        :param faiss_index: Pass an existing FAISS Index, i.e. an empty one that you configured manually
+                            or one with docs that you used in pipelines before and want to load again.
+        :param return_embedding: To return document embedding. Unlike other document stores, FAISS will return normalized embeddings
+        :param index_name: Name of index in document store to use.
+        :param similarity: The similarity function used to compare document vectors. 
+                   In both cases, the returned values in Document.score are normalized to be in range [0,1]:
+                   For `dot_product`: expit(np.asarray(raw_score / 100))
+                   FOr `cosine`: (raw_score + 1) / 2
+        :param embedding_field: Name of field containing an embedding vector.
+        :param progress_bar: Whether to show a tqdm progress bar or not.
+                             Can be helpful to disable in production deployments to keep the logs clean.
+        :param duplicate_documents: Handle duplicates document based on parameter options.
+                                    Parameter options : ( 'skip','overwrite','fail')
+                                    skip: Ignore the duplicates documents
+                                    overwrite: Update any existing documents with the same ID when adding documents.
+                                    fail: an error is raised if the document ID of the document being added already
+                                    exists.
+        :param faiss_index_path: Stored FAISS index file. Can be created via calling `save()`.
+            If specified no other params besides faiss_config_path must be specified.
+        :param faiss_config_path: Stored FAISS initial configuration parameters.
+            Can be created via calling `save()`
+        :param isolation_level: see SQLAlchemy's `isolation_level` parameter for `create_engine()` (https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine.params.isolation_level)
+        """
+        # special case if we want to load an existing index from disk
+        # load init params from disk and run init again
+        if faiss_index_path is not None:
+            sig = signature(self.__class__.__init__)
+            self._validate_params_load_from_disk(sig, locals(), kwargs)
+            init_params = self._load_init_params_from_config(
+                faiss_index_path, faiss_config_path)
+            self.__class__.__init__(self, **init_params)  # pylint: disable=non-parent-init-called
+            return
+
+        # save init parameters to enable export of component config as YAML
+        self.set_config(
+            sql_url=sql_url,
+            vector_dim=vector_dim,
+            embedding_dim=embedding_dim,
+            faiss_index_factory_str=faiss_index_factory_str,
+            return_embedding=return_embedding,
+            duplicate_documents=duplicate_documents,
+            index=index_name,
+            similarity=similarity,
+            embedding_field=embedding_field,
+            progress_bar=progress_bar,
+            isolation_level=isolation_level,
+        )
+
+        if similarity in ("dot_product", "cosine"):
+            self.similarity = similarity
+            self.metric_type = faiss.METRIC_INNER_PRODUCT
+        elif similarity == "l2":
+            self.similarity = similarity
+            self.metric_type = faiss.METRIC_L2
+        else:
+            raise ValueError(
+                "The FAISS document store can currently only support dot_product, cosine and l2 similarity. "
+                "Please set similarity to one of the above.")
+
+        if vector_dim is not None:
+            warnings.warn(
+                "The 'vector_dim' parameter is deprecated, "
+                "use 'embedding_dim' instead.", DeprecationWarning, 2)
+            self.embedding_dim = vector_dim
+        else:
+            self.embedding_dim = embedding_dim
+
+        self.faiss_index_factory_str = faiss_index_factory_str
+        self.faiss_indexes: Dict[str, faiss.swigfaiss.Index] = {}
+        if faiss_index:
+            self.faiss_indexes[index_name] = faiss_index
+        else:
+            self.faiss_indexes[index_name] = self._create_new_index(
+                embedding_dim=self.embedding_dim,
+                index_factory=faiss_index_factory_str,
+                metric_type=self.metric_type,
+                **kwargs,
+            )
+
+        self.return_embedding = return_embedding
+        self.embedding_field = embedding_field
+
+        self.progress_bar = progress_bar
+
+        super().__init__(url=sql_url,
+                         index=index_name,
+                         duplicate_documents=duplicate_documents,
+                         isolation_level=isolation_level)
+
+        self._validate_index_sync()
+
+    def _validate_params_load_from_disk(self, sig: Signature, locals: dict,
+                                        kwargs: dict):
+        allowed_params = [
+            "faiss_index_path", "faiss_config_path", "self", "kwargs"
+        ]
+        invalid_param_set = False
+
+        for param in sig.parameters.values():
+            if param.name not in allowed_params and param.default != locals[
+                    param.name]:
+                invalid_param_set = True
+                break
+
+        if invalid_param_set or len(kwargs) > 0:
+            raise ValueError(
+                "if faiss_index_path is passed no other params besides faiss_config_path are allowed."
+            )
+
+    def _validate_index_sync(self):
+        # This check ensures the correct document database was loaded.
+        # If it fails, make sure you provided the path to the database
+        # used when creating the original FAISS index
+        logger.info(
+            f"document_cnt:{self.get_document_count()}\tembedding_cnt:{self.get_embedding_count()}"
+        )
+        if not self.get_document_count() == self.get_embedding_count():
+            raise ValueError(
+                "The number of documents present in the SQL database does not "
+                "match the number of embeddings in FAISS. Make sure your FAISS "
+                "configuration file correctly points to the same database that "
+                "was used when creating the original index.")
+
+    def _create_new_index(self,
+                          embedding_dim: int,
+                          metric_type,
+                          index_factory: str = "Flat",
+                          **kwargs):
+        if index_factory == "HNSW":
+            # faiss index factory doesn't give the same results for HNSW IP, therefore direct init.
+            n_links = kwargs.get("n_links", 64)
+            index = faiss.IndexHNSWFlat(embedding_dim, n_links, metric_type)
+            index.hnsw.efSearch = kwargs.get("efSearch", 20)  # 20
+            index.hnsw.efConstruction = kwargs.get("efConstruction", 80)  # 80
+            if "ivf" in index_factory.lower(
+            ):  # enable reconstruction of vectors for inverted index
+                self.faiss_indexes[index].set_direct_map_type(
+                    faiss.DirectMap.Hashtable)
+
+            logger.info(
+                f"HNSW params: n_links: {n_links}, efSearch: {index.hnsw.efSearch}, efConstruction: {index.hnsw.efConstruction}"
+            )
+        else:
+            index = faiss.index_factory(embedding_dim, index_factory,
+                                        metric_type)
+        return index
+
+    def write_documents(
+        self,
+        documents: Union[List[dict], List[Document]],
+        index: Optional[str] = None,
+        batch_size: int = 10_000,
+        duplicate_documents: Optional[str] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> None:
+        """
+        Add new documents to the DocumentStore.
+
+        :param documents: List of `Dicts` or List of `Documents`. If they already contain the embeddings, we'll index
+                          them right away in FAISS. If not, you can later call update_embeddings() to create & index them.
+        :param index: (SQL) index name for storing the docs and metadata
+        :param batch_size: When working with large number of documents, batching can help reduce memory footprint.
+        :param duplicate_documents: Handle duplicates document based on parameter options.
+                                    Parameter options : ( 'skip','overwrite','fail')
+                                    skip: Ignore the duplicates documents
+                                    overwrite: Update any existing documents with the same ID when adding documents.
+                                    fail: an error is raised if the document ID of the document being added already
+                                    exists.
+        :raises DuplicateDocumentError: Exception trigger on duplicate document
+        :return: None
+        """
+        if headers:
+            raise NotImplementedError(
+                "FAISSDocumentStore does not support headers.")
+
+        index = index or self.index
+        duplicate_documents = duplicate_documents or self.duplicate_documents
+        assert (
+            duplicate_documents in self.duplicate_documents_options
+        ), f"duplicate_documents parameter must be {', '.join(self.duplicate_documents_options)}"
+
+        if not self.faiss_indexes.get(index):
+            self.faiss_indexes[index] = self._create_new_index(
+                embedding_dim=self.embedding_dim,
+                index_factory=self.faiss_index_factory_str,
+                metric_type=faiss.METRIC_INNER_PRODUCT,
+            )
+
+        field_map = self._create_document_field_map()
+        document_objects = [
+            Document.from_dict(d, field_map=field_map)
+            if isinstance(d, dict) else d for d in documents
+        ]
+        document_objects = self._handle_duplicate_documents(
+            documents=document_objects,
+            index=index,
+            duplicate_documents=duplicate_documents)
+        if len(document_objects) > 0:
+            add_vectors = False if document_objects[
+                0].embedding is None else True
+
+            if self.duplicate_documents == "overwrite" and add_vectors:
+                logger.warning(
+                    "You have to provide `duplicate_documents = 'overwrite'` arg and "
+                    "`FAISSDocumentStore` does not support update in existing `faiss_index`.\n"
+                    "Please call `update_embeddings` method to repopulate `faiss_index`"
+                )
+
+            vector_id = self.faiss_indexes[index].ntotal
+            with tqdm(total=len(document_objects),
+                      disable=not self.progress_bar,
+                      position=0,
+                      desc="Writing Documents") as progress_bar:
+                for i in range(0, len(document_objects), batch_size):
+                    if add_vectors:
+                        embeddings = [
+                            doc.embedding
+                            for doc in document_objects[i:i + batch_size]
+                        ]
+                        embeddings_to_index = np.array(embeddings,
+                                                       dtype="float32")
+
+                        if self.similarity == "cosine":
+                            self.normalize_embedding(embeddings_to_index)
+
+                        self.faiss_indexes[index].add(embeddings_to_index)
+
+                    docs_to_write_in_sql = []
+                    for doc in document_objects[i:i + batch_size]:
+                        meta = doc.meta
+                        if add_vectors:
+                            meta["vector_id"] = vector_id
+                            vector_id += 1
+                        docs_to_write_in_sql.append(doc)
+
+                    super(FAISSDocumentStore, self).write_documents(
+                        docs_to_write_in_sql,
+                        index=index,
+                        duplicate_documents=duplicate_documents,
+                        batch_size=batch_size,
+                    )
+                    progress_bar.update(batch_size)
+            progress_bar.close()
+
+    def _create_document_field_map(self) -> Dict:
+        return {
+            self.index: self.embedding_field,
+        }
+
+    def update_embeddings(
+        self,
+        retriever: "BaseRetriever",
+        index: Optional[str] = None,
+        update_existing_embeddings: bool = True,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in FAISSDocStore
+        batch_size: int = 10_000,
+    ):
+        """
+        Updates the embeddings in the the document store using the encoding model specified in the retriever.
+        This can be useful if want to add or change the embeddings for your documents (e.g. after changing the retriever config).
+
+        :param retriever: Retriever to use to get embeddings for text
+        :param index: Index name for which embeddings are to be updated. If set to None, the default self.index is used.
+        :param update_existing_embeddings: Whether to update existing embeddings of the documents. If set to False,
+                                           only documents without embeddings are processed. This mode can be used for
+                                           incremental updating of embeddings, wherein, only newly indexed documents
+                                           get processed.
+        :param filters: Optional filters to narrow down the documents for which embeddings are to be updated.
+                        Example: {"name": ["some", "more"], "category": ["only_one"]}
+        :param batch_size: When working with large number of documents, batching can help reduce memory footprint.
+        :return: None
+        """
+        index = index or self.index
+
+        if update_existing_embeddings is True:
+            if filters is None:
+                self.faiss_indexes[index].reset()
+                self.reset_vector_ids(index)
+            else:
+                raise Exception(
+                    "update_existing_embeddings=True is not supported with filters."
+                )
+
+        if not self.faiss_indexes.get(index):
+            raise ValueError(
+                "Couldn't find a FAISS index. Try to init the FAISSDocumentStore() again ..."
+            )
+
+        document_count = self.get_document_count(index=index)
+        if document_count == 0:
+            logger.warning(
+                "Calling DocumentStore.update_embeddings() on an empty index")
+            return
+
+        logger.info(f"Updating embeddings for {document_count} docs...")
+        vector_id = sum([index.ntotal for index in self.faiss_indexes.values()])
+
+        # Query texts from SQL.
+        result = self._query(
+            index=index,
+            vector_ids=None,
+            batch_size=batch_size,
+            filters=filters,
+            only_documents_without_embedding=not update_existing_embeddings,
+        )
+        batched_documents = get_batches_from_generator(result, batch_size)
+        with tqdm(total=document_count,
+                  disable=not self.progress_bar,
+                  position=0,
+                  unit=" docs",
+                  desc="Updating Embedding") as progress_bar:
+            for document_batch in batched_documents:
+                embeddings = retriever.embed_documents(
+                    document_batch)  # type: ignore
+                assert len(document_batch) == len(embeddings)
+
+                embeddings_to_index = np.array(embeddings, dtype="float32")
+
+                if self.similarity == "cosine":
+                    self.normalize_embedding(embeddings_to_index)
+
+                self.faiss_indexes[index].add(embeddings_to_index)
+
+                vector_id_map = {}
+                for doc in document_batch:
+                    vector_id_map[str(doc.id)] = str(vector_id)
+                    vector_id += 1
+                self.update_vector_ids(vector_id_map, index=index)
+                progress_bar.set_description_str("Documents Processed")
+                progress_bar.update(batch_size)
+
+    def get_all_documents(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in FAISSDocStore
+        return_embedding: Optional[bool] = None,
+        batch_size: int = 10_000,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+        if headers:
+            raise NotImplementedError(
+                "FAISSDocumentStore does not support headers.")
+
+        result = self.get_all_documents_generator(
+            index=index,
+            filters=filters,
+            return_embedding=return_embedding,
+            batch_size=batch_size)
+        documents = list(result)
+        return documents
+
+    def get_all_documents_generator(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in FAISSDocStore
+        return_embedding: Optional[bool] = None,
+        batch_size: int = 10_000,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Generator[Document, None, None]:
+        """
+        Get all documents from the document store. Under-the-hood, documents are fetched in batches from the
+        document store and yielded as individual documents. This method can be used to iteratively process
+        a large number of documents without having to load all documents in memory.
+
+        :param index: Name of the index to get the documents from. If None, the
+                      DocumentStore's default index (self.index) will be used.
+        :param filters: Optional filters to narrow down the documents to return.
+                        Example: {"name": ["some", "more"], "category": ["only_one"]}
+        :param return_embedding: Whether to return the document embeddings. Unlike other document stores, FAISS will return normalized embeddings
+        :param batch_size: When working with large number of documents, batching can help reduce memory footprint.
+        """
+        if headers:
+            raise NotImplementedError(
+                "FAISSDocumentStore does not support headers.")
+
+        index = index or self.index
+        documents = super(FAISSDocumentStore, self).get_all_documents_generator(
+            index=index,
+            filters=filters,
+            batch_size=batch_size,
+            return_embedding=False)
+        if return_embedding is None:
+            return_embedding = self.return_embedding
+
+        for doc in documents:
+            if return_embedding:
+                if doc.meta and doc.meta.get("vector_id") is not None:
+                    doc.embedding = self.faiss_indexes[index].reconstruct(
+                        int(doc.meta["vector_id"]))
+            yield doc
+
+    def get_documents_by_id(
+        self,
+        ids: List[str],
+        index: Optional[str] = None,
+        batch_size: int = 10_000,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+        if headers:
+            raise NotImplementedError(
+                "FAISSDocumentStore does not support headers.")
+
+        index = index or self.index
+        documents = super(FAISSDocumentStore,
+                          self).get_documents_by_id(ids=ids,
+                                                    index=index,
+                                                    batch_size=batch_size)
+        if self.return_embedding:
+            for doc in documents:
+                if doc.meta and doc.meta.get("vector_id") is not None:
+                    doc.embedding = self.faiss_indexes[index].reconstruct(
+                        int(doc.meta["vector_id"]))
+        return documents
+
+    def get_embedding_count(self,
+                            index: Optional[str] = None,
+                            filters: Optional[Dict[str, Any]] = None) -> int:
+        """
+        Return the count of embeddings in the document store.
+        """
+        if filters:
+            raise Exception(
+                "filters are not supported for get_embedding_count in FAISSDocumentStore"
+            )
+        index = index or self.index
+        return self.faiss_indexes[index].ntotal
+
+    def train_index(
+        self,
+        documents: Optional[Union[List[dict], List[Document]]],
+        embeddings: Optional[np.ndarray] = None,
+        index: Optional[str] = None,
+    ):
+        """
+        Some FAISS indices (e.g. IVF) require initial "training" on a sample of vectors before you can add your final vectors.
+        The train vectors should come from the same distribution as your final ones.
+        You can pass either documents (incl. embeddings) or just the plain embeddings that the index shall be trained on.
+
+        :param documents: Documents (incl. the embeddings)
+        :param embeddings: Plain embeddings
+        :param index: Name of the index to train. If None, the DocumentStore's default index (self.index) will be used.
+        :return: None
+        """
+        index = index or self.index
+        if embeddings and documents:
+            raise ValueError(
+                "Either pass `documents` or `embeddings`. You passed both.")
+        if documents:
+            document_objects = [
+                Document.from_dict(d) if isinstance(d, dict) else d
+                for d in documents
+            ]
+            doc_embeddings = [doc.embedding for doc in document_objects]
+            embeddings_for_train = np.array(doc_embeddings, dtype="float32")
+            self.faiss_indexes[index].train(embeddings_for_train)
+        if embeddings:
+            self.faiss_indexes[index].train(embeddings)
+
+    def delete_all_documents(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in FAISSDocStore
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Delete all documents from the document store.
+        """
+        if headers:
+            raise NotImplementedError(
+                "FAISSDocumentStore does not support headers.")
+
+        logger.warning("""DEPRECATION WARNINGS: 
+                1. delete_all_documents() method is deprecated, please use delete_documents method
+                """)
+        self.delete_documents(index, None, filters)
+
+    def delete_documents(
+        self,
+        index: Optional[str] = None,
+        ids: Optional[List[str]] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in FAISSDocStore
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Delete documents from the document store. All documents are deleted if no filters are passed.
+
+        :param index: Index name to delete the documents from. If None, the
+                      DocumentStore's default index (self.index) will be used.
+        :param ids: Optional list of IDs to narrow down the documents to be deleted.
+        :param filters: Optional filters to narrow down the documents to be deleted.
+            Example filters: {"name": ["some", "more"], "category": ["only_one"]}.
+            If filters are provided along with a list of IDs, this method deletes the
+            intersection of the two query results (documents that match the filters and
+            have their ID in the list).
+        :return: None
+        """
+        if headers:
+            raise NotImplementedError(
+                "FAISSDocumentStore does not support headers.")
+
+        index = index or self.index
+        if index in self.faiss_indexes.keys():
+            if not filters and not ids:
+                self.faiss_indexes[index].reset()
+            else:
+                affected_docs = self.get_all_documents(filters=filters)
+                if ids:
+                    affected_docs = [
+                        doc for doc in affected_docs if doc.id in ids
+                    ]
+                doc_ids = [
+                    doc.meta.get("vector_id") for doc in affected_docs
+                    if doc.meta and doc.meta.get("vector_id") is not None
+                ]
+                self.faiss_indexes[index].remove_ids(
+                    np.array(doc_ids, dtype="int64"))
+
+        super().delete_documents(index=index, ids=ids, filters=filters)
+
+    def query_by_embedding(
+        self,
+        query_emb: np.ndarray,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in FAISSDocStore
+        top_k: int = 10,
+        index: Optional[str] = None,
+        return_embedding: Optional[bool] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+        """
+        Find the document that is most similar to the provided `query_emb` by using a vector similarity metric.
+
+        :param query_emb: Embedding of the query.
+        :param filters: Optional filters to narrow down the search space.
+                        Example: {"name": ["some", "more"], "category": ["only_one"]}
+        :param top_k: How many documents to return
+        :param index: Index name to query the document from.
+        :param return_embedding: To return document embedding. Unlike other document stores, FAISS will return normalized embeddings
+        :return:
+        """
+        if headers:
+            raise NotImplementedError(
+                "FAISSDocumentStore does not support headers.")
+
+        if filters:
+            logger.warning(
+                "Query filters are not implemented for the FAISSDocumentStore.")
+
+        index = index or self.index
+        if not self.faiss_indexes.get(index):
+            raise Exception(
+                f"Index named '{index}' does not exists. Use 'update_embeddings()' to create an index."
+            )
+
+        if return_embedding is None:
+            return_embedding = self.return_embedding
+
+        query_emb = query_emb.reshape(1, -1).astype(np.float32)
+
+        if self.similarity == "cosine":
+            self.normalize_embedding(query_emb)
+
+        score_matrix, vector_id_matrix = self.faiss_indexes[index].search(
+            query_emb, top_k)
+        vector_ids_for_query = [
+            str(vector_id) for vector_id in vector_id_matrix[0]
+            if vector_id != -1
+        ]
+
+        documents = self.get_documents_by_vector_ids(vector_ids_for_query,
+                                                     index=index)
+
+        # assign query score to each document
+        scores_for_vector_ids: Dict[str, float] = {
+            str(v_id): s
+            for v_id, s in zip(vector_id_matrix[0], score_matrix[0])
+        }
+        for doc in documents:
+            raw_score = scores_for_vector_ids[doc.meta["vector_id"]]
+            doc.ann_score = self.finalize_raw_score(raw_score, self.similarity)
+
+            if return_embedding is True:
+                doc.embedding = self.faiss_indexes[index].reconstruct(
+                    int(doc.meta["vector_id"]))
+
+        return documents
+
+    def save(self,
+             index_path: Union[str, Path],
+             config_path: Optional[Union[str, Path]] = None):
+        """
+        Save FAISS Index to the specified file.
+
+        :param index_path: Path to save the FAISS index to.
+        :param config_path: Path to save the initial configuration parameters to.
+            Defaults to the same as the file path, save the extension (.json).
+            This file contains all the parameters passed to FAISSDocumentStore()
+            at creation time (for example the SQL path, embedding_dim, etc), and will be
+            used by the `load` method to restore the index with the appropriate configuration.
+        :return: None
+        """
+        if not config_path:
+            index_path = Path(index_path)
+            config_path = index_path.with_suffix(".json")
+
+        faiss.write_index(self.faiss_indexes[self.index], str(index_path))
+        with open(config_path, "w") as ipp:
+            json.dump(self.pipeline_config["params"], ipp)
+
+    def _load_init_params_from_config(
+            self,
+            index_path: Union[str, Path],
+            config_path: Optional[Union[str, Path]] = None):
+        if not config_path:
+            index_path = Path(index_path)
+            config_path = index_path.with_suffix(".json")
+
+        init_params: dict = {}
+        try:
+            with open(config_path, "r") as ipp:
+                init_params = json.load(ipp)
+        except OSError as e:
+            raise ValueError(
+                f"Can't open FAISS configuration file `{config_path}`. "
+                "Make sure the file exists and the you have the correct permissions "
+                "to access it.") from e
+
+        faiss_index = faiss.read_index(str(index_path))
+
+        # Add other init params to override the ones defined in the init params file
+        init_params["faiss_index"] = faiss_index
+        init_params["embedding_dim"] = faiss_index.d
+
+        return init_params
+
+    @classmethod
+    def load(cls,
+             index_path: Union[str, Path],
+             config_path: Optional[Union[str, Path]] = None):
+        """
+        Load a saved FAISS index from a file and connect to the SQL database.
+        Note: In order to have a correct mapping from FAISS to SQL,
+              make sure to use the same SQL DB that you used when calling `save()`.
+
+        :param index_path: Stored FAISS index file. Can be created via calling `save()`
+        :param config_path: Stored FAISS initial configuration parameters.
+            Can be created via calling `save()`
+        """
+        return cls(faiss_index_path=index_path, faiss_config_path=config_path)

applications/experimental/pipelines/pipelines/document_stores/filter_utils.py

@@ -0,0 +1,778 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Union, List, Dict, Optional, Tuple
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from functools import reduce
+
+from sqlalchemy.sql import select
+from sqlalchemy import and_, or_
+
+from pipelines.document_stores import utils
+
+
+def nested_defaultdict() -> defaultdict:
+    """
+    Data structure that recursively adds a dictionary as value if a key does not exist. Advantage: In nested dictionary
+    structures, we don't need to check if a key already exists (which can become hard to maintain in nested dictionaries
+    with many levels) but access the existing value if a key exists and create an empty dictionary if a key does not
+    exist.
+    """
+    return defaultdict(nested_defaultdict)
+
+
+class LogicalFilterClause(ABC):
+    """
+    Class that is able to parse a filter and convert it to the format that the underlying databases of our
+    DocumentStores require.
+
+    Filters are defined as nested dictionaries. The keys of the dictionaries can be a logical
+    operator (`"$and"`, `"$or"`, `"$not"`), a comparison operator (`"$eq"`, `"$in"`, `"$gt"`, `"$gte"`, `"$lt"`,
+    `"$lte"`) or a metadata field name.
+    Logical operator keys take a dictionary of metadata field names and/or logical operators as
+    value. Metadata field names take a dictionary of comparison operators as value. Comparison
+    operator keys take a single value or (in case of `"$in"`) a list of values as value.
+    If no logical operator is provided, `"$and"` is used as default operation. If no comparison
+    operator is provided, `"$eq"` (or `"$in"` if the comparison value is a list) is used as default
+    operation.
+    Example:
+        ```python
+        filters = {
+            "$and": {
+                "type": {"$eq": "article"},
+                "date": {"$gte": "2015-01-01", "$lt": "2021-01-01"},
+                "rating": {"$gte": 3},
+                "$or": {
+                    "genre": {"$in": ["economy", "politics"]},
+                    "publisher": {"$eq": "nytimes"}
+                }
+            }
+        }
+        # or simpler using default operators
+        filters = {
+            "type": "article",
+            "date": {"$gte": "2015-01-01", "$lt": "2021-01-01"},
+            "rating": {"$gte": 3},
+            "$or": {
+                "genre": ["economy", "politics"],
+                "publisher": "nytimes"
+            }
+        }
+        ```
+
+    To use the same logical operator multiple times on the same level, logical operators take optionally a list of
+    dictionaries as value.
+
+    Example:
+        ```python
+        filters = {
+            "$or": [
+                {
+                    "$and": {
+                        "Type": "News Paper",
+                        "Date": {
+                            "$lt": "2019-01-01"
+                        }
+                    }
+                },
+                {
+                    "$and": {
+                        "Type": "Blog Post",
+                        "Date": {
+                            "$gte": "2019-01-01"
+                        }
+                    }
+                }
+            ]
+        }
+        ```
+
+    """
+
+    def __init__(self, conditions: List[Union["LogicalFilterClause",
+                                              "ComparisonOperation"]]):
+        self.conditions = conditions
+
+    @abstractmethod
+    def evaluate(self, fields) -> bool:
+        pass
+
+    @classmethod
+    def parse(
+        cls, filter_term: Union[dict, List[dict]]
+    ) -> Union["LogicalFilterClause", "ComparisonOperation"]:
+        """
+        Parses a filter dictionary/list and returns a LogicalFilterClause instance.
+
+        :param filter_term: Dictionary or list that contains the filter definition.
+        """
+        conditions: List[Union[LogicalFilterClause, ComparisonOperation]] = []
+
+        if isinstance(filter_term, dict):
+            filter_term = [filter_term]
+        for item in filter_term:
+            for key, value in item.items():
+                if key == "$not":
+                    conditions.append(NotOperation.parse(value))
+                elif key == "$and":
+                    conditions.append(AndOperation.parse(value))
+                elif key == "$or":
+                    conditions.append(OrOperation.parse(value))
+                # Key needs to be a metadata field
+                else:
+                    conditions.extend(ComparisonOperation.parse(key, value))
+
+        if cls == LogicalFilterClause:
+            if len(conditions) == 1:
+                return conditions[0]
+            else:
+                return AndOperation(conditions)
+        else:
+            return cls(conditions)
+
+    @abstractmethod
+    def convert_to_elasticsearch(self):
+        """
+        Converts the LogicalFilterClause instance to an Elasticsearch filter.
+        """
+        pass
+
+    @abstractmethod
+    def convert_to_sql(self, meta_document_orm):
+        """
+        Converts the LogicalFilterClause instance to an SQL filter.
+        """
+        pass
+
+    def convert_to_weaviate(self):
+        """
+        Converts the LogicalFilterClause instance to a Weaviate filter.
+        """
+        pass
+
+    def _merge_es_range_queries(
+            self, conditions: List[Dict]) -> List[Dict[str, Dict]]:
+        """
+        Merges Elasticsearch range queries that perform on the same metadata field.
+        """
+
+        range_conditions = [
+            cond["range"] for cond in filter(
+                lambda condition: "range" in condition, conditions)
+        ]
+        if range_conditions:
+            conditions = [
+                condition for condition in conditions
+                if "range" not in condition
+            ]
+            range_conditions_dict = nested_defaultdict()
+            for condition in range_conditions:
+                field_name = list(condition.keys())[0]
+                operation = list(condition[field_name].keys())[0]
+                comparison_value = condition[field_name][operation]
+                range_conditions_dict[field_name][operation] = comparison_value
+
+            for field_name, comparison_operations in range_conditions_dict.items(
+            ):
+                conditions.append(
+                    {"range": {
+                        field_name: comparison_operations
+                    }})
+
+        return conditions
+
+    @abstractmethod
+    def invert(self) -> Union["LogicalFilterClause", "ComparisonOperation"]:
+        """
+        Inverts the LogicalOperation instance.
+        Necessary for Weaviate as Weaviate doesn't seem to support the 'Not' operator anymore.
+        (https://github.com/semi-technologies/weaviate/issues/1717)
+        """
+        pass
+
+
+class ComparisonOperation(ABC):
+
+    def __init__(self, field_name: str, comparison_value: Union[str, int, float,
+                                                                bool, List]):
+        self.field_name = field_name
+        self.comparison_value = comparison_value
+
+    @abstractmethod
+    def evaluate(self, fields) -> bool:
+        pass
+
+    @classmethod
+    def parse(
+        cls, field_name, comparison_clause: Union[Dict, List, str, float]
+    ) -> List["ComparisonOperation"]:
+        comparison_operations: List[ComparisonOperation] = []
+
+        if isinstance(comparison_clause, dict):
+            for comparison_operation, comparison_value in comparison_clause.items(
+            ):
+                if comparison_operation == "$eq":
+                    comparison_operations.append(
+                        EqOperation(field_name, comparison_value))
+                elif comparison_operation == "$in":
+                    comparison_operations.append(
+                        InOperation(field_name, comparison_value))
+                elif comparison_operation == "$ne":
+                    comparison_operations.append(
+                        NeOperation(field_name, comparison_value))
+                elif comparison_operation == "$nin":
+                    comparison_operations.append(
+                        NinOperation(field_name, comparison_value))
+                elif comparison_operation == "$gt":
+                    comparison_operations.append(
+                        GtOperation(field_name, comparison_value))
+                elif comparison_operation == "$gte":
+                    comparison_operations.append(
+                        GteOperation(field_name, comparison_value))
+                elif comparison_operation == "$lt":
+                    comparison_operations.append(
+                        LtOperation(field_name, comparison_value))
+                elif comparison_operation == "$lte":
+                    comparison_operations.append(
+                        LteOperation(field_name, comparison_value))
+
+        # No comparison operator is given, so we use the default operators "$in" if the comparison value is a list and
+        # "$eq" in every other case
+        elif isinstance(comparison_clause, list):
+            comparison_operations.append(
+                InOperation(field_name, comparison_clause))
+        else:
+            comparison_operations.append((EqOperation(field_name,
+                                                      comparison_clause)))
+
+        return comparison_operations
+
+    @abstractmethod
+    def convert_to_elasticsearch(self):
+        """
+        Converts the ComparisonOperation instance to an Elasticsearch query.
+        """
+        pass
+
+    @abstractmethod
+    def convert_to_sql(self, meta_document_orm):
+        """
+        Converts the ComparisonOperation instance to an SQL filter.
+        """
+        pass
+
+    @abstractmethod
+    def convert_to_weaviate(self):
+        """
+        Converts the ComparisonOperation instance to a Weaviate comparison operator.
+        """
+        pass
+
+    @abstractmethod
+    def invert(self) -> "ComparisonOperation":
+        """
+        Inverts the ComparisonOperation.
+        Necessary for Weaviate as Weaviate doesn't seem to support the 'Not' operator anymore.
+        (https://github.com/semi-technologies/weaviate/issues/1717)
+        """
+        pass
+
+    def _get_weaviate_datatype(
+        self,
+        value: Optional[Union[str, int, float, bool]] = None
+    ) -> Tuple[str, Union[str, int, float, bool]]:
+        """
+        Determines the type of the comparison value and converts it to RFC3339 format if it is as date,
+        as Weaviate requires dates to be in RFC3339 format including the time and timezone.
+
+        """
+        if value is None:
+            assert not isinstance(self.comparison_value,
+                                  list)  # Necessary for mypy
+            value = self.comparison_value
+
+        if isinstance(value, str):
+            # Check if comparison value is a date
+            try:
+                value = utils.convert_date_to_rfc3339(value)
+                data_type = "valueDate"
+            # Comparison value is a plain string
+            except ValueError:
+                data_type = "valueString"
+        elif isinstance(value, int):
+            data_type = "valueInt"
+        elif isinstance(value, float):
+            data_type = "valueNumber"
+        elif isinstance(value, bool):
+            data_type = "valueBoolean"
+        else:
+            raise ValueError(
+                f"Unsupported data type of comparison value for {self.__class__.__name__}."
+                f"Value needs to be of type str, int, float, or bool.")
+
+        return data_type, value
+
+
+class NotOperation(LogicalFilterClause):
+    """
+    Handles conversion of logical 'NOT' operations.
+    """
+
+    def evaluate(self, fields) -> bool:
+        return not any(
+            condition.evaluate(fields) for condition in self.conditions)
+
+    def convert_to_elasticsearch(self) -> Dict[str, Dict]:
+        conditions = [
+            condition.convert_to_elasticsearch()
+            for condition in self.conditions
+        ]
+        conditions = self._merge_es_range_queries(conditions)
+        return {"bool": {"must_not": conditions}}
+
+    def convert_to_sql(self, meta_document_orm):
+        conditions = [
+            meta_document_orm.document_id.in_(
+                condition.convert_to_sql(meta_document_orm))
+            for condition in self.conditions
+        ]
+        return select(meta_document_orm.document_id).filter(~or_(*conditions))
+
+    def convert_to_weaviate(
+            self) -> Dict[str, Union[str, int, float, bool, List[Dict]]]:
+        conditions = [
+            condition.invert().convert_to_weaviate()
+            for condition in self.conditions
+        ]
+        if len(conditions) > 1:
+            # Conditions in self.conditions are by default combined with AND which becomes OR according to DeMorgan
+            return {"operator": "Or", "operands": conditions}
+        else:
+            return conditions[0]
+
+    def invert(self) -> Union[LogicalFilterClause, ComparisonOperation]:
+        # This method is called when a "$not" operation is embedded in another "$not" operation. Therefore, we don't
+        # invert the operations here, as two "$not" operation annihilate each other.
+        # (If we have more than one condition, we return an AndOperation, the default logical operation for combining
+        # multiple conditions.)
+        if len(self.conditions) > 1:
+            return AndOperation(self.conditions)
+        else:
+            return self.conditions[0]
+
+
+class AndOperation(LogicalFilterClause):
+    """
+    Handles conversion of logical 'AND' operations.
+    """
+
+    def evaluate(self, fields) -> bool:
+        return all(condition.evaluate(fields) for condition in self.conditions)
+
+    def convert_to_elasticsearch(self) -> Dict[str, Dict]:
+        conditions = [
+            condition.convert_to_elasticsearch()
+            for condition in self.conditions
+        ]
+        conditions = self._merge_es_range_queries(conditions)
+        return {"bool": {"must": conditions}}
+
+    def convert_to_sql(self, meta_document_orm):
+        conditions = [
+            meta_document_orm.document_id.in_(
+                condition.convert_to_sql(meta_document_orm))
+            for condition in self.conditions
+        ]
+        return select(meta_document_orm.document_id).filter(and_(*conditions))
+
+    def convert_to_weaviate(self) -> Dict[str, Union[str, List[Dict]]]:
+        conditions = [
+            condition.convert_to_weaviate() for condition in self.conditions
+        ]
+        return {"operator": "And", "operands": conditions}
+
+    def invert(self) -> "OrOperation":
+        return OrOperation(
+            [condition.invert() for condition in self.conditions])
+
+
+class OrOperation(LogicalFilterClause):
+    """
+    Handles conversion of logical 'OR' operations.
+    """
+
+    def evaluate(self, fields) -> bool:
+        return any(condition.evaluate(fields) for condition in self.conditions)
+
+    def convert_to_elasticsearch(self) -> Dict[str, Dict]:
+        conditions = [
+            condition.convert_to_elasticsearch()
+            for condition in self.conditions
+        ]
+        conditions = self._merge_es_range_queries(conditions)
+        return {"bool": {"should": conditions}}
+
+    def convert_to_sql(self, meta_document_orm):
+        conditions = [
+            meta_document_orm.document_id.in_(
+                condition.convert_to_sql(meta_document_orm))
+            for condition in self.conditions
+        ]
+        return select(meta_document_orm.document_id).filter(or_(*conditions))
+
+    def convert_to_weaviate(self) -> Dict[str, Union[str, List[Dict]]]:
+        conditions = [
+            condition.convert_to_weaviate() for condition in self.conditions
+        ]
+        return {"operator": "Or", "operands": conditions}
+
+    def invert(self) -> AndOperation:
+        return AndOperation(
+            [condition.invert() for condition in self.conditions])
+
+
+class EqOperation(ComparisonOperation):
+    """
+    Handles conversion of the '$eq' comparison operation.
+    """
+
+    def evaluate(self, fields) -> bool:
+        if self.field_name not in fields:
+            return False
+        return fields[self.field_name] == self.comparison_value
+
+    def convert_to_elasticsearch(
+            self) -> Dict[str, Dict[str, Union[str, int, float, bool]]]:
+        assert not isinstance(
+            self.comparison_value,
+            list), "Use '$in' operation for lists as comparison values."
+        return {"term": {self.field_name: self.comparison_value}}
+
+    def convert_to_sql(self, meta_document_orm):
+        return select([meta_document_orm.document_id]).where(
+            meta_document_orm.name == self.field_name,
+            meta_document_orm.value == self.comparison_value)
+
+    def convert_to_weaviate(
+            self) -> Dict[str, Union[List[str], str, int, float, bool]]:
+        comp_value_type, comp_value = self._get_weaviate_datatype()
+        return {
+            "path": [self.field_name],
+            "operator": "Equal",
+            comp_value_type: comp_value
+        }
+
+    def invert(self) -> "NeOperation":
+        return NeOperation(self.field_name, self.comparison_value)
+
+
+class InOperation(ComparisonOperation):
+    """
+    Handles conversion of the '$in' comparison operation.
+    """
+
+    def evaluate(self, fields) -> bool:
+        if self.field_name not in fields:
+            return False
+        return fields[self.field_name] in self.comparison_value  # type: ignore
+        # is only initialized with lists, but changing the type annotation would mean duplicating __init__
+
+    def convert_to_elasticsearch(self) -> Dict[str, Dict[str, List]]:
+        assert isinstance(
+            self.comparison_value,
+            list), "'$in' operation requires comparison value to be a list."
+        return {"terms": {self.field_name: self.comparison_value}}
+
+    def convert_to_sql(self, meta_document_orm):
+        return select([meta_document_orm.document_id]).where(
+            meta_document_orm.name == self.field_name,
+            meta_document_orm.value.in_(self.comparison_value))
+
+    def convert_to_weaviate(self) -> Dict[str, Union[str, List[Dict]]]:
+        filter_dict: Dict[str, Union[str, List[Dict]]] = {
+            "operator": "Or",
+            "operands": []
+        }
+        assert isinstance(
+            self.comparison_value,
+            list), "'$in' operation requires comparison value to be a list."
+        for value in self.comparison_value:
+            comp_value_type, comp_value = self._get_weaviate_datatype(value)
+            assert isinstance(filter_dict["operands"],
+                              list)  # Necessary for mypy
+            filter_dict["operands"].append({
+                "path": [self.field_name],
+                "operator": "Equal",
+                comp_value_type: comp_value
+            })
+
+        return filter_dict
+
+    def invert(self) -> "NinOperation":
+        return NinOperation(self.field_name, self.comparison_value)
+
+
+class NeOperation(ComparisonOperation):
+    """
+    Handles conversion of the '$ne' comparison operation.
+    """
+
+    def evaluate(self, fields) -> bool:
+        if self.field_name not in fields:
+            return False
+        return fields[self.field_name] != self.comparison_value
+
+    def convert_to_elasticsearch(
+        self
+    ) -> Dict[str, Dict[str, Dict[str, Dict[str, Union[str, int, float,
+                                                       bool]]]]]:
+        assert not isinstance(
+            self.comparison_value,
+            list), "Use '$nin' operation for lists as comparison values."
+        return {
+            "bool": {
+                "must_not": {
+                    "term": {
+                        self.field_name: self.comparison_value
+                    }
+                }
+            }
+        }
+
+    def convert_to_sql(self, meta_document_orm):
+        return select([meta_document_orm.document_id]).where(
+            meta_document_orm.name == self.field_name,
+            meta_document_orm.value != self.comparison_value)
+
+    def convert_to_weaviate(
+            self) -> Dict[str, Union[List[str], str, int, float, bool]]:
+        comp_value_type, comp_value = self._get_weaviate_datatype()
+        return {
+            "path": [self.field_name],
+            "operator": "NotEqual",
+            comp_value_type: comp_value
+        }
+
+    def invert(self) -> "EqOperation":
+        return EqOperation(self.field_name, self.comparison_value)
+
+
+class NinOperation(ComparisonOperation):
+    """
+    Handles conversion of the '$nin' comparison operation.
+    """
+
+    def evaluate(self, fields) -> bool:
+        if self.field_name not in fields:
+            return False
+        return fields[
+            self.field_name] not in self.comparison_value  # type: ignore
+        # is only initialized with lists, but changing the type annotation would mean duplicating __init__
+
+    def convert_to_elasticsearch(
+            self) -> Dict[str, Dict[str, Dict[str, Dict[str, List]]]]:
+        assert isinstance(
+            self.comparison_value,
+            list), "'$nin' operation requires comparison value to be a list."
+        return {
+            "bool": {
+                "must_not": {
+                    "terms": {
+                        self.field_name: self.comparison_value
+                    }
+                }
+            }
+        }
+
+    def convert_to_sql(self, meta_document_orm):
+        return select([meta_document_orm.document_id]).where(
+            meta_document_orm.name == self.field_name,
+            meta_document_orm.value.notin_(self.comparison_value))
+
+    def convert_to_weaviate(self) -> Dict[str, Union[str, List[Dict]]]:
+        filter_dict: Dict[str, Union[str, List[Dict]]] = {
+            "operator": "And",
+            "operands": []
+        }
+        assert isinstance(
+            self.comparison_value,
+            list), "'$nin' operation requires comparison value to be a list."
+        for value in self.comparison_value:
+            comp_value_type, comp_value = self._get_weaviate_datatype(value)
+            assert isinstance(filter_dict["operands"],
+                              list)  # Necessary for mypy
+            filter_dict["operands"].append({
+                "path": [self.field_name],
+                "operator": "NotEqual",
+                comp_value_type: comp_value
+            })
+
+        return filter_dict
+
+    def invert(self) -> "InOperation":
+        return InOperation(self.field_name, self.comparison_value)
+
+
+class GtOperation(ComparisonOperation):
+    """
+    Handles conversion of the '$gt' comparison operation.
+    """
+
+    def evaluate(self, fields) -> bool:
+        if self.field_name not in fields:
+            return False
+        return fields[self.field_name] > self.comparison_value
+
+    def convert_to_elasticsearch(
+            self) -> Dict[str, Dict[str, Dict[str, Union[str, float, int]]]]:
+        assert not isinstance(
+            self.comparison_value,
+            list), "Comparison value for '$gt' operation must not be a list."
+        return {"range": {self.field_name: {"gt": self.comparison_value}}}
+
+    def convert_to_sql(self, meta_document_orm):
+        return select([meta_document_orm.document_id
+                       ]).where(meta_document_orm.name == self.field_name,
+                                meta_document_orm.value > self.comparison_value)
+
+    def convert_to_weaviate(
+            self) -> Dict[str, Union[List[str], str, float, int]]:
+        comp_value_type, comp_value = self._get_weaviate_datatype()
+        assert not isinstance(
+            comp_value,
+            list), "Comparison value for '$gt' operation must not be a list."
+        return {
+            "path": [self.field_name],
+            "operator": "GreaterThan",
+            comp_value_type: comp_value
+        }
+
+    def invert(self) -> "LteOperation":
+        return LteOperation(self.field_name, self.comparison_value)
+
+
+class GteOperation(ComparisonOperation):
+    """
+    Handles conversion of the '$gte' comparison operation.
+    """
+
+    def evaluate(self, fields) -> bool:
+        if self.field_name not in fields:
+            return False
+        return fields[self.field_name] >= self.comparison_value
+
+    def convert_to_elasticsearch(
+            self) -> Dict[str, Dict[str, Dict[str, Union[str, float, int]]]]:
+        assert not isinstance(
+            self.comparison_value,
+            list), "Comparison value for '$gte' operation must not be a list."
+        return {"range": {self.field_name: {"gte": self.comparison_value}}}
+
+    def convert_to_sql(self, meta_document_orm):
+        return select([meta_document_orm.document_id]).where(
+            meta_document_orm.name == self.field_name,
+            meta_document_orm.value >= self.comparison_value)
+
+    def convert_to_weaviate(
+            self) -> Dict[str, Union[List[str], str, float, int]]:
+        comp_value_type, comp_value = self._get_weaviate_datatype()
+        assert not isinstance(
+            comp_value,
+            list), "Comparison value for '$gte' operation must not be a list."
+        return {
+            "path": [self.field_name],
+            "operator": "GreaterThanEqual",
+            comp_value_type: comp_value
+        }
+
+    def invert(self) -> "LtOperation":
+        return LtOperation(self.field_name, self.comparison_value)
+
+
+class LtOperation(ComparisonOperation):
+    """
+    Handles conversion of the '$lt' comparison operation.
+    """
+
+    def evaluate(self, fields) -> bool:
+        if self.field_name not in fields:
+            return False
+        return fields[self.field_name] < self.comparison_value
+
+    def convert_to_elasticsearch(
+            self) -> Dict[str, Dict[str, Dict[str, Union[str, float, int]]]]:
+        assert not isinstance(
+            self.comparison_value,
+            list), "Comparison value for '$lt' operation must not be a list."
+        return {"range": {self.field_name: {"lt": self.comparison_value}}}
+
+    def convert_to_sql(self, meta_document_orm):
+        return select([meta_document_orm.document_id
+                       ]).where(meta_document_orm.name == self.field_name,
+                                meta_document_orm.value < self.comparison_value)
+
+    def convert_to_weaviate(
+            self) -> Dict[str, Union[List[str], str, float, int]]:
+        comp_value_type, comp_value = self._get_weaviate_datatype()
+        assert not isinstance(
+            comp_value,
+            list), "Comparison value for '$lt' operation must not be a list."
+        return {
+            "path": [self.field_name],
+            "operator": "LessThan",
+            comp_value_type: comp_value
+        }
+
+    def invert(self) -> "GteOperation":
+        return GteOperation(self.field_name, self.comparison_value)
+
+
+class LteOperation(ComparisonOperation):
+    """
+    Handles conversion of the '$lte' comparison operation.
+    """
+
+    def evaluate(self, fields) -> bool:
+        if self.field_name not in fields:
+            return False
+        return fields[self.field_name] <= self.comparison_value
+
+    def convert_to_elasticsearch(
+            self) -> Dict[str, Dict[str, Dict[str, Union[str, float, int]]]]:
+        assert not isinstance(
+            self.comparison_value,
+            list), "Comparison value for '$lte' operation must not be a list."
+        return {"range": {self.field_name: {"lte": self.comparison_value}}}
+
+    def convert_to_sql(self, meta_document_orm):
+        return select([meta_document_orm.document_id]).where(
+            meta_document_orm.name == self.field_name,
+            meta_document_orm.value <= self.comparison_value)
+
+    def convert_to_weaviate(
+            self) -> Dict[str, Union[List[str], str, float, int]]:
+        comp_value_type, comp_value = self._get_weaviate_datatype()
+        assert not isinstance(
+            comp_value,
+            list), "Comparison value for '$lte' operation must not be a list."
+        return {
+            "path": [self.field_name],
+            "operator": "LessThanEqual",
+            comp_value_type: comp_value
+        }
+
+    def invert(self) -> "GtOperation":
+        return GtOperation(self.field_name, self.comparison_value)

applications/experimental/pipelines/pipelines/document_stores/sql.py

@@ -0,0 +1,869 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, Union, List, Optional, Generator
+
+import logging
+import itertools
+import numpy as np
+from uuid import uuid4
+
+try:
+    from sqlalchemy import (
+        and_,
+        func,
+        create_engine,
+        Column,
+        String,
+        DateTime,
+        ForeignKey,
+        Boolean,
+        Text,
+        text,
+        JSON,
+        ForeignKeyConstraint,
+    )
+    from sqlalchemy.ext.declarative import declarative_base
+    from sqlalchemy.orm import relationship, sessionmaker
+    from sqlalchemy.sql import case, null
+except (ImportError, ModuleNotFoundError) as ie:
+    from pipelines.utils.import_utils import _optional_component_not_installed
+
+    _optional_component_not_installed(__name__, "sql", ie)
+
+from pipelines.schema import Document, Label, Answer
+from pipelines.document_stores.base import BaseDocumentStore
+
+from pipelines.document_stores.filter_utils import LogicalFilterClause
+
+logger = logging.getLogger(__name__)
+Base = declarative_base()  # type: Any
+
+
+class ORMBase(Base):
+    __abstract__ = True
+
+    id = Column(String(100), default=lambda: str(uuid4()), primary_key=True)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime,
+                        server_default=func.now(),
+                        server_onupdate=func.now())
+
+
+class DocumentORM(ORMBase):
+    __tablename__ = "document"
+
+    content = Column(JSON, nullable=False)
+    content_type = Column(Text, nullable=True)
+    # primary key in combination with id to allow the same doc in different indices
+    index = Column(String(100), nullable=False, primary_key=True)
+    vector_id = Column(String(100), unique=True, nullable=True)
+    # speeds up queries for get_documents_by_vector_ids() by having a single query that returns joined metadata
+    meta = relationship("MetaDocumentORM",
+                        back_populates="documents",
+                        lazy="joined")
+
+
+class MetaDocumentORM(ORMBase):
+    __tablename__ = "meta_document"
+
+    name = Column(String(100), index=True)
+    value = Column(String(1000), index=True)
+    documents = relationship("DocumentORM", back_populates="meta")
+
+    document_id = Column(String(100), nullable=False, index=True)
+    document_index = Column(String(100), nullable=False, index=True)
+    __table_args__ = (
+        ForeignKeyConstraint([document_id, document_index],
+                             [DocumentORM.id, DocumentORM.index],
+                             ondelete="CASCADE",
+                             onupdate="CASCADE"),
+        {},
+    )  # type: ignore
+
+
+class LabelORM(ORMBase):
+    __tablename__ = "label"
+
+    index = Column(String(100), nullable=False, primary_key=True)
+    query = Column(Text, nullable=False)
+    answer = Column(JSON, nullable=True)
+    document = Column(JSON, nullable=False)
+    no_answer = Column(Boolean, nullable=False)
+    origin = Column(String(100), nullable=False)
+    is_correct_answer = Column(Boolean, nullable=False)
+    is_correct_document = Column(Boolean, nullable=False)
+    pipeline_id = Column(String(500), nullable=True)
+
+    meta = relationship("MetaLabelORM", back_populates="labels", lazy="joined")
+
+
+class MetaLabelORM(ORMBase):
+    __tablename__ = "meta_label"
+
+    name = Column(String(100), index=True)
+    value = Column(String(1000), index=True)
+    labels = relationship("LabelORM", back_populates="meta")
+
+    label_id = Column(String(100), nullable=False, index=True)
+    label_index = Column(String(100), nullable=False, index=True)
+    __table_args__ = (
+        ForeignKeyConstraint([label_id, label_index],
+                             [LabelORM.id, LabelORM.index],
+                             ondelete="CASCADE",
+                             onupdate="CASCADE"),
+        {},
+    )  # type: ignore
+
+
+class SQLDocumentStore(BaseDocumentStore):
+
+    def __init__(
+        self,
+        url: str = "sqlite://",
+        index: str = "document",
+        label_index: str = "label",
+        duplicate_documents: str = "overwrite",
+        check_same_thread: bool = False,
+        isolation_level: str = None,
+    ):
+        """
+        An SQL backed DocumentStore. Currently supports SQLite, PostgreSQL and MySQL backends.
+
+        :param url: URL for SQL database as expected by SQLAlchemy. More info here: https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls
+        :param index: The documents are scoped to an index attribute that can be used when writing, querying, or deleting documents.
+                      This parameter sets the default value for document index.
+        :param label_index: The default value of index attribute for the labels.
+        :param duplicate_documents: Handle duplicates document based on parameter options.
+                                    Parameter options : ( 'skip','overwrite','fail')
+                                    skip: Ignore the duplicates documents
+                                    overwrite: Update any existing documents with the same ID when adding documents.
+                                    fail: an error is raised if the document ID of the document being added already
+                                    exists.
+        :param check_same_thread: Set to False to mitigate multithreading issues in older SQLite versions (see https://docs.sqlalchemy.org/en/14/dialects/sqlite.html?highlight=check_same_thread#threading-pooling-behavior)
+        :param isolation_level: see SQLAlchemy's `isolation_level` parameter for `create_engine()` (https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine.params.isolation_level)
+        """
+
+        # save init parameters to enable export of component config as YAML
+        self.set_config(
+            url=url,
+            index=index,
+            label_index=label_index,
+            duplicate_documents=duplicate_documents,
+            check_same_thread=check_same_thread,
+        )
+        create_engine_params = {}
+        if isolation_level:
+            create_engine_params["isolation_level"] = isolation_level
+        if "sqlite" in url:
+            engine = create_engine(
+                url,
+                connect_args={"check_same_thread": check_same_thread},
+                **create_engine_params)
+        else:
+            engine = create_engine(url, **create_engine_params)
+        Base.metadata.create_all(engine)
+        Session = sessionmaker(bind=engine)
+        self.session = Session()
+        self.index: str = index
+        self.label_index = label_index
+        self.duplicate_documents = duplicate_documents
+        if getattr(self, "similarity", None) is None:
+            self.similarity = None
+        self.use_windowed_query = True
+        if "sqlite" in url:
+            import sqlite3
+
+            if sqlite3.sqlite_version < "3.25":
+                self.use_windowed_query = False
+
+    def get_document_by_id(
+            self,
+            id: str,
+            index: Optional[str] = None,
+            headers: Optional[Dict[str, str]] = None) -> Optional[Document]:
+        """Fetch a document by specifying its text id string"""
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        documents = self.get_documents_by_id([id], index)
+        document = documents[0] if documents else None
+        return document
+
+    def get_documents_by_id(
+        self,
+        ids: List[str],
+        index: Optional[str] = None,
+        batch_size: int = 10_000,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+        """Fetch documents by specifying a list of text id strings"""
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        index = index or self.index
+
+        documents = []
+        for i in range(0, len(ids), batch_size):
+            query = self.session.query(DocumentORM).filter(
+                DocumentORM.id.in_(ids[i:i + batch_size]),
+                DocumentORM.index == index)
+            for row in query.all():
+                documents.append(self._convert_sql_row_to_document(row))
+
+        return documents
+
+    def get_documents_by_vector_ids(self,
+                                    vector_ids: List[str],
+                                    index: Optional[str] = None,
+                                    batch_size: int = 10_000):
+        """Fetch documents by specifying a list of text vector id strings"""
+        index = index or self.index
+
+        documents = []
+        for i in range(0, len(vector_ids), batch_size):
+            query = self.session.query(DocumentORM).filter(
+                DocumentORM.vector_id.in_(vector_ids[i:i + batch_size]),
+                DocumentORM.index == index)
+            for row in query.all():
+                documents.append(self._convert_sql_row_to_document(row))
+
+        sorted_documents = sorted(
+            documents, key=lambda doc: vector_ids.index(doc.meta["vector_id"]))
+        return sorted_documents
+
+    def get_all_documents(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in SQLDocStore
+        return_embedding: Optional[bool] = None,
+        batch_size: int = 10_000,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        documents = list(
+            self.get_all_documents_generator(index=index,
+                                             filters=filters,
+                                             return_embedding=return_embedding,
+                                             batch_size=batch_size))
+        return documents
+
+    def get_all_documents_generator(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in SQLDocStore
+        return_embedding: Optional[bool] = None,
+        batch_size: int = 10_000,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Generator[Document, None, None]:
+        """
+        Get documents from the document store. Under-the-hood, documents are fetched in batches from the
+        document store and yielded as individual documents. This method can be used to iteratively process
+        a large number of documents without having to load all documents in memory.
+
+        :param index: Name of the index to get the documents from. If None, the
+                      DocumentStore's default index (self.index) will be used.
+        :param filters: Optional filters to narrow down the documents to return.
+                        Example: {"name": ["some", "more"], "category": ["only_one"]}
+        :param return_embedding: Whether to return the document embeddings.
+        :param batch_size: When working with large number of documents, batching can help reduce memory footprint.
+        """
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        if return_embedding is True:
+            raise Exception(
+                "return_embeddings is not supported by SQLDocumentStore.")
+        result = self._query(
+            index=index,
+            filters=filters,
+            batch_size=batch_size,
+        )
+        yield from result
+
+    def _create_document_field_map(self) -> Dict:
+        """
+        There is no field mapping required
+        """
+        return {}
+
+    def _query(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in SQLDocStore
+        vector_ids: Optional[List[str]] = None,
+        only_documents_without_embedding: bool = False,
+        batch_size: int = 10_000,
+    ):
+        """
+        :param index: Name of the index to get the documents from. If None, the
+                      DocumentStore's default index (self.index) will be used.
+        :param filters: Optional filters to narrow down the documents to return.
+                        Example: {"name": ["some", "more"], "category": ["only_one"]}
+        :param vector_ids: List of vector_id strings to filter the documents by.
+        :param only_documents_without_embedding: return only documents without an embedding.
+        :param batch_size: When working with large number of documents, batching can help reduce memory footprint.
+        """
+        index = index or self.index
+        # Generally ORM objects kept in memory cause performance issue
+        # Hence using directly column name improve memory and performance.
+        # Refer https://stackoverflow.com/questions/23185319/why-is-loading-sqlalchemy-objects-via-the-orm-5-8x-slower-than-rows-via-a-raw-my
+        documents_query = self.session.query(
+            DocumentORM.id, DocumentORM.content, DocumentORM.content_type,
+            DocumentORM.vector_id).filter_by(index=index)
+
+        if filters:
+            parsed_filter = LogicalFilterClause.parse(filters)
+            select_ids = parsed_filter.convert_to_sql(MetaDocumentORM)
+            documents_query = documents_query.filter(
+                DocumentORM.id.in_(select_ids))
+
+        if only_documents_without_embedding:
+            documents_query = documents_query.filter(
+                DocumentORM.vector_id.is_(None))
+        if vector_ids:
+            documents_query = documents_query.filter(
+                DocumentORM.vector_id.in_(vector_ids))
+
+        documents_map = {}
+
+        if self.use_windowed_query:
+            documents_query = self._windowed_query(documents_query,
+                                                   DocumentORM.id, batch_size)
+
+        for i, row in enumerate(documents_query, start=1):
+            documents_map[row.id] = Document.from_dict({
+                "id":
+                row.id,
+                "content":
+                row.content,
+                "content_type":
+                row.content_type,
+                "meta": {} if row.vector_id is None else {
+                    "vector_id": row.vector_id
+                },
+            })
+            if i % batch_size == 0:
+                documents_map = self._get_documents_meta(documents_map)
+                yield from documents_map.values()
+                documents_map = {}
+        if documents_map:
+            documents_map = self._get_documents_meta(documents_map)
+            yield from documents_map.values()
+
+    def _get_documents_meta(self, documents_map):
+        doc_ids = documents_map.keys()
+        meta_query = self.session.query(
+            MetaDocumentORM.document_id, MetaDocumentORM.name,
+            MetaDocumentORM.value).filter(
+                MetaDocumentORM.document_id.in_(doc_ids))
+
+        for row in meta_query.all():
+            documents_map[row.document_id].meta[row.name] = row.value
+        return documents_map
+
+    def get_all_labels(self,
+                       index=None,
+                       filters: Optional[dict] = None,
+                       headers: Optional[Dict[str, str]] = None):
+        """
+        Return all labels in the document store
+        """
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        index = index or self.label_index
+        # TODO: Use batch_size
+        label_rows = self.session.query(LabelORM).filter_by(index=index).all()
+        labels = [self._convert_sql_row_to_label(row) for row in label_rows]
+
+        return labels
+
+    def write_documents(
+        self,
+        documents: Union[List[dict], List[Document]],
+        index: Optional[str] = None,
+        batch_size: int = 10_000,
+        duplicate_documents: Optional[str] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> None:
+        """
+        Indexes documents for later queries.
+
+        :param documents: a list of Python dictionaries or a list of pipelines Document objects.
+                          For documents as dictionaries, the format is {"text": "<the-actual-text>"}.
+                          Optionally: Include meta data via {"text": "<the-actual-text>",
+                          "meta":{"name": "<some-document-name>, "author": "somebody", ...}}
+                          It can be used for filtering and is accessible in the responses of the Finder.
+        :param index: add an optional index attribute to documents. It can be later used for filtering. For instance,
+                      documents for evaluation can be indexed in a separate index than the documents for search.
+        :param batch_size: When working with large number of documents, batching can help reduce memory footprint.
+        :param duplicate_documents: Handle duplicates document based on parameter options.
+                                    Parameter options : ( 'skip','overwrite','fail')
+                                    skip: Ignore the duplicates documents
+                                    overwrite: Update any existing documents with the same ID when adding documents.
+                                    fail: an error is raised if the document ID of the document being added already
+                                    exists.
+
+        :return: None
+        """
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        index = index or self.index
+        duplicate_documents = duplicate_documents or self.duplicate_documents
+        if len(documents) == 0:
+            return
+        # Make sure we comply to Document class format
+        if isinstance(documents[0], dict):
+            document_objects = [
+                Document.from_dict(d) if isinstance(d, dict) else d
+                for d in documents
+            ]
+        else:
+            document_objects = documents
+
+        document_objects = self._handle_duplicate_documents(
+            documents=document_objects,
+            index=index,
+            duplicate_documents=duplicate_documents)
+        for i in range(0, len(document_objects), batch_size):
+            for doc in document_objects[i:i + batch_size]:
+                meta_fields = doc.meta or {}
+                vector_id = meta_fields.pop("vector_id", None)
+                meta_orms = [
+                    MetaDocumentORM(name=key, value=value)
+                    for key, value in meta_fields.items()
+                ]
+                doc_orm = DocumentORM(
+                    id=doc.id,
+                    content=doc.to_dict()["content"],
+                    content_type=doc.content_type,
+                    vector_id=vector_id,
+                    meta=meta_orms,
+                    index=index,
+                )
+                if duplicate_documents == "overwrite":
+                    # First old meta data cleaning is required
+                    self.session.query(MetaDocumentORM).filter_by(
+                        document_id=doc.id).delete()
+                    self.session.merge(doc_orm)
+                else:
+                    self.session.add(doc_orm)
+            try:
+                self.session.commit()
+            except Exception as ex:
+                logger.error(f"Transaction rollback: {ex.__cause__}")
+                # Rollback is important here otherwise self.session will be in inconsistent state and next call will fail
+                self.session.rollback()
+                raise ex
+
+    def write_labels(self,
+                     labels,
+                     index=None,
+                     headers: Optional[Dict[str, str]] = None):
+        """Write annotation labels into document store."""
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        labels = [
+            Label.from_dict(l) if isinstance(l, dict) else l for l in labels
+        ]
+        index = index or self.label_index
+
+        duplicate_ids: list = [
+            label.id
+            for label in self._get_duplicate_labels(labels, index=index)
+        ]
+        if len(duplicate_ids) > 0:
+            logger.warning(
+                f"Duplicate Label IDs: Inserting a Label whose id already exists in this document store."
+                f" This will overwrite the old Label. Please make sure Label.id is a unique identifier of"
+                f" the answer annotation and not the question."
+                f"   Problematic ids: {','.join(duplicate_ids)}")
+        # TODO: Use batch_size
+
+        for label in labels:
+            # TODO As of now, we write documents as part of the Label table as this is consistent with the other
+            #  document stores (e.g. elasticsearch) where "indices" are completely independent.
+            # We should eventually switch to an approach here that writes related documents to the document table if not already existing.
+            # See Issue XXX
+
+            # self.write_documents(documents=[label.document], index=index, duplicate_documents="skip")
+
+            # TODO: Handle label meta data
+            label_orm = LabelORM(
+                id=label.id,
+                no_answer=label.no_answer,
+                # document_id=label.document.id,
+                document=label.document.to_json(),
+                origin=label.origin,
+                query=label.query,
+                is_correct_answer=label.is_correct_answer,
+                is_correct_document=label.is_correct_document,
+                answer=label.answer.to_json(),
+                pipeline_id=label.pipeline_id,
+                index=index,
+            )
+            if label.id in duplicate_ids:
+                self.session.merge(label_orm)
+            else:
+                self.session.add(label_orm)
+
+            # TODO: investigate why test_multilabel() failed when not committing within the loop
+            # Seems that in some cases only the last label get than "committed"
+            self.session.commit()
+
+    def update_vector_ids(self,
+                          vector_id_map: Dict[str, str],
+                          index: Optional[str] = None,
+                          batch_size: int = 10_000):
+        """
+        Update vector_ids for given document_ids.
+
+        :param vector_id_map: dict containing mapping of document_id -> vector_id.
+        :param index: filter documents by the optional index attribute for documents in database.
+        :param batch_size: When working with large number of documents, batching can help reduce memory footprint.
+        """
+        index = index or self.index
+        for chunk_map in self.chunked_dict(vector_id_map, size=batch_size):
+            self.session.query(DocumentORM).filter(
+                DocumentORM.id.in_(chunk_map),
+                DocumentORM.index == index).update(
+                    {
+                        DocumentORM.vector_id:
+                        case(
+                            chunk_map,
+                            value=DocumentORM.id,
+                        )
+                    },
+                    synchronize_session=False,
+                )
+            try:
+                self.session.commit()
+            except Exception as ex:
+                logger.error(f"Transaction rollback: {ex.__cause__}")
+                self.session.rollback()
+                raise ex
+
+    def reset_vector_ids(self, index: Optional[str] = None):
+        """
+        Set vector IDs for all documents as None
+        """
+        index = index or self.index
+        self.session.query(DocumentORM).filter_by(index=index).update(
+            {DocumentORM.vector_id: null()})
+        self.session.commit()
+
+    def update_document_meta(self,
+                             id: str,
+                             meta: Dict[str, str],
+                             index: str = None):
+        """
+        Update the metadata dictionary of a document by specifying its string id
+        """
+        if not index:
+            index = self.index
+        self.session.query(MetaDocumentORM).filter_by(
+            document_id=id, document_index=index).delete()
+        meta_orms = [
+            MetaDocumentORM(name=key,
+                            value=value,
+                            document_id=id,
+                            document_index=index)
+            for key, value in meta.items()
+        ]
+        for m in meta_orms:
+            self.session.add(m)
+        self.session.commit()
+
+    def get_document_count(
+        self,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in SQLDocStore
+        index: Optional[str] = None,
+        only_documents_without_embedding: bool = False,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> int:
+        """
+        Return the number of documents in the document store.
+        """
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        index = index or self.index
+        query = self.session.query(DocumentORM).filter_by(index=index)
+
+        if filters:
+            for key, values in filters.items():
+                query = query.join(MetaDocumentORM, aliased=True).filter(
+                    MetaDocumentORM.name == key,
+                    MetaDocumentORM.value.in_(values),
+                )
+
+        if only_documents_without_embedding:
+            query = query.filter(DocumentORM.vector_id.is_(None))
+
+        count = query.count()
+        return count
+
+    def get_label_count(self,
+                        index: Optional[str] = None,
+                        headers: Optional[Dict[str, str]] = None) -> int:
+        """
+        Return the number of labels in the document store
+        """
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        index = index or self.label_index
+        return self.session.query(LabelORM).filter_by(index=index).count()
+
+    def _convert_sql_row_to_document(self, row) -> Document:
+        doc_dict = {
+            "id": row.id,
+            "content": row.content,
+            "content_type": row.content_type,
+            "meta": {meta.name: meta.value
+                     for meta in row.meta},
+        }
+        document = Document.from_dict(doc_dict)
+
+        if row.vector_id:
+            document.meta["vector_id"] = row.vector_id
+        return document
+
+    def _convert_sql_row_to_label(self, row) -> Label:
+        # doc = self._convert_sql_row_to_document(row.document)
+
+        label = Label(
+            query=row.query,
+            answer=Answer.from_json(row.answer),  # type: ignore
+            document=Document.from_json(row.document),
+            is_correct_answer=row.is_correct_answer,
+            is_correct_document=row.is_correct_document,
+            origin=row.origin,
+            id=row.id,
+            no_answer=row.no_answer,
+            pipeline_id=row.pipeline_id,
+            created_at=row.created_at,
+            updated_at=row.updated_at,
+            meta=row.meta,
+        )
+        return label
+
+    def query_by_embedding(
+        self,
+        query_emb: np.ndarray,
+        filters: Optional[dict] = None,
+        top_k: int = 10,
+        index: Optional[str] = None,
+        return_embedding: Optional[bool] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> List[Document]:
+
+        raise NotImplementedError(
+            "SQLDocumentStore is currently not supporting embedding queries. "
+            "Change the query type (e.g. by choosing a different retriever) "
+            "or change the DocumentStore (e.g. to ElasticsearchDocumentStore)")
+
+    def delete_all_documents(
+        self,
+        index: Optional[str] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in SQLDocStore
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Delete documents in an index. All documents are deleted if no filters are passed.
+
+        :param index: Index name to delete the document from.
+        :param filters: Optional filters to narrow down the documents to be deleted.
+        :return: None
+        """
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        logger.warning("""DEPRECATION WARNINGS: 
+                1. delete_all_documents() method is deprecated, please use delete_documents method
+                """)
+        self.delete_documents(index, None, filters)
+
+    def delete_documents(
+        self,
+        index: Optional[str] = None,
+        ids: Optional[List[str]] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in SQLDocStore
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Delete documents in an index. All documents are deleted if no filters are passed.
+
+        :param index: Index name to delete the document from. If None, the
+                      DocumentStore's default index (self.index) will be used.
+        :param ids: Optional list of IDs to narrow down the documents to be deleted.
+        :param filters: Optional filters to narrow down the documents to be deleted.
+            Example filters: {"name": ["some", "more"], "category": ["only_one"]}.
+            If filters are provided along with a list of IDs, this method deletes the
+            intersection of the two query results (documents that match the filters and
+            have their ID in the list).
+        :return: None
+        """
+        index = index or self.index
+        if not filters and not ids:
+            self.session.query(DocumentORM).filter_by(index=index).delete(
+                synchronize_session=False)
+        else:
+            document_ids_to_delete = self.session.query(
+                DocumentORM.id).filter(DocumentORM.index == index)
+            if filters:
+                for key, values in filters.items():
+                    document_ids_to_delete = document_ids_to_delete.join(
+                        MetaDocumentORM, aliased=True).filter(
+                            MetaDocumentORM.name == key,
+                            MetaDocumentORM.value.in_(values),
+                        )
+            if ids:
+                document_ids_to_delete = document_ids_to_delete.filter(
+                    DocumentORM.id.in_(ids))
+
+            self.session.query(DocumentORM).filter(
+                DocumentORM.id.in_(document_ids_to_delete)).delete(
+                    synchronize_session=False)
+
+        self.session.commit()
+
+    def delete_labels(
+        self,
+        index: Optional[str] = None,
+        ids: Optional[List[str]] = None,
+        filters: Optional[Dict[
+            str,
+            Any]] = None,  # TODO: Adapt type once we allow extended filters in SQLDocStore
+        headers: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Delete labels from the document store. All labels are deleted if no filters are passed.
+
+        :param index: Index name to delete the labels from. If None, the
+                      DocumentStore's default label index (self.label_index) will be used.
+        :param ids: Optional list of IDs to narrow down the labels to be deleted.
+        :param filters: Optional filters to narrow down the labels to be deleted.
+                        Example filters: {"id": ["9a196e41-f7b5-45b4-bd19-5feb7501c159", "9a196e41-f7b5-45b4-bd19-5feb7501c159"]} or {"query": ["question2"]}
+        :return: None
+        """
+        if headers:
+            raise NotImplementedError(
+                "SQLDocumentStore does not support headers.")
+
+        index = index or self.label_index
+        if not filters and not ids:
+            self.session.query(LabelORM).filter_by(index=index).delete(
+                synchronize_session=False)
+        else:
+            label_ids_to_delete = self.session.query(
+                LabelORM.id).filter_by(index=index)
+            if filters:
+                for key, values in filters.items():
+                    label_attribute = getattr(LabelORM, key)
+                    label_ids_to_delete = label_ids_to_delete.filter(
+                        label_attribute.in_(values))
+
+            if ids:
+                label_ids_to_delete = label_ids_to_delete.filter(
+                    LabelORM.id.in_(ids))
+
+            self.session.query(LabelORM).filter(
+                LabelORM.id.in_(label_ids_to_delete)).delete(
+                    synchronize_session=False)
+
+        self.session.commit()
+
+    def _get_or_create(self, session, model, **kwargs):
+        instance = session.query(model).filter_by(**kwargs).first()
+        if instance:
+            return instance
+        else:
+            instance = model(**kwargs)
+            session.add(instance)
+            session.commit()
+            return instance
+
+    def chunked_dict(self, dictionary, size):
+        it = iter(dictionary)
+        for i in range(0, len(dictionary), size):
+            yield {k: dictionary[k] for k in itertools.islice(it, size)}
+
+    def _column_windows(self, session, column, windowsize):
+        """Return a series of WHERE clauses against
+        a given column that break it into windows.
+
+        Result is an iterable of tuples, consisting of
+        ((start, end), whereclause), where (start, end) are the ids.
+
+        The code is taken from: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
+        """
+
+        def int_for_range(start_id, end_id):
+            if end_id:
+                return and_(column >= start_id, column < end_id)
+            else:
+                return column >= start_id
+
+        q = session.query(
+            column,
+            func.row_number().over(
+                order_by=column).label("rownum")).from_self(column)
+        if windowsize > 1:
+            q = q.filter(text("rownum %% %d=1" % windowsize))
+
+        intervals = [id for id, in q]
+
+        while intervals:
+            start = intervals.pop(0)
+            if intervals:
+                end = intervals[0]
+            else:
+                end = None
+            yield int_for_range(start, end)
+
+    def _windowed_query(self, q, column, windowsize):
+        """ "Break a Query into windows on a given column."""
+
+        for whereclause in self._column_windows(q.session, column, windowsize):
+            for row in q.filter(whereclause).order_by(column):
+                yield row

applications/experimental/pipelines/pipelines/document_stores/utils.py

@@ -0,0 +1,451 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+from typing import Dict, List, Optional, Tuple, Union, Generator
+
+import json
+import logging
+from datetime import datetime
+from elasticsearch.helpers import scan
+from tqdm.auto import tqdm
+
+from pipelines.document_stores.filter_utils import LogicalFilterClause
+from pipelines.schema import Document, Label, Answer, Span
+from pipelines.nodes.preprocessor import PreProcessor
+
+if typing.TYPE_CHECKING:
+    # This results in a circular import if we don't use typing.TYPE_CHECKING
+    from pipelines.document_stores.base import BaseDocumentStore
+
+logger = logging.getLogger(__name__)
+
+
+def eval_data_from_json(
+        filename: str,
+        max_docs: Union[int, bool] = None,
+        preprocessor: PreProcessor = None,
+        open_domain: bool = False) -> Tuple[List[Document], List[Label]]:
+    """
+    Read Documents + Labels from a SQuAD-style file.
+    Document and Labels can then be indexed to the DocumentStore and be used for evaluation.
+
+    :param filename: Path to file in SQuAD format
+    :param max_docs: This sets the number of documents that will be loaded. By default, this is set to None, thus reading in all available eval documents.
+    :param open_domain: Set this to True if your file is an open domain dataset where two different answers to the same question might be found in different contexts.
+    """
+    docs: List[Document] = []
+    labels = []
+    problematic_ids = []
+
+    with open(filename, "r", encoding="utf-8") as file:
+        data = json.load(file)
+        if "title" not in data["data"][0]:
+            logger.warning(
+                f"No title information found for documents in QA file: {filename}"
+            )
+
+        for document in data["data"]:
+            if max_docs:
+                if len(docs) > max_docs:
+                    break
+            # Extracting paragraphs and their labels from a SQuAD document dict
+            cur_docs, cur_labels, cur_problematic_ids = _extract_docs_and_labels_from_dict(
+                document, preprocessor, open_domain)
+            docs.extend(cur_docs)
+            labels.extend(cur_labels)
+            problematic_ids.extend(cur_problematic_ids)
+    if len(problematic_ids) > 0:
+        logger.warning(
+            f"Could not convert an answer for {len(problematic_ids)} questions.\n"
+            f"There were conversion errors for question ids: {problematic_ids}")
+    return docs, labels
+
+
+def eval_data_from_jsonl(
+    filename: str,
+    batch_size: Optional[int] = None,
+    max_docs: Union[int, bool] = None,
+    preprocessor: PreProcessor = None,
+    open_domain: bool = False,
+) -> Generator[Tuple[List[Document], List[Label]], None, None]:
+    """
+    Read Documents + Labels from a SQuAD-style file in jsonl format, i.e. one document per line.
+    Document and Labels can then be indexed to the DocumentStore and be used for evaluation.
+
+    This is a generator which will yield one tuple per iteration containing a list
+    of batch_size documents and a list with the documents' labels.
+    If batch_size is set to None, this method will yield all documents and labels.
+
+    :param filename: Path to file in SQuAD format
+    :param max_docs: This sets the number of documents that will be loaded. By default, this is set to None, thus reading in all available eval documents.
+    :param open_domain: Set this to True if your file is an open domain dataset where two different answers to the same question might be found in different contexts.
+    """
+    docs: List[Document] = []
+    labels = []
+    problematic_ids = []
+
+    with open(filename, "r", encoding="utf-8") as file:
+        for document in file:
+            if max_docs:
+                if len(docs) > max_docs:
+                    break
+            # Extracting paragraphs and their labels from a SQuAD document dict
+            document_dict = json.loads(document)
+            cur_docs, cur_labels, cur_problematic_ids = _extract_docs_and_labels_from_dict(
+                document_dict, preprocessor, open_domain)
+            docs.extend(cur_docs)
+            labels.extend(cur_labels)
+            problematic_ids.extend(cur_problematic_ids)
+
+            if batch_size is not None:
+                if len(docs) >= batch_size:
+                    if len(problematic_ids) > 0:
+                        logger.warning(
+                            f"Could not convert an answer for {len(problematic_ids)} questions.\n"
+                            f"There were conversion errors for question ids: {problematic_ids}"
+                        )
+                    yield docs, labels
+                    docs = []
+                    labels = []
+                    problematic_ids = []
+
+    yield docs, labels
+
+
+def _extract_docs_and_labels_from_dict(document_dict: Dict,
+                                       preprocessor: PreProcessor = None,
+                                       open_domain: bool = False):
+    """
+    Set open_domain to True if you are trying to load open_domain labels (i.e. labels without doc id or start idx)
+    """
+    docs = []
+    labels = []
+    problematic_ids = []
+
+    # get all extra fields from document level (e.g. title)
+    meta_doc = {
+        k: v
+        for k, v in document_dict.items() if k not in ("paragraphs", "title")
+    }
+    for paragraph in document_dict["paragraphs"]:
+        ## Create Metadata
+        cur_meta = {"name": document_dict.get("title", None)}
+        # all other fields from paragraph level
+        meta_paragraph = {
+            k: v
+            for k, v in paragraph.items() if k not in ("qas", "context")
+        }
+        cur_meta.update(meta_paragraph)
+        # meta from parent document
+        cur_meta.update(meta_doc)
+
+        ## Create Document
+        cur_full_doc = Document(content=paragraph["context"], meta=cur_meta)
+        if preprocessor is not None:
+            splits_dicts = preprocessor.process(cur_full_doc.to_dict())
+            # we need to pull in _split_id into the document id for unique reference in labels
+            # todo: PreProcessor should work on Documents instead of dicts
+            splits: List[Document] = []
+            offset = 0
+            for d in splits_dicts:
+                id = f"{d['id']}-{d['meta']['_split_id']}"
+                d["meta"]["_split_offset"] = offset
+                offset += len(d["content"])
+                # offset correction based on splitting method
+                if preprocessor.split_by == "word":
+                    offset += 1
+                elif preprocessor.split_by == "passage":
+                    offset += 2
+                else:
+                    raise NotImplementedError
+                mydoc = Document(content=d["content"], id=id, meta=d["meta"])
+                splits.append(mydoc)
+        else:
+            splits = [cur_full_doc]
+        docs.extend(splits)
+
+        ## Assign Labels to corresponding documents
+        for qa in paragraph["qas"]:
+            if not qa.get("is_impossible", False):
+                for answer in qa["answers"]:
+                    ans = answer["text"]
+                    # TODO The following block of code means that answer_start is never calculated
+                    #  and cur_id is always None for open_domain
+                    #  This can be rewritten so that this function could try to calculate offsets
+                    #  and populate id in open_domain mode
+                    if open_domain:
+                        # TODO check with Branden why we want to treat open_domain here differently.
+                        # Shouldn't this be something configured at eval time only?
+                        cur_ans_start = answer.get("answer_start", 0)
+                        # cur_id = '0'
+                        label = Label(
+                            query=qa["question"],
+                            answer=Answer(answer=ans,
+                                          type="extractive",
+                                          score=0.0),
+                            document=None,  # type: ignore
+                            is_correct_answer=True,
+                            is_correct_document=True,
+                            no_answer=qa.get("is_impossible", False),
+                            origin="gold-label",
+                        )
+                        labels.append(label)
+                    else:
+                        ans_position = cur_full_doc.content[
+                            answer["answer_start"]:answer["answer_start"] +
+                            len(ans)]
+                        if ans != ans_position:
+                            # do not use answer
+                            problematic_ids.append(qa.get("id", "missing"))
+                            break
+                        # find corresponding document or split
+                        if len(splits) == 1:
+                            # cur_id = splits[0].id
+                            cur_ans_start = answer["answer_start"]
+                            cur_doc = splits[0]
+                        else:
+                            for s in splits:
+                                # If answer start offset is contained in passage we assign the label to that passage
+                                if (answer["answer_start"] >=
+                                        s.meta["_split_offset"]) and (
+                                            answer["answer_start"] <
+                                            (s.meta["_split_offset"] +
+                                             len(s.content))):
+                                    cur_doc = s
+                                    cur_ans_start = answer[
+                                        "answer_start"] - s.meta["_split_offset"]
+                                    # If a document is splitting an answer we add the whole answer text to the document
+                                    if s.content[cur_ans_start:cur_ans_start +
+                                                 len(ans)] != ans:
+                                        s.content = s.content[:cur_ans_start] + ans
+                                    break
+                        cur_answer = Answer(
+                            answer=ans,
+                            type="extractive",
+                            score=0.0,
+                            context=cur_doc.content,
+                            offsets_in_document=[
+                                Span(start=cur_ans_start,
+                                     end=cur_ans_start + len(ans))
+                            ],
+                            offsets_in_context=[
+                                Span(start=cur_ans_start,
+                                     end=cur_ans_start + len(ans))
+                            ],
+                            document_id=cur_doc.id,
+                        )
+                        label = Label(
+                            query=qa["question"],
+                            answer=cur_answer,
+                            document=cur_doc,
+                            is_correct_answer=True,
+                            is_correct_document=True,
+                            no_answer=qa.get("is_impossible", False),
+                            origin="gold-label",
+                        )
+                        labels.append(label)
+            else:
+                # for no_answer we need to assign each split as not fitting to the question
+                for s in splits:
+                    label = Label(
+                        query=qa["question"],
+                        answer=Answer(
+                            answer="",
+                            type="extractive",
+                            score=0.0,
+                            offsets_in_document=[Span(start=0, end=0)],
+                            offsets_in_context=[Span(start=0, end=0)],
+                        ),
+                        document=s,
+                        is_correct_answer=True,
+                        is_correct_document=True,
+                        no_answer=qa.get("is_impossible", False),
+                        origin="gold-label",
+                    )
+
+                    labels.append(label)
+
+    return docs, labels, problematic_ids
+
+
+def convert_date_to_rfc3339(date: str) -> str:
+    """
+    Converts a date to RFC3339 format, as Weaviate requires dates to be in RFC3339 format including the time and
+    timezone.
+
+    If the provided date string does not contain a time and/or timezone, we use 00:00 as default time
+    and UTC as default time zone.
+
+    This method cannot be part of WeaviateDocumentStore, as this would result in a circular import between weaviate.py
+    and filter_utils.py.
+    """
+    parsed_datetime = datetime.fromisoformat(date)
+    if parsed_datetime.utcoffset() is None:
+        converted_date = parsed_datetime.isoformat() + "Z"
+    else:
+        converted_date = parsed_datetime.isoformat()
+
+    return converted_date
+
+
+def es_index_to_document_store(
+    document_store: "BaseDocumentStore",
+    original_index_name: str,
+    original_content_field: str,
+    original_name_field: Optional[str] = None,
+    included_metadata_fields: Optional[List[str]] = None,
+    excluded_metadata_fields: Optional[List[str]] = None,
+    store_original_ids: bool = True,
+    index: Optional[str] = None,
+    preprocessor: Optional[PreProcessor] = None,
+    batch_size: int = 10_000,
+    host: Union[str, List[str]] = "localhost",
+    port: Union[int, List[int]] = 9200,
+    username: str = "",
+    password: str = "",
+    api_key_id: Optional[str] = None,
+    api_key: Optional[str] = None,
+    aws4auth=None,
+    scheme: str = "http",
+    ca_certs: Optional[str] = None,
+    verify_certs: bool = True,
+    timeout: int = 30,
+    use_system_proxy: bool = False,
+) -> "BaseDocumentStore":
+    """
+    This function provides brownfield support of existing Elasticsearch indexes by converting each of the records in
+    the provided index to pipelines `Document` objects and writing them to the specified `DocumentStore`. It can be used
+    on a regular basis in order to add new records of the Elasticsearch index to the `DocumentStore`.
+
+    :param document_store: The pipelines `DocumentStore` to write the converted `Document` objects to.
+    :param original_index_name: Elasticsearch index containing the records to be converted.
+    :param original_content_field: Elasticsearch field containing the text to be put in the `content` field of the
+        resulting pipelines `Document` objects.
+    :param original_name_field: Optional Elasticsearch field containing the title of the Document.
+    :param included_metadata_fields: List of Elasticsearch fields that shall be stored in the `meta` field of the
+        resulting pipelines `Document` objects. If `included_metadata_fields` and `excluded_metadata_fields` are `None`,
+        all the fields found in the Elasticsearch records will be kept as metadata. You can specify only one of the
+        `included_metadata_fields` and `excluded_metadata_fields` parameters.
+    :param excluded_metadata_fields: List of Elasticsearch fields that shall be excluded from the `meta` field of the
+        resulting pipelines `Document` objects. If `included_metadata_fields` and `excluded_metadata_fields` are `None`,
+        all the fields found in the Elasticsearch records will be kept as metadata. You can specify only one of the
+        `included_metadata_fields` and `excluded_metadata_fields` parameters.
+    :param store_original_ids: Whether to store the ID a record had in the original Elasticsearch index at the
+        `"_original_es_id"` metadata field of the resulting pipelines `Document` objects. This should be set to `True`
+        if you want to continuously update the `DocumentStore` with new records inside your Elasticsearch index. If this
+        parameter was set to `False` on the first call of `es_index_to_document_store`,
+        all the indexed Documents in the `DocumentStore` will be overwritten in the second call.
+    :param index: Name of index in `document_store` to use to store the resulting pipelines `Document` objects.
+    :param preprocessor: Optional PreProcessor that will be applied on the content field of the original Elasticsearch
+        record.
+    :param batch_size: Number of records to process at once.
+    :param host: URL(s) of Elasticsearch nodes.
+    :param port: Ports(s) of Elasticsearch nodes.
+    :param username: Username (standard authentication via http_auth).
+    :param password: Password (standard authentication via http_auth).
+    :param api_key_id: ID of the API key (altenative authentication mode to the above http_auth).
+    :param api_key: Secret value of the API key (altenative authentication mode to the above http_auth).
+    :param aws4auth: Authentication for usage with AWS Elasticsearch
+        (can be generated with the requests-aws4auth package).
+    :param scheme: `"https"` or `"http"`, protocol used to connect to your Elasticsearch instance.
+    :param ca_certs: Root certificates for SSL: it is a path to certificate authority (CA) certs on disk.
+        You can use certifi package with `certifi.where()` to find where the CA certs file is located in your machine.
+    :param verify_certs: Whether to be strict about ca certificates.
+    :param timeout: Number of seconds after which an Elasticsearch request times out.
+    :param use_system_proxy: Whether to use system proxy.
+    """
+    # This import cannot be at the beginning of the file, as this would result in a circular import
+    from pipelines.document_stores.elasticsearch import ElasticsearchDocumentStore
+
+    # Initialize Elasticsearch client
+    es_client = ElasticsearchDocumentStore._init_elastic_client(
+        host=host,
+        port=port,
+        username=username,
+        password=password,
+        api_key=api_key,
+        api_key_id=api_key_id,
+        aws4auth=aws4auth,
+        scheme=scheme,
+        ca_certs=ca_certs,
+        verify_certs=verify_certs,
+        timeout=timeout,
+        use_system_proxy=use_system_proxy,
+    )
+
+    # Get existing original ES IDs inside DocumentStore in order to not reindex the corresponding records
+    existing_ids = [
+        doc.meta["_original_es_id"]
+        for doc in document_store.get_all_documents_generator(index=index)
+        if "_original_es_id" in doc.meta
+    ]
+
+    # Iterate over each individual record
+    query: Dict[str, Dict] = {"query": {"bool": {"must": [{"match_all": {}}]}}}
+    if existing_ids:
+        filters = LogicalFilterClause.parse({
+            "_id": {
+                "$nin": existing_ids
+            }
+        }).convert_to_elasticsearch()
+        query["query"]["bool"]["filter"] = filters
+    records = scan(client=es_client, query=query, index=original_index_name)
+    number_of_records = es_client.count(index=original_index_name,
+                                        body=query)["count"]
+    pipelines_documents: List[Dict] = []
+    for idx, record in enumerate(
+            tqdm(records, total=number_of_records,
+                 desc="Converting ES Records")):
+        # Write batch_size number of documents to pipelines DocumentStore
+        if (idx + 1) % batch_size == 0:
+            document_store.write_documents(pipelines_documents, index=index)
+            pipelines_documents = []
+
+        # Get content and metadata of current record
+        content = record["_source"].pop(original_content_field, "")
+        if content:
+            record_doc = {"content": content, "meta": {}}
+
+            if original_name_field is not None:
+                if original_name_field in record["_source"]:
+                    record_doc["meta"]["name"] = record["_source"].pop(
+                        original_name_field)
+            # Only add selected metadata fields
+            if included_metadata_fields is not None:
+                for metadata_field in included_metadata_fields:
+                    if metadata_field in record["_source"]:
+                        record_doc["meta"][metadata_field] = record["_source"][
+                            metadata_field]
+            # Add all metadata fields except for those in excluded_metadata_fields
+            else:
+                if excluded_metadata_fields is not None:
+                    for metadata_field in excluded_metadata_fields:
+                        record["_source"].pop(metadata_field, None)
+                record_doc["meta"].update(record["_source"])
+
+            if store_original_ids:
+                record_doc["meta"]["_original_es_id"] = record["_id"]
+
+            # Apply preprocessor if provided
+            preprocessed_docs = preprocessor.process(
+                record_doc) if preprocessor is not None else [record_doc]
+
+            pipelines_documents.extend(preprocessed_docs)
+
+    if pipelines_documents:
+        document_store.write_documents(pipelines_documents, index=index)
+
+    return document_store

applications/experimental/pipelines/pipelines/nodes/__init__.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pipelines.utils.import_utils import safe_import
+
+from pipelines.nodes.base import BaseComponent
+from pipelines.nodes.file_classifier import FileTypeClassifier
+from pipelines.nodes.file_converter import (
+    BaseConverter,
+    DocxToTextConverter,
+    ImageToTextConverter,
+    MarkdownConverter,
+    PDFToTextConverter,
+    PDFToTextOCRConverter,
+    TextConverter,
+)
+from pipelines.nodes.preprocessor import BasePreProcessor, PreProcessor
+from pipelines.nodes.ranker import BaseRanker, ErnieRanker
+from pipelines.nodes.reader import BaseReader, ErnieReader
+from pipelines.nodes.retriever import BaseRetriever, DensePassageRetriever

applications/experimental/pipelines/pipelines/nodes/base.py

@@ -0,0 +1,206 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+from typing import Any, Callable, Optional, Dict, List, Tuple, Optional
+
+import io
+from functools import wraps
+from copy import deepcopy
+from abc import abstractmethod
+import inspect
+import logging
+
+from pipelines.schema import Document
+
+logger = logging.getLogger(__name__)
+
+
+class BaseComponent:
+    """
+    A base class for implementing nodes in a Pipeline.
+    """
+
+    outgoing_edges: int
+    subclasses: dict = {}
+    pipeline_config: dict = {}
+    name: Optional[str] = None
+
+    def __init_subclass__(cls, **kwargs):
+        """
+        Automatically keeps track of all available subclasses.
+        Enables generic load() for all specific component implementations.
+        """
+        super().__init_subclass__(**kwargs)
+        cls.subclasses[cls.__name__] = cls
+
+    @classmethod
+    def get_subclass(cls, component_type: str):
+        if component_type not in cls.subclasses.keys():
+            raise Exception(
+                f"pipelines component with the name '{component_type}' does not exist."
+            )
+        subclass = cls.subclasses[component_type]
+        return subclass
+
+    @classmethod
+    def load_from_args(cls, component_type: str, **kwargs):
+        """
+        Load a component instance of the given type using the kwargs.
+
+        :param component_type: name of the component class to load.
+        :param kwargs: parameters to pass to the __init__() for the component.
+        """
+        subclass = cls.get_subclass(component_type)
+        instance = subclass(**kwargs)
+        return instance
+
+    @classmethod
+    def load_from_pipeline_config(cls, pipeline_config: dict,
+                                  component_name: str):
+        """
+        Load an individual component from a YAML config for Pipelines.
+
+        :param pipeline_config: the Pipelines YAML config parsed as a dict.
+        :param component_name: the name of the component to load.
+        """
+        if pipeline_config:
+            all_component_configs = pipeline_config["components"]
+            all_component_names = [
+                comp["name"] for comp in all_component_configs
+            ]
+            component_config = next(comp for comp in all_component_configs
+                                    if comp["name"] == component_name)
+            component_params = component_config["params"]
+
+            for key, value in component_params.items():
+                if value in all_component_names:  # check if the param value is a reference to another component
+                    component_params[key] = cls.load_from_pipeline_config(
+                        pipeline_config, value)
+
+            component_instance = cls.load_from_args(component_config["type"],
+                                                    **component_params)
+        else:
+            component_instance = cls.load_from_args(component_name)
+        return component_instance
+
+    @abstractmethod
+    def run(
+        self,
+        query: Optional[str] = None,
+        file_paths: Optional[List[str]] = None,
+        documents: Optional[List[Document]] = None,
+        meta: Optional[dict] = None,
+    ) -> Tuple[Dict, str]:
+        """
+        Method that will be executed when the node in the graph is called.
+
+        The argument that are passed can vary between different types of nodes
+        (e.g. retriever nodes expect different args than a reader node)
+
+
+        See an example for an implementation in pipelines/reader/base/BaseReader.py
+        :return:
+        """
+        pass
+
+    def _dispatch_run(self, **kwargs) -> Tuple[Dict, str]:
+        """
+        The Pipelines call this method which in turn executes the run() method of Component.
+
+        It takes care of the following:
+          - inspect run() signature to validate if all necessary arguments are available
+          - pop `debug` and sets them on the instance to control debug output
+          - call run() with the corresponding arguments and gather output
+          - collate `_debug` information if present
+          - merge component output with the preceding output and pass it on to the subsequent Component in the Pipeline
+        """
+        arguments = deepcopy(kwargs)
+        params = arguments.get("params") or {}
+
+        run_signature_args = inspect.signature(self.run).parameters.keys()
+
+        run_params: Dict[str, Any] = {}
+        for key, value in params.items():
+            if key == self.name:  # targeted params for this node
+                if isinstance(value, dict):
+
+                    # Extract debug attributes
+                    if "debug" in value.keys():
+                        self.debug = value.pop("debug")
+
+                    for _k, _v in value.items():
+                        if _k not in run_signature_args:
+                            raise Exception(
+                                f"Invalid parameter '{_k}' for the node '{self.name}'."
+                            )
+
+                run_params.update(**value)
+            elif key in run_signature_args:  # global params
+                run_params[key] = value
+
+        run_inputs = {}
+        for key, value in arguments.items():
+            if key in run_signature_args:
+                run_inputs[key] = value
+
+        output, stream = self.run(**run_inputs, **run_params)
+
+        # Collect debug information
+        debug_info = {}
+        if getattr(self, "debug", None):
+            # Include input
+            debug_info["input"] = {**run_inputs, **run_params}
+            debug_info["input"]["debug"] = self.debug
+            # Include output
+            filtered_output = {
+                key: value
+                for key, value in output.items() if key != "_debug"
+            }  # Exclude _debug to avoid recursion
+            debug_info["output"] = filtered_output
+        # Include custom debug info
+        custom_debug = output.get("_debug", {})
+        if custom_debug:
+            debug_info["runtime"] = custom_debug
+
+        # append _debug information from nodes
+        all_debug = arguments.get("_debug", {})
+        if debug_info:
+            all_debug[self.name] = debug_info
+        if all_debug:
+            output["_debug"] = all_debug
+
+        # add "extra" args that were not used by the node
+        for k, v in arguments.items():
+            if k not in output.keys():
+                output[k] = v
+
+        output["params"] = params
+        return output, stream
+
+    def set_config(self, **kwargs):
+        """
+        Save the init parameters of a component that later can be used with exporting
+        YAML configuration of a Pipeline.
+
+        :param kwargs: all parameters passed to the __init__() of the Component.
+        """
+        if not self.pipeline_config:
+            self.pipeline_config = {"params": {}, "type": type(self).__name__}
+            for k, v in kwargs.items():
+                if isinstance(v, BaseComponent):
+                    self.pipeline_config["params"][k] = v.pipeline_config
+                elif v is not None:
+                    self.pipeline_config["params"][k] = v

applications/experimental/pipelines/pipelines/nodes/file_classifier/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pipelines.nodes.file_classifier.file_type import FileTypeClassifier

applications/experimental/pipelines/pipelines/nodes/file_classifier/file_type.py

@@ -13,10 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from pathlib import Path
+from multiprocessing.sharedctypes import Value
 from typing import List, Union
-
+from pathlib import Path
 from pipelines.nodes.base import BaseComponent
 
 DEFAULT_TYPES = ["txt", "pdf", "md", "docx", "html", "png", "jpg"]
@@ -61,11 +60,13 @@ class FileTypeClassifier(BaseComponent):
 
         for path in file_paths:
             if path.suffix != extension:
-                raise ValueError("Multiple file types are not allowed at once.")
+                raise ValueError(
+                    f"Multiple file types are not allowed at once.")
 
         return extension.lstrip(".")
 
-    def run(self, file_paths: Union[Path, List[Path], str, List[str], List[Union[Path, str]]]):  # type: ignore
+    def run(self, file_paths: Union[Path, List[Path], str, List[str],
+                                    List[Union[Path, str]]]):  # type: ignore
         """
         Sends out files on a different output edge depending on their extension.
 
@@ -85,6 +86,5 @@ class FileTypeClassifier(BaseComponent):
                 f"Files of type '{extension}' are not supported. "
                 f"The supported types are: {self.supported_types}. "
                 "Consider using the 'supported_types' parameter to "
-                "change the types accepted by this node."
-            )
+                "change the types accepted by this node.")
         return output, f"output_{index}"

applications/experimental/pipelines/pipelines/nodes/file_converter/__init__.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pipelines.nodes.file_converter.base import BaseConverter
+
+from pipelines.utils.import_utils import safe_import
+
+MarkdownConverter = safe_import("pipelines.nodes.file_converter.markdown",
+                                "MarkdownConverter",
+                                "preprocessing")  # Has optional dependencies
+ImageToTextConverter = safe_import("pipelines.nodes.file_converter.image",
+                                   "ImageToTextConverter",
+                                   "ocr")  # Has optional dependencies
+PDFToTextConverter = safe_import("pipelines.nodes.file_converter.pdf",
+                                 "PDFToTextConverter",
+                                 "ocr")  # Has optional dependencies
+PDFToTextOCRConverter = safe_import("pipelines.nodes.file_converter.pdf",
+                                    "PDFToTextOCRConverter",
+                                    "ocr")  # Has optional dependencies
+
+from pipelines.nodes.file_converter.docx import DocxToTextConverter
+from pipelines.nodes.file_converter.txt import TextConverter
+from pipelines.nodes.file_converter.pdf import PDFToTextConverter
+from pipelines.nodes.file_converter.image import ImageToTextConverter

applications/experimental/pipelines/pipelines/nodes/file_converter/base.py

@@ -0,0 +1,133 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional, Dict, Any, Union
+
+from abc import abstractmethod
+from pathlib import Path
+import langdetect
+
+from pipelines.nodes.base import BaseComponent
+
+
+class BaseConverter(BaseComponent):
+    """
+    Base class for implementing file converts to transform input documents to text format for ingestion in DocumentStore.
+    """
+
+    outgoing_edges = 1
+
+    def __init__(
+        self,
+        remove_numeric_tables: bool = False,
+        valid_languages: Optional[List[str]] = None,
+    ):
+        """
+        :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
+                                      The tabular structures in documents might be noise for the reader model if it
+                                      does not have table parsing capability for finding answers. However, tables
+                                      may also have long strings that could possible candidate for searching answers.
+                                      The rows containing strings are thus retained in this option.
+        :param valid_languages: validate languages from a list of languages specified in the ISO 639-1
+                                (https://en.wikipedia.org/wiki/ISO_639-1) format.
+                                This option can be used to add test for encoding errors. If the extracted text is
+                                not one of the valid languages, then it might likely be encoding error resulting
+                                in garbled text.
+        """
+
+        # save init parameters to enable export of component config as YAML
+        self.set_config(remove_numeric_tables=remove_numeric_tables,
+                        valid_languages=valid_languages)
+
+        self.remove_numeric_tables = remove_numeric_tables
+        self.valid_languages = valid_languages
+
+    @abstractmethod
+    def convert(
+        self,
+        file_path: Path,
+        meta: Optional[Dict[str, str]],
+        remove_numeric_tables: Optional[bool] = None,
+        valid_languages: Optional[List[str]] = None,
+        encoding: Optional[str] = "utf-8",
+    ) -> List[Dict[str, Any]]:
+        """
+        Convert a file to a dictionary containing the text and any associated meta data.
+
+        File converters may extract file meta like name or size. In addition to it, user
+        supplied meta data like author, url, external IDs can be supplied as a dictionary.
+
+        :param file_path: path of the file to convert
+        :param meta: dictionary of meta data key-value pairs to append in the returned document.
+        :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
+                                      The tabular structures in documents might be noise for the reader model if it
+                                      does not have table parsing capability for finding answers. However, tables
+                                      may also have long strings that could possible candidate for searching answers.
+                                      The rows containing strings are thus retained in this option.
+        :param valid_languages: validate languages from a list of languages specified in the ISO 639-1
+                                (https://en.wikipedia.org/wiki/ISO_639-1) format.
+                                This option can be used to add test for encoding errors. If the extracted text is
+                                not one of the valid languages, then it might likely be encoding error resulting
+                                in garbled text.
+        :param encoding: Select the file encoding (default is `utf-8`)
+        """
+        pass
+
+    def validate_language(self,
+                          text: str,
+                          valid_languages: Optional[List[str]] = None) -> bool:
+        """
+        Validate if the language of the text is one of valid languages.
+        """
+        if valid_languages is None:
+            valid_languages = self.valid_languages
+
+        if not valid_languages:
+            return True
+
+        try:
+            lang = langdetect.detect(text)
+        except langdetect.lang_detect_exception.LangDetectException:
+            lang = None
+
+        return lang in valid_languages
+
+    def run(  # type: ignore
+            self,
+            file_paths: Union[Path, List[Path]],  # type: ignore
+            meta: Optional[Union[Dict[str, str],
+                                 List[Dict[str, str]]]] = None,  # type: ignore
+            remove_numeric_tables: Optional[bool] = None,  # type: ignore
+            valid_languages: Optional[List[str]] = None,  # type: ignore
+    ):
+
+        if isinstance(file_paths, Path):
+            file_paths = [file_paths]
+
+        if meta is None or isinstance(meta, dict):
+            meta = [meta] * len(file_paths)  # type: ignore
+
+        documents: list = []
+        for file_path, file_meta in zip(file_paths, meta):
+            for doc in self.convert(
+                    file_path=file_path,
+                    meta=file_meta,
+                    remove_numeric_tables=remove_numeric_tables,
+                    valid_languages=valid_languages,
+            ):
+                documents.append(doc)
+
+        result = {"documents": documents}
+        return result, "output_1"

applications/experimental/pipelines/pipelines/nodes/file_converter/docx.py

@@ -0,0 +1,75 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Optional, Any, List
+
+import logging
+from pathlib import Path
+import docx
+
+from pipelines.nodes.file_converter import BaseConverter
+
+logger = logging.getLogger(__name__)
+
+
+class DocxToTextConverter(BaseConverter):
+
+    def convert(
+        self,
+        file_path: Path,
+        meta: Optional[Dict[str, str]] = None,
+        remove_numeric_tables: Optional[bool] = None,
+        valid_languages: Optional[List[str]] = None,
+        encoding: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        Extract text from a .docx file.
+        Note: As docx doesn't contain "page" information, we actually extract and return a list of paragraphs here.
+        For compliance with other converters we nevertheless opted for keeping the methods name.
+
+        :param file_path: Path to the .docx file you want to convert
+        :param meta: dictionary of meta data key-value pairs to append in the returned document.
+        :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
+                                      The tabular structures in documents might be noise for the reader model if it
+                                      does not have table parsing capability for finding answers. However, tables
+                                      may also have long strings that could possible candidate for searching answers.
+                                      The rows containing strings are thus retained in this option.
+        :param valid_languages: validate languages from a list of languages specified in the ISO 639-1
+                                (https://en.wikipedia.org/wiki/ISO_639-1) format.
+                                This option can be used to add test for encoding errors. If the extracted text is
+                                not one of the valid languages, then it might likely be encoding error resulting
+                                in garbled text.
+        :param encoding: Not applicable
+        """
+        if remove_numeric_tables is None:
+            remove_numeric_tables = self.remove_numeric_tables
+        if valid_languages is None:
+            valid_languages = self.valid_languages
+        if remove_numeric_tables is True:
+            raise Exception(
+                "'remove_numeric_tables' is not supported by DocxToTextConverter."
+            )
+        if valid_languages is True:
+            raise Exception(
+                "Language validation using 'valid_languages' is not supported by DocxToTextConverter."
+            )
+
+        file = docx.Document(file_path)  # Creating word reader object.
+        paragraphs = [para.text for para in file.paragraphs]
+        documents = []
+        for page in paragraphs:
+            document = {"content": page, "content_type": "text", "meta": meta}
+            documents.append(document)
+        return documents

applications/experimental/pipelines/pipelines/nodes/file_converter/image.py

@@ -13,18 +13,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
-from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import List, Optional, Dict, Any, Union
 
-import paddle
+import logging
+import subprocess
+from pathlib import Path
 from paddleocr import PaddleOCR
+try:
+    from PIL.PpmImagePlugin import PpmImageFile
+    from PIL import Image
+except (ImportError, ModuleNotFoundError) as ie:
+    from pipelines.utils.import_utils import _optional_component_not_installed
+
+    _optional_component_not_installed(__name__, "ocr", ie)
+
 from pipelines.nodes.file_converter import BaseConverter
 
 logger = logging.getLogger(__name__)
 
 
 class ImageToTextConverter(BaseConverter):
+
     def __init__(
         self,
         remove_numeric_tables: bool = False,
@@ -46,11 +55,11 @@ class ImageToTextConverter(BaseConverter):
         """
 
         # save init parameters to enable export of component config as YAML
-        self.set_config(remove_numeric_tables=remove_numeric_tables, valid_languages=valid_languages)
-        use_gpu = True if "gpu" in paddle.device.get_device() else False
-        self.recognize = PaddleOCR(use_angle_cls=True, lang="ch", use_gpu=use_gpu)
-
-        super().__init__(remove_numeric_tables=remove_numeric_tables, valid_languages=valid_languages)
+        self.set_config(remove_numeric_tables=remove_numeric_tables,
+                        valid_languages=valid_languages)
+        self.recognize = PaddleOCR(use_angle_cls=True, lang='ch')
+        super().__init__(remove_numeric_tables=remove_numeric_tables,
+                         valid_languages=valid_languages)
 
     def convert(
         self,
@@ -59,7 +68,6 @@ class ImageToTextConverter(BaseConverter):
         remove_numeric_tables: Optional[bool] = None,
         valid_languages: Optional[List[str]] = None,
         encoding: Optional[str] = "utf-8",
-        **kwargs: Any,
     ) -> List[Dict[str, Any]]:
         """
         Extract text from image file using the pytesseract library (https://github.com/madmaze/pytesseract)
@@ -90,11 +98,14 @@ class ImageToTextConverter(BaseConverter):
             cleaned_lines = []
             for line in lines:
                 words = line.split()
-                digits = [word for word in words if any(i.isdigit() for i in word)]
+                digits = [
+                    word for word in words if any(i.isdigit() for i in word)
+                ]
 
                 # remove lines having > 40% of words as digits AND not ending with a period(.)
                 if remove_numeric_tables:
-                    if words and len(digits) / len(words) > 0.4 and not line.strip().endswith("."):
+                    if words and len(digits) / len(
+                            words) > 0.4 and not line.strip().endswith("."):
                         logger.debug(f"Removing line '{line}' from file")
                         continue
                 cleaned_lines.append(line)
@@ -105,8 +116,7 @@ class ImageToTextConverter(BaseConverter):
             if not self.validate_language(document_text, valid_languages):
                 logger.warning(
                     f"The language for image is not one of {valid_languages}. The file may not have "
-                    f"been decoded in the correct text format."
-                )
+                    f"been decoded in the correct text format.")
         documents = []
         for page in cleaned_pages:
             document = {"content": page, "meta": meta}
@@ -122,7 +132,7 @@ class ImageToTextConverter(BaseConverter):
         img_path = str(img_path)
         result = self.recognize.ocr(img_path, cls=True)
         texts = []
-        for line in result[0]:
+        for line in result:
             texts.append(line[-1][0])
-        texts = ["".join(texts)]
+        texts = [''.join(texts)]
         return texts

applications/experimental/pipelines/pipelines/nodes/file_converter/markdown.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+try:
+    from bs4 import BeautifulSoup
+    from markdown import markdown
+except (ImportError, ModuleNotFoundError) as ie:
+    from pipelines.utils.import_utils import _optional_component_not_installed
+
+    _optional_component_not_installed(__name__, "preprocessing", ie)
+
+from pipelines.nodes.file_converter import BaseConverter
+
+logger = logging.getLogger(__name__)
+
+
+class MarkdownConverter(BaseConverter):
+
+    def convert(
+        self,
+        file_path: Path,
+        meta: Optional[Dict[str, str]] = None,
+        remove_numeric_tables: Optional[bool] = None,
+        valid_languages: Optional[List[str]] = None,
+        encoding: Optional[str] = "utf-8",
+    ) -> List[Dict[str, Any]]:
+        """
+        Reads text from a txt file and executes optional preprocessing steps.
+
+        :param file_path: path of the file to convert
+        :param meta: dictionary of meta data key-value pairs to append in the returned document.
+        :param encoding: Select the file encoding (default is `utf-8`)
+        :param remove_numeric_tables: Not applicable
+        :param valid_languages: Not applicable
+
+        :return: Dict of format {"text": "The text from file", "meta": meta}}
+        """
+        with open(file_path, encoding=encoding, errors="ignore") as f:
+            markdown_text = f.read()
+        text = self.markdown_to_text(markdown_text)
+        document = {"content": text, "content_type": "text", "meta": meta}
+        return [document]
+
+    # Following code snippet is copied from https://gist.github.com/lorey/eb15a7f3338f959a78cc3661fbc255fe
+    @staticmethod
+    def markdown_to_text(markdown_string: str) -> str:
+        """
+        Converts a markdown string to plaintext
+
+        :param markdown_string: String in markdown format
+        """
+        # md -> html -> text since BeautifulSoup can extract text cleanly
+        html = markdown(markdown_string)
+
+        # remove code snippets
+        html = re.sub(r"<pre>(.*?)</pre>", " ", html)
+        html = re.sub(r"<code>(.*?)</code >", " ", html)
+
+        # extract text
+        soup = BeautifulSoup(html, "html.parser")
+        text = "".join(soup.findAll(text=True))
+
+        return text

applications/experimental/pipelines/pipelines/nodes/file_converter/pdf.py

@@ -0,0 +1,245 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional, Dict, Any
+
+import os
+import logging
+import tempfile
+import subprocess
+from pathlib import Path
+import pdfplumber
+
+from pipelines.nodes.file_converter import BaseConverter, ImageToTextConverter
+
+logger = logging.getLogger(__name__)
+
+
+class PDFToTextConverter(BaseConverter):
+
+    def __init__(
+        self,
+        remove_numeric_tables: bool = False,
+        valid_languages: Optional[List[str]] = None,
+    ):
+        """
+        :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
+                                      The tabular structures in documents might be noise for the reader model if it
+                                      does not have table parsing capability for finding answers. However, tables
+                                      may also have long strings that could possible candidate for searching answers.
+                                      The rows containing strings are thus retained in this option.
+        :param valid_languages: validate languages from a list of languages specified in the ISO 639-1
+                                (https://en.wikipedia.org/wiki/ISO_639-1) format.
+                                This option can be used to add test for encoding errors. If the extracted text is
+                                not one of the valid languages, then it might likely be encoding error resulting
+                                in garbled text.
+        """
+        # save init parameters to enable export of component config as YAML
+        self.set_config(remove_numeric_tables=remove_numeric_tables,
+                        valid_languages=valid_languages)
+
+        verify_installation = subprocess.run(["pdftotext -v"], shell=True)
+        if verify_installation.returncode == 127:
+            raise Exception(
+                """pdftotext is not installed. It is part of xpdf or poppler-utils software suite.
+                
+                   Installation on Linux:
+                   wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-linux-4.03.tar.gz &&
+                   tar -xvf xpdf-tools-linux-4.03.tar.gz && sudo cp xpdf-tools-linux-4.03/bin64/pdftotext /usr/local/bin
+                   
+                   Installation on MacOS:
+                   brew install xpdf
+                   
+                   You can find more details here: https://www.xpdfreader.com
+                """)
+
+        super().__init__(remove_numeric_tables=remove_numeric_tables,
+                         valid_languages=valid_languages)
+
+    def convert(
+        self,
+        file_path: Path,
+        meta: Optional[Dict[str, str]] = None,
+        remove_numeric_tables: Optional[bool] = None,
+        valid_languages: Optional[List[str]] = None,
+        encoding: Optional[str] = "Latin1",
+    ) -> List[Dict[str, Any]]:
+        """
+        Extract text from a .pdf file using the pdftotext library (https://www.xpdfreader.com/pdftotext-man.html)
+
+        :param file_path: Path to the .pdf file you want to convert
+        :param meta: Optional dictionary with metadata that shall be attached to all resulting documents.
+                     Can be any custom keys and values.
+        :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
+                                      The tabular structures in documents might be noise for the reader model if it
+                                      does not have table parsing capability for finding answers. However, tables
+                                      may also have long strings that could possible candidate for searching answers.
+                                      The rows containing strings are thus retained in this option.
+        :param valid_languages: validate languages from a list of languages specified in the ISO 639-1
+                                (https://en.wikipedia.org/wiki/ISO_639-1) format.
+                                This option can be used to add test for encoding errors. If the extracted text is
+                                not one of the valid languages, then it might likely be encoding error resulting
+                                in garbled text.
+        :param encoding: Encoding that will be passed as -enc parameter to pdftotext. "Latin 1" is the default encoding
+                         of pdftotext. While this works well on many PDFs, it might be needed to switch to "UTF-8" or
+                         others if your doc contains special characters (e.g. German Umlauts, Cyrillic characters ...).
+                         Note: With "UTF-8" we experienced cases, where a simple "fi" gets wrongly parsed as
+                         "xef\xac\x81c" (see test cases). That's why we keep "Latin 1" as default here.
+                         (See list of available encodings by running `pdftotext -listenc` in the terminal)
+        """
+
+        pages = self._read_pdf(file_path, layout=False, encoding=encoding)
+        if remove_numeric_tables is None:
+            remove_numeric_tables = self.remove_numeric_tables
+        if valid_languages is None:
+            valid_languages = self.valid_languages
+
+        cleaned_pages = []
+        for page in pages:
+            # pdftotext tool provides an option to retain the original physical layout of a PDF page. This behaviour
+            # can be toggled by using the layout param.
+            #  layout=True
+            #      + table structures get retained better
+            #      - multi-column pages(eg, research papers) gets extracted with text from multiple columns on same line
+            #  layout=False
+            #      + keeps strings in content stream order, hence multi column layout works well
+            #      - cells of tables gets split across line
+            #
+            #  Here, as a "safe" default, layout is turned off.
+            lines = page.splitlines()
+            cleaned_lines = []
+            for line in lines:
+                words = line.split()
+                digits = [
+                    word for word in words if any(i.isdigit() for i in word)
+                ]
+
+                # remove lines having > 40% of words as digits AND not ending with a period(.)
+                if remove_numeric_tables:
+                    if words and len(digits) / len(
+                            words) > 0.4 and not line.strip().endswith("."):
+                        logger.debug(f"Removing line '{line}' from {file_path}")
+                        continue
+                cleaned_lines.append(line)
+            cleaned_pages.extend(cleaned_lines)
+
+        if valid_languages:
+            document_text = "".join(cleaned_pages)
+            if not self.validate_language(document_text, valid_languages):
+                logger.warning(
+                    f"The language for {file_path} is not one of {valid_languages}. The file may not have "
+                    f"been decoded in the correct text format.")
+
+        documents = []
+        for page in cleaned_pages:
+            document = {"content": page, "content_type": "text", "meta": meta}
+            documents.append(document)
+        return documents
+
+    def _read_pdf(self,
+                  file_path: Path,
+                  layout: bool,
+                  encoding: Optional[str] = "Latin1") -> List[str]:
+        """
+        Extract pages from the pdf file at file_path.
+
+        :param file_path: path of the pdf file
+        :param layout: whether to retain the original physical layout for a page. If disabled, PDF pages are read in
+                       the content stream order.
+        """
+        pdf = pdfplumber.open(file_path)
+        page_text = []
+        for page in pdf.pages:
+            paragraphs = page.extract_text()
+            page_text.append(paragraphs)
+        return page_text
+
+
+class PDFToTextOCRConverter(BaseConverter):
+
+    def __init__(
+        self,
+        remove_numeric_tables: bool = False,
+        valid_languages: Optional[List[str]] = ["eng"],
+    ):
+        """
+        Extract text from image file using the pytesseract library (https://github.com/madmaze/pytesseract)
+
+        :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
+                                      The tabular structures in documents might be noise for the reader model if it
+                                      does not have table parsing capability for finding answers. However, tables
+                                      may also have long strings that could possible candidate for searching answers.
+                                      The rows containing strings are thus retained in this option.
+        :param valid_languages: validate languages from a list of languages supported by tessarect
+                                (https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html).
+                                This option can be used to add test for encoding errors. If the extracted text is
+                                not one of the valid languages, then it might likely be encoding error resulting
+                                in garbled text.
+        """
+        # init image to text instance
+        self.image_2_text = ImageToTextConverter(remove_numeric_tables,
+                                                 valid_languages)
+
+        # save init parameters to enable export of component config as YAML
+        self.set_config(remove_numeric_tables=remove_numeric_tables,
+                        valid_languages=valid_languages)
+        super().__init__(remove_numeric_tables=remove_numeric_tables,
+                         valid_languages=valid_languages)
+
+    def convert(
+        self,
+        file_path: Path,
+        meta: Optional[Dict[str, str]] = None,
+        remove_numeric_tables: Optional[bool] = None,
+        valid_languages: Optional[List[str]] = None,
+        encoding: Optional[str] = "utf-8",
+    ) -> List[Dict[str, Any]]:
+        """
+        Convert a file to a dictionary containing the text and any associated meta data.
+
+        File converters may extract file meta like name or size. In addition to it, user
+        supplied meta data like author, url, external IDs can be supplied as a dictionary.
+
+        :param file_path: path of the file to convert
+        :param meta: dictionary of meta data key-value pairs to append in the returned document.
+        :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
+                                      The tabular structures in documents might be noise for the reader model if it
+                                      does not have table parsing capability for finding answers. However, tables
+                                      may also have long strings that could possible candidate for searching answers.
+                                      The rows containing strings are thus retained in this option.
+        :param valid_languages: validate languages from a list of languages specified in the ISO 639-1
+                                (https://en.wikipedia.org/wiki/ISO_639-1) format.
+                                This option can be used to add test for encoding errors. If the extracted text is
+                                not one of the valid languages, then it might likely be encoding error resulting
+                                in garbled text.
+        :param encoding: Select the file encoding (default is `utf-8`)
+        """
+        pages = []
+        try:
+            images = convert_from_path(file_path)
+            for image in images:
+                temp_img = tempfile.NamedTemporaryFile(dir=os.path.dirname(
+                    os.path.realpath(__file__)),
+                                                       suffix=".jpeg")
+                image.save(temp_img.name)
+                pages.append(
+                    self.image_2_text.convert(temp_img.name)[0]["content"])
+        except Exception as exception:
+            logger.error(f"File {file_path} has an error \n {exception}")
+
+        raw_text = "\f".join(pages)
+        document = {"content": raw_text, "meta": meta}
+
+        return [document]

applications/experimental/pipelines/pipelines/nodes/file_converter/txt.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2021 deepset GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional, Any, Dict
+
+import logging
+from pathlib import Path
+from pipelines.nodes.file_converter import BaseConverter
+
+logger = logging.getLogger(__name__)
+
+
+class TextConverter(BaseConverter):
+
+    def convert(
+        self,
+        file_path: Path,
+        meta: Optional[Dict[str, str]] = None,
+        remove_numeric_tables: Optional[bool] = None,
+        valid_languages: Optional[List[str]] = None,
+        encoding: Optional[str] = "utf-8",
+    ) -> List[Dict[str, Any]]:
+        """
+        Reads text from a txt file and executes optional preprocessing steps.
+
+        :param file_path: path of the file to convert
+        :param meta: dictionary of meta data key-value pairs to append in the returned document.
+        :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
+                                      The tabular structures in documents might be noise for the reader model if it
+                                      does not have table parsing capability for finding answers. However, tables
+                                      may also have long strings that could possible candidate for searching answers.
+                                      The rows containing strings are thus retained in this option.
+        :param valid_languages: validate languages from a list of languages specified in the ISO 639-1
+                                (https://en.wikipedia.org/wiki/ISO_639-1) format.
+                                This option can be used to add test for encoding errors. If the extracted text is
+                                not one of the valid languages, then it might likely be encoding error resulting
+                                in garbled text.
+        :param encoding: Select the file encoding (default is `utf-8`)
+
+        :return: Dict of format {"text": "The text from file", "meta": meta}}
+
+        """
+        if remove_numeric_tables is None:
+            remove_numeric_tables = self.remove_numeric_tables
+        if valid_languages is None:
+            valid_languages = self.valid_languages
+
+        with open(file_path, encoding=encoding, errors="ignore") as f:
+            text = f.read()
+            pages = text.split("\n\n")
+
+        cleaned_pages = []
+        for page in pages:
+            lines = page.splitlines()
+            cleaned_lines = []
+            for line in lines:
+                words = line.split()
+                digits = [
+                    word for word in words if any(i.isdigit() for i in word)
+                ]
+
+                # remove lines having > 40% of words as digits AND not ending with a period(.)
+                if remove_numeric_tables:
+                    if words and len(digits) / len(
+                            words) > 0.4 and not line.strip().endswith("."):
+                        logger.debug(f"Removing line '{line}' from {file_path}")
+                        continue
+
+                cleaned_lines.append(line)
+
+            page = "\n".join(cleaned_lines)
+            cleaned_pages.append(page)
+
+        if valid_languages:
+            document_text = "".join(cleaned_pages)
+            if not self.validate_language(document_text, valid_languages):
+                logger.warning(
+                    f"The language for {file_path} is not one of {valid_languages}. The file may not have "
+                    f"been decoded in the correct text format.")
+        documents = []
+        for page in cleaned_pages:
+            document = {"content": page, "content_type": "text", "meta": meta}
+            documents.append(document)
+        return documents