[cherry-pick] Support zero dataset split, fix fp32 dist dataloader (#7265)

* Support zero dataset splits for pre-training (#7259)

* support empty splits

* Fix fp32 dist dataloader (#7261)

.copyright.hook

@@ -71,7 +71,7 @@ RE_SHEBANG = re.compile(r"^[ \t\v]*#[ \t]?\!")
 def _check_copyright(path):
     head=[]
     try:
-        with open(path, encoding="utf-8") as f:
+        with open(path) as f:
             head = [next(f) for x in range(4)]
     except StopIteration:
         pass

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,18 +1,4 @@
 <!-- Demo: https://github.com/PaddlePaddle/PaddleNLP/pull/26 -->
-#### Before submitting
-
-- [ ] Lint code. If there are lint issues, please format the code first.
-
-```shell
-# Install and register `pre-commit` in the project folder
-pip install pre-commit && pre-commit install
-
-# Process previous code files separately
-pre-commit run --file XXXX.py
-```
-
-- [ ] Add test cases into `tests` folder. If there are codecov issues, please add tests cases first.
-
 ### PR types
 <!-- One of [ New features | Bug fixes | Function optimization | Performance optimization | Breaking changes | Others ] -->
 

.github/actions/check-bypass/action.yml

@@ -1,41 +0,0 @@
-name: "Check bypass"
-description: "A custom action to encapsulate PFCCLab/ci-bypass"
-inputs:
-  github-token:
-    description: "GitHub token"
-    required: true
-  workflow-name:
-    description: "Workflow name"
-    required: true
-outputs:
-  can-skip:
-    description: "Whether the workflow can be skipped."
-    value: ${{ steps.check-bypass.outputs.can-skip }}
-
-runs:
-  using: "composite"
-  steps:
-    - id: check-bypass
-      name: Check Bypass
-      env:
-        CI_TEAM_MEMBERS: '["tianshuo78520a", "swgu98", "risemeup1", "XieYunshen","luotao1","From00"]'
-      uses: PFCCLab/ci-bypass@v1
-      with:
-        github-token: ${{ inputs.github-token }}
-        non-pull-request-event-strategy: 'never-skipped'
-        type: 'composite'
-        composite-rule: |
-          {
-            "any": [
-              {
-                "type": "labeled",
-                "label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
-                "username": ${{ env.CI_TEAM_MEMBERS }}
-              },
-              {
-                "type": "commented",
-                "comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
-                "username": ${{ env.CI_TEAM_MEMBERS }}
-              }
-            ]
-          }

.github/actions/rerun-workflow/action.yml

@@ -1,30 +0,0 @@
-name: 'Rerun Workflow'
-description: 'Re-run GitHub Actions workflow for a given Pull Request'
-inputs:
-  GITHUB_TOKEN:
-    description: 'GitHub token with repo scope'
-    required: true
-  OWNER:
-    description: 'Repository owner'
-    required: true
-  REPO:
-    description: 'Repository name'
-    required: true
-  PR_ID:
-    description: 'Pull Request ID'
-    required: true
-  JOB_NAME:
-    description: 'Job name to rerun'
-    required: true
-
-runs:
-  using: 'composite'
-  steps:
-    - run: bash ./.github/actions/rerun-workflow/rerun.sh
-      shell: bash
-      env:
-        GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }}
-        OWNER: ${{ inputs.OWNER }}
-        REPO: ${{ inputs.REPO }}
-        PR_ID: ${{ inputs.PR_ID }}
-        JOB_NAME: ${{ inputs.JOB_NAME }}

.github/actions/rerun-workflow/rerun.sh

@@ -1,77 +0,0 @@
-# Copyright (c) 2025 PaddleNLP Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-COMMIT_SHA=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
-  "https://api.github.com/repos/$OWNER/$REPO/pulls/$PR_ID" | jq -r '.head.sha')
-
-echo "Commit SHA: $COMMIT_SHA"
-
-response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
-  "https://api.github.com/repos/$OWNER/$REPO/actions/runs?head_sha=$COMMIT_SHA&per_page=100")
-
-echo "Response: $response"
-
-run_ids=$(echo "$response" | jq -r '.workflow_runs[].id')
-
-if [ -n "$run_ids" ]; then
-  echo "Found run_ids for commit $COMMIT_SHA: $run_ids"
-
-  for run_id in $run_ids; do
-    if [ "$JOB_NAME" = "all-failed" ]; then
-      echo "Rerunning all failed jobs for run_id: $run_id"
-
-      rerun_response=$(curl -X POST -s -w "%{http_code}" -o /dev/null \
-        -H "Accept: application/vnd.github.v3+json" \
-        -H "Authorization: Bearer $GITHUB_TOKEN" \
-        "https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/rerun-failed-jobs")
-      if [ "$rerun_response" -eq 201 ]; then
-        echo "Successfully requested rerun for all blocked jobs in run_id: $run_id"
-      else
-        echo "Failed to request rerun for run_id: $run_id with status code $rerun_response"
-      fi
-
-    else
-      jobs_response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
-      "https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/jobs")
-
-      echo "Jobs Response for run_id $run_id: $jobs_response"
-
-      # if [[ "$JOB_NAME" == *"bypass"* ]]; then
-        block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
-        '.jobs[] | select(.name == $job_name) | .id')
-      # else
-      #   block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
-      #   '.jobs[] | select(.name == $job_name and .conclusion != "success") | .id')
-      # fi
-
-      if [ -n "$block_jobs" ]; then
-        echo "Found block jobs for run_id $run_id: $block_jobs"
-
-        for job_id in $block_jobs; do
-          echo "Rerunning job_id: $job_id"
-          curl -X POST -H "Accept: application/vnd.github.v3+json" \
-            -H "Authorization: token $GITHUB_TOKEN" \
-            "https://api.github.com/repos/$OWNER/$REPO/actions/jobs/$job_id/rerun"
-        done
-      else
-        echo "No block jobs found for run_id $run_id with name $JOB_NAME."
-      fi
-    fi
-  done
-else
-  echo "No matching workflow runs found for commit $COMMIT_SHA."
-  exit 1
-fi

.github/codecov.yml

@@ -1,13 +1,8 @@
-codecov:
-  notify:
-    wait_for_ci: false
-
 coverage:
   status:
     project:
-      default: 
-        target: 58% # overall project Coverage 
-        threshold: 1% # Allow the coverage to drop by 1%, and posting a success status.
+      default:
+        informational: true
     patch:
-      default: 
-        target: 80% # lines adjusted  Coverage < 80%  CI will fail
+      default:
+        informational: true

.github/workflows/PaddleFormers-PR-Check.yml

@@ -1,47 +0,0 @@
-name: PaddleFormers PR Check
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-
-env:
-  BRANCH: ${{ github.base_ref }}
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  token: ${{ vars.ACTION_GITHUB_TOKEN }}
-
-jobs:
-  check-approvers:
-    name: Check approval
-    runs-on: 
-      group: APPROVAL
-    steps:
-      - name: Cleanup
-        run: |
-          rm -rf * .[^.]*
-      - name: Update paddle
-        run: |
-          wget -q --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleNLP/PaddleNLP-develop.tar.gz --no-check-certificate
-          tar zxf PaddleNLP-develop.tar.gz --strip-components=1 >/dev/null
-          rm -rf PaddleNLP-develop.tar.gz >/dev/null
-          git fetch origin pull/${PR_ID}/head
-          git checkout -b origin_pr FETCH_HEAD
-          git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-          git fetch upstream
-          git checkout -b test_pr upstream/${BRANCH}
-          git merge --no-edit origin_pr
-          git log --pretty=oneline -10
-
-      - name: Check bypass
-        id: check-bypass
-        uses: ./.github/actions/check-bypass
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          workflow-name: approval
-        
-      - name: Display Required Approvers
-        if: steps.check-bypass.outputs.can-skip != 'true'
-        run: |
-          cd scripts/ci_approval
-          bash run_ci_approval_cherry-pick.sh

.github/workflows/approval.yml

@@ -1,47 +0,0 @@
-name: Approval
-
-on:
-  pull_request:
-    types: [opened, synchronize]
-
-env:
-  BRANCH: ${{ github.base_ref }}
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  token: ${{ vars.ACTION_GITHUB_TOKEN }}
-
-jobs:
-  check-approvers:
-    name: Check approval
-    runs-on: 
-      group: APPROVAL
-    steps:
-      - name: Cleanup
-        run: |
-          rm -rf * .[^.]*
-      - name: Update paddle
-        run: |
-          wget -q --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleNLP/PaddleNLP-develop.tar.gz --no-check-certificate
-          tar zxf PaddleNLP-develop.tar.gz --strip-components=1 >/dev/null
-          rm -rf PaddleNLP-develop.tar.gz >/dev/null
-          git fetch origin pull/${PR_ID}/head
-          git checkout -b origin_pr FETCH_HEAD
-          git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-          git fetch upstream
-          git checkout -b test_pr upstream/${BRANCH}
-          git merge --no-edit origin_pr
-          git log --pretty=oneline -10
-
-      - name: Check bypass
-        id: check-bypass
-        uses: ./.github/actions/check-bypass
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          workflow-name: approval
-        
-      - name: Display Required Approvers
-        if: steps.check-bypass.outputs.can-skip != 'true'
-        run: |
-          cd scripts/ci_approval
-          bash -x run_ci_approval.sh

.github/workflows/check-bypass.yml

@@ -1,52 +0,0 @@
-on:
-  workflow_call:
-    inputs:
-      workflow-name:
-        required: true
-        type: string
-    secrets:
-      github-token:
-        required: true
-    outputs:
-      can-skip:
-        description: "Whether the workflow can be skipped."
-        value: ${{ jobs.check-bypass.outputs.can-skip }}
-
-jobs:
-  check-bypass:
-    name: Check bypass
-    runs-on:
-      group: APPROVAL
-    permissions:
-      contents: read
-    env:
-      CI_TEAM_MEMBERS: '["tianshuo78520a", "swgu98", "risemeup1" , "XieYunshen","luotao1","From00"]'
-    outputs:
-      can-skip: ${{ steps.check-bypass.outputs.can-skip }}
-    steps:
-      - name: Cleanup
-        run: |
-          rm -rf * .[^.]*
-
-      - id: check-bypass
-        name: Check Bypass
-        uses: PFCCLab/ci-bypass@v1
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          non-pull-request-event-strategy: 'never-skipped'
-          type: 'composite'
-          composite-rule: |
-            {
-              "any": [
-                {
-                  "type": "labeled",
-                  "label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
-                  "username": ${{ env.CI_TEAM_MEMBERS }}
-                },
-                {
-                  "type": "commented",
-                  "comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
-                  "username": ${{ env.CI_TEAM_MEMBERS }}
-                }
-              ]
-            }

.github/workflows/fast_tokenizer.yml

@@ -0,0 +1,48 @@
+name: FastTokenizer
+
+on:
+  push:
+    paths:
+      - 'fast_tokenizer/*'
+  pull_request:
+    paths:
+      - 'fast_tokenizer/*'
+
+jobs:
+  fast_tokenizer_cpp:
+    name: fast_tokenizer_cpp
+    runs-on: ubuntu-22.04
+    permissions:
+      pull-requests: write
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: compile
+        working-directory: ./fast_tokenizer
+        run: make fast_tokenizer_cpp_compile
+      - name: test
+        working-directory: ./fast_tokenizer
+        run: make fast_tokenizer_cpp_test
+  fast_tokenizer_python38:
+      name: fast_tokenizer_python38
+      runs-on: ubuntu-22.04
+      permissions:
+        pull-requests: write
+        contents: read
+        id-token: write
+      steps:
+        - uses: actions/checkout@v3
+        - uses: actions/setup-python@v1
+          with:
+            python-version: 3.8
+        - name: install
+          working-directory: ./fast_tokenizer
+          run: make fast_tokenizer_python_install
+        - name: compile
+          working-directory: ./fast_tokenizer
+          run: make fast_tokenizer_python_compile
+        - name: test
+          working-directory: ./fast_tokenizer
+          run: make fast_tokenizer_python_test
+  

.github/workflows/lint.yml

@@ -1,80 +1,17 @@
-name: Codestyle Check
+name: Lint
 
 on: [push, pull_request]
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-env:
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  BRANCH: ${{ github.event.pull_request.base.ref }}
-  TASK: PaddleNLP-CI-Lint-${{ github.event.pull_request.number }}
-
 jobs:
-  check-bypass:
-    name: Check bypass
-    uses: ./.github/workflows/check-bypass.yml
-    with:
-      workflow-name: 'lint'
-    secrets:
-      github-token: ${{ secrets.GITHUB_TOKEN }}
-
   Lint:
     name: Lint
-    needs: check-bypass
-    if: ${{ needs.check-bypass.outputs.can-skip != 'true' }}
-    runs-on: [self-hosted, ernie-cpu]
+    runs-on: ubuntu-20.04
     steps:
-      - name: Run Container
-        env:
-          work_dir: ${{ github.workspace }}
-          python_version: "3.10"
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: checkout develop
         run: |
-          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
-          echo "container_name=${container_name}" >> ${{ github.env }}
-          docker_image="iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu20.04-gcc12.2-cuda12.3-cudnn9.0-nccl2.20.3.1-openmpi4.1.5-latest"
-          docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
-            -v $work_dir/../../..:$work_dir/../../.. \
-            -v $work_dir:/workspace \
-            -v /home/.cache/pip:/home/.cache/pip \
-            -e BRANCH \
-            -e PR_ID \
-            -e COMMIT_ID \
-            -e work_dir \
-            -e no_proxy \
-            -e python_version \
-            -w /workspace ${docker_image}
-
-      - name: Download Code
-        env:
-          work_dir: ${{ github.workspace }}
-        run: |
-          docker exec -t ${container_name} /bin/bash -c '
-          rm -rf * .[^.]*
-          echo "Downloading PaddleNLP.tar"
-          wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
-          echo "Extracting PaddleNLP.tar"
-          tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
-          source $work_dir/../../../proxy
-          cd PaddleNLP
-          git config --global user.name "PaddleCI"
-          git config --global user.email "paddle_ci@example.com"
-          git pull
-          git submodule update --init --recursive --force
-          if [ -n "${PR_ID}" ]; then
-            git fetch origin pull/${PR_ID}/head
-            git checkout -b PR_${PR_ID} FETCH_HEAD
-            git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-            git fetch upstream ${BRANCH}
-            git merge ${BRANCH} --no-edit
-            git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
-          else
-            echo "Not in a pull_request event. Skipping PR-specific operations."
-          fi
-          git log --pretty=oneline -10
-
           if ! git show-ref --quiet refs/heads/develop; then \
               echo "local develop branch is missing, creating local develop branch that tracks remote develop branch"
               git fetch origin develop
@@ -82,32 +19,14 @@ jobs:
           else
             echo "local develop branch exist, skipping"
           fi
-          '
-          
-      - name: Setup Environment
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+          cache: 'pip' # caching pip dependencies
+      - name: Install dependencies
         run: |
-          docker exec -t $container_name /bin/bash -c '
-          unlink /usr/local/bin/python
-          ln -sf $(which python${python_version}) /usr/local/bin/python
-          set -e
-          python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-          python -m pip config set global.cache-dir "/home/.cache/pip"
-          source $work_dir/../../../proxy
           python -m pip install --upgrade pip
-          cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
           make install
-          '
-
-      - name: Test
-        run: |
-          docker exec -t $container_name /bin/bash -c '
-          set -e
-          source $work_dir/../../../proxy
-          cd /workspace/PaddleNLP
-          make lint
-          '
-      
-      - name: Terminate And Delete the Container
-        if: always()
-        run: |
-          docker rm -f $container_name 2>/dev/null || true
+      - name: run the command
+        run: make lint
+    

.github/workflows/llm.yml

@@ -1,207 +0,0 @@
-# Disabled LLM CI workflow - manually enabled when needed
-name: LLM CI (DISABLED)
-
-#on:
-#  pull_request:
-#    types: [opened, synchronize, reopened]
-#    branches: [develop]
-#    schedule:
-#    - cron: "2 0 * * *"
-#  workflow_call:
-#    inputs:
-#      run_downstream:
-#        required: true
-#        type: string
-#      image_name:
-#        required: true
-#        type: string
-
-#concurrency:
-#  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-#  cancel-in-progress: true
-
-#env:
-#  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  TASK: paddlenlp-CI-${{ github.event.pull_request.number }}-llm
-  ci_scripts: /workspace/PaddleNLP/scripts/regression
-  BRANCH: ${{ github.event.pull_request.base.ref }}
-  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
-  CI_name: llm-ci
-  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
-  HF_ENDPOINT: https://hf-mirror.com
-  STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
-  PPNLP_HOME: /ssd1/paddlenlp
-  HF_DATASETS_CACHE: /ssd1/paddlenlp/huggingface/datasets
-  TRANSFORMERS_CACHE: /ssd1/paddlenlp/huggingface
-  CCACHE_DIR: /home/data/gzcfs/.ccache/gpubox
-  RUN_DOWNSTREAM: ${{ inputs.run_downstream }}
-
-#defaults:
-#  run:
-#    shell: bash
-
-#jobs:
-#  llm-ci:
-#    name: llm-ci (DISABLED)
-    runs-on: [self-hosted, ernie-8gpu]
-    steps:
-      - name: Determine Image Name
-        env:
-          IMAGE_NAME: ${{ inputs.image_name }}
-        run: |
-          if [[ -n "${IMAGE_NAME}" ]]; then
-            echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
-          else
-            echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-paddlenlp-latest" >> "$GITHUB_ENV"
-          fi
-
-      - name: Run Container
-        env:
-          work_dir: ${{ github.workspace }}
-          CACHE_DIR: /home/data/cfs/.cache
-          FLAGS_dynamic_static_unified_comm: "True"
-          python_version: "3.10"
-          paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
-        run: |
-          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
-          echo "container_name=${container_name}" >> "$GITHUB_ENV"
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
-              -v $work_dir/../../..:$work_dir/../../.. \
-              -v $work_dir:/workspace \
-              -v /home/.cache/pip:/home/.cache/pip \
-              -v /ssd1/paddlenlp:/ssd1/paddlenlp \
-              -v /home/data/gzcfs/.ccache/gpubox:/home/data/gzcfs/.ccache/gpubox \
-              -e BRANCH \
-              -e AGILE_COMPILE_BRANCH \
-              -e PR_ID \
-              -e COMMIT_ID \
-              -e work_dir \
-              -e ci_scripts \
-              -e no_proxy \
-              -e CI_name \
-              -e paddle_whl \
-              -e HF_ENDPOINT \
-              -e STUDIO_GIT_HOST \
-              -e PPNLP_HOME \
-              -e HF_DATASETS_CACHE \
-              -e TRANSFORMERS_CACHE \
-              -e CACHE_DIR \
-              -e FLAGS_dynamic_static_unified_comm \
-              -e python_version \
-              -w /workspace --runtime=nvidia $IMAGE_NAME
-          fi
-
-      - name: Download Code
-        env:
-          work_dir: ${{ github.workspace }}
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping.."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            rm -rf * .[^.]*
-            echo "Downloading PaddleNLP.tar.gz"
-            wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
-            echo "Extracting PaddleNLP.tar.gz"
-            tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
-            source $work_dir/../../../proxy
-            cd PaddleNLP
-            git config --global user.name "PaddleCI"
-            git config --global user.email "paddle_ci@example.com"
-            git pull
-            git submodule update --init --recursive --force
-            if [ -n "${PR_ID}" ]; then
-              git fetch origin pull/${PR_ID}/head
-              git checkout -b PR_${PR_ID} FETCH_HEAD
-              git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-              git fetch upstream ${BRANCH}
-              git merge ${BRANCH} --no-edit
-              git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
-            else
-              echo "Not in a pull_request event. Skipping PR-specific operations."
-            fi
-            git log --pretty=oneline -10
-            '
-          fi
-
-      - name: Skip For Bug
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            cd /workspace/PaddleNLP
-            git revert f2477c07272d04244cd3287d1f21c70482a4a85f --no-edit  # 套件PR#10413引入bug-待修复
-            git revert d74c950e15a35b7a100d1688c89318195cc83bca --no-edit  # 框架升级后配合修改，当前因框架bug固定commit适配
-            '
-          fi
-      
-      - name: Test
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            ldconfig
-            unlink /usr/bin/python3
-            ln -sf $(which python${python_version}) /usr/bin/python3
-            pip config set global.cache-dir "/home/.cache/pip"
-            set -e
-            source $work_dir/../../../proxy
-            cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
-            echo "Paddle PR#73283  PR#74484 import bug, set paddle_commit=8ae742 to skip "
-            export paddle_whl=https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/8ae7423e99b2ea96e410968a0ebb3f1795e37205/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
-            timeout 2h bash scripts/regression/run_ci.sh python${python_version} ${paddle_whl}
-            '
-          fi
-
-      - name: Upload Allure-reports & Logs
-        if: always()
-        env:
-          home_path: ${{ github.workspace }}/../../..
-          bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
-          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            unset http_proxy && unset https_proxy
-            if [ ! -f "${{ env.bos_file }}" ]; then
-              wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
-              mkdir ${{ env.home_path }}/bos
-              tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
-            fi
-            if [ ! -f "${{ env.allure_file }}" ]; then
-              wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
-              unzip -q ${{ env.home_path }}/allure-2.19.0.zip -d ${{ env.home_path }}/
-            fi
-            if [[ "${{ env.RUN_DOWNSTREAM }}" == "" && -n "${PR_ID}" ]]; then
-              bos_prefix="${PR_ID}/${COMMIT_ID}"
-            elif [[ "${{ env.RUN_DOWNSTREAM }}" == "true" && -n "${PR_ID}" ]]; then
-              bos_prefix="${PR_ID}/${COMMIT_ID}/test_build"
-            else
-              bos_prefix="schedule/$(date +%Y%m%d)"
-            fi
-            cd /workspace/PaddleNLP/model_logs
-            for FILE in /workspace/PaddleNLP/model_logs/*; do
-              file=$(basename "$FILE")
-              python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleNLP/llm/${bos_prefix}/logs
-              echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/llm/${bos_prefix}/logs/$file"
-            done
-            cd /workspace/PaddleNLP/
-            ${{ env.allure_file }} generate result -o report
-            tar -czf products.tar.gz report model_logs
-            python ${{ env.bos_file }} products.tar.gz paddle-github-action/PR/PaddleNLP/llm/${bos_prefix}/logs
-            echo "products: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/llm/${bos_prefix}/logs/products.tar.gz"
-            '
-          fi
-
-      - name: Terminate And Delete the Container
-        if: always()
-        run: |
-          docker rm -f ${{ env.container_name }} 2>/dev/null || true

.github/workflows/pipelines.yml

@@ -3,28 +3,28 @@ name: Pipelines-Test
 on:
   push:
     paths:
-      - 'slm/pipelines/*'
+      - 'pipelines/*'
   pull_request:
     paths:
-      - 'slm/pipelines/*'
+      - 'pipelines/*'
 
 
 jobs:
   Pipelines-Test:
     name: Pipelines-Test
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: 3.8
           cache: 'pip' # caching pip dependencies
       - name: Install dependencies
-        working-directory: ./slm/pipelines
+        working-directory: ./pipelines
         run: |
           python -m pip install --upgrade pip
           make install
           pip install -r tests/requirements.txt
       - name: run the command
-        working-directory: ./slm/pipelines
+        working-directory: ./pipelines
         run: make test

.github/workflows/release.yml

@@ -16,12 +16,12 @@ on:
 jobs:
   Release:
     name: Release
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
-          python-version: '3.10'
+          python-version: 3.8
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -35,14 +35,14 @@ jobs:
       - name: Check Branch
         id: check-branch
         run: |
-          if [[ ${{ github.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          if [[ ${{ github.ref }} =~ ^refs/heads/(develop)$ ]]; then
               echo "match=true" >> $GITHUB_OUTPUT
           fi  # See: https://stackoverflow.com/a/58869470/1123955
 
       - name: Is A Publish Branch
         if: steps.check-branch.outputs.match == 'true' && steps.should-deploy.outputs.should_deploy == 'true'
         env:
-          TWINE_USERNAME: __token__
+          TWINE_USERNAME: paddle-dev
           TWINE_PASSWORD: ${{ secrets.paddlenlp }}
           PACKAGE: ${{ inputs.package }}
           PADDLENLP_STABLE_VERSION: "true"

.github/workflows/rerun.yml

@@ -1,73 +0,0 @@
-name: Re-run
-
-on:
-  issue_comment:
-    types: [created]
-
-jobs:
-  re-run:
-    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/re-run')  && github.event.comment.user.login == github.event.issue.user.login }}
-    runs-on: [self-hosted, ernie-cpu]
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: Rerun all failed jobs
-        if: ${{ contains(github.event.comment.body, 'all-failed') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'all-failed'
-
-      - name: Rerun LLM
-        if: ${{ contains(github.event.comment.body, 'LLM') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'LLM CI / llm-ci'
-
-      - name: Unittest GPU
-        if: ${{ contains(github.event.comment.body, 'Unittest GPU') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'Unittest GPU CI / unittest-gpu-ci'
-
-      - name: Rerun Unittest CPU
-        if: ${{ contains(github.event.comment.body, 'Unittest CPU') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'Unittest CPU CI / unit-cpu-ci'
-
-      - name: Rerun Distribute (V100)
-        if: ${{ contains(github.event.comment.body, 'Distribute (V100)') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'Distribute CI (V100)/ distribute-v100-ci'
-
-      - name: Rerun Codestyle Check
-        if: ${{ contains(github.event.comment.body, 'Codestyle Check') }}
-        uses: ./.github/actions/rerun-workflow
-        with:
-          PR_ID: ${{ github.event.issue.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OWNER: ${{ github.repository_owner }}
-          REPO: ${{ github.event.repository.name }}
-          JOB_NAME: 'Codestyle Check / Lint'

.github/workflows/stale.yml

@@ -13,7 +13,7 @@ permissions:
 
 jobs:
   stale:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-20.04
 
     permissions:
       issues: write

.github/workflows/tests.yml

@@ -0,0 +1,27 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  Test:
+    name: Test
+    runs-on: ubuntu-20.04
+    permissions:
+      pull-requests: write
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+          cache: 'pip' # caching pip dependencies
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tests/requirements.txt
+          make install
+      - name: run the command
+        run: make test
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3

.github/workflows/unittest-cpu.yml

@@ -1,120 +0,0 @@
-# Disabled unittest CPU workflow - manually enabled when needed
-name: Unittest CPU CI (DISABLED)
-
-#on: [pull_request]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-env:
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  BRANCH: ${{ github.event.pull_request.base.ref }}
-  TASK: PaddleNLP-CI-${{ github.event.pull_request.number }}-unittest-cpu
-  HF_ENDPOINT: https://hf-mirror.com
-  STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
-  PPNLP_HOME: /home/disk1/cache
-  HF_DATASETS_CACHE: /home/disk1/cache/huggingface/datasets
-  TRANSFORMERS_CACHE: /home/disk1/cache/huggingface
-
-#jobs:
-#  unittest-cpu-ci:
-#    name: unittest-cpu-ci (DISABLED)
-    runs-on: [self-hosted, ernie-cpu]
-    permissions:
-      pull-requests: write
-      contents: read
-      id-token: write
-    steps:
-      - name: Run Container
-        env:
-          work_dir: ${{ github.workspace }}
-          python_version: "3.10"
-        run: |
-          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
-          echo "container_name=${container_name}" >> ${{ github.env }}
-          docker_image="iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu20.04-gcc12.2-cuda12.3-cudnn9.0-nccl2.20.3.1-openmpi4.1.5-latest"
-          docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
-            -v $work_dir/../../..:$work_dir/../../.. \
-            -v $work_dir:/workspace \
-            -v /home/.cache/pip:/home/.cache/pip \
-            -v /home/disk1/cache:/home/disk1/cache \
-            -e BRANCH \
-            -e PR_ID \
-            -e COMMIT_ID \
-            -e work_dir \
-            -e no_proxy \
-            -e python_version \
-            -e HF_ENDPOINT \
-            -e STUDIO_GIT_HOST \
-            -e PPNLP_HOME \
-            -e HF_DATASETS_CACHE \
-            -e TRANSFORMERS_CACHE \
-            -w /workspace ${docker_image}
-
-      - name: Download Code
-        env:
-          work_dir: ${{ github.workspace }}
-        run: |
-          docker exec -t ${container_name} /bin/bash -c '
-          rm -rf * .[^.]*
-          echo "Downloading PaddleNLP.tar"
-          wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
-          echo "Extracting PaddleNLP.tar"
-          tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
-          source $work_dir/../../../proxy
-          cd PaddleNLP
-          git config --global user.name "PaddleCI"
-          git config --global user.email "paddle_ci@example.com"
-          git pull
-          git submodule update --init --recursive --force
-          if [ -n "${PR_ID}" ]; then
-            git fetch origin pull/${PR_ID}/head
-            git checkout -b PR_${PR_ID} FETCH_HEAD
-            git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-            git fetch upstream ${BRANCH}
-            git merge ${BRANCH} --no-edit
-            git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
-          else
-            echo "Not in a pull_request event. Skipping PR-specific operations."
-          fi
-          git log --pretty=oneline -10
-          '
-
-      - name: Setup Environment
-        run: |
-          docker exec -t $container_name /bin/bash -c '
-          unlink /usr/bin/python3
-          ln -sf $(which python${python_version}) /usr/bin/python3
-          set -e
-          python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-          python -m pip config set global.cache-dir "/home/.cache/pip"
-          source $work_dir/../../../proxy
-          python -m pip install --upgrade pip
-          cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
-          pip install -r tests/requirements.txt
-          make install
-          '
-
-      - name: Test
-        run: |
-          docker exec -t $container_name /bin/bash -c '
-          source $work_dir/../../../proxy
-          cd /workspace/PaddleNLP
-          set -e
-          make test
-          '
-
-      # - name: Upload Coverage To Codecov
-      #   uses: codecov/codecov-action@v3
-      #   env:
-      #     CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-      #   with:
-      #     token: ${{ secrets.CODECOV_TOKEN }}
-      #     files: ./PaddleNLP/coverage.xml
-      
-      - name: Terminate And Delete the Container
-        if: always()
-        run: |
-          docker rm -f $container_name 2>/dev/null || true

.github/workflows/unittest-gpu.yml

@@ -1,204 +0,0 @@
-# Disabled unittest GPU workflow - manually enabled when needed
-name: Unittest GPU CI (DISABLED)
-
-#on:
-#  pull_request:
-#    types: [opened, synchronize, reopened]
-#    branches: [develop]
-#  schedule:
-#    - cron: "3 0 * * *"
-#  workflow_call:
-#    inputs:
-#      run_downstream:
-#        required: true
-#        type: string
-#      image_name:
-#        required: true
-#        type: string
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-env:
-  PR_ID: ${{ github.event.pull_request.number }}
-  COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  TASK: paddlenlp-CI-${{ github.event.pull_request.number }}-unit-gpu
-  ci_scripts: /workspace/PaddleNLP/scripts/unit_test
-  BRANCH: ${{ github.event.pull_request.base.ref }}
-  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
-  CI_name: unittest-gpu-ci
-  no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
-  HF_ENDPOINT: https://hf-mirror.com
-  STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
-  PPNLP_HOME: /ssd1/paddlenlp
-  HF_DATASETS_CACHE: /ssd1/paddlenlp/huggingface/datasets
-  TRANSFORMERS_CACHE: /ssd1/paddlenlp/huggingface
-  CCACHE_DIR: /home/data/gzcfs/.ccache/gpubox
-  RUN_DOWNSTREAM: ${{ inputs.run_downstream }}
-
-defaults:
-  run:
-    shell: bash
-
-#jobs:
-#  unittest-gpu-ci:
-#    name: unittest-gpu-ci (DISABLED)
-    runs-on: [self-hosted, ernie-8gpu]
-    steps:
-      - name: Determine Image Name
-        env:
-          IMAGE_NAME: ${{ inputs.image_name }}
-        run: |
-          if [[ -n "${IMAGE_NAME}" ]]; then
-            echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
-          else
-            echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-paddlenlp-latest" >> "$GITHUB_ENV"
-          fi
-      
-      - name: Run Container
-        env:
-          work_dir: ${{ github.workspace }}
-          CACHE_DIR: /home/data/cfs/.cache
-          FLAGS_dynamic_static_unified_comm: "True"
-          python_version: "3.10"
-          paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
-        run: |
-          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
-          echo "container_name=${container_name}" >> "$GITHUB_ENV"
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
-              -v $work_dir/../../..:$work_dir/../../.. \
-              -v $work_dir:/workspace \
-              -v /home/.cache/pip:/home/.cache/pip \
-              -v /ssd1/paddlenlp:/ssd1/paddlenlp \
-              -v /home/data/gzcfs/.ccache/gpubox:/home/data/gzcfs/.ccache/gpubox \
-              -e BRANCH \
-              -e AGILE_COMPILE_BRANCH \
-              -e PR_ID \
-              -e COMMIT_ID \
-              -e work_dir \
-              -e ci_scripts \
-              -e no_proxy \
-              -e CI_name \
-              -e paddle_whl \
-              -e HF_ENDPOINT \
-              -e STUDIO_GIT_HOST \
-              -e PPNLP_HOME \
-              -e HF_DATASETS_CACHE \
-              -e TRANSFORMERS_CACHE \
-              -e CACHE_DIR \
-              -e FLAGS_dynamic_static_unified_comm \
-              -e python_version \
-              -w /workspace --runtime=nvidia $IMAGE_NAME
-          fi
-          
-      - name: Download Code
-        env:
-          work_dir: ${{ github.workspace }}
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping.."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            rm -rf * .[^.]*
-            echo "Downloading PaddleNLP.tar.gz"
-            wget -q --no-proxy  https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
-            echo "Extracting PaddleNLP.tar.gz"
-            tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
-            source $work_dir/../../../proxy
-            cd PaddleNLP
-            git config --global user.name "PaddleCI"
-            git config --global user.email "paddle_ci@example.com"
-            git pull
-            git submodule update --init --recursive --force
-            if [ -n "${PR_ID}" ]; then
-              git fetch origin pull/${PR_ID}/head
-              git checkout -b PR_${PR_ID} FETCH_HEAD
-              git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
-              git fetch upstream ${BRANCH}
-              git merge ${BRANCH} --no-edit
-              git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
-            else
-              echo "Not in a pull_request event. Skipping PR-specific operations."
-            fi
-            git log --pretty=oneline -10
-            '
-          fi
-      
-      - name: Skip For Bug
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            cd /workspace/PaddleNLP
-            echo "no skip for bug"
-            '
-          fi
-      
-      - name: Test
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            ldconfig
-            unlink /usr/bin/python3
-            ln -sf $(which python${python_version}) /usr/bin/python3
-            pip config set global.cache-dir "/home/.cache/pip"
-            source $work_dir/../../../proxy
-            set -e
-            cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
-            timeout 50m bash scripts/unit_test/ci_unit.sh ${paddle_whl}
-            '
-          fi
-
-      - name: Upload Allure-reports & Logs
-        if: always()
-        env:
-          home_path: ${{ github.workspace }}/../../..
-          bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
-          allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
-        run: |
-          if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
-            echo "Not in a pull_request or test_build event. Skipping..."
-          else
-            docker exec -t $container_name /bin/bash -c '
-            unset http_proxy && unset https_proxy
-            if [ ! -f "${{ env.bos_file }}" ]; then
-              wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
-              mkdir ${{ env.home_path }}/bos
-              tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
-            fi
-            if [ ! -f "${{ env.allure_file }}" ]; then
-              wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
-              unzip -q ${{ env.home_path }}/allure-2.19.0.zip -d ${{ env.home_path }}/
-            fi
-            if [[ "${{ env.RUN_DOWNSTREAM }}" == "" && -n "${PR_ID}" ]]; then
-              bos_prefix="${PR_ID}/${COMMIT_ID}"
-            elif [[ "${{ env.RUN_DOWNSTREAM }}" == "true" && -n "${PR_ID}" ]]; then
-              bos_prefix="${PR_ID}/${COMMIT_ID}/test_build"
-            else
-              bos_prefix="schedule/$(date +%Y%m%d)"
-            fi
-            cd /workspace/PaddleNLP/unittest_logs
-            for FILE in /workspace/PaddleNLP/unittest_logs/*; do
-              file=$(basename "$FILE")
-              python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs
-              echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs/$file"
-            done
-            cd /workspace/PaddleNLP/
-            ${{ env.allure_file }} generate result -o report
-            tar -czf products.tar.gz report unittest_logs
-            python ${{ env.bos_file }} products.tar.gz paddle-github-action/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs
-            echo "report: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs/products.tar.gz"
-            '
-          fi
-
-      - name: Terminate And Delete the Container
-        if: always()
-        run: |
-          docker rm -f $container_name 2>/dev/null || true

.gitignore

@@ -123,21 +123,4 @@ FETCH_HEAD
 
 # vscode
 .vscode
-./ppdiffusers/ppdiffusers/version.py
-
-# third party
-csrc/third_party/
-dataset/
-output/
-
-# gen codes
-autogen/
-
-# cutlass kernel
-!csrc/gpu/cutlass_kernels/gemm/collective/builders
-
-
-#fp8
-ops/csrc/fp8/deep_gemm/include/cutlass
-ops/csrc/fp8/deep_gemm/include/cute
-.ccls-cache
+./ppdiffusers/ppdiffusers/version.py

.gitmodules

@@ -1,6 +0,0 @@
-[submodule "csrc/third_party/cutlass"]
-	path = csrc/third_party/cutlass
-	url = https://github.com/NVIDIA/cutlass.git
-[submodule "csrc/third_party/nlohmann_json"]
-	path = csrc/third_party/nlohmann_json
-	url = https://github.com/nlohmann/json.git

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: 'slm/model_zoo/gpt-3;csrc/third_party'
+exclude: 'model_zoo/gpt-3'
 repos:
 # For Python files
 -   repo: https://github.com/psf/black.git
@@ -44,30 +44,3 @@ repos:
         entry: python .copyright.hook
         language: system
         files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|xpu|kps|py|sh)$
-# For Markdown files
--   repo: local
-    hooks:
-    -   id: add-spaces-between-chinese-and-english
-        name: Add spaces between Chinese and English characters
-        entry: python scripts/codestyle/check_spaces.py
-        language: python
-        files: \.(md|markdown)$
-        pass_filenames: true
-# For dead links
--   repo: local
-    hooks:
-    -   id: check-dead-links
-        name: Check dead links
-        entry: python scripts/codestyle/check_dead_links.py
-        language: python
-        files: \.(md|markdown|rst)$
-        pass_filenames: true
-
--   repo: local
-    hooks:
-    -   id: clang-format
-        name: clang-format
-        description: Format files with ClangFormat.
-        entry: bash ./tools/codestyle/clang_format.sh -i
-        language: system
-        files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$

.readthedocs.yaml

@@ -7,7 +7,7 @@ version: 2
 build:
   os: "ubuntu-20.04"
   tools:
-    python: "3.10"
+    python: "3.8"
 
 submodules:
   include: all
@@ -15,7 +15,7 @@ submodules:
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
-   configuration: docs/zh/conf.py
+   configuration: docs/conf.py
 
 # Optionally build your docs in additional formats such as PDF
 #formats:

CONTRIBUTING.md

@@ -9,30 +9,30 @@
 
 我们欢迎不同的向`PaddleNLP`做出贡献的方式，例如：
 
-- 修复已知的 Issue
-- 提交新的 Issue，例如提出功能需求或者 bug 报告
+- 修复已知的Issue
+- 提交新的Issue，例如提出功能需求或者bug报告
 - 实现新的模型结构
 
-如果您不知道从哪里开始，请查看 Issues 板块中的`Good First Issue`标签。它为您提供一个对初学者友好的已知 Issue 列表，可以降低贡献的门槛，帮助您开始为开源做出贡献。您只需在您想处理的 Issue 中告知我们您想负责此 Issue 即可。
+如果您不知道从哪里开始，请查看Issues板块中的`Good First Issue`标签。它为您提供一个对初学者友好的已知Issue列表，可以降低贡献的门槛，帮助您开始为开源做出贡献。您只需在您想处理的Issue中告知我们您想负责此Issue即可。
 
 ## 开发流程
 
 PaddleNLP 使用 [Git 分支模型](http://nvie.com/posts/a-successful-git-branching-model/)。对于常见的开源贡献，我们有以下的贡献流程：
 
-### 1. Fork
+#### 1. Fork
 
-   因为 PaddleNLP 的开发社区一直在发展，如果每位贡献者都直接向官方 Repo 提交 commit 将会难以管理。因此，请从您的分支中提交 Pull Requests。建议您通过 GitHub 的[“Fork”按钮](https://help.github.com/articles/fork-a-repo/)来创建您的 Fork 分支。
+   因为PaddleNLP的开发社区一直在发展，如果每位贡献者都直接向官方Repo提交commit将会难以管理。因此，请从您的分支中提交 Pull Requests。建议您通过GitHub的[“Fork”按钮](https://help.github.com/articles/fork-a-repo/)来创建您的Fork分支。
 
-### 2. Clone
+#### 2. Clone
 
-   请运行一下命令将您的分支 clone 到本地
+   请运行一下命令将您的分支clone到本地
 
    ```bash
    git clone https://github.com/<your-github-account>/PaddleNLP
    cd PaddleNLP
    ```
 
-### 3. 创建本地开发分支
+#### 3. 创建本地开发分支
 
    对于添加新功能或修复错误等日常工作，请在开发前创建您的本地开发分支：
 
@@ -40,8 +40,7 @@ PaddleNLP 使用 [Git 分支模型](http://nvie.com/posts/a-successful-git-branc
    git checkout -b my-cool-feature
    ```
 
-### 4. 配置开发环境
-
+#### 4. 配置开发环境
    在开始编码之前，您需要设置开发环境。我们强烈建议您在虚拟环境中进行所有开发，例如[venv](https://docs.python.org/3/library/venv.html)或[conda](https://docs.conda.io/en/latest/)。
    请您设置并激活虚拟环境后，运行以下命令：
 
@@ -55,7 +54,8 @@ PaddleNLP 使用 [Git 分支模型](http://nvie.com/posts/a-successful-git-branc
    如果在虚拟环境中已经安装 `PaddleNLP` ，请使用 `pip uninstall paddlenlp` 将其删除，然后以可编辑模式重新安装它
    `pip install -e .`
 
-### 5. 开发
+
+#### 5. 开发
 
    当您开发时，请确保您新增的代码会被单元测试所覆盖。我们所有的单元测试都可以在 `tests` 目录下找到。
    您可以修改现有单元测试以覆盖新功能，也可以从头开始创建新测试。
@@ -65,14 +65,14 @@ PaddleNLP 使用 [Git 分支模型](http://nvie.com/posts/a-successful-git-branc
    pytest tests/<test_to_run>.py
    ```
 
-### 6. Commit
+#### 6. Commit
 
    我们使用 [`pre-commit`](http://pre-commit.com/)工具（包括[black](https://black.readthedocs.io/en/stable/)、[isort](https:/ /pycqa.github.io/isort/) 和
    [flake8](https://flake8.pycqa.org/en/latest/)）来检查每次提交中的代码和文档的风格。当你运行 `git commit` 时，你会看到
    类似于以下内容：
 
-   ```text
-    ➜  (my-virtual-env) git commit -m "committing my cool feature"
+   ```
+    ➜  (my-virtual-env) git commit -m "commiting my cool feature"
     black....................................................................Passed
     isort....................................................................Passed
     flake8...................................................................Passed
@@ -89,9 +89,8 @@ PaddleNLP 使用 [Git 分支模型](http://nvie.com/posts/a-successful-git-branc
    ```
 
    但大多数时候事情并没有那么顺利。当您的代码或文档不符合标准时，`pre-commit` 检查将失败。
-
-   ```text
-    ➜  (my-virtual-env) git commit -m "committing my cool feature"
+   ```
+    ➜  (my-virtual-env) git commit -m "commiting my cool feature"
     black....................................................................Passed
     isort....................................................................Failed
     - hook id: isort
@@ -113,34 +112,34 @@ PaddleNLP 使用 [Git 分支模型](http://nvie.com/posts/a-successful-git-branc
    ```
 
    我们的工具将自动修复大部分样式错误，但是有些错误需要手动解决。幸运的是，错误信息一般通俗易懂，很容易修复。
-   解决错误后，您可以再次运行 `git add <files>` 和 `git commit`，这将再次触发 pre-commit 。
+   解决错误后，您可以再次运行 git add <files> 和 git commit ，这将再次触发 pre-commit 。
    一旦 pre-commit 检查通过，您就可以推送代码了。
 
-   [Google](https://google.com/) 或 [StackOverflow](https://stackoverflow.com/) 是帮助您了解代码风格错误的好工具。
+   [Google][http://google.com/] 或 [StackOverflow](https://stackoverflow.com/) 是帮助您了解代码风格错误的好工具。
    如果您仍然无法弄清楚，请不要担心。您可以使用 `git commit -m "style error" --no-verify` 提交，我们很乐意在您创建 Pull Request 后帮助您。
 
-### 7. git pull 与代码冲突
+#### 7. git pull与代码冲突
 
-   有经验的 Git 用户经常从官方 Repo 中 git pull。因为这样子他们会及早注意到与其他人的代码冲突，并且让代码冲突更容易解决
+   有经验的 Git 用户经常从官方Repo中git pull。因为这样子他们会及早注意到与其他人的代码冲突，并且让代码冲突更容易解决
 
    ```bash
    git remote add upstream https://github.com/PaddlePaddle/PaddleNLP
    git pull upstream develop
    ```
 
-### 8. git push 与提交 Pull Request
+#### 8. git push与提交Pull Request
 
-   您可以将您的本地开发分支中的工作 push 到您的 fork 的分支中：
+   您可以将您的本地开发分支中的工作 push 到您的fork的分支中：
 
    ```bash
    git push origin my-cool-stuff
    ```
 
-   git push 之后，您可以提交 Pull Request，请求[官方 repo](https://github.com/PaddlePaddle/PaddleNLP) 采纳您的开发工作。请您依照[这些步骤](https://help.github.com/articles/creating-a-pull-request/)创建 Pull Request。
+   git push之后，您可以提交Pull Request，请求[官方repo](https://github.com/PaddlePaddle/PaddleNLP) 采纳您的开发工作。请您依照[这些步骤](https://help.github.com/articles/creating-a-pull-request/)创建Pull Request。
 
-### 9. 删除已经合入的本地和远程分支
+#### 9. 删除已经合入的本地和远程分支
 
-   为了保持您本地的工作区和 fork 分支的干净整洁，建议您在 Pull Request 合入之后删除本地的残余分支：
+   为了保持您本地的工作区和fork分支的干净整洁，建议您在Pull Request合入之后删除本地的残余分支：
 
    ```bash
    git push origin my-cool-stuff
@@ -149,10 +148,10 @@ PaddleNLP 使用 [Git 分支模型](http://nvie.com/posts/a-successful-git-branc
    git branch -d my-cool-stuff
    ```
 
-## 代码 Review
+## 代码Review
 
-- 在您的 Pull Request 能够顺利通过本地测试以及 CI 的情况下，您可以在 Pull Request 中 @ 相关的 Reviewer，提醒他们尽快对您的 Pull Request 进行 Review。
+- 在您的Pull Request能够顺利通过本地测试以及CI的情况下，您可以在Pull Request中 @ 相关的Reviewer，提醒他们尽快对您的Pull Request进行Review。
 
-- 请处理 Reviewer 的每一条评论。如果您已按照评论修改，请回复“完成”；否则，可以在评论下展开讨论。
+- 请处理Reviewer的每一条评论。如果您已按照评论修改，请回复“完成”；否则，可以在评论下展开讨论。
 
-- 如果您不希望您的 Reviewer 被电子邮件通知淹没，您可以[批量回复](https://help.github.com/articles/reviewing-proposed-changes-in-a-pull-request/)。
+- 如果您不希望您的Reviewer被电子邮件通知淹没，您可以[批量回复](https://help.github.com/articles/reviewing-proposed-changes-in-a-pull-request/)。

Makefile

@@ -37,7 +37,6 @@ test: unit-test
 unit-test:
 	PYTHONPATH=$(shell pwd) pytest -v \
 		-n auto \
-		--retries 1 --retry-delay 1 \
 		--durations 20 \
 		--cov paddlenlp \
 		--cov-report xml:coverage.xml
@@ -46,7 +45,6 @@ unit-test:
 
 .PHONY: install
 install:
-	pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/
 	pip install -r requirements-dev.txt
 	pip install -r requirements.txt
 	pip install -r paddlenlp/experimental/autonlp/requirements.txt

README.md

@@ -7,7 +7,7 @@
 ------------------------------------------------------------------------------------------
 
 <p align="center">
-    <a href="https://paddlenlp.readthedocs.io/en/latest/?badge=latest"><img src="https://readthedocs.org/projects/paddlenlp/badge/?version=latest">
+    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/PaddleNLP?color=ffa"></a>
     <a href=""><img src="https://img.shields.io/badge/python-3.7+-aff.svg"></a>
     <a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
@@ -16,181 +16,35 @@
     <a href="https://pypi.org/project/paddlenlp/"><img src="https://img.shields.io/pypi/dm/paddlenlp?color=9cf"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/PaddleNLP?color=9cc"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/PaddleNLP?color=ccf"></a>
-    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
 </p>
 
+
 <h4 align="center">
-  <a href=#特性> 特性 </a> |
-  <a href=#模型支持> 模型支持 </a> |
   <a href=#安装> 安装 </a> |
   <a href=#快速开始> 快速开始 </a> |
+  <a href=#特性> 特性 </a> |
   <a href=#社区交流> 社区交流 </a>
 </h4>
 
-**PaddleNLP**是一款基于飞桨深度学习框架的大语言模型(LLM)开发套件，支持在多种硬件上进行高效的大模型训练、无损压缩以及高性能推理。PaddleNLP 具备**简单易用**和**性能极致**的特点，致力于助力开发者实现高效的大模型产业级应用。
-
-<a href="https://trendshift.io/repositories/2246" target="_blank"><img src="https://trendshift.io/api/badge/repositories/2246" alt="PaddlePaddle%2FPaddleNLP | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+**PaddleNLP**是一款**简单易用**且**功能强大**的自然语言处理和大语言模型(LLM)开发库。聚合业界**优质预训练模型**并提供**开箱即用**的开发体验，覆盖NLP多场景的模型库搭配**产业实践范例**可满足开发者**灵活定制**的需求。
 
 ## News 📢
 
-* **2025.04.29 PaddleNLP 现已支持 Qwen3 系列模型**: Qwen3 系列模型支持持两种思考模式，预训练约 36 万亿个 token、119 种语言和方言。包括六个 Dense 模型, Qwen3-32B、Qwen3-14B、Qwen3-8B、Qwen3-4B、Qwen3-1.7B 和 Qwen3-0.6B。两个 MoE 模型的权重：Qwen3-235B-A22B，Qwen3-30B-A3B。
+* **2023.8.15 [PaddleNLP v2.6](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.6.0)**： 发布[全流程大模型工具链](./llm)，涵盖预训练，精调，压缩，推理以及部署等各个环节，为用户提供端到端的大模型方案和一站式的开发体验；内置[4D并行分布式Trainer](./docs/trainer.md)，[高效微调算法LoRA/Prefix Tuning](./llm#33-lora), [自研INT8/INT4量化算法](./llm#6-量化)等等；全面支持[LLaMA 1/2](./llm/llama), [BLOOM](.llm/bloom), [ChatGLM 1/2](./llm/chatglm), [GLM](./llm/glm), [OPT](./llm/opt)等主流大模型
 
-* **2025.03.12 [PaddleNLP v3.0 Beta4](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta4)**：全面支持 DeepSeek V3/R1/R1-Distill, 及 QwQ-32B 等热门思考模型。**DeepSeek V3/R1完整版支持 FP8、INT8、4-bit 量化推理，MTP 投机解码**。单机 FP8推理输出超**1000 tokens/s**; 4-bit 推理输出超**2100 tokens/s**! 发布新版推理部署镜像，热门模型[一键部署](https://paddlenlp.readthedocs.io/zh/latest/llm/server/docs/general_model_inference.html)。推理部署[使用文档](https://paddlenlp.readthedocs.io/zh/latest/llm/docs/predict/index.html)全面更新，体验全面提升！自研下一代通用信息抽取模型 PP-UIE [全新发布](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/application/information_extraction)，支持8K 长度信息抽取。新增大模型 Embedding 训练，支持 INF-CL 超大 batch size 训练。新增[MergeKit](https://paddlenlp.readthedocs.io/zh/latest/llm/docs/mergekit.html)模型融合工具，缓解对齐代价。低资源训练全面优化，16G 小显存可以流畅训练。
-
-
-* **2025.02.10 PaddleNLP 现已支持 DeepSeek-R1系列模型，[在线使用](https://aistudio.baidu.com/projectdetail/8775758)**：依托全新的 PaddleNLP 3.0套件，DeepSeek-R1系列模型现已全面支持。凭借数据并行、数据分组切分并行、模型并行、流水线并行以及专家并行等一系列先进的分布式训练能力，结合 Paddle 框架独有的列稀疏注意力掩码表示技术——FlashMask 方法，DeepSeek-R1系列模型在训练过程中显著降低了显存消耗，同时取得了卓越的训练性能提升。
-
-<details><summary> <b>点击展开</b> </summary><div>
-
-* **2025.03.17 《DeepSeek-R1满血版单机部署实测》** 🔥🔥🔥 飞桨框架3.0大模型推理部署全面升级，支持多款主流大模型，DeepSeek-R1满血版实现单机部署，吞吐提升一倍！欢迎广大用户开箱体验～现已开启有奖活动：完成 DeepSeek-R1-MTP 单机部署任务、提交高质量测评 blog，即可实时赢取奖金！💰💰💰
-报名[地址](https://www.wjx.top/vm/OlzzmbG.aspx#)， 活动详情：https://github.com/PaddlePaddle/PaddleNLP/issues/10166 ， 参考文档：https://github.com/PaddlePaddle/PaddleNLP/issues/10157 。
-
-* **2025.03.06 PaddleNLP 现已支持 Qwen/QwQ-32B 模型**: 其模型参数仅有 32B，但其数学推理、编程能力和通用能力可与具备 671B 参数（其中 37B 被激活）的 DeepSeek-R1 媲美。借助 PaddleNLP 3.0套件，现可实现多种并行策略[微调训练](./llm/README.md)、[高性能推理、低比特量化](./llm/docs/predict/qwen.md)和[服务化部署](./llm/server/README.md)。
-
-* **2025.02.20 🔥🔥《PP-UIE 信息抽取智能引擎全新升级》** 强化零样本学习能力，支持极少甚至零标注数据实现高效冷启动与迁移学习，显著降低数据标注成本；具备处理长文本能力，支持 8192 个 Token 长度文档信息抽取，实现跨段落识别关键信息，形成完整理解；提供完整可定制化的训练和推理全流程，训练效率相较于 LLama-Factory 实现了1.8倍的提升。
-2月26日（周三）19：00为您深度解析全新 PP-UIE 技术方案及在部署方面的功能、优势与技巧。报名链接：https://www.wjx.top/vm/mBKC6pb.aspx?udsid=606418
-
-* **2024.12.16 [PaddleNLP v3.0 Beta3](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta3)**：大模型功能全新升级，新增了 Llama-3.2、DeepSeekV2模型，升级了 TokenizerFast，快速分词，重构了 SFTTrainer，一键开启 SFT 训练。此外，PaddleNLP 还支持了优化器状态的卸载和重载功能，实现了精细化的重新计算，训练性能提升7%。在 Unified Checkpoint 方面，进一步优化了异步保存逻辑，新增 Checkpoint 压缩功能，可节省78.5%存储空间。
-最后，在大模型推理方面，升级 Append Attention，支持了 FP8量化，支持投机解码。
-
-* **2024.12.13 📚《飞桨大模型套件 Unified Checkpoint 技术》**，加速模型存储95%，节省空间78%。支持全分布式策略调整自适应转换，提升模型训练的灵活性与可扩展性。训练-压缩-推理统一存储协议，无需手动转换提升全流程体验。Checkpoint 无损压缩结合异步保存，实现秒级存储并降低模型存储成本。适用于智能制造、指挥交通、医疗健康、金融服务等产业实际场景。12月24日（周二）19：00直播为您详细解读该技术如何优化大模型训练流程。报名链接：https://www.wjx.top/vm/huZkHn9.aspx?udsid=787976
-
-* **2024.11.28 📚《FlashRAG-Paddle | 基于 PaddleNLP 的高效开发与评测 RAG 框架》**，为文本更快更好构建准确嵌入表示、加速推理生成速度。PaddleNLP 支持超大 Batch 嵌入表示学习与多硬件高性能推理，涵盖 INT8/INT4量化技术及多种高效注意力机制优化与 TensorCore 深度优化。内置全环节算子融合技术，使得 FlashRAG 推理性能相比 transformers 动态图提升70%以上，结合检索增强知识输出结果更加准确，带来敏捷高效的使用体验。直播时间：12月3日（周二）19：00。报名链接：https://www.wjx.top/vm/eaBa1vA.aspx?udsid=682361
-
-* **2024.08.08 📚《飞桨产业级大语言模型开发利器 PaddleNLP 3.0 重磅发布》**，训压推全流程贯通，主流模型全覆盖。大模型自动并行，千亿模型训推全流程开箱即用。提供产业级高性能精调与对齐解决方案，压缩推理领先，多硬件适配。覆盖产业级智能助手、内容创作、知识问答、关键信息抽取等应用场景。直播时间：8月22日（周四）19：00。报名链接：https://www.wjx.top/vm/Y2f7FFY.aspx?udsid=143844
-
-* **2024.06.27 [PaddleNLP v3.0 Beta](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta0)**：拥抱大模型，体验全升级。统一大模型套件，实现国产计算芯片全流程接入；全面支持飞桨4D 并行配置、高效精调策略、高效对齐算法、高性能推理等大模型产业级应用流程；自研极致收敛的 RsLoRA+算法、自动扩缩容存储机制 Unified Checkpoint 和通用化支持的 FastFFN、FusedQKV 助力大模型训推；主流模型持续支持更新，提供高效解决方案。
-
-* **2024.04.24 [PaddleNLP v2.8](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.8.0)**：自研极致收敛的 RsLoRA+算法，大幅提升 PEFT 训练收敛速度以及训练效果；引入高性能生成加速到 RLHF PPO 算法，打破 PPO 训练中生成速度瓶颈，PPO 训练性能大幅领先。通用化支持 FastFFN、FusedQKV 等多个大模型训练性能优化方式，大模型训练更快、更稳定。
-</div></details>
-
-## 特性
-
-### <a href=#多硬件训推一体> 🔧 多硬件训推一体 </a>
-
-支持英伟达 GPU、昆仑 XPU、昇腾 NPU、燧原 GCU 和海光 DCU 等多个硬件的大模型和自然语言理解模型训练和推理，套件接口支持硬件快速切换，大幅降低硬件切换研发成本。
-当前支持的自然语言理解模型：[多硬件自然语言理解模型列表](./docs/zh/model_zoo/model_list_multy_device.md)
-
-### <a href=#高效易用的预训练> 🚀 高效易用的预训练 </a>
-
-支持纯数据并行策略、分组参数切片的数据并行策略、张量模型并行策略和流水线模型并行策略的4D 高性能训练，Trainer 支持分布式策略配置化，降低复杂分布式组合带来的使用成本；
-[Unified Checkpoint 大模型存储工具](./llm/docs/unified_checkpoint.md)可以使得训练断点支持机器资源动态扩缩容恢复。此外，异步保存，模型存储可加速95%，Checkpoint 压缩，可节省78.5%存储空间。
-
-### <a href=#高效精调> 🤗 高效精调 </a>
-
-精调算法深度结合零填充数据流和 [FlashMask](./llm/docs/flashmask.md) 高性能算子，降低训练无效数据填充和计算，大幅提升精调训练吞吐。
-
-### <a href=#无损压缩和高性能推理> 🎛️ 无损压缩和高性能推理 </a>
-
-大模型套件高性能推理模块内置动态插入和全环节算子融合策略，极大加快并行推理速度。底层实现细节封装化，实现开箱即用的高性能并行推理能力。
-
-## 文档
-更多详细文档, 请访问 [PaddleNLP Documentation](https://paddlenlp.readthedocs.io/).
-
-------------------------------------------------------------------------------------------
-
-## 模型支持
-
-* 模型参数已支持 LLaMA 系列、Baichuan 系列、Bloom 系列、ChatGLM 系列、Gemma 系列、Mistral 系列、OPT 系列和 Qwen 系列，详细列表👉【LLM】模型参数支持列表如下：
-
-|                                                模型系列                                                 | 模型名称                                                                                                                                                                                                                                                                                                                                                                                      |
-|:-------------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [PP-UIE](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/application/information_extraction) | paddlenlp/PP-UIE-0.5B, paddlenlp/PP-UIE-1.5B, paddlenlp/PP-UIE-7B, paddlenlp/PP-UIE-14B                                                                                                                                                                                                                                                                                                       |
-|            [LLaMA](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)             | facebook/llama-7b, facebook/llama-13b, facebook/llama-30b, facebook/llama-65b                                                                                                                                                                                                                                                                                                                 |
-|            [Llama2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)            | meta-llama/Llama-2-7b, meta-llama/Llama-2-7b-chat, meta-llama/Llama-2-13b, meta-llama/Llama-2-13b-chat, meta-llama/Llama-2-70b, meta-llama/Llama-2-70b-chat                                                                                                                                                                                                                                   |
-|            [Llama3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)            | meta-llama/Meta-Llama-3-8B, meta-llama/Meta-Llama-3-8B-Instruct, meta-llama/Meta-Llama-3-70B, meta-llama/Meta-Llama-3-70B-Instruct                                                                                                                                                                                                                                                            |
-|           [Llama3.1](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)           | meta-llama/Meta-Llama-3.1-8B, meta-llama/Meta-Llama-3.1-8B-Instruct, meta-llama/Meta-Llama-3.1-70B, meta-llama/Meta-Llama-3.1-70B-Instruct, meta-llama/Meta-Llama-3.1-405B, meta-llama/Meta-Llama-3.1-405B-Instruct, meta-llama/Llama-Guard-3-8B                                                                                                                                              |
-|           [Llama3.2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)           | meta-llama/Llama-3.2-1B, meta-llama/Llama-3.2-1B-Instruct, meta-llama/Llama-3.2-3B, meta-llama/Llama-3.2-3B-Instruct, meta-llama/Llama-Guard-3-1B                                                                                                                                                                                                                                             |
-|           [Llama3.3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama)           | meta-llama/Llama-3.3-70B-Instruct                                                                                                                                                                                                                                                                                                                                                             |
-|         [Baichuan](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/baichuan)          | baichuan-inc/Baichuan-7B, baichuan-inc/Baichuan-13B-Base, baichuan-inc/Baichuan-13B-Chat                                                                                                                                                                                                                                                                                                      |
-|         [Baichuan2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/baichuan)         | baichuan-inc/Baichuan2-7B-Base, baichuan-inc/Baichuan2-7B-Chat, baichuan-inc/Baichuan2-13B-Base, baichuan-inc/Baichuan2-13B-Chat                                                                                                                                                                                                                                                              |
-|            [Bloom](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/bloom)             | bigscience/bloom-560m, bigscience/bloom-560m-bf16, bigscience/bloom-1b1, bigscience/bloom-3b, bigscience/bloom-7b1, bigscience/bloomz-560m, bigscience/bloomz-1b1, bigscience/bloomz-3b, bigscience/bloomz-7b1-mt, bigscience/bloomz-7b1-p3, bigscience/bloomz-7b1, bellegroup/belle-7b-2m                                                                                                    |
-|          [ChatGLM](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/chatglm/)          | THUDM/chatglm-6b, THUDM/chatglm-6b-v1.1                                                                                                                                                                                                                                                                                                                                                       |
-|         [ChatGLM2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/chatglm2)          | THUDM/chatglm2-6b                                                                                                                                                                                                                                                                                                                                                                             |
-|         [ChatGLM3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/chatglm2)          | THUDM/chatglm3-6b                                                                                                                                                                                                                                                                                                                                                                             |
-|       [DeepSeekV2](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/config/deepseek-v2)       | deepseek-ai/DeepSeek-V2, deepseek-ai/DeepSeek-V2-Chat, deepseek-ai/DeepSeek-V2-Lite, deepseek-ai/DeepSeek-V2-Lite-Chat, deepseek-ai/DeepSeek-Coder-V2-Base, deepseek-ai/DeepSeek-Coder-V2-Instruct, deepseek-ai/DeepSeek-Coder-V2-Lite-Base, deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct                                                                                                      |
-|       [DeepSeekV3](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/config/deepseek-v2)       | deepseek-ai/DeepSeek-V3, deepseek-ai/DeepSeek-V3-Base                                                                                                                                                                                                                                                                                                                                         |
-|      [DeepSeek-R1](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/config/deepseek-v2)       | deepseek-ai/DeepSeek-R1, deepseek-ai/DeepSeek-R1-Zero, deepseek-ai/DeepSeek-R1-Distill-Llama-70B, deepseek-ai/DeepSeek-R1-Distill-Llama-8B, deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, deepseek-ai/DeepSeek-R1-Distill-Qwen-7B                                                                            |
-|            [Gemma](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/gemma)             | google/gemma-7b, google/gemma-7b-it, google/gemma-2b, google/gemma-2b-it                                                                                                                                                                                                                                                                                                                      |
-|          [Mistral](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/mistral)           | mistralai/Mistral-7B-Instruct-v0.3, mistralai/Mistral-7B-v0.1                                                                                                                                                                                                                                                                                                                                 |
-|          [Mixtral](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/mixtral)           | mistralai/Mixtral-8x7B-Instruct-v0.1                                                                                                                                                                                                                                                                                                                                                          |
-|              [OPT](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/opt)               | facebook/opt-125m, facebook/opt-350m, facebook/opt-1.3b, facebook/opt-2.7b, facebook/opt-6.7b, facebook/opt-13b, facebook/opt-30b, facebook/opt-66b, facebook/opt-iml-1.3b, opt-iml-max-1.3b                                                                                                                                                                                                  |
-|             [Qwen](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)             | qwen/qwen-7b, qwen/qwen-7b-chat, qwen/qwen-14b, qwen/qwen-14b-chat, qwen/qwen-72b, qwen/qwen-72b-chat,                                                                                                                                                                                                                                                                                        |
-|           [Qwen1.5](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)            | Qwen/Qwen1.5-0.5B, Qwen/Qwen1.5-0.5B-Chat, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-1.8B-Chat, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-4B-Chat, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-7B-Chat, Qwen/Qwen1.5-14B, Qwen/Qwen1.5-14B-Chat, Qwen/Qwen1.5-32B, Qwen/Qwen1.5-32B-Chat, Qwen/Qwen1.5-72B, Qwen/Qwen1.5-72B-Chat, Qwen/Qwen1.5-110B, Qwen/Qwen1.5-110B-Chat, Qwen/Qwen1.5-MoE-A2.7B, Qwen/Qwen1.5-MoE-A2.7B-Chat |
-|            [Qwen2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)             | Qwen/Qwen2-0.5B, Qwen/Qwen2-0.5B-Instruct, Qwen/Qwen2-1.5B, Qwen/Qwen2-1.5B-Instruct, Qwen/Qwen2-7B, Qwen/Qwen2-7B-Instruct, Qwen/Qwen2-72B, Qwen/Qwen2-72B-Instruct, Qwen/Qwen2-57B-A14B, Qwen/Qwen2-57B-A14B-Instruct                                                                                                                                                                       |
-|          [Qwen2-Math](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)          | Qwen/Qwen2-Math-1.5B, Qwen/Qwen2-Math-1.5B-Instruct, Qwen/Qwen2-Math-7B, Qwen/Qwen2-Math-7B-Instruct, Qwen/Qwen2-Math-72B, Qwen/Qwen2-Math-72B-Instruct, Qwen/Qwen2-Math-RM-72B                                                                                                                                                                                                               |
-|           [Qwen2.5](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)            | Qwen/Qwen2.5-0.5B, Qwen/Qwen2.5-0.5B-Instruct, Qwen/Qwen2.5-1.5B, Qwen/Qwen2.5-1.5B-Instruct, Qwen/Qwen2.5-3B, Qwen/Qwen2.5-3B-Instruct, Qwen/Qwen2.5-7B, Qwen/Qwen2.5-7B-Instruct, Qwen/Qwen2.5-7B-Instruct-1M, Qwen/Qwen2.5-14B, Qwen/Qwen2.5-14B-Instruct, Qwen/Qwen2.5-14B-Instruct-1M, Qwen/Qwen2.5-32B, Qwen/Qwen2.5-32B-Instruct, Qwen/Qwen2.5-72B, Qwen/Qwen2.5-72B-Instruct          |
-|         [Qwen2.5-Math](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)         | Qwen/Qwen2.5-Math-1.5B, Qwen/Qwen2.5-Math-1.5B-Instruct, Qwen/Qwen2.5-Math-7B, Qwen/Qwen2.5-Math-7B-Instruct, Qwen/Qwen2.5-Math-72B, Qwen/Qwen2.5-Math-72B-Instruct, Qwen/Qwen2.5-Math-RM-72B                                                                                                                                                                                                 |
-|        [Qwen2.5-Coder](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)         | Qwen/Qwen2.5-Coder-1.5B, Qwen/Qwen2.5-Coder-1.5B-Instruct, Qwen/Qwen2.5-Coder-7B, Qwen/Qwen2.5-Coder-7B-Instruct                                                                                                                                                                                                                                                                              |
-|           [Qwen3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)              | Qwen/Qwen3-0.6B, Qwen/Qwen3-1.7B, Qwen/Qwen3-4B, Qwen/Qwen3-8B, Qwen/Qwen3-14B, Qwen/Qwen3-32B, Qwen/Qwen3-30B-A3B, Qwen/Qwen3-235B-A22B, Qwen/Qwen3-0.6B-Base, Qwen/Qwen3-1.7B-Base, Qwen/Qwen3-4B-Base, Qwen/Qwen3-8B-Base, Qwen/Qwen3-14B-Base, Qwen/Qwen3-30B-A3B-Base                                                                                                                    |
-|             [QwQ](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/)              | Qwen/QwQ-32B, Qwen/QwQ-32B-Preview                                                                                                                                                                                                                                                                                                                                                            |
-|            [Yuan2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/yuan/)             | IEITYuan/Yuan2-2B, IEITYuan/Yuan2-51B, IEITYuan/Yuan2-102B                                                                                                                                                                                                                                                                                                                                    |
-
-* 4D 并行和算子优化已支持 LLaMA 系列、Baichuan 系列、Bloom 系列、ChatGLM 系列、Gemma 系列、Mistral 系列、OPT 系列和 Qwen 系列，【LLM】模型4D 并行和算子支持列表如下：
-
-| 模型名称/并行能力支持 | 数据并行 | 张量模型并行 |          | 参数分片并行 |        |        | 流水线并行 |
-|:---------------------:|:--------:|:------------:|:--------:|:------------:|:------:|:------:|:----------:|
-|                       |          |   基础能力   | 序列并行 |    stage1    | stage2 | stage3 |            |
-|         Llama         |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|         Qwen          |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|        Qwen1.5        |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|         Qwen2         |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|     Mixtral(moe)      |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     🚧     |
-|        Mistral        |    ✅     |      ✅       |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|       Baichuan        |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|       Baichuan2       |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|        ChatGLM        |    ✅     |      ✅       |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|       ChatGLM2        |    ✅     |      🚧      |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|       ChatGLM3        |    ✅     |      🚧      |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|         Bloom         |    ✅     |      ✅       |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|      GPT-2/GPT-3      |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|          OPT          |    ✅     |      ✅       |    🚧    |      ✅       |   ✅    |   ✅    |     🚧     |
-|         Gemma         |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     ✅      |
-|         Yuan2         |    ✅     |      ✅       |    ✅     |      ✅       |   ✅    |   ✅    |     🚧     |
-
-* 大模型预训练、精调（包含 SFT、PEFT 技术）、对齐、量化已支持 LLaMA 系列、Baichuan 系列、Bloom 系列、ChatGLM 系列、Mistral 系列、OPT 系列和 Qwen 系列，【LLM】模型预训练、精调、对齐、量化支持列表如下：
-
-
-| Model                                      | Pretrain | SFT | LoRA | FlashMask | Prefix Tuning | DPO/SimPO/ORPO/KTO | RLHF | Mergekit | Quantization |
-|--------------------------------------------|:--------:|:---:|:----:|:---------:|:-------------:|:------------------:|:----:|:--------:|:------------:|
-| [Llama](./llm/config/llama)                |    ✅     |  ✅  |  ✅   |     ✅     |       ✅       |         ✅          |  ✅   |    ✅     |      ✅       |
-| [Qwen](./llm/config/qwen)                  |    ✅     |  ✅  |  ✅   |     ✅     |       ✅       |         ✅          |  🚧  |    ✅     |      🚧      |
-| [Mixtral](./llm/config/mixtral)            |    ✅     |  ✅  |  ✅   |    🚧     |      🚧       |         ✅          |  🚧  |    ✅     |      🚧      |
-| [Mistral](./llm/config/mistral)            |    ✅     |  ✅  |  ✅   |    🚧     |       ✅       |         ✅          |  🚧  |    ✅     |      🚧      |
-| [Baichuan/Baichuan2](./llm/config/llama)   |    ✅     |  ✅  |  ✅   |     ✅     |       ✅       |         ✅          |  🚧  |    ✅     |      ✅       |
-| [ChatGLM-6B](./llm/config/chatglm)         |    ✅     |  ✅  |  ✅   |    🚧     |       ✅       |         🚧         |  🚧  |    ✅     |      ✅       |
-| [ChatGLM2/ChatGLM3](./llm/config/chatglm2) |    ✅     |  ✅  |  ✅   |    🚧     |       ✅       |         ✅          |  🚧  |    ✅     |      ✅       |
-| [Bloom](./llm/config/bloom)                |    ✅     |  ✅  |  ✅   |    🚧     |       ✅       |         🚧         |  🚧  |    ✅     |      ✅       |
-| [GPT-3](./llm/config/gpt-3)                |    ✅     |  ✅  |  🚧  |    🚧     |      🚧       |         🚧         |  🚧  |    ✅     |      🚧      |
-| [OPT](./llm/config/opt)                    |    ✅     |  ✅  |  ✅   |    🚧     |      🚧       |         🚧         |  🚧  |    ✅     |      🚧      |
-| [Gemma](./llm/config/gemma)                |    ✅     |  ✅  |  ✅   |    🚧     |      🚧       |         ✅          |  🚧  |    ✅     |      🚧      |
-| [Yuan](./llm/config/yuan)                  |    ✅     |  ✅  |  ✅   |    🚧     |      🚧       |         ✅          |  🚧  |    ✅     |      🚧      |
-* [大模型推理](./llm/docs/predict/inference.md)已支持 LLaMA 系列、Qwen 系列、DeepSeek 系列、Mistral 系列、ChatGLM 系列、Bloom 系列和 Baichuan 系列，支持 Weight Only INT8及 INT4推理，支持 WAC（权重、激活、Cache KV）进行 INT8、FP8量化的推理，【LLM】模型推理支持列表如下：
-
-|           模型名称/量化类型支持            | FP16/BF16 | WINT8 | WINT4 | INT8-A8W8 | FP8-A8W8 | INT8-A8W8C8 |
-|:------------------------------------------:|:---------:|:-----:|:-----:|:---------:|:--------:|:-----------:|
-|    [LLaMA](./llm/docs/predict/llama.md)    |     ✅     |   ✅   |   ✅   |     ✅     |    ✅     |      ✅      |
-|     [Qwen](./llm/docs/predict/qwen.md)     |     ✅     |   ✅   |   ✅   |     ✅     |    ✅     |      ✅      |
-| [DeepSeek](./llm/docs/predict/deepseek.md) |     ✅     |   ✅   |   ✅   |    🚧     |    ✅     |     🚧      |
-|   [Qwen-Moe](./llm/docs/predict/qwen.md)   |     ✅     |   ✅   |   ✅   |    🚧     |    🚧    |     🚧      |
-|  [Mixtral](./llm/docs/predict/mixtral.md)  |     ✅     |   ✅   |   ✅   |    🚧     |    🚧    |     🚧      |
-|                  ChatGLM                   |     ✅     |   ✅   |   ✅   |    🚧     |    🚧    |     🚧      |
-|                   Bloom                    |     ✅     |   ✅   |   ✅   |    🚧     |    🚧    |     🚧      |
-|                  BaiChuan                  |     ✅     |   ✅   |   ✅   |     ✅     |    ✅     |     🚧      |
 
 ## 安装
 
 ### 环境依赖
 
-* python >= 3.8
-* paddlepaddle >= 3.0.0rc1
+- python >= 3.7
+- paddlepaddle >= 2.5.1
+- 如需大模型功能，请使用 paddlepaddle-gpu >= 2.5.1
 
-如果您尚未安装 PaddlePaddle，请参考 [飞桨官网](https://www.paddlepaddle.org.cn/) 进行安装。
-
-### pip 安装
+### pip安装
 
 ```shell
-pip install --upgrade paddlenlp==3.0.0b4
+pip install --upgrade paddlenlp
 ```
 
 或者可通过以下命令安装最新 develop 分支代码：
@@ -199,88 +53,281 @@ pip install --upgrade paddlenlp==3.0.0b4
 pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html
 ```
 
-更多关于 PaddlePaddle 和 PaddleNLP 安装的详细教程请查看[Installation](./docs/zh/get_started/installation.rst)。
-
-------------------------------------------------------------------------------------------
+更多关于PaddlePaddle和PaddleNLP安装的详细教程请查看[Installation](./docs/get_started/installation.rst)。
 
 ## 快速开始
 
+
 ### 大模型文本生成
 
-PaddleNLP 提供了方便易用的 Auto API，能够快速的加载模型和 Tokenizer。这里以使用 `Qwen/Qwen2-0.5B` 模型做文本生成为例：
+PaddleNLP提供了方便易用的Auto API，能够快速的加载模型和Tokenizer。这里以使用 `linly-ai/chinese-llama-2-7b` 大模型做文本生成为例：
 
 ```python
-from paddlenlp.transformers import AutoTokenizer, AutoModelForCausalLM
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B")
-# if using CPU, please change float16 to float32
-model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B", dtype="float16")
-input_features = tokenizer("你好！请自我介绍一下。", return_tensors="pd")
-outputs = model.generate(**input_features, max_new_tokens=128)
-print(tokenizer.batch_decode(outputs[0], skip_special_tokens=True))
-# ['我是一个AI语言模型，我可以回答各种问题，包括但不限于：天气、新闻、历史、文化、科学、教育、娱乐等。请问您有什么需要了解的吗？']
+>>> from paddlenlp.transformers import AutoTokenizer, AutoModelForCausalLM
+>>> tokenizer = AutoTokenizer.from_pretrained("linly-ai/chinese-llama-2-7b")
+>>> model = AutoModelForCausalLM.from_pretrained("linly-ai/chinese-llama-2-7b", dtype="float16")
+>>> input_features = tokenizer("你好！请自我介绍一下。", return_tensors="pd")
+>>> outputs = model.generate(**input_features, max_length=128)
+>>> tokenizer.batch_decode(outputs[0])
+['\n你好！我是一个AI语言模型，可以回答你的问题和提供帮助。']
 ```
 
-### 大模型预训练
+### 一键UIE预测
 
-```shell
-git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # 如已clone或下载PaddleNLP可跳过
-mkdir -p llm/data && cd llm/data
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
-cd .. # change folder to PaddleNLP/llm
-# 如需使用use_fused_rms_norm=true，需要前往slm/model_zoo/gpt-3/external_ops安装fused_ln
-python -u run_pretrain.py ./config/qwen/pretrain_argument_0p5b.json
-```
-
-### 大模型 SFT 精调
-
-```shell
-git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # 如已clone或下载PaddleNLP可跳过
-mkdir -p llm/data && cd llm/data
-wget https://bj.bcebos.com/paddlenlp/datasets/examples/AdvertiseGen.tar.gz && tar -zxvf AdvertiseGen.tar.gz
-cd .. # change folder to PaddleNLP/llm
-python -u run_finetune.py ./config/qwen/sft_argument_0p5b.json
-```
-
-更多大模型全流程步骤，请参考[飞桨大模型套件](./llm)介绍。
-另外我们还提供了快速微调方式, 无需 clone 源代码：
+PaddleNLP提供[一键预测功能](./docs/model_zoo/taskflow.md)，无需训练，直接输入数据即可开放域抽取结果。这里以信息抽取-命名实体识别任务，UIE模型为例：
 
 ```python
-from paddlenlp.trl import SFTConfig, SFTTrainer
-from datasets import load_dataset
+>>> from pprint import pprint
+>>> from paddlenlp import Taskflow
 
-dataset = load_dataset("ZHUI/alpaca_demo", split="train")
-
-training_args = SFTConfig(output_dir="Qwen/Qwen2.5-0.5B-SFT", device="gpu")
-trainer = SFTTrainer(
-    args=training_args,
-    model="Qwen/Qwen2.5-0.5B-Instruct",
-    train_dataset=dataset,
-)
-trainer.train()
+>>> schema = ['时间', '选手', '赛事名称'] # Define the schema for entity extraction
+>>> ie = Taskflow('information_extraction', schema=schema)
+>>> pprint(ie("2月8日上午北京冬奥会自由式滑雪女子大跳台决赛中中国选手谷爱凌以188.25分获得金牌！"))
+[{'时间': [{'end': 6,
+          'probability': 0.9857378532924486,
+          'start': 0,
+          'text': '2月8日上午'}],
+  '赛事名称': [{'end': 23,
+            'probability': 0.8503089953268272,
+            'start': 6,
+            'text': '北京冬奥会自由式滑雪女子大跳台决赛'}],
+  '选手': [{'end': 31,
+          'probability': 0.8981548639781138,
+          'start': 28,
+          'text': '谷爱凌'}]}]
 ```
 
-更多 PaddleNLP 内容可参考：
+更多PaddleNLP内容可参考：
+- [大模型全流程工具链](./llm)，包含主流中文大模型的全流程方案。
+- [精选模型库](./model_zoo)，包含优质预训练模型的端到端全流程使用。
+- [多场景示例](./examples)，了解如何使用PaddleNLP解决NLP多种技术问题，包含基础技术、系统应用与拓展应用。
+- [交互式教程](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)，在🆓免费算力平台AI Studio上快速学习PaddleNLP。
 
-* [精选模型库](./slm/model_zoo)，包含优质预训练模型的端到端全流程使用。
-* [多场景示例](./slm/examples)，了解如何使用 PaddleNLP 解决 NLP 多种技术问题，包含基础技术、系统应用与拓展应用。
-* [交互式教程](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)，在🆓免费算力平台 AI Studio 上快速学习 PaddleNLP。
 
-------------------------------------------------------------------------------------------
+## 特性
+
+#### <a href=#开箱即用的nlp工具集> 📦 开箱即用的NLP工具集 </a>
+
+#### <a href=#丰富完备的中文模型库> 🤗 丰富完备的中文模型库 </a>
+
+#### <a href=#产业级端到端系统范例> 🎛️ 产业级端到端系统范例 </a>
+
+#### <a href=#高性能分布式训练与推理> 🚀 高性能分布式训练与推理 </a>
+
+
+### 开箱即用的NLP工具集
+
+Taskflow提供丰富的**📦开箱即用**的产业级NLP预置模型，覆盖自然语言理解与生成两大场景，提供**💪产业级的效果**与**⚡️极致的推理性能**。
+
+![taskflow1](https://user-images.githubusercontent.com/11793384/159693816-fda35221-9751-43bb-b05c-7fc77571dd76.gif)
+
+更多使用方法可参考[Taskflow文档](./docs/model_zoo/taskflow.md)。
+### 丰富完备的中文模型库
+
+#### 🀄 业界最全的中文预训练模型
+
+精选 45+ 个网络结构和 500+ 个预训练模型参数，涵盖业界最全的中文预训练模型：既包括文心NLP大模型的ERNIE、PLATO等，也覆盖BERT、GPT、RoBERTa、T5等主流结构。通过`AutoModel` API一键⚡**高速下载**⚡。
+
+```python
+from paddlenlp.transformers import *
+
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+bert = AutoModel.from_pretrained('bert-wwm-chinese')
+albert = AutoModel.from_pretrained('albert-chinese-tiny')
+roberta = AutoModel.from_pretrained('roberta-wwm-ext')
+electra = AutoModel.from_pretrained('chinese-electra-small')
+gpt = AutoModelForPretraining.from_pretrained('gpt-cpm-large-cn')
+```
+
+针对预训练模型计算瓶颈，可以使用API一键使用文心ERNIE-Tiny全系列轻量化模型，降低预训练模型部署难度。
+
+```python
+# 6L768H
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+# 6L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-mini-zh')
+# 4L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-micro-zh')
+# 4L312H
+ernie = AutoModel.from_pretrained('ernie-3.0-nano-zh')
+```
+
+对预训练模型应用范式如语义表示、文本分类、句对匹配、序列标注、问答等，提供统一的API体验。
+
+```python
+import paddle
+from paddlenlp.transformers import *
+
+tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
+text = tokenizer('自然语言处理')
+
+# 语义表示
+model = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+sequence_output, pooled_output = model(input_ids=paddle.to_tensor([text['input_ids']]))
+# 文本分类 & 句对匹配
+model = AutoModelForSequenceClassification.from_pretrained('ernie-3.0-medium-zh')
+# 序列标注
+model = AutoModelForTokenClassification.from_pretrained('ernie-3.0-medium-zh')
+# 问答
+model = AutoModelForQuestionAnswering.from_pretrained('ernie-3.0-medium-zh')
+```
+
+#### 💯 全场景覆盖的应用示例
+
+覆盖从学术到产业的NLP应用示例，涵盖NLP基础技术、NLP系统应用以及拓展应用。全面基于飞桨核心框架2.0全新API体系开发，为开发者提供飞桨文本领域的最佳实践。
+
+精选预训练模型示例可参考[Model Zoo](./model_zoo)，更多场景示例文档可参考[examples目录](./examples)。更有免费算力支持的[AI Studio](https://aistudio.baidu.com)平台的[Notbook交互式教程](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)提供实践。
+
+<details><summary> PaddleNLP预训练模型适用任务汇总（<b>点击展开详情</b>）</summary><div>
+
+| Model              | Sequence Classification | Token Classification | Question Answering | Text Generation | Multiple Choice |
+| :----------------- | ----------------------- | -------------------- | ------------------ | --------------- | --------------- |
+| ALBERT             | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BART               | ✅                       | ✅                    | ✅                  | ✅               | ❌               |
+| BERT               | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BigBird            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BlenderBot         | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| ChineseBERT        | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ConvBERT           | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| CTRL               | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| DistilBERT         | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ELECTRA            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| ERNIE              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| ERNIE-CTM          | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| ERNIE-Doc          | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ERNIE-GEN          | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| ERNIE-Gram         | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ERNIE-M            | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| FNet               | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| Funnel-Transformer | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| GPT                | ✅                       | ✅                    | ❌                  | ✅               | ❌               |
+| LayoutLM           | ✅                       | ✅                    | ❌                  | ❌               | ❌               |
+| LayoutLMv2         | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| LayoutXLM          | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| LUKE               | ❌                       | ✅                    | ✅                  | ❌               | ❌               |
+| mBART              | ✅                       | ❌                    | ✅                  | ❌               | ✅               |
+| MegatronBERT       | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| MobileBERT         | ✅                       | ❌                    | ✅                  | ❌               | ❌               |
+| MPNet              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| NEZHA              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| PP-MiniLM          | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| ProphetNet         | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| Reformer           | ✅                       | ❌                    | ✅                  | ❌               | ❌               |
+| RemBERT            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| RoBERTa            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| RoFormer           | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| SKEP               | ✅                       | ✅                    | ❌                  | ❌               | ❌               |
+| SqueezeBERT        | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| T5                 | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| TinyBERT           | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| UnifiedTransformer | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| XLNet              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+
+</div></details>
+
+可参考[Transformer 文档](/docs/model_zoo/index.rst) 查看目前支持的预训练模型结构、参数和详细用法。
+
+### 产业级端到端系统范例
+
+PaddleNLP针对信息抽取、语义检索、智能问答、情感分析等高频NLP场景，提供了端到端系统范例，打通*数据标注*-*模型训练*-*模型调优*-*预测部署*全流程，持续降低NLP技术产业落地门槛。更多详细的系统级产业范例使用说明请参考[Applications](./applications)。
+
+#### 🔍 语义检索系统
+
+针对无监督数据、有监督数据等多种数据情况，结合SimCSE、In-batch Negatives、ERNIE-Gram单塔模型等，推出前沿的语义检索方案，包含召回、排序环节，打通训练、调优、高效向量检索引擎建库和查询全流程。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514909-8817d79a-72c4-4be1-8080-93d1f682bb46.gif" width="400">
+</div>
+
+
+更多使用说明请参考[语义检索系统](./applications/neural_search)。
+
+#### ❓ 智能问答系统
+
+基于[🚀RocketQA](https://github.com/PaddlePaddle/RocketQA)技术的检索式问答系统，支持FAQ问答、说明书问答等多种业务场景。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514868-1babe981-c675-4f89-9168-dd0a3eede315.gif" width="400">
+</div>
+
+
+更多使用说明请参考[智能问答系统](./applications/question_answering)与[文档智能问答](./applications/document_intelligence/doc_vqa)
+
+#### 💌 评论观点抽取与情感分析
+
+基于情感知识增强预训练模型SKEP，针对产品评论进行评价维度和观点抽取，以及细粒度的情感分析。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407260-b7f92800-861c-4207-98f3-2291e0102bbe.png" width="400">
+</div>
+
+更多使用说明请参考[情感分析](./applications/sentiment_analysis)。
+
+#### 🎙️ 智能语音指令解析
+
+集成了[PaddleSpeech](https://github.com/PaddlePaddle/PaddleSpeech)和[百度开放平台](https://ai.baidu.com/)的语音识别和[UIE](./model_zoo/uie)通用信息抽取等技术，打造智能一体化的语音指令解析系统范例，该方案可应用于智能语音填单、智能语音交互、智能语音检索等场景，提高人机交互效率。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/16698950/168589100-a6c6f346-97bb-47b2-ac26-8d50e71fddc5.png" width="400">
+</div>
+
+更多使用说明请参考[智能语音指令解析](./applications/speech_cmd_analysis)。
+
+### 高性能分布式训练与推理
+
+#### ⚡ FastTokenizer：高性能文本处理库
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407921-b4395b1d-44bd-41a0-8c58-923ba2b703ef.png" width="400">
+</div>
+
+```python
+AutoTokenizer.from_pretrained("ernie-3.0-medium-zh", use_fast=True)
+```
+
+为了实现更极致的模型部署性能，安装FastTokenizers后只需在`AutoTokenizer` API上打开 `use_fast=True`选项，即可调用C++实现的高性能分词算子，轻松获得超Python百余倍的文本处理加速，更多使用说明可参考[FastTokenizer文档](./fast_tokenizer)。
+
+#### ⚡️ FastGeneration：高性能生成加速库
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407831-914dced0-3a5a-40b8-8a65-ec82bf13e53c.gif" width="400">
+</div>
+
+```python
+model = GPTLMHeadModel.from_pretrained('gpt-cpm-large-cn')
+...
+outputs, _ = model.generate(
+    input_ids=inputs_ids, max_length=10, decode_strategy='greedy_search',
+    use_fast=True)
+```
+
+简单地在`generate()`API上打开`use_fast=True`选项，轻松在Transformer、GPT、BART、PLATO、UniLM等生成式预训练模型上获得5倍以上GPU加速，更多使用说明可参考[FastGeneration文档](./fast_generation)。
+
+#### 🚀 Fleet：飞桨4D混合并行分布式训练技术
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168515134-513f13e0-9902-40ef-98fa-528271dcccda.png" width="300">
+</div>
+
+
+更多关于千亿级AI模型的分布式训练使用说明可参考[GPT-3](./examples/language_model/gpt-3)。
 
 ## 社区交流
 
-* 微信扫描二维码并填写问卷，即可加入交流群与众多社区开发者以及官方团队深度交流.
+- 微信扫描二维码并填写问卷，回复小助手关键词（NLP）之后，即可加入交流群领取福利
 
-<div align="center">
-    <img src="https://github.com/user-attachments/assets/3a58cc9f-69c7-4ccb-b6f5-73e966b8051a" width="150" height="150" />
-</div>
+  - 与众多社区开发者以及官方团队深度交流。
+  - 10G重磅NLP学习大礼包！
+
+  <div align="center">
+  <img src="https://user-images.githubusercontent.com/11987277/245085922-0aa68d24-00ff-442e-9c53-2f1e898151ce.png" width="150" height="150" />
+  </div>
 
 ## Citation
 
-如果 PaddleNLP 对您的研究有帮助，欢迎引用
+如果PaddleNLP对您的研究有帮助，欢迎引用
 
-```bibtex
+```
 @misc{=paddlenlp,
     title={PaddleNLP: An Easy-to-use and High Performance NLP Library},
     author={PaddleNLP Contributors},
@@ -291,8 +338,8 @@ trainer.train()
 
 ## Acknowledge
 
-我们借鉴了 Hugging Face 的[Transformers](https://github.com/huggingface/transformers)🤗关于预训练模型使用的优秀设计，在此对 Hugging Face 作者及其开源社区表示感谢。
+我们借鉴了Hugging Face的[Transformers](https://github.com/huggingface/transformers)🤗关于预训练模型使用的优秀设计，在此对Hugging Face作者及其开源社区表示感谢。
 
 ## License
 
-PaddleNLP 遵循[Apache-2.0开源协议](./LICENSE)。
+PaddleNLP遵循[Apache-2.0开源协议](./LICENSE)。

README_en.md

@@ -1,13 +1,12 @@
+
 [简体中文🀄](./README.md) | **English🌎**
 
-<p align="center">
-  <img src="https://user-images.githubusercontent.com/1371212/175816733-8ec25eb0-9af3-4380-9218-27c154518258.png" align="middle"  width="500" />
-</p>
+<p align="center"> <img src="https://user-images.githubusercontent.com/1371212/175816733-8ec25eb0-9af3-4380-9218-27c154518258.png" align="middle"  width="500" /> </p>
 
 ------------------------------------------------------------------------------------------
 
 <p align="center">
-    <a href="https://paddlenlp.readthedocs.io/en/latest/?badge=latest"><img src="https://readthedocs.org/projects/paddlenlp/badge/?version=latest">
+    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/PaddleNLP?color=ffa"></a>
     <a href=""><img src="https://img.shields.io/badge/python-3.7+-aff.svg"></a>
     <a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
@@ -16,63 +15,261 @@
     <a href="https://pypi.org/project/paddlenlp/"><img src="https://img.shields.io/pypi/dm/paddlenlp?color=9cf"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/PaddleNLP?color=9cc"></a>
     <a href="https://github.com/PaddlePaddle/PaddleNLP/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/PaddleNLP?color=ccf"></a>
-    <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
 </p>
 
-<h4 align="center">
-    <a href=#Features> Features </a> |
-    <a href=#Support-Models> Supported Models </a> |
-    <a href=#Installation> Installation </a> |
-    <a href=#Quick-start> Quick Start </a> |
-    <a href=#community> Community </a>
-</h4>
+<h4 align="center"> <a href=#features> Features </a> | <a href=#installation> Installation </a> | <a href=#quick-start> Quick Start </a> | <a href=#api-reference> API Reference </a> | <a href=#community> Community </a>
 
-**PaddleNLP** is a Large Language Model (LLM) development suite based on the PaddlePaddle deep learning framework, supporting efficient large model training, lossless compression, and high-performance inference on various hardware devices. With its **simplicity** and **ultimate performance**, PaddleNLP is dedicated to helping developers achieve efficient industrial applications of large models.
+**PaddleNLP** is a NLP library that is both **easy to use** and **powerful**. It aggregates high-quality pretrained models in the industry and provides a **plug-and-play** development experience, covering a model library for various NLP scenarios. With practical examples from industry practices, PaddleNLP can meet the needs of developers who require **flexible customization**.
 
 ## News 📢
 
-* **2024.06.27 [PaddleNLP v3.0 Beta](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta0)**：Embrace large models and experience a complete upgrade. With a unified large model suite, we achieve full-process access to domestically produced computing chips. We fully support industrial-level application processes for large models, such as PaddlePaddle's 4D parallel configuration, efficient fine-tuning strategies, efficient alignment algorithms, and high-performance reasoning. Our developed RsLoRA+ algorithm, full checkpoint storage mechanism Unified Checkpoint, and generalized support for FastFNN and FusedQKV all contribute to the training and inference of large models. We continuously support updates to mainstream models for providing efficient solutions.
+* **2023.6.12: [Release of PaddleNLP v2.6rc](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.6.0rc)**
+  * 🔨 LLM Tools：Introduces comprehensive examples of open-source LLM training and inference, including [Bloom](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/bloom), [ChatGLM](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/chatglm), [GLM](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/glm), [Llama](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/llama) and [OPT](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/opt). Added Tensor Parallel capability to [Trainer API](./docs/trainer.md) for distributed LLM trainin. Also released [Parameter-Efficient Finetuning](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/paddlenlp/peft),which enables training LLMs on consumer hardware.
+
+* **2023.1.12: [Release of PaddleNLP v2.5](<https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.5.0>)**
+
+    * 🔨 NLP Tools: [PPDiffusers](./ppdiffusers), our cross-modal diffusion model toolbox based on PaddlePaddle, has been released! It provides a complete training process for diffusion models, and supports FastDeploy inference acceleration and multi-hardware deployment (supports Ascend chips and Kunlun core deployment).
+    * 💎 Industrial Applications: Information extraction, text classification, sentiment analysis, and intelligent question answering have all been newly upgraded. New releases include document information extraction [UIE-X](./applications/information_extraction/document), unified text classification [UTC](./applications/zero_shot_text_classification), unified sentiment analysis [UIE-Senta](./applications/sentiment_analysis/unified_sentiment_extraction) , and [unsupervised QA application](./applications/question_answering/unsupervised_qa). At the same time, the [ERNIE 3.0 Tiny v2](./model_zoo/ernie-tiny) series of pretrained small models have been released, which are more effective with low-resource and foreign data. They provide open-source end-to-end deployment solutions such as model pruning, model quantization, FastDeploy inference acceleration, and edge-side deployment to reduce the difficulty of pretrained model deployment.
+    * 💪 Framework Upgrade: Pretrained model [parameter configuration unification](./paddlenlp/transformers/configuration_utils.py), saving and loading custom parameter configurations no longer requires additional development; [Trainer API](./docs/trainer.md) has added BF16 training, recompute recalculations, sharding, and other distributed capabilities. Large-scale pre-training model training can easily be accomplished through simple configuration. [Model Compression API](./docs/compression.md) supports quantization training, vocabulary compression, and other functions. The compressed model has smaller accuracy loss, and the memory consumption of model deployment is greatly reduced. [Data Augmentation API](./docs/dataaug.md) has been comprehensively upgraded to support three granularities of data augmentation strategy: character, word, and sentence, making it easy to customize data augmentation strategies.
+    * 🤝 Community: 🤗Huggingface hub officially supports PaddleNLP pretrained models, supporting PaddleNLP Model and Tokenizer downloads and uploads directly from the 🤗Huggingface hub. Everyone is welcome to try out PaddleNLP pretrained models on the 🤗Huggingface hub [here](https://huggingface.co/PaddlePaddle).
+
+* **September 6, 2022: [Release of PaddleNLP v2.4](<https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.4.0>)**
+
+    * 🔨 NLP Tools: [NLP Pipeline System Pipelines](./pipelines) has been released, supporting the rapid construction of search engines and question-answering systems, and can be extended to support various NLP systems, making it easy, flexible, and efficient to solve NLP tasks like building blocks!
+    * 💎 Industrial Applications: A new [text classification full-process application solution](./applications/text_classification) has been added, covering various scenarios such as multi-classification, multi-label, and hierarchical classification, supporting small-sample learning and TrustAI trustworthy computing model training and tuning.
+    * 🍭 AIGC: The SOTA model [CodeGen](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/examples/code_generation/codegen) for code generation in various programming languages has been added.
+    * 💪 Framework Upgrade: [Automatic Model Compression API](./docs/compression.md) has been released, which automatically cuts and quantizes models, greatly reducing the threshold for using model compression technology. [Few-shot Prompt](./applications/text_classification/multi_class/few-shot) capability has been released, integrating classic algorithms such as PET, P-Tuning, and RGL.
+
+
+
+
 
-* **2024.04.24 [PaddleNLP v2.8](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.8.0)**：Our self-developed RsLoRA+ algorithm with extreme convergence significantly improves the convergence speed and training effectiveness of PEFT training. By introducing high-performance generation acceleration into the RLHF PPO algorithm, we have broken through the generation speed bottleneck in PPO training, achieving a significant lead in PPO training performance. We generally support multiple large model training performance optimization methods such as FastFFN and FusedQKV, making large model training faster and more stable.
 
 ## Features
 
-### <a href=#Integrated training and inference on multiple hardware platforms> 🔧 Integrated training and inference on multiple hardware platforms </a>
+#### <a href=#out-of-box-nlp-toolset> 📦 Out-of-Box NLP Toolset </a>
 
-Our development suit supports large model training and inference on multiple hardware platforms, including NVIDIA GPUs, Kunlun XPUs, Ascend NPUs, Enflame GCUs, and Hygon DCUs. The toolkit's interface allows for quick hardware switching, significantly reducing research and development costs associated with hardware transitions.
+#### <a href=#awesome-chinese-model-zoo> 🤗 Awesome Chinese Model Zoo </a>
 
-### <a href=Efficient and easy-to-use pre-training> 🚀 Efficient and easy-to-use pre-training </a>
+#### <a href=#industrial-end-to-end-system> 🎛️ Industrial End-to-end System </a>
 
-We support 4D high-performance training with data parallelism, sharding parallelism, tensor parallelism, and pipeline parallelism. The Trainer supports configurable distributed strategies, reducing the cost associated with complex distributed combinations. The Unified Checkpoint large model storage format supports dynamic scaling of model parameter distribution during training, thereby reducing the migration cost caused by hardware switching.
+#### <a href=#high-performance-distributed-training-and-inference> 🚀 High Performance Distributed Training and Inference </a>
 
-### <a href=#Efficient fine-tuning> 🤗 Efficient fine-tuning </a>
 
-The fine-tuning algorithms are deeply integrated with zero-padding data streams and high-performance FlashMask operators, reducing invalid data padding and computation during training, and significantly improving the throughput of fine-tuning training.
+### Out-of-Box NLP Toolset
 
-### <a href=#Lossless compression and high-performance inference> 🎛️ Lossless compression and high-performance inference </a>
+Taskflow aims to provide off-the-shelf NLP pre-built task covering NLU and NLG technique, in the meanwhile with extremely fast inference satisfying industrial scenario.
 
-The high-performance inference module of the large model toolkit incorporates dynamic insertion and operator fusion strategies throughout the entire process, greatly accelerating parallel inference speed. The underlying implementation details are encapsulated, enabling out-of-the-box high-performance parallel inference capabilities.
+![taskflow1](https://user-images.githubusercontent.com/11793384/159693816-fda35221-9751-43bb-b05c-7fc77571dd76.gif)
 
-## Documentation
-For detailed documentation, visit the [PaddleNLP Documentation](https://paddlenlp.readthedocs.io/).
+For more usage please refer to [Taskflow Docs](./docs/model_zoo/taskflow.md).
 
-------------------------------------------------------------------------------------------
+### Awesome Chinese Model Zoo
 
-## Support Models
+#### 🀄 Comprehensive Chinese Transformer Models
+
+We provide **45+** network architectures and over **500+** pretrained models. Not only includes all the SOTA model like ERNIE, PLATO and SKEP released by Baidu, but also integrates most of the high-quality Chinese pretrained model developed by other organizations. Use `AutoModel` API to **⚡SUPER FAST⚡** download pretrained models of different architecture. We welcome all developers to contribute your Transformer models to PaddleNLP!
+
+```python
+from paddlenlp.transformers import *
+
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+bert = AutoModel.from_pretrained('bert-wwm-chinese')
+albert = AutoModel.from_pretrained('albert-chinese-tiny')
+roberta = AutoModel.from_pretrained('roberta-wwm-ext')
+electra = AutoModel.from_pretrained('chinese-electra-small')
+gpt = AutoModelForPretraining.from_pretrained('gpt-cpm-large-cn')
+```
+
+Due to the computation limitation, you can use the ERNIE-Tiny light models to accelerate the deployment of pretrained models.
+```python
+# 6L768H
+ernie = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+# 6L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-mini-zh')
+# 4L384H
+ernie = AutoModel.from_pretrained('ernie-3.0-micro-zh')
+# 4L312H
+ernie = AutoModel.from_pretrained('ernie-3.0-nano-zh')
+```
+Unified API experience for NLP task like semantic representation, text classification, sentence matching, sequence labeling, question answering, etc.
+
+```python
+import paddle
+from paddlenlp.transformers import *
+
+tokenizer = AutoTokenizer.from_pretrained('ernie-3.0-medium-zh')
+text = tokenizer('natural language processing')
+
+# Semantic Representation
+model = AutoModel.from_pretrained('ernie-3.0-medium-zh')
+sequence_output, pooled_output = model(input_ids=paddle.to_tensor([text['input_ids']]))
+# Text Classificaiton and Matching
+model = AutoModelForSequenceClassification.from_pretrained('ernie-3.0-medium-zh')
+# Sequence Labeling
+model = AutoModelForTokenClassification.from_pretrained('ernie-3.0-medium-zh')
+# Question Answering
+model = AutoModelForQuestionAnswering.from_pretrained('ernie-3.0-medium-zh')
+```
+
+#### Wide-range NLP Task Support
+
+PaddleNLP provides rich examples covering mainstream NLP task to help developers accelerate problem solving. You can find our powerful transformer [Model Zoo](./model_zoo), and wide-range NLP application [examples](./examples) with detailed instructions.
+
+Also you can run our interactive [Notebook tutorial](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995) on AI Studio, a powerful platform with **FREE** computing resource.
+
+<details><summary> PaddleNLP Transformer model summary (<b>click to show details</b>) </summary><div>
+
+| Model              | Sequence Classification | Token Classification | Question Answering | Text Generation | Multiple Choice |
+| :----------------- | ----------------------- | -------------------- | ------------------ | --------------- | --------------- |
+| ALBERT             | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BART               | ✅                       | ✅                    | ✅                  | ✅               | ❌               |
+| BERT               | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BigBird            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| BlenderBot         | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| ChineseBERT        | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ConvBERT           | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| CTRL               | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| DistilBERT         | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ELECTRA            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| ERNIE              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| ERNIE-CTM          | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| ERNIE-Doc          | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ERNIE-GEN          | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| ERNIE-Gram         | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| ERNIE-M            | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| FNet               | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| Funnel-Transformer | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| GPT                | ✅                       | ✅                    | ❌                  | ✅               | ❌               |
+| LayoutLM           | ✅                       | ✅                    | ❌                  | ❌               | ❌               |
+| LayoutLMv2         | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| LayoutXLM          | ❌                       | ✅                    | ❌                  | ❌               | ❌               |
+| LUKE               | ❌                       | ✅                    | ✅                  | ❌               | ❌               |
+| mBART              | ✅                       | ❌                    | ✅                  | ❌               | ✅               |
+| MegatronBERT       | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| MobileBERT         | ✅                       | ❌                    | ✅                  | ❌               | ❌               |
+| MPNet              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| NEZHA              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| PP-MiniLM          | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| ProphetNet         | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| Reformer           | ✅                       | ❌                    | ✅                  | ❌               | ❌               |
+| RemBERT            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| RoBERTa            | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+| RoFormer           | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| SKEP               | ✅                       | ✅                    | ❌                  | ❌               | ❌               |
+| SqueezeBERT        | ✅                       | ✅                    | ✅                  | ❌               | ❌               |
+| T5                 | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| TinyBERT           | ✅                       | ❌                    | ❌                  | ❌               | ❌               |
+| UnifiedTransformer | ❌                       | ❌                    | ❌                  | ✅               | ❌               |
+| XLNet              | ✅                       | ✅                    | ✅                  | ❌               | ✅               |
+
+</div></details>
+
+For more pretrained model usage, please refer to [Transformer API Docs](./docs/model_zoo/index.rst).
+
+### Industrial End-to-end System
+
+We provide high value scenarios including information extraction, semantic retrieval, question answering high-value.
+
+For more details industrial cases please refer to [Applications](./applications).
+
+
+#### 🔍 Neural Search System
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514909-8817d79a-72c4-4be1-8080-93d1f682bb46.gif" width="400">
+</div>
+
+
+For more details please refer to [Neural Search](./applications/neural_search).
+
+#### ❓ Question Answering System
+
+We provide question answering pipeline which can support FAQ system, Document-level Visual Question answering system based on [🚀RocketQA](https://github.com/PaddlePaddle/RocketQA).
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514868-1babe981-c675-4f89-9168-dd0a3eede315.gif" width="400">
+</div>
+
+
+For more details please refer to [Question Answering](./applications/question_answering) and [Document VQA](./applications/document_intelligence/doc_vqa).
+
+
+#### 💌 Opinion Extraction and Sentiment Analysis
+
+We build an opinion extraction system for product review and fine-grained sentiment analysis based on [SKEP](https://arxiv.org/abs/2005.05635) Model.
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407260-b7f92800-861c-4207-98f3-2291e0102bbe.png" width="300">
+</div>
+
+
+For more details please refer to [Sentiment Analysis](./applications/sentiment_analysis).
+
+#### 🎙️ Speech Command Analysis
+
+Integrated ASR Model, Information Extraction, we provide a speech command analysis pipeline that show how to use PaddleNLP and [PaddleSpeech](https://github.com/PaddlePaddle/PaddleSpeech) to solve Speech + NLP real scenarios.
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168412618-04897a47-79c9-4fe7-a054-5dc1f6a1f75c.png" width="500">
+</div>
+
+
+For more details please refer to [Speech Command Analysis](./applications/speech_cmd_analysis).
+
+### High Performance Distributed Training and Inference
+
+#### ⚡ FastTokenizer: High Performance Text Preprocessing Library
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407921-b4395b1d-44bd-41a0-8c58-923ba2b703ef.png" width="400">
+</div>
+
+```python
+AutoTokenizer.from_pretrained("ernie-3.0-medium-zh", use_fast=True)
+```
+
+Set `use_fast=True` to use C++ Tokenizer kernel to achieve 100x faster on text pre-processing. For more usage please refer to [FastTokenizer](./fast_tokenizer).
+
+#### ⚡ FastGeneration: High Performance Generation Library
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168407831-914dced0-3a5a-40b8-8a65-ec82bf13e53c.gif" width="400">
+</div>
+
+```python
+model = GPTLMHeadModel.from_pretrained('gpt-cpm-large-cn')
+...
+outputs, _ = model.generate(
+    input_ids=inputs_ids, max_length=10, decode_strategy='greedy_search',
+    use_fast=True)
+```
+
+Set `use_fast=True` to achieve 5x speedup for Transformer, GPT, BART, PLATO, UniLM text generation. For more usage please refer to [FastGeneration](./fast_generation).
+
+#### 🚀 Fleet: 4D Hybrid Distributed Training
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168515134-513f13e0-9902-40ef-98fa-528271dcccda.png" width="300">
+</div>
+
+
+For more super large-scale model pre-training details please refer to [GPT-3](./examples/language_model/gpt-3).
 
-Detailed list 👉 [Supported Model List](https://github.com/PaddlePaddle/PaddleNLP/issues/8663)
 
 ## Installation
 
 ### Prerequisites
 
-* python >= 3.8
-* paddlepaddle >= 3.0.0b0
+* python >= 3.7
+* paddlepaddle >= 2.3
 
-### Pip Installation
+More information about PaddlePaddle installation please refer to [PaddlePaddle's Website](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/conda/linux-conda.html).
 
-```shell
-pip install --upgrade paddlenlp==3.0.0b3
+### Python pip Installation
+
+```
+pip install --upgrade paddlenlp
 ```
 
 or you can install the latest develop branch code with the following command:
@@ -81,56 +278,46 @@ or you can install the latest develop branch code with the following command:
 pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html
 ```
 
-More information about PaddlePaddle installation please refer to [PaddlePaddle's Website](https://www.paddlepaddle.org.cn).
-
-------------------------------------------------------------------------------------------
-
 ## Quick Start
 
-### Text generation with large language model
-
-PaddleNLP provides a convenient and easy-to-use Auto API, which can quickly load models and Tokenizers. Here, we use the `Qwen/Qwen2-0.5B` large model as an example for text generation:
+**Taskflow** aims to provide off-the-shelf NLP pre-built task covering NLU and NLG scenario, in the meanwhile with extremely fast inference satisfying industrial applications.
 
 ```python
->>> from paddlenlp.transformers import AutoTokenizer, AutoModelForCausalLM
->>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B")
->>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B", dtype="float16")
->>> input_features = tokenizer("你好！请自我介绍一下。", return_tensors="pd")
->>> outputs = model.generate(**input_features, max_length=128)
->>> print(tokenizer.batch_decode(outputs[0], skip_special_tokens=True))
-['我是一个AI语言模型，我可以回答各种问题，包括但不限于：天气、新闻、历史、文化、科学、教育、娱乐等。请问您有什么需要了解的吗？']
+from paddlenlp import Taskflow
+
+# Chinese Word Segmentation
+seg = Taskflow("word_segmentation")
+seg("第十四届全运会在西安举办")
+>>> ['第十四届', '全运会', '在', '西安', '举办']
+
+# POS Tagging
+tag = Taskflow("pos_tagging")
+tag("第十四届全运会在西安举办")
+>>> [('第十四届', 'm'), ('全运会', 'nz'), ('在', 'p'), ('西安', 'LOC'), ('举办', 'v')]
+
+# Named Entity Recognition
+ner = Taskflow("ner")
+ner("《孤女》是2010年九州出版社出版的小说，作者是余兼羽")
+>>> [('《', 'w'), ('孤女', '作品类_实体'), ('》', 'w'), ('是', '肯定词'), ('2010年', '时间类'), ('九州出版社', '组织机构类'), ('出版', '场景事件'), ('的', '助词'), ('小说', '作品类_概念'), ('，', 'w'), ('作者', '人物类_概念'), ('是', '肯定词'), ('余兼羽', '人物类_实体')]
+
+# Dependency Parsing
+ddp = Taskflow("dependency_parsing")
+ddp("9月9日上午纳达尔在亚瑟·阿什球场击败俄罗斯球员梅德韦杰夫")
+>>> [{'word': ['9月9日', '上午', '纳达尔', '在', '亚瑟·阿什球场', '击败', '俄罗斯', '球员', '梅德韦杰夫'], 'head': [2, 6, 6, 5, 6, 0, 8, 9, 6], 'deprel': ['ATT', 'ADV', 'SBV', 'MT', 'ADV', 'HED', 'ATT', 'ATT', 'VOB']}]
+
+# Sentiment Analysis
+senta = Taskflow("sentiment_analysis")
+senta("这个产品用起来真的很流畅，我非常喜欢")
+>>> [{'text': '这个产品用起来真的很流畅，我非常喜欢', 'label': 'positive', 'score': 0.9938690066337585}]
 ```
 
-### Pre-training for large language model
+## API Reference
 
-```shell
-git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # if cloned or downloaded, can skip this step
-mkdir -p llm/data && cd llm/data
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
-wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
-cd .. # change folder to PaddleNLP/llm
-python -u -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" run_pretrain.py ./config/llama/pretrain_argument.json
-```
+- Support [LUGE](https://www.luge.ai/) dataset loading and compatible with Hugging Face [Datasets](https://huggingface.co/datasets). For more details please refer to [Dataset API](https://paddlenlp.readthedocs.io/zh/latest/data_prepare/dataset_list.html).
+- Using Hugging Face style API to load 500+ selected transformer models and download with fast speed. For more information please refer to [Transformers API](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/index.html).
+- One-line of code to load pre-trained word embedding. For more usage please refer to [Embedding API](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/embeddings.html).
 
-### SFT finetuning forlarge language model
-
-```shell
-git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # if cloned or downloaded, can skip this step
-mkdir -p llm/data && cd llm/data
-wget https://bj.bcebos.com/paddlenlp/datasets/examples/AdvertiseGen.tar.gz && tar -zxvf AdvertiseGen.tar.gz
-cd .. # change folder to PaddleNLP/llm
-python -u -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" run_finetune.py ./config/llama/sft_argument.json
-```
-
-For more steps in the entire large model process, please refer to the[Large Model Full-Process Suite](./llm).
-
-For more PaddleNLP content, please refer to:
-
-* [Model Library](./slm/model_zoo)，which includes end-to-end usage of high-quality pre-trained models.
-* [Multi-scenario Examples](./slm/examples)，to understand how to use PaddleNLP to solve various NLP technical problems, including basic techniques, system applications, and extended applications.
-* [Interactive Tutorial](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)，to quickly learn PaddleNLP on the free computing platform AI Studio.
-
-------------------------------------------------------------------------------------------
+Please find all PaddleNLP API Reference from our [readthedocs](https://paddlenlp.readthedocs.io/).
 
 ## Community
 
@@ -143,14 +330,15 @@ To connect with other users and contributors, welcome to join our [Slack channel
 Scan the QR code below with your Wechat⬇️. You can access to official technical exchange group. Look forward to your participation.
 
 <div align="center">
-    <img src="https://github.com/user-attachments/assets/3a58cc9f-69c7-4ccb-b6f5-73e966b8051a" width="150" height="150" />
+<img src="https://user-images.githubusercontent.com/11987277/245085922-0aa68d24-00ff-442e-9c53-2f1e898151ce.png" width="150" height="150" />
 </div>
 
+
+
 ## Citation
 
-If you find PaddleNLP useful in your research, please consider citing
-
-```bibtext
+If you find PaddleNLP useful in your research, please consider cite
+```
 @misc{=paddlenlp,
     title={PaddleNLP: An Easy-to-use and High Performance NLP Library},
     author={PaddleNLP Contributors},

applications/README.md

@@ -0,0 +1,130 @@
+# 产业级端到端系统范例
+
+## 1、简介
+
+PaddleNLP 从预训练模型库出发，提供了经典预训练模型在主流 NLP 任务上丰富的[应用示例](../examples)，满足了大量开发者的学习科研与基础应用需求。
+
+针对更广泛的产业落地需求、更复杂的 NLP 场景任务，PaddleNLP 推出**产业级端到端系统范例库**（下文简称产业范例），提供单个模型之上的产业解决方案。
+
+- 最强模型与实践———产业范例针对具体业务场景，提供最佳模型（组合），兼顾模型精度与性能，降低开发者模型选型成本；
+- 全流程———打通数据标注-模型训练-模型调优-模型压缩—预测部署全流程，帮助开发者更低成本得完成产业落地。
+
+## 2、基于 Pipelines 构建产业范例，加速落地
+
+在面向不同场景任务建设一系列产业方案的过程中，不难发现，从技术基础设施角度看：
+
+（1）NLP系统都可以抽象为由多个基础组件串接而成的流水线系统；
+（2）多个NLP流水线系统可共享使用相同的基础组件。
+
+因此，PaddleNLP 逐渐孵化出了一套 NLP 流水线系统 [Pipelines](../pipelines)，将各个 NLP 复杂系统的通用模块抽象封装为标准组件，支持开发者通过配置文件对标准组件进行组合，仅需几分钟即可定制化构建智能系统，让解决NLP任务像搭积木一样便捷、灵活、高效。同时，Pipelines 中预置了前沿的预训练模型和算法，在研发效率、模型效果和性能方面提供多重保障。因此，Pipelines 能够大幅加快开发者使用飞桨落地的效率。
+
+
+<div>
+    <img src="https://user-images.githubusercontent.com/11793384/212836991-d9132e46-b5bf-4389-80e1-4f9dee32f1fe.png" width="90%" length="90%">
+</div>
+
+<br>
+
+**PaddleNLP 提供了多个版本的产业范例:**
+
+- 如果你希望快速体验、直接应用、从零搭建一套完整系统，推荐使用 **Pipelines 版本**。这里集成了训练好的模型，无需关心模型训练细节；提供 Docker 环境，可快速一键部署端到端系统；打通前端 Demo 界面，便于直观展示、分析、调试效果。
+- 如果你希望使用自己的业务数据进行二次开发，推荐使用`./applications`目录下的**可定制版本**，训练好的模型可以直接集成进 Pipelines 中进行使用。
+- 也可以使用 [AI Studio](https://aistudio.baidu.com/aistudio/index) 在线 Jupyter Notebook 快速体验，有 GPU 算力哦。
+
+| 场景任务   | Pipelines版本地址 | 可定制版本地址 | Notebook |
+| :--------------- | ------- | ------- | ------- |
+| **检索**| [字面+语义检索](../pipelines/examples/semantic-search) | [语义检索](./neural_search) | [基于Pipelines搭建检索系统](https://aistudio.baidu.com/aistudio/projectdetail/4442670)<br>[二次开发语义检索](https://aistudio.baidu.com/aistudio/projectdetail/3351784) |
+| **问答** | [FAQ问答](../pipelines/examples/FAQ/)<br>[无监督检索式问答](../pipelines/examples/unsupervised-question-answering)<br>[有监督检索式问答](../pipelines/examples/question-answering) | [FAQ问答](./question_answering/supervised_qa)<br>[无监督检索式问答](./question_answering/unsupervised_qa) | [基于Pipelines搭建FAQ问答系统](https://aistudio.baidu.com/aistudio/projectdetail/4465498)<br>[基于Pipelines搭建抽取式问答系统](https://aistudio.baidu.com/aistudio/projectdetail/4442857)<br>[FAQ政务问答](https://aistudio.baidu.com/aistudio/projectdetail/3678873)<br>[FAQ保险问答](https://aistudio.baidu.com/aistudio/projectdetail/3882519) |
+| **文本分类**| 暂无 | [文本分类](./text_classification)  | [对话意图识别](https://aistudio.baidu.com/aistudio/projectdetail/2017202)<br>[法律文本多标签分类](https://aistudio.baidu.com/aistudio/projectdetail/3996601)<br>[层次分类](https://aistudio.baidu.com/aistudio/projectdetail/4568985) |
+| **通用文本分类** | 暂无 | [通用文本分类](./zero_shot_text_classification) |  |
+| **通用信息抽取** | 暂无 | [通用信息抽取](./information_extraction) | [UIE快速体验](https://aistudio.baidu.com/aistudio/projectdetail/3914778)<br>[UIE微调实体抽取](https://aistudio.baidu.com/aistudio/projectdetail/4038499)<br>[UIE微调关系抽取](https://aistudio.baidu.com/aistudio/projectdetail/4371345)<br>[UIE-X快速体验](https://aistudio.baidu.com/aistudio/projectdetail/5017442)<br>[UIE-X微调](https://aistudio.baidu.com/aistudio/projectdetail/5261592) |
+| **情感分析**  | [情感分析](../pipelines/examples/sentiment_analysis)  | [情感分析](./sentiment_analysis) |  [情感分析](https://aistudio.baidu.com/aistudio/projectdetail/5318177)|
+| **文档智能**  | [文档抽取问答](../pipelines/examples/document-intelligence) |  [跨模态文档问答](./document_intelligence/doc_vqa)| [文档抽取问答](https://aistudio.baidu.com/aistudio/projectdetail/4881278)<br>[汽车说明书问答](https://aistudio.baidu.com/aistudio/projectdetail/4049663)  |
+| **文生图**  | [文生图系统](../pipelines/examples/text_to_image)  | 可参考[PPDiffusers](../ppdiffusers) |   |
+| **语音指令解析**  | 暂无 | [语音指令解析](./speech_cmd_analysis) | [语音指令解析](https://aistudio.baidu.com/aistudio/projectdetail/4399703) |
+| **文本摘要**  | 暂无 | [文本摘要](./text_summarization) | [文本摘要](https://aistudio.baidu.com/aistudio/projectdetail/4903667) |
+
+## 3、典型范例介绍
+
+#### 📄 通用信息抽取系统
+
+- 首个产业级通用信息抽取方案 UIE，面向纯文本，实现多任务统一建模，提供强大的零样本抽取和少样本快速迁移能力；
+- 首个兼具文本及文档抽取能力、多语言、开放域的信息抽取方案 UIE-X，基于 [ERNIE-Layout](../model_zoo/ernie-layout) 跨模态布局增强预训练模型，集成 [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR) 的 PP-OCR、PP-Structure 版面分析能力，小样本文档信息抽取效果领先。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/213365046-69967745-b4a8-4435-98fb-c34f68cd22e9.png" width="60%" length="60%">
+</div>
+
+
+详细使用说明请参考[通用信息抽取系统](./information_extraction)，更多：[UIE 解读](https://mp.weixin.qq.com/s/-hHz8knHIKKqKCBTke7i5A)、[UIE-X 解读](https://zhuanlan.zhihu.com/p/592422623)。
+
+#### 🔍 语义检索系统
+
+- 前沿算法———基于 SimCSE、In-batch Negatives、ERNIE Pairwise、RocketQA Pointwise 等提供针对无监督、有监督等多种数据情况的多样化方案；
+- 全流程———覆盖召回、排序环节，集成主流 ANN 引擎，同时兼容 ElasticSearch 字面检索模式，提供多路召回方案。打通训练、调优、高效向量检索引擎建库和查询全流程。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/213134465-30cae5fd-4cd1-4e5b-a1cb-fa55c72980a7.gif" width="60%" length="60%">
+</div>
+
+详细使用说明请参考[语义检索系统](./neural_search)。
+
+#### ❓ 智能问答系统
+
+- 端到端问答技术 [🚀RocketQA](https://github.com/PaddlePaddle/RocketQA)，首个中文端到端问答模型，基于知识增强的预训练模型ERNIE和百万量级的人工标注数据集DuReader训练得到，效果优异；
+- 覆盖有监督（如 FAQ 问答）、无监督（自动生成 QA 对，生成的问答对语料可以通过无监督的方式构建检索式问答系统）等多种情况，适用各类业务场景。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/168514868-1babe981-c675-4f89-9168-dd0a3eede315.gif" width="60%" length="60%">
+</div>
+
+
+详细使用说明请参考[智能问答系统](./question_answering)与[文档智能问答](./document_intelligence/doc_vqa)。
+
+#### 📚 通用文本分类
+
+- 基于“任务架构统一、通用能力共享”的通用文本分类技术 UTC，实了良好的零/少样本迁移能力，实现大一统诸多任务的开放域分类，可支持情感分析、意图识别、语义匹配、蕴含推理等各种可转换为分类问题的 NLU 任务。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/11793384/213347595-e9c08bd1-3d32-4519-9a52-31fb69b841e8.png" width="60%" length="60%">
+</div>
+
+<br>
+
+详细使用说明请参考[通用文本分类](./zero_shot_text_classification)，更多：[文章解读](https://mp.weixin.qq.com/s/VV-nYv4y1r7oipJnURRL5w)。
+
+
+#### 🗂 文本分类
+
+- 场景方案全覆盖––––开源预训练模型-微调、提示学习、基于语义索引等多种分类技术方案，满足不同场景需求，涵盖多分类（multi-class）、多标签（multi-label）、层次分类（hierarchical）三类任务；
+- 模型高效调优––––强强结合数据增强能力与可信增强技术，解决脏数据、标注数据欠缺、数据不平衡等问题，大幅提升模型效果。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/63761690/186378697-630d3590-4e67-49a0-8d5f-7cabd9daa894.png" width="60%" length="60%">
+</div>
+
+<br>
+
+详细使用说明请参考[文本分类](./text_classification)，更多：[文章解读](https://mp.weixin.qq.com/s/tas7yM8vapxwtlJt-MRZdg)。
+
+#### 💌 评论观点抽取与情感分析
+
+- 经典方案：基于情感知识增强预训练模型SKEP，两阶段式抽取和分类，首先通过序列标注的方式定位属性词和观点词，然后进行属性集情感分类；
+- 前沿方案：基于UIE的情感分析方案采用 Prompt Learning 的方式进行情感信息抽取，精度更高。支持语句级和属性级情感分析，解决同义属性聚合、隐性观点抽取难点，并提供可视化分析能力。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/35913314/200259473-434888f7-c0ac-4253-ab23-ede1628e6ba2.png" width="60%" length="60%">
+</div>
+<br>
+
+详细使用说明请参考[情感分析](./sentiment_analysis)，更多：[文章解读](https://mp.weixin.qq.com/s/QAHjIRG9zxpYfM6YPRQ-9w)。
+
+#### 🎙️ 智能语音指令解析
+
+- 集成了[PaddleSpeech](https://github.com/PaddlePaddle/PaddleSpeech)和[百度开放平台](https://ai.baidu.com/)的语音识别和[UIE](./model_zoo/uie)通用信息抽取等技术，打造智能一体化的语音指令解析系统范例，该方案可应用于智能语音填单、智能语音交互、智能语音检索等场景，提高人机交互效率。
+
+<div align="center">
+    <img src="https://user-images.githubusercontent.com/16698950/168589100-a6c6f346-97bb-47b2-ac26-8d50e71fddc5.png" width="400">
+</div>
+
+详细使用说明请参考[智能语音指令解析](./applications/speech_cmd_analysis)。

applications/document_intelligence/README.md

@@ -0,0 +1,188 @@
+# 文档智能应用
+
+**目录**
+- [1. 文档智能应用简介](#文档智能应用简介)
+- [2. 技术特色介绍](#技术特色介绍)
+  - [2.1 多语言跨模态训练基座](#多语言跨模态训练基座)
+  - [2.2 多场景覆盖](#多场景覆盖)
+- [3. 快速开始](#快速开始)
+  - [3.1 开箱即用](#开箱即用)
+  - [3.2 产业级流程方案](#产业级流程方案)
+
+## 1. 文档智能应用简介
+
+文档智能（DI, Document Intelligence）主要指**对于网页、数字文档或扫描文档所包含的文本以及丰富的排版格式等信息，通过人工智能技术进行理解、分类、提取以及信息归纳**的过程。文档智能技术广泛应用于金融、保险、能源、物流、医疗等行业，常见的应用场景包括财务报销单、招聘简历、企业财报、合同文书、动产登记证、法律判决书、物流单据等多模态文档的关键信息抽取、文档解析、文档比对等。
+
+在实际应用中，需要解决文档格式繁杂、布局多样、信息模态多样、需求开放、业务数据少等多重难题。针对文档智能领域的痛点和难点，PaddleNLP将持续开源一系列产业实践范例，解决开发者们实际应用难题。
+
+<div align="center">
+    <img width="1000" height="270" alt="文档智能技术一般流程" src="https://user-images.githubusercontent.com/40840292/196361583-6b1c66d1-6a9b-4193-949a-71e2d420a82a.png">
+</div>
+
+<a name="技术特色介绍"></a>
+
+## 2. 技术特色介绍
+
+<a name="多语言跨模态训练基座"></a>
+
+### 2.1 多语言跨模态训练基座
+
+近期，百度文心文档智能，基于多语言跨模态布局增强的文档智能大模型[ERNIE-Layout](http://arxiv.org/abs/2210.06155)，刷新了五类11项文档智能任务效果。依托文心ERNIE大模型，基于布局知识增强技术，融合文本、图像、布局等信息进行联合建模，能够对多模态文档（如文档图片、PDF 文件、扫描件等）进行深度理解与分析，为各类上层应用提供SOTA模型底座。
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/196373896-597f6178-4c78-41a1-bb12-796546644b32.png width="600"/>
+</div>
+
+<a name="多场景覆盖"></a>
+
+### 2.2 多场景覆盖
+
+以下是文档智能技术的一些应用场景展示：
+
+- 发票抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/196118171-fd3e49a0-b9f1-4536-a904-c48f709a2dec.png height=350 width=1000 hspace='10'/>
+</div>
+
+- 海报抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195610368-04230855-62de-439e-b708-2c195b70461f.png height=600 width=1000 hspace='15'/>
+</div>
+
+- 网页抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195611613-bdbe692e-d7f2-4a2b-b548-1a933463b0b9.png height=350 width=1000 hspace='10'/>
+</div>
+
+
+- 表格抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195610692-8367f1c8-32c2-4b5d-9514-a149795cf609.png height=350 width=1000 hspace='10'/>
+</div>
+
+
+- 试卷抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195823294-d891d95a-2ef8-4519-be59-0fedb96c00de.png height=700 width=1000 hspace='10'/>
+</div>
+
+
+- 英文票据多语种（中、英、日、泰、西班牙、俄语）抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195610820-7fb88608-b317-45fc-a6ab-97bf3b20a4ac.png height=400 width=1000 hspace='15'/>
+</div>
+
+- 中文票据多语种（中简、中繁、英、日、法语）抽取问答
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195611075-9323ce9f-134b-4657-ab1c-f4892075d909.png height=350 width=1000 hspace='15'/>
+</div>
+
+- Demo图片可在此[下载](https://bj.bcebos.com/paddlenlp/taskflow/document_intelligence/demo.zip)
+
+<a name="快速开始"></a>
+
+## 3. 快速开始
+
+<a name="开箱即用"></a>
+
+### 3.1 开箱即用
+
+开源DocPrompt开放文档抽取问答模型，以ERNIE-Layout为底座，可精准理解图文信息，推理学习附加知识，准备捕捉图片、PDF等多模态文档中的每个细节。
+
+🧾 通过[Huggingface网页](https://huggingface.co/spaces/PaddlePaddle/ERNIE-Layout)体验DocPrompt功能：
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/195749427-864d7744-1fd1-455e-99c6-53a260776483.jpg height=700 width=1100 hspace='10'/>
+</div>
+
+#### Taskflow
+
+通过``paddlenlp.Taskflow``三行代码调用DocPrompt功能，具备多语言文档抽取问答能力，部分应用场景展示如下：
+
+- 输入格式
+
+```
+[
+  {"doc": "./invoice.jpg", "prompt": ["发票号码是多少?", "校验码是多少?"]},
+  {"doc": "./resume.png", "prompt": ["五百丁本次想要担任的是什么职位?", "五百丁是在哪里上的大学?", "大学学的是什么专业?"]}
+]
+```
+
+默认使用PaddleOCR进行OCR识别，同时支持用户通过``word_boxes``传入自己的OCR结果，格式为``List[str, List[float, float, float, float]]``。
+
+```
+[
+  {"doc": doc_path, "prompt": prompt, "word_boxes": word_boxes}
+]
+```
+
+- 支持单条、批量预测
+
+  - 支持本地图片路径输入
+
+  <div align="center">
+      <img src=https://user-images.githubusercontent.com/40840292/194748579-f9e8aa86-7f65-4827-bfae-824c037228b3.png height=800 hspace='20'/>
+  </div>
+
+  ```python
+  >>> from pprint import pprint
+  >>> from paddlenlp import Taskflow
+
+  >>> docprompt = Taskflow("document_intelligence")
+  >>> pprint(docprompt([{"doc": "./resume.png", "prompt": ["五百丁本次想要担任的是什么职位?", "五百丁是在哪里上的大学?", "大学学的是什么专业?"]}]))
+  [{'prompt': '五百丁本次想要担任的是什么职位?',
+    'result': [{'end': 7, 'prob': 1.0, 'start': 4, 'value': '客户经理'}]},
+  {'prompt': '五百丁是在哪里上的大学?',
+    'result': [{'end': 37, 'prob': 1.0, 'start': 31, 'value': '广州五百丁学院'}]},
+  {'prompt': '大学学的是什么专业?',
+    'result': [{'end': 44, 'prob': 0.82, 'start': 38, 'value': '金融学(本科）'}]}]
+  ```
+
+  - http图片链接输入
+
+  <div align="center">
+      <img src=https://user-images.githubusercontent.com/40840292/194748592-e20b2a5f-d36b-46fb-8057-86755d188af0.jpg height=400 hspace='10'/>
+  </div>
+
+  ```python
+  >>> from pprint import pprint
+  >>> from paddlenlp import Taskflow
+
+  >>> docprompt = Taskflow("document_intelligence")
+  >>> pprint(docprompt([{"doc": "https://bj.bcebos.com/paddlenlp/taskflow/document_intelligence/images/invoice.jpg", "prompt": ["发票号码是多少?", "校验码是多少?"]}]))
+  [{'prompt': '发票号码是多少?',
+    'result': [{'end': 2, 'prob': 0.74, 'start': 2, 'value': 'No44527206'}]},
+  {'prompt': '校验码是多少?',
+    'result': [{'end': 233,
+                'prob': 1.0,
+                'start': 231,
+                'value': '01107 555427109891646'}]}]
+  ```
+
+- 可配置参数说明
+  * `batch_size`：批处理大小，请结合机器情况进行调整，默认为1。
+  * `lang`：选择PaddleOCR的语言，`ch`可在中英混合的图片中使用，`en`在英文图片上的效果更好，默认为`ch`。
+  * `topn`: 如果模型识别出多个结果，将返回前n个概率值最高的结果，默认为1。
+
+<a name="产业级流程方案"></a>
+
+### 3.2 产业级流程方案
+
+针对文档智能领域的痛点和难点，PaddleNLP将持续开源一系列文档智能产业实践范例，解决开发者们实际应用难题。
+
+- 👉 [汽车说明书跨模态智能问答](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/applications/document_intelligence/doc_vqa#readme)
+
+更多：百度TextMind智能文档分析平台可提供包括文档信息抽取、文本内容审查、企业文档管理、文档格式解析、文档内容比对等全方位一站式的文档智能服务，已形成一套完整的企业文档场景化解决方案，满足银行、券商、法律、能源、传媒、通信、物流等不同行业和场景的文档处理需求，以AI助力企业的办公智能化升级和数字化转型。欢迎深度交流与商业合作，了解详情：https://ai.baidu.com/tech/nlp/Textanalysis
+
+## References
+
+- [文档智能：数据集、模型和应用](http://jcip.cipsc.org.cn/CN/abstract/abstract3331.shtml)
+
+- [ERNIE-Layout: Layout-Knowledge Enhanced Multi-modal Pre-training for Document Understanding](http://arxiv.org/abs/2210.06155)

applications/document_intelligence/doc_vqa/.gitignore

@@ -0,0 +1,17 @@
+OCR_process/*.json
+*.png
+*.json
+answers/*
+checkpoints/*
+__pycache__/*
+OCR_process/demo_pics/*
+Rerank/log/*
+Rerank/checkpoints/*
+Rerank/data/*
+Rerank/output/*
+Rerank/__pycache__/*
+Extraction/log/*
+Extraction/checkpoints/*
+Extraction/data/*
+Extraction/output/*
+Extraction/__pycache__/*

applications/document_intelligence/doc_vqa/Extraction/change_to_mrc.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import json
+import numpy as np
+
+
+def get_top1_from_ranker(path):
+    with open(path, "r", encoding="utf-8") as f:
+        scores = [float(line.strip()) for line in f.readlines()]
+        top_id = np.argmax(scores)
+
+    return top_id
+
+
+def get_ocr_result_by_id(path, top_id):
+    with open(path, "r", encoding="utf-8") as f:
+        reses = f.readlines()
+        res = reses[top_id]
+    return json.loads(res)
+
+
+def write_to_file(doc, path):
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(doc, f, ensure_ascii=False)
+        f.write("\n")
+
+
+if __name__ == "__main__":
+    question = sys.argv[1]
+    ranker_result_path = "../Rerank/data/demo.score"
+    ocr_result_path = "../OCR_process/demo_ocr_res.json"
+    save_path = "data/demo_test.json"
+    top_id = get_top1_from_ranker(ranker_result_path)
+    doc = get_ocr_result_by_id(ocr_result_path, top_id)
+    doc["question"] = question
+    doc["img_id"] = str(top_id + 1)
+
+    write_to_file(doc, save_path)

applications/document_intelligence/doc_vqa/Extraction/docvqa.py

@@ -0,0 +1,359 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import json
+import sys
+
+import numpy as np
+import paddle
+from paddle.io import Dataset
+from tqdm import tqdm
+
+sys.path.insert(0, "../")
+
+
+class DocVQAExample(object):
+    def __init__(self, question, doc_tokens, doc_boxes=[], answer=None, labels=None, image=None):
+        self.question = question
+        self.doc_tokens = doc_tokens
+        self.doc_boxes = doc_boxes
+        self.image = image
+        self.answer = answer
+        self.labels = labels
+
+
+class DocVQAFeatures(object):
+    """A single set of features of data."""
+
+    def __init__(self, example_index, input_ids, input_mask, segment_ids, boxes=None, label=None):
+        self.example_index = example_index
+        self.input_ids = input_ids
+        self.input_mask = input_mask
+        self.segment_ids = segment_ids
+        self.boxes = boxes
+        self.label = label
+
+
+class DocVQA(Dataset):
+    def __init__(
+        self, args, tokenizer, label2id_map, max_seq_len=512, max_query_length=20, max_doc_length=512, max_span_num=1
+    ):
+        super(DocVQA, self).__init__()
+        self.tokenizer = tokenizer
+        self.label2id_map = label2id_map
+        self.max_seq_len = max_seq_len
+        self.max_query_length = max_query_length
+        self.max_doc_length = max_doc_length
+        self.max_span_num = max_span_num
+        self.sample_list = None
+        self.args = args
+
+        self.docvqa_inputs = self.docvqa_input()
+
+    def check_is_max_context(self, doc_spans, cur_span_index, position):
+        """Check if this is the 'max context' doc span for the token."""
+
+        # Because of the sliding window approach taken to scoring documents, a single
+        # token can appear in multiple documents. E.g.
+        #  Doc: the man went to the store and bought a gallon of milk
+        #  Span A: the man went to the
+        #  Span B: to the store and bought
+        #  Span C: and bought a gallon of
+        #  ...
+        #
+        # Now the word 'bought' will have two scores from spans B and C. We only
+        # want to consider the score with "maximum context", which we define as
+        # the *minimum* of its left and right context (the *sum* of left and
+        # right context will always be the same, of course).
+        #
+        # In the example the maximum context for 'bought' would be span C since
+        # it has 1 left context and 3 right context, while span B has 4 left context
+        # and 0 right context.
+        best_score = None
+        best_span_index = None
+        for (span_index, doc_span) in enumerate(doc_spans):
+            end = doc_span.start + doc_span.length - 1
+            if position < doc_span.start:
+                continue
+            if position > end:
+                continue
+            num_left_context = position - doc_span.start
+            num_right_context = end - position
+            score = min(num_left_context, num_right_context) + 0.01 * doc_span.length
+            if best_score is None or score > best_score:
+                best_score = score
+            best_span_index = span_index
+
+        return cur_span_index == best_span_index
+
+    def convert_examples_to_features(
+        self, examples, tokenizer, label_map, max_seq_length, max_span_num, max_doc_length, max_query_length
+    ):
+
+        if "[CLS]" in self.tokenizer.get_vocab():
+            start_token = "[CLS]"
+            end_token = "[SEP]"
+        else:
+            start_token = "<s>"
+            end_token = "</s>"
+
+        features = []
+        for (example_index, example) in enumerate(examples):
+            query_tokens = tokenizer.tokenize(example.question)
+            if len(query_tokens) > max_query_length:
+                query_tokens = query_tokens[0:max_query_length]
+
+            all_doc_tokens = example.doc_tokens
+            all_doc_boxes_tokens = example.doc_boxes
+
+            cls_token_box = [0, 0, 0, 0]
+            sep_token_box = [1000, 1000, 1000, 1000]
+            pad_token_box = [0, 0, 0, 0]
+            ques_token_box = [0, 0, 0, 0]
+
+            # The -3 accounts for [CLS], [SEP] and [SEP]
+            max_tokens_for_doc = max_seq_length - len(query_tokens) - 3
+
+            # We can have documents that are longer than the maximum sequence length.
+            # To deal with this we do a sliding window approach, where we take chunks
+            # of the up to our max length with a stride of `doc_stride`.
+            _DocSpan = collections.namedtuple("DocSpan", ["start", "length"])
+            doc_spans = []
+            start_offset = 0
+            while start_offset < len(all_doc_tokens):
+                length = len(all_doc_tokens) - start_offset
+                if length > max_tokens_for_doc:
+                    length = max_tokens_for_doc
+                doc_spans.append(_DocSpan(start=start_offset, length=length))
+                if start_offset + length == len(all_doc_tokens):
+                    break
+                start_offset += length
+
+            spans_input_ids = []
+            spans_input_mask = []
+            spans_segment_ids = []
+            spans_boxes_tokens = []
+            for (doc_span_index, doc_span) in enumerate(doc_spans):
+                if doc_span_index == max_span_num:
+                    break
+                tokens = []
+                boxes_tokens = []
+                token_is_max_context = {}
+                segment_ids = []
+                tokens.append(start_token)
+                boxes_tokens.append(cls_token_box)
+                segment_ids.append(0)
+                for token in query_tokens:
+                    tokens.append(token)
+                    boxes_tokens.append(ques_token_box)
+                    segment_ids.append(0)
+                tokens.append(end_token)
+                boxes_tokens.append(sep_token_box)
+                segment_ids.append(0)
+                for i in range(doc_span.length):
+                    split_token_index = doc_span.start + i
+                    is_max_context = self.check_is_max_context(doc_spans, doc_span_index, split_token_index)
+                    token_is_max_context[len(tokens)] = is_max_context
+                    tokens.append(all_doc_tokens[split_token_index])
+                    boxes_tokens.append(all_doc_boxes_tokens[split_token_index])
+                    segment_ids.append(0)
+
+                tokens.append(end_token)
+                boxes_tokens.append(sep_token_box)
+                segment_ids.append(0)
+                input_ids = tokenizer.convert_tokens_to_ids(tokens)
+                # The mask has 1 for real tokens and 0 for padding tokens. Only real
+                # tokens are attended to.
+                input_mask = [1] * len(input_ids)
+                # Zero-pad up to the sequence length.
+                while len(input_ids) < max_seq_length:
+                    input_ids.append(0)
+                    input_mask.append(0)
+                    segment_ids.append(0)
+                    boxes_tokens.append(pad_token_box)
+                assert len(input_ids) == max_seq_length
+                assert len(input_mask) == max_seq_length
+                assert len(segment_ids) == max_seq_length
+                assert len(boxes_tokens) == max_seq_length
+
+                spans_input_ids.append(input_ids)
+                spans_input_mask.append(input_mask)
+                spans_segment_ids.append(segment_ids)
+                spans_boxes_tokens.append(boxes_tokens)
+
+            # Padding
+            # padding spans
+            # max_span_num: max_seg_num
+            # spans_input_ids: the tokens in each segment
+            if len(spans_input_ids) > max_span_num:
+                spans_input_ids = spans_input_ids[0:max_span_num]
+                spans_input_mask = spans_input_mask[0:max_span_num]
+                spans_segment_ids = spans_segment_ids[0:max_span_num]
+                spans_boxes_tokens = spans_boxes_tokens[0:max_span_num]
+            while len(spans_input_ids) < max_span_num:
+                tokens = []
+                boxes_tokens = []
+                segment_ids = []
+                tokens.append(start_token)
+                boxes_tokens.append(cls_token_box)
+                segment_ids.append(0)
+                for token in query_tokens:
+                    tokens.append(token)
+                    boxes_tokens.append(ques_token_box)
+                    segment_ids.append(0)
+                tokens.append(end_token)
+                boxes_tokens.append(sep_token_box)
+                segment_ids.append(0)
+                tokens.append(end_token)
+                boxes_tokens.append(sep_token_box)
+                segment_ids.append(0)
+                input_ids = tokenizer.convert_tokens_to_ids(tokens)
+                input_mask = [1] * len(input_ids)
+                while len(input_ids) < max_seq_length:
+                    input_ids.append(0)
+                    input_mask.append(0)
+                    segment_ids.append(0)
+                    boxes_tokens.append(pad_token_box)
+                spans_input_ids.append(input_ids)
+                spans_input_mask.append(input_mask)
+                spans_segment_ids.append(segment_ids)
+                spans_boxes_tokens.append(boxes_tokens)
+
+            # padding labels
+            labels = example.labels
+            sep_id = tokenizer.convert_tokens_to_ids(end_token)
+            labels = ["O"] * (spans_input_ids[0].index(sep_id) + 1) + labels
+            if len(labels) > 512:
+                labels = labels[:512]
+
+            if len(labels) < 512:
+                labels += ["O"] * (512 - len(labels))
+            assert len(spans_input_ids[0]) == len(labels)
+
+            label_ids = []
+            for lid, l in enumerate(labels):
+                if l not in label_map:
+                    label_ids.append(0)
+                else:
+                    label_ids.append(label_map[l])
+
+            feature = DocVQAFeatures(
+                example_index=example_index,
+                input_ids=spans_input_ids,
+                input_mask=spans_input_mask,
+                segment_ids=spans_segment_ids,
+                boxes=spans_boxes_tokens,
+                label=label_ids,
+            )
+            features.append(feature)
+        return features
+
+    def create_examples(self, data, is_test=False):
+        """Creates examples for the training and dev sets."""
+        examples = []
+        for sample in tqdm(data, total=len(data)):
+            question = sample["question"]
+            doc_tokens = sample["document"]
+            doc_boxes = sample["document_bbox"]
+            labels = sample["labels"] if not is_test else []
+
+            x_min, y_min = min(doc_boxes, key=lambda x: x[0])[0], min(doc_boxes, key=lambda x: x[2])[2]
+            x_max, y_max = max(doc_boxes, key=lambda x: x[1])[1], max(doc_boxes, key=lambda x: x[3])[3]
+            width = x_max - x_min
+            height = y_max - y_min
+
+            if max(width, height) < 1000:
+                scale_x = 1
+                scale_y = 1
+            else:
+                scale_x = 1000 / max(width, height)
+                scale_y = 1000 / max(width, height)
+
+            scaled_doc_boxes = [
+                [
+                    round((b[0] - x_min) * scale_x),
+                    round((b[2] - y_min) * scale_y),
+                    round((b[1] - x_min) * scale_x),
+                    round((b[3] - y_min) * scale_y),
+                ]
+                for b in doc_boxes
+            ]
+
+            for box, oribox in zip(scaled_doc_boxes, doc_boxes):
+                if box[0] < 0:
+                    print(box, oribox)
+                if box[2] - box[0] < 0:
+                    print(box, oribox)
+                if box[3] - box[1] < 0:
+                    print(box, oribox)
+                for pos in box:
+                    if pos > 1000:
+                        print(width, height, box, oribox)
+
+            example = DocVQAExample(
+                question=question, doc_tokens=doc_tokens, doc_boxes=scaled_doc_boxes, labels=labels
+            )
+            examples.append(example)
+        return examples
+
+    def docvqa_input(self):
+        data = []
+        if self.args.do_train:
+            dataset = self.args.train_file
+        elif self.args.do_test:
+            dataset = self.args.test_file
+        with open(dataset, "r", encoding="utf8") as f:
+            for index, line in enumerate(f):
+                data.append(json.loads(line.strip()))
+
+            # read the examples from train/test xlm files
+            examples = self.create_examples(data, is_test=self.args.do_test)
+
+        features = self.convert_examples_to_features(
+            examples,
+            self.tokenizer,
+            self.label2id_map,
+            max_seq_length=self.max_seq_len,
+            max_doc_length=self.max_doc_length,
+            max_span_num=self.max_span_num,
+            max_query_length=self.max_query_length,
+        )
+
+        all_input_ids = paddle.to_tensor([f.input_ids for f in features], dtype="int64")
+        all_input_mask = paddle.to_tensor([f.input_mask for f in features], dtype="int64")
+        all_segment_ids = paddle.to_tensor([f.segment_ids for f in features], dtype="int64")
+        all_bboxes = paddle.to_tensor([f.boxes for f in features], dtype="int64")
+        all_labels = paddle.to_tensor([f.label for f in features], dtype="int64")
+        self.sample_list = [
+            np.array(all_input_ids),
+            np.array(all_input_mask),
+            np.array(all_segment_ids),
+            np.array(all_bboxes),
+            np.array(all_labels),
+        ]
+
+    def __getitem__(self, idx):
+        return (
+            self.sample_list[0][idx],
+            self.sample_list[1][idx],
+            self.sample_list[2][idx],
+            self.sample_list[3][idx],
+            self.sample_list[4][idx],
+        )
+
+    def __len__(
+        self,
+    ):
+        return self.sample_list[0].shape[0]

applications/document_intelligence/doc_vqa/Extraction/model.py

@@ -0,0 +1,206 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle
+import paddle.fluid as fluid
+import paddle.nn as nn
+
+from paddlenlp.transformers import LayoutXLMPretrainedModel
+
+
+class Crf_decoding(paddle.fluid.dygraph.Layer):
+    def __init__(self, param_attr, size=None, is_test=True, dtype="float32"):
+        super(Crf_decoding, self).__init__()
+
+        self._dtype = dtype
+        self._size = size
+        self._is_test = is_test
+        self._param_attr = param_attr
+        self._transition = self.create_parameter(
+            attr=self._param_attr, shape=[self._size + 2, self._size], dtype=self._dtype
+        )
+
+    @property
+    def weight(self):
+        return self._transition
+
+    @weight.setter
+    def weight(self, value):
+        self._transition = value
+
+    def forward(self, input, label=None, length=None):
+
+        viterbi_path = self._helper.create_variable_for_type_inference(dtype=self._dtype)
+        this_inputs = {"Emission": [input], "Transition": self._transition, "Label": label}
+        if length is not None:
+            this_inputs["Length"] = [length]
+        self._helper.append_op(
+            type="crf_decoding",
+            inputs=this_inputs,
+            outputs={"ViterbiPath": [viterbi_path]},
+            attrs={
+                "is_test": self._is_test,
+            },
+        )
+        return viterbi_path
+
+
+class Chunk_eval(paddle.fluid.dygraph.Layer):
+    def __init__(self, num_chunk_types, chunk_scheme, excluded_chunk_types=None):
+        super(Chunk_eval, self).__init__()
+        self.num_chunk_types = num_chunk_types
+        self.chunk_scheme = chunk_scheme
+        self.excluded_chunk_types = excluded_chunk_types
+
+    def forward(self, input, label, seq_length=None):
+
+        precision = self._helper.create_variable_for_type_inference(dtype="float32")
+        recall = self._helper.create_variable_for_type_inference(dtype="float32")
+        f1_score = self._helper.create_variable_for_type_inference(dtype="float32")
+        num_infer_chunks = self._helper.create_variable_for_type_inference(dtype="int64")
+        num_label_chunks = self._helper.create_variable_for_type_inference(dtype="int64")
+        num_correct_chunks = self._helper.create_variable_for_type_inference(dtype="int64")
+
+        this_input = {"Inference": [input], "Label": [label]}
+        if seq_length is not None:
+            this_input["SeqLength"] = [seq_length]
+
+        self._helper.append_op(
+            type="chunk_eval",
+            inputs=this_input,
+            outputs={
+                "Precision": [precision],
+                "Recall": [recall],
+                "F1-Score": [f1_score],
+                "NumInferChunks": [num_infer_chunks],
+                "NumLabelChunks": [num_label_chunks],
+                "NumCorrectChunks": [num_correct_chunks],
+            },
+            attrs={
+                "num_chunk_types": self.num_chunk_types,
+                "chunk_scheme": self.chunk_scheme,
+                "excluded_chunk_types": self.excluded_chunk_types or [],
+            },
+        )
+        return (precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks)
+
+
+class Linear_chain_crf(paddle.fluid.dygraph.Layer):
+    def __init__(self, param_attr, size=None, is_test=False, dtype="float32"):
+        super(Linear_chain_crf, self).__init__()
+
+        self._param_attr = param_attr
+        self._dtype = dtype
+        self._size = size
+        self._is_test = is_test
+        self._transition = self.create_parameter(
+            attr=self._param_attr, shape=[self._size + 2, self._size], dtype=self._dtype
+        )
+
+    @property
+    def weight(self):
+        return self._transition
+
+    @weight.setter
+    def weight(self, value):
+        self._transition = value
+
+    def forward(self, input, label, length=None):
+
+        alpha = self._helper.create_variable_for_type_inference(dtype=self._dtype)
+        emission_exps = self._helper.create_variable_for_type_inference(dtype=self._dtype)
+        transition_exps = self._helper.create_variable_for_type_inference(dtype=self._dtype)
+        log_likelihood = self._helper.create_variable_for_type_inference(dtype=self._dtype)
+        this_inputs = {"Emission": [input], "Transition": self._transition, "Label": [label]}
+        if length is not None:
+            this_inputs["Length"] = [length]
+        self._helper.append_op(
+            type="linear_chain_crf",
+            inputs=this_inputs,
+            outputs={
+                "Alpha": [alpha],
+                "EmissionExps": [emission_exps],
+                "TransitionExps": transition_exps,
+                "LogLikelihood": log_likelihood,
+            },
+            attrs={
+                "is_test": self._is_test,
+            },
+        )
+        return log_likelihood
+
+
+class LayoutXLMForTokenClassification_with_CRF(LayoutXLMPretrainedModel):
+    def __init__(self, layoutxlm, num_classes, dropout=None):
+        super(LayoutXLMForTokenClassification_with_CRF, self).__init__()
+        self.num_classes = num_classes
+        self.layoutxlm = layoutxlm
+        self.dropout = nn.Dropout(dropout if dropout is not None else self.layoutxlm.config["hidden_dropout_prob"])
+        self.emission_classifier = nn.Linear(self.layoutxlm.config["hidden_size"], self.num_classes)
+        self.emission_classifier.apply(self.init_weights)
+        self.linear_chain_crf = Linear_chain_crf(
+            size=self.num_classes, param_attr=paddle.fluid.ParamAttr(name="liner_chain_crfw")
+        )
+        self.crf_decoding = Crf_decoding(param_attr=paddle.fluid.ParamAttr(name="crfw_decode"), size=self.num_classes)
+        self.crf_decoding.weight = self.linear_chain_crf.weight
+        self.crfw = fluid.layers.create_parameter(
+            shape=[self.num_classes + 2, self.num_classes], dtype="float32", name="crfw"
+        )
+        self.mask_crfw = fluid.layers.create_parameter(
+            shape=[self.num_classes + 2, self.num_classes], dtype="float32", name="mask_matrix"
+        )
+
+    def get_input_embeddings(self):
+        return self.layoutxlm.embeddings.word_embeddings
+
+    def forward(
+        self,
+        input_ids=None,
+        bbox=None,
+        attention_mask=None,
+        token_type_ids=None,
+        labels=None,
+        image=None,
+        position_ids=None,
+        head_mask=None,
+        is_train=False,
+    ):
+
+        input_ids = input_ids.squeeze(axis=1)
+        bbox = bbox.squeeze(axis=1)
+        attention_mask = attention_mask.squeeze(axis=1)
+        token_type_ids = token_type_ids.squeeze(axis=1)
+        outputs = self.layoutxlm(
+            input_ids=input_ids,
+            bbox=bbox,
+            image=image,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+        )
+        seq_length = input_ids.shape[1]
+        # sequence out and image out
+        sequence_logits, _ = outputs[0][:, :seq_length], outputs[0][:, seq_length:]
+        emission = self.emission_classifier(sequence_logits)
+        length = paddle.sum(attention_mask, axis=1)
+        labels = labels.reshape([-1, seq_length, 1])
+
+        # standard crf loss
+        crf_cost = self.linear_chain_crf(input=emission, label=labels, length=length)
+        crf_decode = self.crf_decoding(input=emission, length=length)
+        if is_train:
+            return [crf_cost]
+        else:
+            return [crf_cost, crf_decode]

applications/document_intelligence/doc_vqa/Extraction/run_docvqa.py

@@ -0,0 +1,457 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import logging
+import os
+import random
+import warnings
+from collections import Counter
+
+import numpy as np
+import paddle
+from docvqa import DocVQA
+from model import LayoutXLMForTokenClassification_with_CRF
+
+from paddlenlp.transformers import LayoutXLMModel, LayoutXLMTokenizer
+
+warnings.filterwarnings("ignore")
+logger = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    # yapf: disable
+    parser.add_argument("--model_name_or_path", default=None, type=str, required=True)
+    parser.add_argument("--do_train", default=False, type=bool, required=False)
+    parser.add_argument("--do_test", default=False, type=bool, required=False)
+    parser.add_argument("--test_file", default=None, type=str, required=False)
+    parser.add_argument("--train_file", default=None, type=str, required=False)
+    parser.add_argument("--output_dir", default=None, type=str, required=True)
+    parser.add_argument("--max_seq_len", default=512, type=int)
+    parser.add_argument("--max_query_length", default=20, type=int)
+    parser.add_argument("--max_doc_length", default=512, type=int)
+    parser.add_argument("--max_span_num", default=1, type=int)
+    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.")
+    parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for eval.")
+    parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
+    parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
+    parser.add_argument("--num_train_epochs", default=3, type=int, help="Total number of training epochs to perform.")
+    parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")
+    parser.add_argument("--eval_steps", type=int, default=10, help="eval every X updates steps.")
+    parser.add_argument("--save_steps", type=int, default=50, help="Save checkpoint every X updates steps.")
+    parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
+    parser.add_argument("--init_checkpoint", type=str, default=None, help="the initialized checkpoint")
+    parser.add_argument("--save_path", type=str, default=None, help="the initialized checkpoint")
+    # yapf: enable
+    args = parser.parse_args()
+    return args
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    paddle.seed(args.seed)
+
+
+def get_label_maps():
+    labels = ["O", "I-ans", "B-ans", "E-ans"]
+    label2id_map = {label: idx for idx, label in enumerate(labels)}
+    id2label_map = {idx: label for idx, label in enumerate(labels)}
+    return label2id_map, id2label_map
+
+
+def main(args):
+    os.makedirs(args.output_dir, exist_ok=True)
+    logging.basicConfig(
+        filename=os.path.join(args.output_dir, "train.log") if paddle.distributed.get_rank() == 0 else None,
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO if paddle.distributed.get_rank() == 0 else logging.WARN,
+    )
+
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    logger.addHandler(ch)
+
+    label2id_map, id2label_map = get_label_maps()
+    pad_token_label_id = paddle.nn.CrossEntropyLoss().ignore_index
+
+    # dist mode
+    if paddle.distributed.get_world_size() > 1:
+        paddle.distributed.init_parallel_env()
+
+    tokenizer = LayoutXLMTokenizer.from_pretrained(args.model_name_or_path)
+
+    if args.do_test:
+        model = LayoutXLMForTokenClassification_with_CRF.from_pretrained(args.init_checkpoint)
+        evaluate(args, model, tokenizer, label2id_map, id2label_map, pad_token_label_id, global_step=0)
+        exit(0)
+
+    if args.init_checkpoint:
+        logger.info("Init checkpoint from {}".format(args.init_checkpoint))
+        model = LayoutXLMForTokenClassification_with_CRF.from_pretrained(args.init_checkpoint)
+    else:
+        base_model = LayoutXLMModel.from_pretrained(args.model_name_or_path)
+        model = LayoutXLMForTokenClassification_with_CRF(base_model, num_classes=len(label2id_map), dropout=None)
+
+    # dist mode
+    if paddle.distributed.get_world_size() > 1:
+        model = paddle.DataParallel(model)
+
+    train_dataset = DocVQA(
+        args,
+        tokenizer,
+        label2id_map,
+        max_seq_len=args.max_seq_len,
+        max_query_length=args.max_query_length,
+        max_doc_length=args.max_doc_length,
+        max_span_num=args.max_span_num,
+    )
+
+    train_sampler = paddle.io.DistributedBatchSampler(
+        train_dataset, batch_size=args.per_gpu_train_batch_size, shuffle=False
+    )
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, paddle.distributed.get_world_size())
+
+    train_dataloader = paddle.io.DataLoader(
+        train_dataset, batch_sampler=train_sampler, num_workers=0, use_shared_memory=True, collate_fn=None
+    )
+
+    t_total = len(train_dataloader) * args.num_train_epochs
+    # build linear decay with warmup lr sch
+    lr_scheduler = paddle.optimizer.lr.PolynomialDecay(
+        learning_rate=args.learning_rate, decay_steps=t_total, end_lr=0.0, power=1.0
+    )
+    if args.warmup_steps > 0:
+        lr_scheduler = paddle.optimizer.lr.LinearWarmup(
+            lr_scheduler, args.warmup_steps, start_lr=0, end_lr=args.learning_rate
+        )
+
+    optimizer = paddle.optimizer.AdamW(
+        learning_rate=lr_scheduler,
+        parameters=model.parameters(),
+        epsilon=args.adam_epsilon,
+        weight_decay=args.weight_decay,
+    )
+
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info(
+        "  Total train batch size (w. parallel, distributed) = %d",
+        args.train_batch_size * paddle.distributed.get_world_size(),
+    )
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss = 0.0
+    set_seed(args)
+    for epoch_id in range(args.num_train_epochs):
+        print("epoch id:{}".format(epoch_id))
+        for step, batch in enumerate(train_dataloader):
+            model.train()
+            input_ids, input_mask, segment_ids, bboxes, labels = batch
+            if input_ids.shape[0] != args.per_gpu_train_batch_size:
+                continue
+            outputs = model(
+                input_ids=input_ids,
+                bbox=bboxes,
+                attention_mask=input_mask,
+                token_type_ids=segment_ids,
+                labels=labels,
+                is_train=True,
+            )
+            # model outputs are always tuple in paddlenlp (see doc)
+            loss = outputs[0]
+            loss = loss.mean()
+            if global_step % 50 == 0:
+                logger.info(
+                    "[epoch {}/{}][iter: {}/{}] lr: {:.5f}, train loss: {:.5f}, ".format(
+                        epoch_id,
+                        args.num_train_epochs,
+                        step,
+                        len(train_dataloader),
+                        lr_scheduler.get_lr(),
+                        float(loss),
+                    )
+                )
+
+            loss.backward()
+            tr_loss += loss.item()
+            optimizer.step()
+            lr_scheduler.step()  # Update learning rate schedule
+            optimizer.clear_grad()
+            global_step += 1
+
+            if paddle.distributed.get_rank() == 0 and args.save_steps > 0 and global_step % args.save_steps == 0:
+                # Save model checkpoint
+                output_dir = os.path.join(args.output_dir, "checkpoint-{}".format(global_step))
+                os.makedirs(output_dir, exist_ok=True)
+                if paddle.distributed.get_rank() == 0:
+                    model.save_pretrained(output_dir)
+                    tokenizer.save_pretrained(output_dir)
+                    paddle.save(args, os.path.join(output_dir, "training_args.bin"))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+
+def _tokenize_chinese_chars(text):
+    """
+    :param text: input text, unicode string
+    :return:
+        tokenized text, list
+    """
+
+    def _is_chinese_char(cp):
+        """Checks whether CP is the codepoint of a CJK character."""
+        # This defines a "chinese character" as anything in the CJK Unicode block:
+        #     https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+        #
+        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+        # despite its name. The modern Korean Hangul alphabet is a different block,
+        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+        # space-separated words, so they are not treated specially and handled
+        # like the all of the other languages.
+        if (
+            (cp >= 0x4E00 and cp <= 0x9FFF)
+            or (cp >= 0x3400 and cp <= 0x4DBF)  #
+            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
+            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
+            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
+            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
+            or (cp >= 0xF900 and cp <= 0xFAFF)
+            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
+        ):  #
+            return True
+
+        return False
+
+    output = []
+    buff = ""
+    for char in text:
+        cp = ord(char)
+        if _is_chinese_char(cp) or char == "=":
+            if buff != "":
+                output.append(buff)
+                buff = ""
+            output.append(char)
+        else:
+            buff += char
+
+    if buff != "":
+        output.append(buff)
+
+    return output
+
+
+def fast_f1(text1, text2):
+    common_char = Counter(text1) & Counter(text2)
+    len_seq1 = len(text1)
+    len_seq2 = len(text2)
+    len_common = sum(common_char.values())
+    if len_common == 0:
+        return 0.0
+    precision = 1.0 * len_common / len_seq2
+    recall = 1.0 * len_common / len_seq1
+    return (2.0 * precision * recall) / (precision + recall)
+
+
+def _normalize(in_str):
+    """
+    normalize the input unicode string
+    """
+    in_str = in_str.lower()
+    sp_char = [
+        ":",
+        "_",
+        "`",
+        "，",
+        "。",
+        "：",
+        "？",
+        "！",
+        "(",
+        ")",
+        "“",
+        "”",
+        "；",
+        "’",
+        "《",
+        "》",
+        "……",
+        "·",
+        "、",
+        ",",
+        "「",
+        "」",
+        "（",
+        "）",
+        "－",
+        "～",
+        "『",
+        "』",
+        "|",
+    ]
+    out_segs = []
+    for char in in_str:
+        if char in sp_char:
+            continue
+        else:
+            out_segs.append(char)
+    return "".join(out_segs)
+
+
+def calc_f1_score(answer, prediction):
+    ans_segs = _tokenize_chinese_chars(_normalize(answer))
+    prediction_segs = _tokenize_chinese_chars(_normalize(prediction))
+    f1 = fast_f1(prediction_segs, ans_segs)
+    return f1
+
+
+def decode(tokenizer, res):
+    sep_id = tokenizer._convert_token_to_id("</s>")
+    text_res = []
+    all_f1 = []
+    save_f1 = []
+    for i in range(len(res)):
+        input_ids, label_ids, predict_ids, bbox = res[i]
+        remove_pos = (
+            len(" ".join([str(x) for x in input_ids]).split("2 6 ")[0].strip(" ").split(" ")) + 2
+        )  # remove the question bbox and sep bbox
+        start_pos = input_ids.index(sep_id)
+        query_text = []
+        for idx in range(1, start_pos):
+            input_id = input_ids[idx]
+            query_text.append(tokenizer._convert_id_to_token(int(input_id)))
+
+        # label texts and predict texts
+        text_label, text_predict = [], []
+        label_bbox_index, predict_bbox_index = [], []
+        for idx in range(start_pos + 1, len(input_ids)):
+            input_id, label_id, predict_id = input_ids[idx], label_ids[idx], predict_ids[idx]
+
+            if label_id in [1, 2, 3]:
+                text_label.append(tokenizer._convert_id_to_token(int(input_id)))
+                label_bbox_index.append(idx - remove_pos + 1)
+            if predict_id in [1, 2, 3]:
+                text_predict.append(tokenizer._convert_id_to_token(int(input_id)))
+                predict_bbox_index.append(idx - remove_pos + 1)
+        text_res.append(
+            ["".join(query_text), "".join(text_label), "".join(text_predict), label_bbox_index, predict_bbox_index]
+        )
+
+        f1 = calc_f1_score("".join(text_label), "".join(text_predict))
+        save_f1.append(f1)
+
+        if len("".join(text_label)) > 10:
+            all_f1.append(f1)
+    if len(all_f1) > 0:
+        print("F1: ", sum(all_f1) / len(all_f1))
+
+    assert len(text_res) == len(save_f1)
+    return text_res
+
+
+def evaluate(args, model, tokenizer, label2id_map, id2label_map, pad_token_label_id, prefix="", global_step=0):
+
+    eval_dataset = DocVQA(
+        args, tokenizer, label2id_map, max_seq_len=512, max_query_length=20, max_doc_length=512, max_span_num=1
+    )
+
+    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, paddle.distributed.get_world_size())
+
+    eval_dataloader = paddle.io.DataLoader(
+        eval_dataset, batch_size=args.eval_batch_size, num_workers=0, use_shared_memory=True, collate_fn=None
+    )
+
+    # Eval!
+    logger.info("***** Running evaluation %s *****", prefix)
+    logger.info("  Num examples = %d", len(eval_dataset))
+    logger.info("  Batch size = %d", args.eval_batch_size)
+    model.eval()
+    res = []
+    for idx, batch in enumerate(eval_dataloader):
+        with paddle.no_grad():
+            input_ids, input_mask, segment_ids, bboxes, labels = batch
+
+            if input_ids.shape[0] != args.eval_batch_size:
+                continue
+            outputs = model(
+                input_ids=input_ids,
+                bbox=bboxes,
+                attention_mask=input_mask,
+                token_type_ids=segment_ids,
+                labels=labels,
+                is_train=False,
+            )
+            labels = labels.numpy()
+            crf_decode = outputs[1].numpy()
+            bboxes = bboxes.squeeze().numpy()
+            input_ids = input_ids.squeeze(axis=1).numpy()
+
+            for index in range(input_ids.shape[0]):
+                res.append([list(input_ids[index]), list(labels[index]), list(crf_decode[index]), bboxes[index]])
+
+    origin_inputs = []
+    with open(args.test_file, "r", encoding="utf8") as f:
+        for line in f:
+            line = json.loads(line.strip())
+            origin_inputs.append(
+                {
+                    "img_name": line["img_name"],
+                    "question": line["question"],
+                    "bboxes": line["document_bbox"],
+                    "img_id": line["img_id"],
+                }
+            )
+
+    text_res = decode(tokenizer, res)
+
+    with open(args.save_path, "w", encoding="utf8") as f:
+        for line_res, line_text, line_label in zip(res, text_res, origin_inputs):
+            line_json = {}
+            line_json["img_name"] = line_label["img_name"]
+            line_json["img_id"] = line_label["img_id"]
+            line_json["question"] = line_label["question"]
+            line_json["label_answer"] = line_text[1]
+            line_json["predict_answer"] = line_text[2]
+            label_bbox_index, predict_bbox_index = line_text[3], line_text[4]
+            label_bboxes, predict_bboxes = [], []
+            for i in range(len(line_label["bboxes"])):
+                if i in label_bbox_index:
+                    label_bboxes.append(line_label["bboxes"][i])
+                if i in predict_bbox_index:
+                    predict_bboxes.append(line_label["bboxes"][i])
+            line_json["label_bboxes"] = label_bboxes
+            line_json["predict_bboxes"] = predict_bboxes
+            json.dump(line_json, f, ensure_ascii=False)
+            f.write("\n")
+
+
+def print_arguments(args):
+    """print arguments"""
+    print("-----------  Configuration Arguments -----------")
+    for arg, value in sorted(vars(args).items()):
+        print("%s: %s" % (arg, value))
+    print("------------------------------------------------")
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    print_arguments(args)
+    main(args)

applications/document_intelligence/doc_vqa/Extraction/run_test.sh

@@ -0,0 +1,38 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export CUDA_VISIBLE_DEVICES=0
+
+QUESTION=$1
+
+python3 change_to_mrc.py ${QUESTION}
+
+python3  ./run_docvqa.py \
+    --model_name_or_path "layoutxlm-base-uncased" \
+    --max_seq_len 512 \
+    --do_test true \
+	--test_file "data/demo_test.json" \
+	--num_train_epochs 100 \
+    --eval_steps 6000 \
+    --save_steps 6000 \
+    --output_dir "output/" \
+    --save_path "data/decode_res.json" \
+	--init_checkpoint "./checkpoints/layoutxlm/" \
+    --learning_rate 3e-5 \
+    --warmup_steps 12000 \
+    --per_gpu_train_batch_size 4 \
+    --per_gpu_eval_batch_size 1 \
+    --seed 2048
+
+python3 view.py

applications/document_intelligence/doc_vqa/Extraction/run_train.sh

@@ -0,0 +1,32 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export CUDA_VISIBLE_DEVICES=0
+
+python3 ./run_docvqa.py \
+    --model_name_or_path "layoutxlm-base-uncased" \
+    --max_seq_len 512 \
+    --train_file "data/train.json" \
+    --init_checkpoint "checkpoints/base_model" \
+	--do_train true \
+    --num_train_epochs 50 \
+    --eval_steps 24000 \
+    --save_steps 40 \
+    --output_dir "output" \
+    --save_path "data/decode_res.json" \
+    --learning_rate 3e-5 \
+    --warmup_steps 40 \
+    --per_gpu_train_batch_size 4 \
+    --per_gpu_eval_batch_size 4 \
+    --seed 2048

applications/document_intelligence/doc_vqa/Extraction/view.py

@@ -0,0 +1,67 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import cv2
+import json
+import numpy as np
+
+
+def view_ocr_result(img_path, bboxes, opath):
+    image = cv2.imread(img_path)
+    for char_bbox in bboxes:
+        x_min, x_max, y_min, y_max = char_bbox
+        cv2.rectangle(image, (x_min, y_min), (x_max, y_max), (0, 0, 255), 1)
+    cv2.imwrite(opath, image)
+
+
+def _highlight_bbox(img, bbox):
+    x = bbox[0]
+    w = bbox[1] - x
+    y = bbox[2]
+    h = bbox[3] - y
+    sub_img = img[y : y + h, x : x + w]
+    colored_rect = np.zeros(sub_img.shape, dtype=np.uint8)
+    colored_rect[:, :, 2] = 255
+    colored_rect[:, :, 1] = 255
+    res = cv2.addWeighted(sub_img, 0.5, colored_rect, 0.5, 1.0)
+    img[y : y + h, x : x + w] = res
+
+
+def highlight_ans(source_img_path, output_img_path, ans_bbox):
+    image = cv2.imread(source_img_path)
+    for bbox in ans_bbox:
+        _highlight_bbox(image, bbox)
+    cv2.imwrite(output_img_path, image)
+
+
+def highlight_img(source_img_path, output_img_path):
+    image = cv2.imread(source_img_path)
+    height = image.shape[0]
+    width = image.shape[1]
+    bbox = [0, width - 1, 0, height - 1]
+    _highlight_bbox(image, bbox)
+    cv2.imwrite(output_img_path, image)
+
+
+if __name__ == "__main__":
+    res_path = "./data/decode_res.json"
+    result = {}
+    with open(res_path, "r", encoding="utf-8") as f:
+        line = f.readline()
+        result = json.loads(line.strip())
+
+    img_path = "../OCR_process/demo_pics/demo_{}.png".format(result["img_id"])
+    img_save_path = "../answer.png"
+    highlight_ans(img_path, img_save_path, result["predict_bboxes"])
+    print("extraction result has been saved to answer.png")

applications/document_intelligence/doc_vqa/OCR_process/ocr_process.py

@@ -0,0 +1,272 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+
+from paddleocr import PaddleOCR
+
+from paddlenlp.transformers import LayoutXLMTokenizer
+
+tokenizer = LayoutXLMTokenizer.from_pretrained("layoutxlm-base-uncased")
+
+
+def get_all_chars(tokenizer):
+    all_chr = []
+    for i in range(30000):
+        tok_chr = tokenizer.tokenize(chr(i))
+        tok_chr = [tc.replace("▁", "") for tc in tok_chr]
+        while "" in tok_chr:
+            tok_chr.remove("")
+        tok_chr = "".join(tok_chr)
+        if len(tok_chr) != 1:
+            all_chr.append(i)
+    return all_chr
+
+
+def merge_bbox(tok_bboxes):
+    min_gx = min([box[0] for box in tok_bboxes])
+    max_gx = max([box[1] for box in tok_bboxes])
+    min_gy = min([box[2] for box in tok_bboxes])
+    max_gy = max([box[3] for box in tok_bboxes])
+    height_g = max_gy - min_gy
+    width_g = max_gx - min_gx
+    height_m = 0
+    width_m = 0
+    for box in tok_bboxes:
+        x_min, x_max, y_min, y_max = box
+        height_m += y_max - y_min
+        width_m += x_max - x_min
+    height_m = height_m / len(tok_bboxes)
+    if (height_g - height_m) < 0.5 * height_m and width_g - width_m < 0.1 * width_m:
+        return False, [min_gx, max_gx, min_gy, max_gy]
+    else:
+        return True, tok_bboxes[0]
+
+
+def xlm_parse(ocr_res, tokenizer):
+    doc_bboxes = []
+    all_chr = get_all_chars(tokenizer)
+
+    try:
+        new_tokens, new_token_boxes = [], []
+        for item in ocr_res:
+            new_tokens.extend(item["tokens"])
+            new_token_boxes.extend(item["token_box"])
+
+        # get layoutxlm tokenizer results and get the final results
+        temp_span_text = "".join(new_tokens)
+        temp_span_bbox = new_token_boxes
+        span_text = ""
+        span_bbox = []
+        # drop blank space
+        for text, bbox in zip(temp_span_text, temp_span_bbox):
+            if text == " ":
+                continue
+            else:
+                span_text += text
+                span_bbox += [bbox]
+
+        # span_tokens starts with "_"
+        span_tokens = tokenizer.tokenize(span_text)
+        span_tokens[0] = span_tokens[0].replace("▁", "")
+        while "" in span_tokens:
+            span_tokens.remove("")
+
+        doc_bboxes = []
+        i = 0
+        for tid, tok in enumerate(span_tokens):
+            tok = tok.replace("▁", "")
+            if tok == "":
+                doc_bboxes.append(span_bbox[i])
+                continue
+            if tok == "<unk>":
+                if tid + 1 == len(span_tokens):
+                    tok_len = 1
+                else:
+                    if span_tokens[tid + 1] == "<unk>":
+                        tok_len = 1
+                    else:
+                        for j in range(i, len(span_text)):
+                            if span_text[j].lower() == span_tokens[tid + 1][0]:
+                                break
+                        tok_len = j - i
+            elif ord(span_text[i]) in all_chr:
+                if tid + 1 == len(span_tokens):
+                    tok_len = 1
+                elif "°" in tok and "C" in span_tokens[tid + 1]:
+                    tok_len = len(tok) - 1
+                    if tok_len == 0:
+                        doc_bboxes.append(span_bbox[i])
+                        continue
+                elif span_text[i] == "ⅱ":
+                    if tok == "ii":
+                        if span_text[i + 1] != "i":
+                            tok_len = len(tok) - 1
+                        else:
+                            tok_len = len(tok)
+                    elif tok == "i":
+                        tok_len = len(tok) - 1
+                        if tok_len == 0:
+                            doc_bboxes.append(span_bbox[i])
+                            continue
+                elif "m" in tok and "2" == span_tokens[tid + 1][0]:
+                    tok_len = len(tok) - 1
+                    if tok_len == 0:
+                        doc_bboxes.append(span_bbox[i])
+                        continue
+                elif ord(span_text[i + 1]) in all_chr:
+                    tok_len = 1
+                else:
+                    for j in range(i, len(span_text)):
+                        if span_text[j].lower() == span_tokens[tid + 1][0]:
+                            break
+                        if span_text[j].lower() == "，" and span_tokens[tid + 1][0] == ",":
+                            break
+                        if span_text[j].lower() == "；" and span_tokens[tid + 1][0] == ";":
+                            break
+                        if span_text[j].lower() == "）" and span_tokens[tid + 1][0] == ")":
+                            break
+                        if span_text[j].lower() == "（" and span_tokens[tid + 1][0] == "(":
+                            break
+                        if span_text[j].lower() == "￥" and span_tokens[tid + 1][0] == "¥":
+                            break
+
+                    tok_len = j - i
+
+            else:
+                if "<EFBFBD>" == span_text[i]:
+                    tok_len = len(tok) + 1
+                elif tok == "......" and "…" in span_text[i : i + 6]:
+                    tok_len = len(tok) - 2
+                elif "ⅱ" in span_text[i + len(tok) - 1]:
+                    if tok == "i":
+                        tok_len = 1
+                    else:
+                        tok_len = len(tok) - 1
+                elif "°" in tok and "C" in span_tokens[tid + 1]:
+                    tok_len = len(tok) - 1
+                else:
+                    tok_len = len(tok)
+
+            assert i + tok_len <= len(span_bbox)
+            tok_bboxes = span_bbox[i : i + tok_len]
+            _, merged_bbox = merge_bbox(tok_bboxes)
+
+            doc_bboxes.append(merged_bbox)
+            i = i + tok_len
+    except Exception:
+        print("Error")
+        span_tokens = ["▁"] * 512
+        doc_bboxes = [[0, 0, 0, 0]] * 512
+
+    return span_tokens, doc_bboxes
+
+
+def tokenize_ocr_res(ocr_reses):
+    """
+    input:
+        ocr_res: the ocr result of the image
+    return:
+        new_reses: {
+            pid: {
+                "text": all text in each ocr_res,
+                "bounding_box": the bounding box of the ocr_res,
+                "tokens": all chars in ocr_res,
+                "token_box: bounding box of each chars in ocr_res
+                }
+        }
+    """
+    new_reses = []
+    for img_name, ocr_res in ocr_reses:
+        new_res = []
+        for para in ocr_res:
+            text = para["text"]
+            text_box = para["bbox"]
+            x_min, y_min = [int(min(idx)) for idx in zip(*text_box)]
+            x_max, y_max = [int(max(idx)) for idx in zip(*text_box)]
+            text_chars = list(text.lower())
+            char_num = 0
+            for char in text_chars:
+                if re.match("[^\x00-\xff]", char):
+                    char_num += 2
+                else:
+                    char_num += 1
+            width = x_max - x_min
+            shift = x_min
+            new_token_boxes, new_tokens = [], []
+            for char in text_chars:
+                if re.match("[^\x00-\xff]", char):
+                    tok_x_max = shift + width / char_num * 2
+                else:
+                    tok_x_max = shift + width / char_num * 1
+                tok_x_min = shift
+                tok_y_min = y_min
+                tok_y_max = y_max
+
+                shift = tok_x_max
+                new_token_boxes.append([round(tok_x_min), round(tok_x_max), tok_y_min, tok_y_max])
+                new_tokens.append(char)
+            new_res.append(
+                {
+                    "text": para["text"],
+                    "bounding_box": para["bbox"],
+                    "tokens": new_tokens,
+                    "token_box": new_token_boxes,
+                }
+            )
+        new_reses.append((img_name, new_res))
+    return new_reses
+
+
+def process_input(ocr_reses, tokenizer, save_ocr_path):
+    ocr_reses = tokenize_ocr_res(ocr_reses)
+
+    examples = []
+    for img_name, ocr_res in ocr_reses:
+        doc_tokens, doc_bboxes = xlm_parse(ocr_res, tokenizer)
+        doc_tokens.insert(0, "▁")
+        doc_bboxes.insert(0, doc_bboxes[0])
+        example = {"img_name": img_name, "document": doc_tokens, "document_bbox": doc_bboxes}
+        examples.append(example)
+
+    with open(save_ocr_path, "w", encoding="utf8") as f:
+        for example in examples:
+            json.dump(example, f, ensure_ascii=False)
+            f.write("\n")
+
+    print(f"ocr parsing results has been save to: {save_ocr_path}")
+
+
+def ocr_preprocess(img_dir):
+    ocr = PaddleOCR(use_angle_cls=True, lang="ch", use_gpu=True)
+    ocr_reses = []
+    img_names = sorted(os.listdir(img_dir), key=lambda x: int(x.split("_")[1].split(".")[0]))
+    for img_name in img_names:
+        img_path = os.path.join(img_dir, img_name)
+        parsing_res = ocr.ocr(img_path, cls=True)[0]
+        ocr_res = []
+        for para in parsing_res:
+            ocr_res.append({"text": para[1][0], "bbox": para[0]})
+        ocr_reses.append((img_name, ocr_res))
+
+    return ocr_reses
+
+
+if __name__ == "__main__":
+    img_dir = "./demo_pics"
+    save_path = "./demo_ocr_res.json"
+    ocr_results = ocr_preprocess(img_dir)
+    process_input(ocr_results, tokenizer, save_path)

applications/document_intelligence/doc_vqa/README.md

@@ -0,0 +1,131 @@
+# 汽车说明书跨模态智能问答
+
+## 1. 项目说明
+
+**跨模态文档问答** 是跨模态的文档抽取任务，要求文档智能模型在文档中抽取能够回答文档相关问题的答案，需要模型在抽取和理解文档中文本信息的同时，还能充分利用文档的布局、字体、颜色等视觉信息，这比单一模态的信息抽取任务更具挑战性。
+
+这种基于跨模态文档阅读理解技术的智能问答能力，可以深度解析非结构化文档中排版复杂的图文/图表内容，直接定位问题答案。
+
+本项目将基于跨模态文档问答技术实现**汽车说明书问答系统**，该系统能够对用户提出的问题，自动从汽车说明书中寻找答案并进行回答。
+
+如下图所示， 用户提出问题："如何更换前风窗玻璃的刮水片"，跨模态文档问答引擎将从库中寻找相关的文档，然后通过跨模态阅读理解模型抽取出相应的答案，并进行了高亮展示。
+
+<center><img width="883" alt="image" src="https://user-images.githubusercontent.com/35913314/169781111-0734729d-3c7b-400d-8e92-e56548bb7dc5.png"></center>
+
+通过使用汽车说明书问答系统，能够极大地解决传统汽车售后的压力：
+- 用户：用户没有耐心查阅说明书，打客服电话需要等待
+- 售后客服：需要配置大量客服人员，且客服专业知识培训周期长
+- 构建问题库：需要投入大量人力整理常见问题库，并且固定的问题库难以覆盖灵活多变的提问
+
+对于用户来说，汽车说明书问答系统能够支持通过车机助手/APP/小程序为用户提供即问即答的功能。对于常见问题，用户不再需要查阅说明书，也无需打客服电话，从而缓解了人工客服的压力。
+
+对于客服来讲，汽车说明书问答系统帮助客服人员快速定位答案，高效查阅文档，提高客服的专业水平，同时也能够缩短客服的培训周期。
+
+## 2. 安装说明
+
+#### 环境要求
+
+- paddlepaddle == 2.3.2
+- paddlenlp == 2.5.2
+- paddleocr == 2.6.1.3
+
+安装相关问题可参考[PaddlePaddle](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/pip/linux-pip.html)和[PaddleNLP](https://paddlenlp.readthedocs.io/zh/latest/get_started/installation.html)文档。
+
+
+## 3. 整体流程
+
+汽车说明书问答系统针对用户提出的汽车使用相关问题，智能化地在汽车说明书中找出对应答案，并返回给用户。本项目提供的汽车说明书问答系统的使用流程如下图所示。本项目提供的汽车说明书问答系统主要包括 3 个模块：OCR处理模块、排序模块和跨模态阅读理解模块。
+
+在使用汽车说明书问答模型回答问题之前，需要先使用PaddleOCR对离线提供的汽车说明书文档进解析，并将解析结果保存下来，以备后续排序模块使用。
+
+对于用户提问的问题，首先会被传入排序模块，排序模块会针对该问题对解析的文档进行排序打分，其结果将会被传入跨模态阅读理解模块。阅读理解模块将从分数最高的说明书文档中，抽取用户问题的答案，并返回给用户。
+
+<center><img width="864" alt="image" src="https://user-images.githubusercontent.com/35913314/170222662-c438ff2a-a1df-44e5-8a83-f14dc0814b9d.png"></center>
+
+下面将具体介绍各个模块的功能。
+
+## 4. OCR处理模块
+
+本项目提供了包含10张图片的汽车说明书，为方便后续处理，首先需要通过 PaddleOCR 对汽车说明书进行识别，记录汽车说明书上的文字和文字布局信息， 以方便后续使用计算机视觉和自然语言处理方面的技术进行问答任务。
+
+本项目提供的汽车说明书图片可点击[这里](https://paddlenlp.bj.bcebos.com/images/applications/automobile.tar.gz)进行下载，下载后解压放至 `./OCR_process/demo_pics` 目录下，然后通过如下命令，使用 PaddleOCR 对图片进行解析。
+
+```shell
+cd OCR_process/
+python3 ocr_process.py
+cd ..
+```
+
+解析后的结果存放至 `./OCR_process/demo_ocr_res.json` 中。
+
+## 5. 排序模块
+对于用户提出的问题，如果从所有的汽车说明书图片中去寻找答案会比较耗时且耗费资源。因此这里使用了一个基于[RocketQA](https://arxiv.org/pdf/2010.08191.pdf)的排序模块，该模块将根据用户提出的问题对汽车说明书的不同图片进行打分排序，这样便可以获取和问题最相关的图片，并使用跨模态阅读理解模块在该问题上进行抽取答案。
+
+本项目提供了140条汽车说明书相关的训练样本，用于排序模型的训练， 同时也提供了一个基于RocketQA的预先训练好的基线模型 base_model。 本模块可以使用 base_model 在汽车说明书训练样本上进一步微调。
+
+其中，汽车说明书的训练集可点击[这里](https://paddlenlp.bj.bcebos.com/data/automobile_rerank_train.tsv) 进行下载，下载后将其重命名为 `train.tsv` ，存放至 `./Rerank/data/` 目录下。
+
+同时，base_model 是 [Dureader retrieval](https://arxiv.org/abs/2203.10232) 数据集训练的排序模型， 可点击[这里](https://paddlenlp.bj.bcebos.com/models/base_ranker.tar.gz) 进行下载，解压后可获得包含模型的目录 `base_model`，将其放至 `./Rerank/checkpoints` 目录下。
+
+可使用如下代码进行训练：
+
+```shell
+cd Rerank
+bash run_train.sh ./data/train.tsv ./checkpoints/base_model 50 1
+cd ..
+```
+其中，参数依次为训练数据地址，base_model 地址，训练轮次，节点数。
+
+在模型训练完成后，可将模型重命名为 `ranker` 存放至 `./checkpoints/` 目录下，接下来便可以使用如下命令，根据给定的汽车说明书相关问题，对汽车说明书的图片进行打分。代码如下：
+
+```shell
+cd Rerank
+bash run_test.sh 后备箱怎么开
+cd ..
+```
+
+其中，后一项为用户问题，命令执行完成后，分数文件将会保存至 `./Rerank/data/demo.score` 中。
+
+
+## 6. 跨模态阅读理解模块
+本项目首先获取排序模块输出的结果中评分最高的图片，然后将会使用跨模态的语言模型 LayoutXLM 从该图片中去抽取用户提问的答案。在获取答案后，将会对答案在该图片中进行高亮显示并返回用户。
+
+本项目提供了28条汽车说明书相关的训练样本，用于跨模态阅读理解模型的训练， 同时也提供了一个预先训练好的基线模型 base_model。 本模块可以使用 base_model 在汽车说明书训练样本上进一步微调，增强模型对汽车说明书领域的理解。
+
+其中，汽车说明书的阅读理解训练集可点击[这里](https://paddlenlp.bj.bcebos.com/data/automobile_mrc_train.json) 进行下载，下载后将其重命名为 `train.json`，存放至 `./Extraction/data/` 目录下。
+
+同时，base_model 是 [Dureader VIS](https://aclanthology.org/2022.findings-acl.105.pdf) 数据集训练的跨模态阅读理解模型， 可点击[这里](https://paddlenlp.bj.bcebos.com/models/base_mrc.tar.gz) 进行下载，解压后可获得包含模型的目录 `base_model`，将其放至 `./Extraction/checkpoints` 目录下。
+
+可使用如下代码进行训练：
+
+```shell
+cd Extraction
+bash run_train.sh
+cd ..
+```
+
+在模型训练完成后，可将模型重命名为 `layoutxlm` 存放至 `./checkpoints/` 目录下，接下来便可以使用如下命令，根据给定的汽车说明书相关问题，从得分最高的汽车说明书图片中抽取答案。代码如下：
+
+```shell
+cd Extraction
+bash run_test.sh 后备箱怎么开
+cd ..
+```
+
+其中，后一项为用户问题，命令执行完成后，最终结果将会保存至 `./answer.png` 中。
+
+## 7. 全流程预测
+本项目提供了全流程预测的功能，可通过如下命令进行一键式预测：
+
+```shell
+bash run_test.sh 后备箱怎么开
+```
+
+其中，后一项参数为用户问题，最终结果将会保存至 `./answer.png` 中。
+
+**备注**：在运行命令前，请确保已使用第4节介绍的命令对原始汽车说明书图片完成了文档解析。
+
+
+下图展示了用户提问的三个问题："后备箱怎么开"，"钥匙怎么充电" 和 "NFC解锁注意事项"， 可以看到，本项目的汽车说明书问答系统能够精准地找到答案并进行高亮显示。
+
+<center><img src="https://user-images.githubusercontent.com/35913314/169012902-1a42bd14-976f-4da8-b5b5-d8e7352b68df.png"/></center>

applications/document_intelligence/doc_vqa/Rerank/change_to_rerank.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import json
+
+question = sys.argv[1]
+
+with open("../OCR_process/demo_ocr_res.json", "r", encoding="utf8") as f:
+    paras = []
+    for line in f:
+        line = json.loads(line.strip())
+        document = line["document"]
+        para = []
+        for token in document:
+            token = token.replace("▁", "")
+            para.append(token)
+        paras.append("".join(para))
+
+with open("./data/demo.tsv", "w", encoding="utf8") as f:
+    for para in paras:
+        f.write("{}\t\t{}\t0\n".format(question, para))

applications/document_intelligence/doc_vqa/Rerank/config/ernie_base_1.0_CN/ernie_config.json

@@ -0,0 +1,12 @@
+{
+  "attention_probs_dropout_prob": 0.1, 
+  "hidden_act": "relu", 
+  "hidden_dropout_prob": 0.1, 
+  "hidden_size": 768, 
+  "initializer_range": 0.02, 
+  "max_position_embeddings": 513, 
+  "num_attention_heads": 12, 
+  "num_hidden_layers": 12, 
+  "type_vocab_size": 2, 
+  "vocab_size": 18000
+}

applications/document_intelligence/doc_vqa/Rerank/run_test.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export CUDA_VISIBLE_DEVICES=0
+
+QUESTION=$1
+
+if [ ! -d output ]; then
+    mkdir output
+fi
+if [ ! -d log ]; then
+    mkdir log
+fi
+
+python3 change_to_rerank.py ${QUESTION}
+
+python3 -u ./src/train_ce.py \
+                   --use_cuda true \
+                   --verbose true \
+                   --do_train false \
+                   --do_val false \
+                   --do_test true \
+                   --batch_size 128 \
+                   --init_checkpoint "./checkpoints/ranker" \
+                   --test_set "./data/demo.tsv" \
+                   --test_save "data/demo.score" \
+                   --max_seq_len 384 \
+                   --for_cn true \
+                   --vocab_path "config/ernie_base_1.0_CN/vocab.txt" \
+                   --ernie_config_path "config/ernie_base_1.0_CN/ernie_config.json"
+                   1>>log/train.log 2>&1
+

applications/document_intelligence/doc_vqa/Rerank/run_train.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export CUDA_VISIBLE_DEVICES=0
+
+if [ $# != 4 ];then
+    echo "USAGE: sh run_train.sh \$TRAIN_SET \$MODEL_PATH \$epoch \$nodes_count"
+    exit 1
+fi
+
+TRAIN_SET=$1
+MODEL_PATH=$2
+epoch=$3
+node=$4
+
+CHECKPOINT_PATH=output
+if [ ! -d output ]; then
+    mkdir output
+fi
+if [ ! -d log ]; then
+    mkdir log
+fi
+
+lr=1e-5
+batch_size=32
+train_exampls=`cat $TRAIN_SET | wc -l`
+save_steps=$[$train_exampls/$batch_size/$node]
+data_size=$[$save_steps*$batch_size*$node]
+new_save_steps=$[$save_steps*$epoch/2]
+
+python3 -m paddle.distributed.launch \
+    --log_dir log \
+    ./src/train_ce.py \
+                   --use_cuda true \
+                   --verbose true \
+                   --do_train true \
+                   --do_val false \
+                   --do_test false \
+                   --use_mix_precision false \
+                   --train_data_size ${data_size} \
+                   --batch_size ${batch_size} \
+                   --init_pretraining_params ${MODEL_PATH} \
+                   --train_set ${TRAIN_SET} \
+                   --save_steps ${new_save_steps} \
+                   --validation_steps ${new_save_steps} \
+                   --checkpoints ${CHECKPOINT_PATH} \
+                   --weight_decay  0.01 \
+                   --warmup_proportion 0.0 \
+                   --epoch $epoch \
+                   --max_seq_len 384 \
+                   --for_cn true \
+                   --vocab_path config/ernie_base_1.0_CN/vocab.txt \
+                   --ernie_config_path config/ernie_base_1.0_CN/ernie_config.json \
+                   --learning_rate ${lr} \
+                   --skip_steps 10 \
+                   --num_iteration_per_drop_scope 1 \
+                   --num_labels 2 \
+                   --random_seed 1 
+

applications/document_intelligence/doc_vqa/Rerank/src/batching.py

@@ -0,0 +1,69 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Mask, padding and batching."""
+
+import numpy as np
+
+
+def pad_batch_data(
+    insts,
+    pad_idx=0,
+    return_pos=False,
+    return_input_mask=False,
+    return_max_len=False,
+    return_num_token=False,
+    return_seq_lens=False,
+):
+    """
+    Pad the instances to the max sequence length in batch, and generate the
+    corresponding position data and attention bias.
+    """
+    return_list = []
+    max_len = max(len(inst) for inst in insts)
+    # Any token included in dict can be used to pad, since the paddings' loss
+    # will be masked out by weights and make no effect on parameter gradients.
+
+    inst_data = np.array([inst + list([pad_idx] * (max_len - len(inst))) for inst in insts])
+    return_list += [inst_data.astype("int64").reshape([-1, max_len, 1])]
+
+    # position data
+    if return_pos:
+        inst_pos = np.array([list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) for inst in insts])
+
+        return_list += [inst_pos.astype("int64").reshape([-1, max_len, 1])]
+
+    if return_input_mask:
+        # This is used to avoid attention on paddings.
+        input_mask_data = np.array([[1] * len(inst) + [0] * (max_len - len(inst)) for inst in insts])
+        input_mask_data = np.expand_dims(input_mask_data, axis=-1)
+        return_list += [input_mask_data.astype("float32")]
+
+    if return_max_len:
+        return_list += [max_len]
+
+    if return_num_token:
+        num_token = 0
+        for inst in insts:
+            num_token += len(inst)
+        return_list += [num_token]
+
+    if return_seq_lens:
+        seq_lens = np.array([len(inst) for inst in insts])
+        return_list += [seq_lens.astype("int64").reshape([-1, 1])]
+
+    return return_list if len(return_list) > 1 else return_list[0]
+
+
+if __name__ == "__main__":
+    pass

applications/document_intelligence/doc_vqa/Rerank/src/cross_encoder.py

@@ -0,0 +1,333 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model for classifier."""
+
+import logging
+import time
+
+import numpy as np
+import paddle.fluid as fluid
+from model.ernie import ErnieModel
+from scipy.stats import pearsonr, spearmanr
+
+log = logging.getLogger(__name__)
+
+
+def create_model(args, pyreader_name, ernie_config, is_prediction=False, task_name=""):
+    pyreader = fluid.layers.py_reader(
+        capacity=50,
+        shapes=[
+            [-1, args.max_seq_len, 1],
+            [-1, args.max_seq_len, 1],
+            [-1, args.max_seq_len, 1],
+            [-1, args.max_seq_len, 1],
+            [-1, args.max_seq_len, 1],
+            [-1, 1],
+            [-1, 1],
+        ],
+        dtypes=["int64", "int64", "int64", "int64", "float32", "int64", "int64"],
+        lod_levels=[0, 0, 0, 0, 0, 0, 0],
+        name=task_name + "_" + pyreader_name,
+        use_double_buffer=True,
+    )
+
+    (src_ids, sent_ids, pos_ids, task_ids, input_mask, labels, qids) = fluid.layers.read_file(pyreader)
+
+    def _model(is_noise=False):
+        ernie = ErnieModel(
+            src_ids=src_ids,
+            position_ids=pos_ids,
+            sentence_ids=sent_ids,
+            task_ids=task_ids,
+            input_mask=input_mask,
+            config=ernie_config,
+            is_noise=is_noise,
+        )
+
+        cls_feats = ernie.get_pooled_output()
+        if not is_noise:
+            cls_feats = fluid.layers.dropout(x=cls_feats, dropout_prob=0.1, dropout_implementation="upscale_in_train")
+        logits = fluid.layers.fc(
+            input=cls_feats,
+            size=args.num_labels,
+            param_attr=fluid.ParamAttr(
+                name=task_name + "_cls_out_w", initializer=fluid.initializer.TruncatedNormal(scale=0.02)
+            ),
+            bias_attr=fluid.ParamAttr(name=task_name + "_cls_out_b", initializer=fluid.initializer.Constant(0.0)),
+        )
+        """
+        if is_prediction:
+            probs = fluid.layers.softmax(logits)
+            feed_targets_name = [
+                src_ids.name, sent_ids.name, pos_ids.name, input_mask.name
+            ]
+            if ernie_version == "2.0":
+                feed_targets_name += [task_ids.name]
+            return pyreader, probs, feed_targets_name
+        """
+
+        num_seqs = fluid.layers.create_tensor(dtype="int64")
+        # add focal loss
+        ce_loss, probs = fluid.layers.softmax_with_cross_entropy(logits=logits, label=labels, return_softmax=True)
+        loss = fluid.layers.mean(x=ce_loss)
+        accuracy = fluid.layers.accuracy(input=probs, label=labels, total=num_seqs)
+        graph_vars = {
+            "loss": loss,
+            "probs": probs,
+            "accuracy": accuracy,
+            "labels": labels,
+            "num_seqs": num_seqs,
+            "qids": qids,
+        }
+        return graph_vars
+
+    if not is_prediction:
+        graph_vars = _model(is_noise=True)
+        old_loss = graph_vars["loss"]
+        token_emb = fluid.default_main_program().global_block().var("word_embedding")
+        token_emb.stop_gradient = False
+        token_gradient = fluid.gradients(old_loss, token_emb)[0]
+        token_gradient.stop_gradient = False
+        epsilon = 1e-8
+        norm = fluid.layers.sqrt(fluid.layers.reduce_sum(fluid.layers.square(token_gradient)) + epsilon)
+        gp = (0.01 * token_gradient) / norm
+        gp.stop_gradient = True
+        fluid.layers.assign(token_emb + gp, token_emb)
+        graph_vars = _model()
+        fluid.layers.assign(token_emb - gp, token_emb)
+    else:
+        graph_vars = _model()
+
+    return pyreader, graph_vars
+
+
+def evaluate_mrr(preds):
+    last_qid = None
+    total_mrr = 0.0
+    qnum = 0.0
+    rank = 0.0
+    correct = False
+    for qid, score, label in preds:
+        if qid != last_qid:
+            rank = 0.0
+            qnum += 1
+            correct = False
+            last_qid = qid
+
+        rank += 1
+        if not correct and label != 0:
+            total_mrr += 1.0 / rank
+            correct = True
+
+    return total_mrr / qnum
+
+
+def evaluate(
+    exe, test_program, test_pyreader, graph_vars, eval_phase, use_multi_gpu_test=False, metric="simple_accuracy"
+):
+    train_fetch_list = [graph_vars["loss"].name, graph_vars["accuracy"].name, graph_vars["num_seqs"].name]
+
+    if eval_phase == "train":
+        if "learning_rate" in graph_vars:
+            train_fetch_list.append(graph_vars["learning_rate"].name)
+        outputs = exe.run(fetch_list=train_fetch_list, program=test_program)
+        ret = {"loss": np.mean(outputs[0]), "accuracy": np.mean(outputs[1])}
+        if "learning_rate" in graph_vars:
+            ret["learning_rate"] = float(outputs[3][0])
+        return ret
+
+    test_pyreader.start()
+    total_cost = 0.0
+    total_acc = 0.0
+    total_num_seqs = 0.0
+    total_label_pos_num = 0.0
+    total_pred_pos_num = 0.0
+    total_correct_num = 0.0
+    qids, labels, scores, preds = [], [], [], []
+    time_begin = time.time()
+
+    fetch_list = [
+        graph_vars["loss"].name,
+        graph_vars["accuracy"].name,
+        graph_vars["probs"].name,
+        graph_vars["labels"].name,
+        graph_vars["num_seqs"].name,
+        graph_vars["qids"].name,
+    ]
+    while True:
+        try:
+            if use_multi_gpu_test:
+                np_loss, np_acc, np_probs, np_labels, np_num_seqs, np_qids = exe.run(fetch_list=fetch_list)
+            else:
+                np_loss, np_acc, np_probs, np_labels, np_num_seqs, np_qids = exe.run(
+                    program=test_program, fetch_list=fetch_list
+                )
+            total_cost += np.sum(np_loss * np_num_seqs)
+            total_acc += np.sum(np_acc * np_num_seqs)
+            total_num_seqs += np.sum(np_num_seqs)
+            labels.extend(np_labels.reshape((-1)).tolist())
+            if np_qids is None:
+                np_qids = np.array([])
+            qids.extend(np_qids.reshape(-1).tolist())
+            scores.extend(np_probs[:, 1].reshape(-1).tolist())
+            np_preds = np.argmax(np_probs, axis=1).astype(np.float32)
+            preds.extend(np_preds)
+            total_label_pos_num += np.sum(np_labels)
+            total_pred_pos_num += np.sum(np_preds)
+            total_correct_num += np.sum(np.dot(np_preds, np_labels))
+        except fluid.core.EOFException:
+            test_pyreader.reset()
+            break
+    time_end = time.time()
+    cost = total_cost / total_num_seqs
+    elapsed_time = time_end - time_begin
+
+    evaluate_info = ""
+    if metric == "acc_and_f1":
+        ret = acc_and_f1(preds, labels)
+        evaluate_info = "[%s evaluation] ave loss: %f, ave_acc: %f, f1: %f, data_num: %d, elapsed time: %f s" % (
+            eval_phase,
+            cost,
+            ret["acc"],
+            ret["f1"],
+            total_num_seqs,
+            elapsed_time,
+        )
+    elif metric == "matthews_corrcoef":
+        ret = matthews_corrcoef(preds, labels)
+        evaluate_info = "[%s evaluation] ave loss: %f, matthews_corrcoef: %f, data_num: %d, elapsed time: %f s" % (
+            eval_phase,
+            cost,
+            ret,
+            total_num_seqs,
+            elapsed_time,
+        )
+    elif metric == "pearson_and_spearman":
+        ret = pearson_and_spearman(scores, labels)
+        evaluate_info = (
+            "[%s evaluation] ave loss: %f, pearson:%f, spearman:%f, corr:%f, data_num: %d, elapsed time: %f s"
+            % (eval_phase, cost, ret["pearson"], ret["spearman"], ret["corr"], total_num_seqs, elapsed_time)
+        )
+    elif metric == "simple_accuracy":
+        ret = simple_accuracy(preds, labels)
+        evaluate_info = "[%s evaluation] ave loss: %f, acc:%f, data_num: %d, elapsed time: %f s" % (
+            eval_phase,
+            cost,
+            ret,
+            total_num_seqs,
+            elapsed_time,
+        )
+    elif metric == "acc_and_f1_and_mrr":
+        ret_a = acc_and_f1(preds, labels)
+        preds = sorted(zip(qids, scores, labels), key=lambda elem: (elem[0], -elem[1]))
+        ret_b = evaluate_mrr(preds)
+        evaluate_info = "[%s evaluation] ave loss: %f, acc: %f, f1: %f, mrr: %f, data_num: %d, elapsed time: %f s" % (
+            eval_phase,
+            cost,
+            ret_a["acc"],
+            ret_a["f1"],
+            ret_b,
+            total_num_seqs,
+            elapsed_time,
+        )
+    else:
+        raise ValueError("unsupported metric {}".format(metric))
+    return evaluate_info
+
+
+def matthews_corrcoef(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+    tp = np.sum((labels == 1) & (preds == 1))
+    tn = np.sum((labels == 0) & (preds == 0))
+    fp = np.sum((labels == 0) & (preds == 1))
+    fn = np.sum((labels == 1) & (preds == 0))
+
+    mcc = ((tp * tn) - (fp * fn)) / np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
+    return mcc
+
+
+def f1_score(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+
+    tp = np.sum((labels == 1) & (preds == 1))
+    fp = np.sum((labels == 0) & (preds == 1))
+    fn = np.sum((labels == 1) & (preds == 0))
+    p = tp / (tp + fp)
+    r = tp / (tp + fn)
+    f1 = (2 * p * r) / (p + r + 1e-8)
+    return f1
+
+
+def pearson_and_spearman(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+
+    pearson_corr = pearsonr(preds, labels)[0]
+    spearman_corr = spearmanr(preds, labels)[0]
+    return {
+        "pearson": pearson_corr,
+        "spearmanr": spearman_corr,
+        "corr": (pearson_corr + spearman_corr) / 2,
+    }
+
+
+def acc_and_f1(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+
+    acc = simple_accuracy(preds, labels)
+    f1 = f1_score(preds, labels)
+    return {
+        "acc": acc,
+        "f1": f1,
+        "acc_and_f1": (acc + f1) / 2,
+    }
+
+
+def simple_accuracy(preds, labels):
+    preds = np.array(preds)
+    labels = np.array(labels)
+    return (preds == labels).mean()
+
+
+def predict(exe, test_program, test_pyreader, graph_vars, dev_count=1):
+    test_pyreader.start()
+    qids, probs = [], []
+    preds = []
+
+    fetch_list = [graph_vars["probs"].name, graph_vars["qids"].name]
+
+    while True:
+        try:
+            if dev_count == 1:
+                np_probs, np_qids = exe.run(program=test_program, fetch_list=fetch_list)
+            else:
+                np_probs, np_qids = exe.run(fetch_list=fetch_list)
+
+            if np_qids is None:
+                np_qids = np.array([])
+            qids.extend(np_qids.reshape(-1).tolist())
+            np_preds = np.argmax(np_probs, axis=1).astype(np.float32)
+            preds.extend(np_preds)
+            probs.append(np_probs)
+
+        except fluid.core.EOFException:
+            test_pyreader.reset()
+            break
+
+    probs = np.concatenate(probs, axis=0).reshape([len(preds), -1])
+
+    return qids, preds, probs

applications/document_intelligence/doc_vqa/Rerank/src/finetune_args.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+
+from src.utils.args import ArgumentGroup
+
+# yapf: disable
+parser = argparse.ArgumentParser(__doc__)
+model_g = ArgumentGroup(parser, "model", "model configuration and paths.")
+model_g.add_arg("ernie_config_path", str, None, "Path to the json file for ernie model config.")
+model_g.add_arg("init_checkpoint", str, None, "Init checkpoint to resume training from.")
+model_g.add_arg("init_pretraining_params", str, None, "Init pre-training params which preforms fine-tuning from. If the arg 'init_checkpoint' has been set, this argument wouldn't be valid.")
+model_g.add_arg("checkpoints", str, "checkpoints", "Path to save checkpoints.")
+
+model_g.add_arg("is_classify", bool, True, "is_classify")
+model_g.add_arg("is_regression", bool, False, "is_regression")
+model_g.add_arg("task_id", int, 0, "task id")
+
+train_g = ArgumentGroup(parser, "training", "training options.")
+train_g.add_arg("epoch", int, 3, "Number of epoches for fine-tuning.")
+train_g.add_arg("learning_rate", float, 5e-5, "Learning rate used to train with warmup.")
+train_g.add_arg("lr_scheduler", str, "linear_warmup_decay", "scheduler of learning rate.", choices=['linear_warmup_decay', 'noam_decay'])
+train_g.add_arg("weight_decay", float, 0.01, "Weight decay rate for L2 regularizer.")
+train_g.add_arg("warmup_proportion", float, 0.1, "Proportion of training steps to perform linear learning rate warmup for.")
+train_g.add_arg("save_steps", int, 10000, "The steps interval to save checkpoints.")
+train_g.add_arg("validation_steps", int, 1000, "The steps interval to evaluate model performance.")
+train_g.add_arg("use_recompute", bool, False, "Whether to use recompute optimizer for training.")
+train_g.add_arg("use_mix_precision", bool, False, "Whether to use mix-precision optimizer for training.")
+train_g.add_arg("use_cross_batch", bool, False, "Whether to use cross-batch for training.")
+train_g.add_arg("use_lamb", bool, False, "Whether to use LambOptimizer for training.")
+train_g.add_arg("use_dynamic_loss_scaling", bool, True, "Whether to use dynamic loss scaling.")
+
+train_g.add_arg("test_save", str, "./checkpoints/test_result", "test_save")
+train_g.add_arg("metric", str, "simple_accuracy", "metric")
+train_g.add_arg("incr_every_n_steps", int, 100, "Increases loss scaling every n consecutive.")
+train_g.add_arg("decr_every_n_nan_or_inf", int, 2, "Decreases loss scaling every n accumulated steps with nan or inf gradients.")
+train_g.add_arg("incr_ratio", float, 2.0, "The multiplier to use when increasing the loss scaling.")
+train_g.add_arg("decr_ratio", float, 0.8, "The less-than-one-multiplier to use when decreasing.")
+
+log_g = ArgumentGroup(parser, "logging", "logging related.")
+log_g.add_arg("skip_steps", int, 10, "The steps interval to print loss.")
+log_g.add_arg("verbose", bool, False, "Whether to output verbose log.")
+
+data_g = ArgumentGroup(parser, "data", "Data paths, vocab paths and data processing options")
+data_g.add_arg("tokenizer", str, "FullTokenizer", "ATTENTION: the INPUT must be splited by Word with blank while using SentencepieceTokenizer or WordsegTokenizer")
+data_g.add_arg("train_set", str, None, "Path to training data.")
+data_g.add_arg("test_set", str, None, "Path to test data.")
+data_g.add_arg("dev_set", str, None, "Path to validation data.")
+data_g.add_arg("vocab_path", str, None, "Vocabulary path.")
+data_g.add_arg("max_seq_len", int, 512, "Number of words of the longest seqence.")
+data_g.add_arg("q_max_seq_len", int, 32, "Number of words of the longest seqence.")
+data_g.add_arg("p_max_seq_len", int, 256, "Number of words of the longest seqence.")
+data_g.add_arg("train_data_size", int, 0, "Number of training data's total examples. Set for distribute.")
+data_g.add_arg("batch_size", int, 32, "Total examples' number in batch for training. see also --in_tokens.")
+data_g.add_arg("predict_batch_size", int, None, "Total examples' number in batch for predict. see also --in_tokens.")
+data_g.add_arg("in_tokens", bool, False, "If set, the batch size will be the maximum number of tokens in one batch. Otherwise, it will be the maximum number of examples in one batch.")
+data_g.add_arg("do_lower_case", bool, True, "Whether to lower case the input text. Should be True for uncased models and False for cased models.")
+data_g.add_arg("random_seed", int, None, "Random seed.")
+data_g.add_arg("label_map_config", str, None, "label_map_path.")
+data_g.add_arg("num_labels", int, 2, "label number")
+data_g.add_arg("diagnostic", str, None, "GLUE Diagnostic Dataset")
+data_g.add_arg("diagnostic_save", str, None, "GLUE Diagnostic save f")
+data_g.add_arg("max_query_length", int, 64, "Max query length.")
+data_g.add_arg("max_answer_length", int, 100, "Max answer length.")
+data_g.add_arg("doc_stride", int, 128, "When splitting up a long document into chunks, how much stride to take between chunks.")
+data_g.add_arg("n_best_size", int, 20, "The total number of n-best predictions to generate in the nbest_predictions.json output file.")
+data_g.add_arg("chunk_scheme", type=str, default="IOB", choices=["IO", "IOB", "IOE", "IOBES"], help="chunk scheme")
+
+run_type_g = ArgumentGroup(parser, "run_type", "running type options.")
+run_type_g.add_arg("use_cuda", bool, True, "If set, use GPU for training.")
+run_type_g.add_arg("is_distributed", bool, False, "If set, then start distributed training.")
+run_type_g.add_arg("use_fast_executor", bool, False, "If set, use fast parallel executor (in experiment).")
+run_type_g.add_arg("num_iteration_per_drop_scope", int, 10, "Iteration intervals to drop scope.")
+run_type_g.add_arg("do_train", bool, True, "Whether to perform training.")
+run_type_g.add_arg("do_val", bool, True, "Whether to perform evaluation on dev data set.")
+run_type_g.add_arg("do_test", bool, True, "Whether to perform evaluation on test data set.")
+run_type_g.add_arg("output_item", int, 3, "Test output format.")
+run_type_g.add_arg("output_file_name", str, None, "Test output file name")
+run_type_g.add_arg("test_data_cnt", int, 1110000 , "total cnt of testset")
+run_type_g.add_arg("use_multi_gpu_test", bool, False, "Whether to perform evaluation using multiple gpu cards")
+run_type_g.add_arg("metrics", bool, True, "Whether to perform evaluation on test data set.")
+run_type_g.add_arg("shuffle", bool, True, "")
+run_type_g.add_arg("for_cn", bool, False, "model train for cn or for other langs.")

applications/document_intelligence/doc_vqa/Rerank/src/index_search.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import faiss
+import numpy as np
+
+
+def read_embed(file_name, dim=768, bs=3000):
+    if file_name.endswith("npy"):
+        i = 0
+        emb_np = np.load(file_name)
+        while i < len(emb_np):
+            vec_list = emb_np[i : i + bs]
+            i += bs
+            yield vec_list
+    else:
+        vec_list = []
+        with open(file_name) as inp:
+            for line in inp:
+                data = line.strip()
+                vector = [float(item) for item in data.split(" ")]
+                assert len(vector) == dim
+                vec_list.append(vector)
+                if len(vec_list) == bs:
+                    yield vec_list
+                    vec_list = []
+            if vec_list:
+                yield vec_list
+
+
+def load_qid(file_name):
+    qid_list = []
+    with open(file_name) as inp:
+        for line in inp:
+            line = line.strip()
+            qid = line.split("\t")[0]
+            qid_list.append(qid)
+    return qid_list
+
+
+def search(index, emb_file, qid_list, outfile, top_k):
+    q_idx = 0
+    with open(outfile, "w") as out:
+        for batch_vec in read_embed(emb_file):
+            q_emb_matrix = np.array(batch_vec)
+            res_dist, res_p_id = index.search(q_emb_matrix.astype("float32"), top_k)
+            for i in range(len(q_emb_matrix)):
+                qid = qid_list[q_idx]
+                for j in range(top_k):
+                    pid = res_p_id[i][j]
+                    score = res_dist[i][j]
+                    out.write("%s\t%s\t%s\t%s\n" % (qid, pid, j + 1, score))
+                q_idx += 1
+
+
+def main():
+    part = sys.argv[1]
+    topk = int(sys.argv[2])
+    q_text_file = sys.argv[3]
+    outfile = "output/res.top%s-part%s" % (topk, part)
+
+    qid_list = load_qid(q_text_file)
+
+    engine = faiss.read_index("output/para.index.part%s" % part)
+    emb_file = "output/query.emb.npy"
+    search(engine, emb_file, qid_list, outfile, topk)
+
+
+if __name__ == "__main__":
+    main()

applications/document_intelligence/doc_vqa/Rerank/src/merge.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+shift = int(sys.argv[1])
+top = int(sys.argv[2])
+total_part = int(sys.argv[3])
+
+f_list = []
+for part in range(total_part):
+    f0 = open("output/res.top%s-part%s" % (top, part))
+    f_list.append(f0)
+
+line_list = []
+for part in range(total_part):
+    line = f_list[part].readline()
+    line_list.append(line)
+
+out = open("output/dev.res.top%s" % top, "w")
+last_q = ""
+ans_list = {}
+while line_list[-1]:
+    cur_list = []
+    for line in line_list:
+        sub = line.strip().split("\t")
+        cur_list.append(sub)
+
+    if last_q == "":
+        last_q = cur_list[0][0]
+    if cur_list[0][0] != last_q:
+        rank = sorted(ans_list.items(), key=lambda a: a[1], reverse=True)
+        for i in range(top):
+            out.write("%s\t%s\t%s\t%s\n" % (last_q, rank[i][0], i + 1, rank[i][1]))
+        ans_list = {}
+    for i, sub in enumerate(cur_list):
+        ans_list[int(sub[1]) + shift * i] = float(sub[-1])
+    last_q = cur_list[0][0]
+
+    line_list = []
+    for f0 in f_list:
+        line = f0.readline()
+        line_list.append(line)
+
+rank = sorted(ans_list.items(), key=lambda a: a[1], reverse=True)
+for i in range(top):
+    out.write("%s\t%s\t%s\t%s\n" % (last_q, rank[i][0], i + 1, rank[i][1]))
+out.close()
+
+print("output/dev.res.top%s" % top)

applications/document_intelligence/doc_vqa/Rerank/src/model/ernie.py

@@ -0,0 +1,259 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Ernie model."""
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+import logging
+from io import open
+
+import paddle
+import paddle.fluid as fluid
+import six
+from model.transformer_encoder import encoder, pre_process_layer
+
+log = logging.getLogger(__name__)
+
+
+class ErnieConfig(object):
+    def __init__(self, config_path):
+        self._config_dict = self._parse(config_path)
+
+    def _parse(self, config_path):
+        try:
+            with open(config_path, "r", encoding="utf8") as json_file:
+                config_dict = json.load(json_file)
+        except Exception:
+            raise IOError("Error in parsing Ernie model config file '%s'" % config_path)
+        else:
+            return config_dict
+
+    def __getitem__(self, key):
+        return self._config_dict.get(key, None)
+
+    def print_config(self):
+        for arg, value in sorted(six.iteritems(self._config_dict)):
+            log.info("%s: %s" % (arg, value))
+        log.info("------------------------------------------------")
+
+
+class ErnieModel(object):
+    def __init__(
+        self,
+        src_ids,
+        position_ids,
+        sentence_ids,
+        task_ids,
+        input_mask,
+        config,
+        weight_sharing=True,
+        model_name="",
+        is_noise=False,
+    ):
+
+        self._emb_size = config["hidden_size"]
+        self._n_layer = config["num_hidden_layers"]
+        self._n_head = config["num_attention_heads"]
+        self._voc_size = config["vocab_size"]
+        self._max_position_seq_len = config["max_position_embeddings"]
+        if config["sent_type_vocab_size"]:
+            self._sent_types = config["sent_type_vocab_size"]
+        else:
+            self._sent_types = config["type_vocab_size"]
+
+        self._use_task_id = config["use_task_id"]
+        if self._use_task_id:
+            self._task_types = config["task_type_vocab_size"]
+        self._hidden_act = config["hidden_act"]
+        self._prepostprocess_dropout = config["hidden_dropout_prob"]
+        self._attention_dropout = config["attention_probs_dropout_prob"]
+        if is_noise:
+            self._prepostprocess_dropout = 0
+            self._attention_dropout = 0
+        self._weight_sharing = weight_sharing
+        self.checkpoints = []
+
+        self._word_emb_name = "word_embedding"
+        self._pos_emb_name = "pos_embedding"
+        self._sent_emb_name = "sent_embedding"
+        self._task_emb_name = "task_embedding"
+        self._emb_dtype = "float32"
+
+        # Initialize all weights by truncated normal initializer, and all biases
+        # will be initialized by constant zero by default.
+        self._param_initializer = fluid.initializer.TruncatedNormal(scale=config["initializer_range"])
+
+        self._build_model(model_name, src_ids, position_ids, sentence_ids, task_ids, input_mask)
+
+    def _build_model(self, model_name, src_ids, position_ids, sentence_ids, task_ids, input_mask):
+        # padding id in vocabulary must be set to 0
+        emb_out = fluid.layers.embedding(
+            input=src_ids,
+            size=[self._voc_size, self._emb_size],
+            dtype=self._emb_dtype,
+            param_attr=fluid.ParamAttr(name=model_name + self._word_emb_name, initializer=self._param_initializer),
+            is_sparse=False,
+        )
+
+        position_emb_out = fluid.layers.embedding(
+            input=position_ids,
+            size=[self._max_position_seq_len, self._emb_size],
+            dtype=self._emb_dtype,
+            param_attr=fluid.ParamAttr(name=model_name + self._pos_emb_name, initializer=self._param_initializer),
+        )
+
+        sent_emb_out = fluid.layers.embedding(
+            sentence_ids,
+            size=[self._sent_types, self._emb_size],
+            dtype=self._emb_dtype,
+            param_attr=fluid.ParamAttr(name=model_name + self._sent_emb_name, initializer=self._param_initializer),
+        )
+
+        emb_out = emb_out + position_emb_out
+        emb_out = emb_out + sent_emb_out
+
+        if self._use_task_id:
+            task_emb_out = fluid.layers.embedding(
+                task_ids,
+                size=[self._task_types, self._emb_size],
+                dtype=self._emb_dtype,
+                param_attr=fluid.ParamAttr(name=model_name + self._task_emb_name, initializer=self._param_initializer),
+            )
+
+            emb_out = emb_out + task_emb_out
+
+        emb_out = pre_process_layer(emb_out, "nd", self._prepostprocess_dropout, name=model_name + "pre_encoder")
+
+        self_attn_mask = paddle.matmul(x=input_mask, y=input_mask, transpose_y=True)
+
+        self_attn_mask = fluid.layers.scale(x=self_attn_mask, scale=10000.0, bias=-1.0, bias_after_scale=False)
+        n_head_self_attn_mask = fluid.layers.stack(x=[self_attn_mask] * self._n_head, axis=1)
+        n_head_self_attn_mask.stop_gradient = True
+
+        self._enc_out, self.checkpoints = encoder(
+            enc_input=emb_out,
+            attn_bias=n_head_self_attn_mask,
+            n_layer=self._n_layer,
+            n_head=self._n_head,
+            d_key=self._emb_size // self._n_head,
+            d_value=self._emb_size // self._n_head,
+            d_model=self._emb_size,
+            d_inner_hid=self._emb_size * 4,
+            prepostprocess_dropout=self._prepostprocess_dropout,
+            attention_dropout=self._attention_dropout,
+            relu_dropout=0,
+            hidden_act=self._hidden_act,
+            preprocess_cmd="",
+            postprocess_cmd="dan",
+            param_initializer=self._param_initializer,
+            model_name=model_name,
+            name=model_name + "encoder",
+        )
+
+    def get_sequence_output(self):
+        return self._enc_out
+
+    def get_cls_output(self):
+        """Get the first feature of each sequence for classification"""
+        cls_output = fluid.layers.slice(input=self._enc_out, axes=[1], starts=[0], ends=[1])
+        cls_output = fluid.layers.squeeze(cls_output, axes=[1])
+        return cls_output
+
+    def get_pooled_output(self):
+        """Get the first feature of each sequence for classification"""
+        next_sent_feat = fluid.layers.slice(input=self._enc_out, axes=[1], starts=[0], ends=[1])
+        next_sent_feat = fluid.layers.fc(
+            input=next_sent_feat,
+            size=self._emb_size,
+            act="tanh",
+            param_attr=fluid.ParamAttr(name="pooled_fc.w_0", initializer=self._param_initializer),
+            bias_attr="pooled_fc.b_0",
+        )
+        return next_sent_feat
+
+    def get_lm_output(self, mask_label, mask_pos):
+        """Get the loss & accuracy for pretraining"""
+
+        mask_pos = fluid.layers.cast(x=mask_pos, dtype="int32")
+
+        # extract the first token feature in each sentence
+        self.next_sent_feat = self.get_pooled_output()
+        reshaped_emb_out = fluid.layers.reshape(x=self._enc_out, shape=[-1, self._emb_size])
+        # extract masked tokens' feature
+        mask_feat = fluid.layers.gather(input=reshaped_emb_out, index=mask_pos)
+
+        # transform: fc
+        mask_trans_feat = fluid.layers.fc(
+            input=mask_feat,
+            size=self._emb_size,
+            act=self._hidden_act,
+            param_attr=fluid.ParamAttr(name="mask_lm_trans_fc.w_0", initializer=self._param_initializer),
+            bias_attr=fluid.ParamAttr(name="mask_lm_trans_fc.b_0"),
+        )
+
+        # transform: layer norm
+        mask_trans_feat = fluid.layers.layer_norm(
+            mask_trans_feat,
+            begin_norm_axis=len(mask_trans_feat.shape) - 1,
+            param_attr=fluid.ParamAttr(
+                name="mask_lm_trans_layer_norm_scale", initializer=fluid.initializer.Constant(1.0)
+            ),
+            bias_attr=fluid.ParamAttr(
+                name="mask_lm_trans_layer_norm_bias", initializer=fluid.initializer.Constant(1.0)
+            ),
+        )
+        # transform: layer norm
+        # mask_trans_feat = pre_process_layer(
+        #    mask_trans_feat, 'n', name='mask_lm_trans')
+
+        mask_lm_out_bias_attr = fluid.ParamAttr(
+            name="mask_lm_out_fc.b_0", initializer=fluid.initializer.Constant(value=0.0)
+        )
+        if self._weight_sharing:
+            fc_out = paddle.matmul(
+                x=mask_trans_feat,
+                y=fluid.default_main_program().global_block().var(self._word_emb_name),
+                transpose_y=True,
+            )
+            fc_out += fluid.layers.create_parameter(
+                shape=[self._voc_size], dtype=self._emb_dtype, attr=mask_lm_out_bias_attr, is_bias=True
+            )
+
+        else:
+            fc_out = fluid.layers.fc(
+                input=mask_trans_feat,
+                size=self._voc_size,
+                param_attr=fluid.ParamAttr(name="mask_lm_out_fc.w_0", initializer=self._param_initializer),
+                bias_attr=mask_lm_out_bias_attr,
+            )
+
+        mask_lm_loss = fluid.layers.softmax_with_cross_entropy(logits=fc_out, label=mask_label)
+        mean_mask_lm_loss = fluid.layers.mean(mask_lm_loss)
+
+        return mean_mask_lm_loss
+
+    def get_task_output(self, task, task_labels):
+        task_fc_out = fluid.layers.fc(
+            input=self.next_sent_feat,
+            size=task["num_labels"],
+            param_attr=fluid.ParamAttr(name=task["task_name"] + "_fc.w_0", initializer=self._param_initializer),
+            bias_attr=task["task_name"] + "_fc.b_0",
+        )
+        task_loss, task_softmax = fluid.layers.softmax_with_cross_entropy(
+            logits=task_fc_out, label=task_labels, return_softmax=True
+        )
+        task_acc = fluid.layers.accuracy(input=task_softmax, label=task_labels)
+        mean_task_loss = fluid.layers.mean(task_loss)
+        return mean_task_loss, task_acc

applications/document_intelligence/doc_vqa/Rerank/src/model/transformer_encoder.py

@@ -0,0 +1,318 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Transformer encoder."""
+
+from __future__ import absolute_import, division, print_function
+
+from functools import partial
+
+import paddle
+import paddle.fluid as fluid
+import paddle.fluid.layers as layers
+
+
+def multi_head_attention(
+    queries,
+    keys,
+    values,
+    attn_bias,
+    d_key,
+    d_value,
+    d_model,
+    n_head=1,
+    dropout_rate=0.0,
+    cache=None,
+    param_initializer=None,
+    name="multi_head_att",
+):
+    """
+    Multi-Head Attention. Note that attn_bias is added to the logit before
+    computing softmax activation to mask certain selected positions so that
+    they will not considered in attention weights.
+    """
+    keys = queries if keys is None else keys
+    values = keys if values is None else values
+
+    if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3):
+        raise ValueError("Inputs: queries, keys and values should all be 3-D tensors.")
+
+    def __compute_qkv(queries, keys, values, n_head, d_key, d_value):
+        """
+        Add linear projection to queries, keys, and values.
+        """
+        q = layers.fc(
+            input=queries,
+            size=d_key * n_head,
+            num_flatten_dims=2,
+            param_attr=fluid.ParamAttr(name=name + "_query_fc.w_0", initializer=param_initializer),
+            bias_attr=name + "_query_fc.b_0",
+        )
+        k = layers.fc(
+            input=keys,
+            size=d_key * n_head,
+            num_flatten_dims=2,
+            param_attr=fluid.ParamAttr(name=name + "_key_fc.w_0", initializer=param_initializer),
+            bias_attr=name + "_key_fc.b_0",
+        )
+        v = layers.fc(
+            input=values,
+            size=d_value * n_head,
+            num_flatten_dims=2,
+            param_attr=fluid.ParamAttr(name=name + "_value_fc.w_0", initializer=param_initializer),
+            bias_attr=name + "_value_fc.b_0",
+        )
+        return q, k, v
+
+    def __split_heads(x, n_head):
+        """
+        Reshape the last dimension of input tensor x so that it becomes two
+        dimensions and then transpose. Specifically, input a tensor with shape
+        [bs, max_sequence_length, n_head * hidden_dim] then output a tensor
+        with shape [bs, n_head, max_sequence_length, hidden_dim].
+        """
+        hidden_size = x.shape[-1]
+        # The value 0 in shape attr means copying the corresponding dimension
+        # size of the input as the output dimension size.
+        reshaped = layers.reshape(x=x, shape=[0, 0, n_head, hidden_size // n_head], inplace=True)
+
+        # permuate the dimensions into:
+        # [batch_size, n_head, max_sequence_len, hidden_size_per_head]
+        return layers.transpose(x=reshaped, perm=[0, 2, 1, 3])
+
+    def __combine_heads(x):
+        """
+        Transpose and then reshape the last two dimensions of input tensor x
+        so that it becomes one dimension, which is reverse to __split_heads.
+        """
+        if len(x.shape) == 3:
+            return x
+        if len(x.shape) != 4:
+            raise ValueError("Input(x) should be a 4-D Tensor.")
+
+        trans_x = layers.transpose(x, perm=[0, 2, 1, 3])
+        # The value 0 in shape attr means copying the corresponding dimension
+        # size of the input as the output dimension size.
+        return layers.reshape(x=trans_x, shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], inplace=True)
+
+    def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate):
+        """
+        Scaled Dot-Product Attention
+        """
+        scaled_q = layers.scale(x=q, scale=d_key**-0.5)
+        product = paddle.matmul(x=scaled_q, y=k, transpose_y=True)
+        if attn_bias:
+            product += attn_bias
+        weights = layers.softmax(product)
+        if dropout_rate:
+            weights = layers.dropout(
+                weights, dropout_prob=dropout_rate, dropout_implementation="upscale_in_train", is_test=False
+            )
+        out = paddle.matmul(weights, v)
+        return out
+
+    q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value)
+
+    if cache is not None:  # use cache and concat time steps
+        # Since the inplace reshape in __split_heads changes the shape of k and
+        # v, which is the cache input for next time step, reshape the cache
+        # input from the previous time step first.
+        k = cache["k"] = layers.concat([layers.reshape(cache["k"], shape=[0, 0, d_model]), k], axis=1)
+        v = cache["v"] = layers.concat([layers.reshape(cache["v"], shape=[0, 0, d_model]), v], axis=1)
+
+    q = __split_heads(q, n_head)
+    k = __split_heads(k, n_head)
+    v = __split_heads(v, n_head)
+
+    ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate)
+
+    out = __combine_heads(ctx_multiheads)
+
+    # Project back to the model size.
+    proj_out = layers.fc(
+        input=out,
+        size=d_model,
+        num_flatten_dims=2,
+        param_attr=fluid.ParamAttr(name=name + "_output_fc.w_0", initializer=param_initializer),
+        bias_attr=name + "_output_fc.b_0",
+    )
+    return proj_out
+
+
+def positionwise_feed_forward(x, d_inner_hid, d_hid, dropout_rate, hidden_act, param_initializer=None, name="ffn"):
+    """
+    Position-wise Feed-Forward Networks.
+    This module consists of two linear transformations with a ReLU activation
+    in between, which is applied to each position separately and identically.
+    """
+    hidden = layers.fc(
+        input=x,
+        size=d_inner_hid,
+        num_flatten_dims=2,
+        act=hidden_act,
+        param_attr=fluid.ParamAttr(name=name + "_fc_0.w_0", initializer=param_initializer),
+        bias_attr=name + "_fc_0.b_0",
+    )
+    if dropout_rate:
+        hidden = layers.dropout(
+            hidden, dropout_prob=dropout_rate, dropout_implementation="upscale_in_train", is_test=False
+        )
+    out = layers.fc(
+        input=hidden,
+        size=d_hid,
+        num_flatten_dims=2,
+        param_attr=fluid.ParamAttr(name=name + "_fc_1.w_0", initializer=param_initializer),
+        bias_attr=name + "_fc_1.b_0",
+    )
+    return out
+
+
+def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.0, name=""):
+    """
+    Add residual connection, layer normalization and dropout to the out tensor
+    optionally according to the value of process_cmd.
+    This will be used before or after multi-head attention and position-wise
+    feed-forward networks.
+    """
+    for cmd in process_cmd:
+        if cmd == "a":  # add residual connection
+            out = out + prev_out if prev_out else out
+        elif cmd == "n":  # add layer normalization
+            out_dtype = out.dtype
+            if out_dtype == fluid.core.VarDesc.VarType.FP16:
+                out = layers.cast(x=out, dtype="float32")
+            out = layers.layer_norm(
+                out,
+                begin_norm_axis=len(out.shape) - 1,
+                param_attr=fluid.ParamAttr(
+                    name=name + "_layer_norm_scale", initializer=fluid.initializer.Constant(1.0)
+                ),
+                bias_attr=fluid.ParamAttr(name=name + "_layer_norm_bias", initializer=fluid.initializer.Constant(0.0)),
+            )
+            if out_dtype == fluid.core.VarDesc.VarType.FP16:
+                out = layers.cast(x=out, dtype="float16")
+        elif cmd == "d":  # add dropout
+            if dropout_rate:
+                out = layers.dropout(
+                    out, dropout_prob=dropout_rate, dropout_implementation="upscale_in_train", is_test=False
+                )
+    return out
+
+
+pre_process_layer = partial(pre_post_process_layer, None)
+post_process_layer = pre_post_process_layer
+
+
+def encoder_layer(
+    enc_input,
+    attn_bias,
+    n_head,
+    d_key,
+    d_value,
+    d_model,
+    d_inner_hid,
+    prepostprocess_dropout,
+    attention_dropout,
+    relu_dropout,
+    hidden_act,
+    preprocess_cmd="n",
+    postprocess_cmd="da",
+    param_initializer=None,
+    name="",
+):
+    """The encoder layers that can be stacked to form a deep encoder.
+    This module consists of a multi-head (self) attention followed by
+    position-wise feed-forward networks and both the two components companied
+    with the post_process_layer to add residual connection, layer normalization
+    and dropout.
+    """
+    attn_output = multi_head_attention(
+        pre_process_layer(enc_input, preprocess_cmd, prepostprocess_dropout, name=name + "_pre_att"),
+        None,
+        None,
+        attn_bias,
+        d_key,
+        d_value,
+        d_model,
+        n_head,
+        attention_dropout,
+        param_initializer=param_initializer,
+        name=name + "_multi_head_att",
+    )
+    attn_output = post_process_layer(
+        enc_input, attn_output, postprocess_cmd, prepostprocess_dropout, name=name + "_post_att"
+    )
+    ffd_output = positionwise_feed_forward(
+        pre_process_layer(attn_output, preprocess_cmd, prepostprocess_dropout, name=name + "_pre_ffn"),
+        d_inner_hid,
+        d_model,
+        relu_dropout,
+        hidden_act,
+        param_initializer=param_initializer,
+        name=name + "_ffn",
+    )
+    return (
+        post_process_layer(attn_output, ffd_output, postprocess_cmd, prepostprocess_dropout, name=name + "_post_ffn"),
+        ffd_output,
+    )
+
+
+def encoder(
+    enc_input,
+    attn_bias,
+    n_layer,
+    n_head,
+    d_key,
+    d_value,
+    d_model,
+    d_inner_hid,
+    prepostprocess_dropout,
+    attention_dropout,
+    relu_dropout,
+    hidden_act,
+    preprocess_cmd="n",
+    postprocess_cmd="da",
+    param_initializer=None,
+    model_name="",
+    name="",
+):
+    """
+    The encoder is composed of a stack of identical layers returned by calling
+    encoder_layer.
+    """
+    checkpoints = []
+    for i in range(n_layer):
+        enc_output, cp = encoder_layer(
+            enc_input,
+            attn_bias,
+            n_head,
+            d_key,
+            d_value,
+            d_model,
+            d_inner_hid,
+            prepostprocess_dropout,
+            attention_dropout,
+            relu_dropout,
+            hidden_act,
+            preprocess_cmd,
+            postprocess_cmd,
+            param_initializer=param_initializer,
+            name=name + "_layer_" + str(i),
+        )
+        checkpoints.append(cp)
+        enc_input = enc_output
+    enc_output = pre_process_layer(
+        enc_output, preprocess_cmd, prepostprocess_dropout, name=model_name + "post_encoder"
+    )
+
+    return enc_output, checkpoints

applications/document_intelligence/doc_vqa/Rerank/src/optimization.py

@@ -0,0 +1,121 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Optimization and learning rate scheduling."""
+
+import paddle.fluid as fluid
+from paddle.fluid.incubate.fleet.collective import fleet
+
+
+def linear_warmup_decay(learning_rate, warmup_steps, num_train_steps):
+    """Applies linear warmup of learning rate from 0 and decay to 0."""
+    with fluid.default_main_program()._lr_schedule_guard():
+        lr = fluid.layers.tensor.create_global_var(
+            shape=[1], value=0.0, dtype="float32", persistable=True, name="scheduled_learning_rate"
+        )
+
+        global_step = fluid.layers.learning_rate_scheduler._decay_step_counter()
+
+        with fluid.layers.control_flow.Switch() as switch:
+            with switch.case(global_step < warmup_steps):
+                warmup_lr = learning_rate * (global_step / warmup_steps)
+                fluid.layers.tensor.assign(warmup_lr, lr)
+            with switch.default():
+                decayed_lr = fluid.layers.learning_rate_scheduler.polynomial_decay(
+                    learning_rate=learning_rate,
+                    decay_steps=num_train_steps,
+                    end_learning_rate=0.0,
+                    power=1.0,
+                    cycle=False,
+                )
+                fluid.layers.tensor.assign(decayed_lr, lr)
+
+        return lr
+
+
+def optimization(
+    loss,
+    warmup_steps,
+    num_train_steps,
+    learning_rate,
+    train_program,
+    startup_prog,
+    weight_decay,
+    scheduler="linear_warmup_decay",
+    use_dynamic_loss_scaling=False,
+    incr_every_n_steps=1000,
+    decr_every_n_nan_or_inf=2,
+    incr_ratio=2.0,
+    decr_ratio=0.8,
+    dist_strategy=None,
+    use_lamb=False,
+):
+    if warmup_steps > 0:
+        if scheduler == "noam_decay":
+            scheduled_lr = fluid.layers.learning_rate_scheduler.noam_decay(
+                1 / (warmup_steps * (learning_rate**2)), warmup_steps
+            )
+        elif scheduler == "linear_warmup_decay":
+            scheduled_lr = linear_warmup_decay(learning_rate, warmup_steps, num_train_steps)
+        else:
+            raise ValueError("Unknown learning rate scheduler, should be " "'noam_decay' or 'linear_warmup_decay'")
+        if use_lamb:
+            optimizer = fluid.optimizer.LambOptimizer(learning_rate=scheduled_lr)
+        else:
+            optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr)
+    else:
+        scheduled_lr = fluid.layers.create_global_var(
+            name=fluid.unique_name.generate("learning_rate"),
+            shape=[1],
+            value=learning_rate,
+            dtype="float32",
+            persistable=True,
+        )
+        if use_lamb:
+            optimizer = fluid.optimizer.LambOptimizer(learning_rate=scheduled_lr)
+        else:
+            optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr)
+        optimizer._learning_rate_map[fluid.default_main_program()] = scheduled_lr
+
+    fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0))
+
+    def exclude_from_weight_decay(name):
+        if name.find("layer_norm") > -1:
+            return True
+        bias_suffix = ["_bias", "_b", ".b_0"]
+        for suffix in bias_suffix:
+            if name.endswith(suffix):
+                return True
+        return False
+
+    param_list = dict()
+
+    for param in train_program.global_block().all_parameters():
+        param_list[param.name] = param * 1.0
+        param_list[param.name].stop_gradient = True
+
+    if dist_strategy is not None:
+        # use fleet api
+        optimizer = fleet.distributed_optimizer(optimizer, strategy=dist_strategy)
+
+    _, param_grads = optimizer.minimize(loss)
+
+    if weight_decay > 0:
+        for param, grad in param_grads:
+            if exclude_from_weight_decay(param.name):
+                continue
+            with param.block.program._optimized_guard([param, grad]), fluid.framework.name_scope("weight_decay"):
+                updated_param = param - param_list[param.name] * weight_decay * scheduled_lr
+                fluid.layers.assign(output=param, input=updated_param)
+
+    return scheduled_lr

applications/document_intelligence/doc_vqa/Rerank/src/reader_ce.py

@@ -0,0 +1,314 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import sys
+from collections import namedtuple
+from io import open
+
+import numpy as np
+import six
+import tokenization
+from batching import pad_batch_data
+
+log = logging.getLogger(__name__)
+
+if six.PY3:
+    import io
+
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
+    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8")
+
+
+def csv_reader(fd, delimiter="\t", trainer_id=0, trainer_num=1):
+    def gen():
+        for i, line in enumerate(fd):
+            if i % trainer_num == trainer_id:
+                slots = line.rstrip("\n").split(delimiter)
+                if len(slots) == 1:
+                    yield slots,
+                else:
+                    yield slots
+
+    return gen()
+
+
+class BaseReader(object):
+    def __init__(
+        self,
+        vocab_path,
+        label_map_config=None,
+        max_seq_len=512,
+        total_num=0,
+        do_lower_case=True,
+        in_tokens=False,
+        is_inference=False,
+        random_seed=None,
+        tokenizer="FullTokenizer",
+        for_cn=True,
+        task_id=0,
+    ):
+        self.max_seq_len = max_seq_len
+        self.tokenizer = tokenization.FullTokenizer(vocab_file=vocab_path, do_lower_case=do_lower_case)
+        self.vocab = self.tokenizer.vocab
+        self.pad_id = self.vocab["[PAD]"]
+        self.cls_id = self.vocab["[CLS]"]
+        self.sep_id = self.vocab["[SEP]"]
+        self.in_tokens = in_tokens
+        self.is_inference = is_inference
+        self.for_cn = for_cn
+        self.task_id = task_id
+
+        np.random.seed(random_seed)
+
+        self.current_example = 0
+        self.current_epoch = 0
+        self.num_examples = 0
+        self.total_num = total_num
+
+        if label_map_config:
+            with open(label_map_config, encoding="utf8") as f:
+                self.label_map = json.load(f)
+        else:
+            self.label_map = None
+
+    def get_train_progress(self):
+        """Gets progress for training phase."""
+        return self.current_example, self.current_epoch
+
+    def _read_tsv(self, input_file, quotechar=None):
+        """Reads a tab separated value file."""
+        with open(input_file, "r", encoding="utf8") as f:
+            reader = csv_reader(f)
+            headers = next(reader)
+            Example = namedtuple("Example", headers)
+
+            examples = []
+            for line in reader:
+                example = Example(*line)
+                examples.append(example)
+            return examples
+
+    def _truncate_seq_pair(self, tokens_a, tokens_b, max_length):
+        """Truncates a sequence pair in place to the maximum length."""
+
+        # This is a simple heuristic which will always truncate the longer sequence
+        # one token at a time. This makes more sense than truncating an equal percent
+        # of tokens from each, since if one sequence is very short then each token
+        # that's truncated likely contains more information than a longer sequence.
+        while True:
+            total_length = len(tokens_a) + len(tokens_b)
+            if total_length <= max_length:
+                break
+            if len(tokens_a) > len(tokens_b):
+                tokens_a.pop()
+            else:
+                tokens_b.pop()
+
+    def _convert_example_to_record(self, example, max_seq_length, tokenizer):
+        """Converts a single `Example` into a single `Record`."""
+
+        query = tokenization.convert_to_unicode(example.query)
+        tokens_a = tokenizer.tokenize(query)
+        tokens_b = None
+
+        title = tokenization.convert_to_unicode(example.title)
+        tokens_b = tokenizer.tokenize(title)
+
+        para = tokenization.convert_to_unicode(example.para)
+        tokens_para = tokenizer.tokenize(para)
+
+        tokens_b.extend(tokens_para)
+
+        self._truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
+
+        # The convention in BERT/ERNIE is:
+        # (a) For sequence pairs:
+        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
+        #  type_ids: 0     0  0    0    0     0       0 0     1  1  1  1   1 1
+        # (b) For single sequences:
+        #  tokens:   [CLS] the dog is hairy . [SEP]
+        #  type_ids: 0     0   0   0  0     0 0
+        #
+        # Where "type_ids" are used to indicate whether this is the first
+        # sequence or the second sequence. The embedding vectors for `type=0` and
+        # `type=1` were learned during pre-training and are added to the wordpiece
+        # embedding vector (and position vector). This is not *strictly* necessary
+        # since the [SEP] token unambiguously separates the sequences, but it makes
+        # it easier for the model to learn the concept of sequences.
+        #
+        # For classification tasks, the first vector (corresponding to [CLS]) is
+        # used as the "sentence vector". Note that this only makes sense because
+        # the entire model is fine-tuned.
+        tokens = []
+        text_type_ids = []
+        tokens.append("[CLS]")
+        text_type_ids.append(0)
+        for token in tokens_a:
+            tokens.append(token)
+            text_type_ids.append(0)
+        tokens.append("[SEP]")
+        text_type_ids.append(0)
+
+        if tokens_b:
+            for token in tokens_b:
+                tokens.append(token)
+                text_type_ids.append(1)
+            tokens.append("[SEP]")
+            text_type_ids.append(1)
+
+        token_ids = tokenizer.convert_tokens_to_ids(tokens)
+        position_ids = list(range(len(token_ids)))
+
+        if self.is_inference:
+            Record = namedtuple("Record", ["token_ids", "text_type_ids", "position_ids"])
+            record = Record(token_ids=token_ids, text_type_ids=text_type_ids, position_ids=position_ids)
+        else:
+            if self.label_map:
+                label_id = self.label_map[example.label]
+            else:
+                label_id = example.label
+
+            Record = namedtuple("Record", ["token_ids", "text_type_ids", "position_ids", "label_id", "qid"])
+
+            qid = None
+            if "qid" in example._fields:
+                qid = example.qid
+
+            record = Record(
+                token_ids=token_ids, text_type_ids=text_type_ids, position_ids=position_ids, label_id=label_id, qid=qid
+            )
+        return record
+
+    def _prepare_batch_data(self, examples, batch_size, phase=None):
+        """generate batch records"""
+        batch_records, max_len = [], 0
+        for index, example in enumerate(examples):
+            if phase == "train":
+                self.current_example = index
+            record = self._convert_example_to_record(example, self.max_seq_len, self.tokenizer)
+            max_len = max(max_len, len(record.token_ids))
+            if self.in_tokens:
+                to_append = (len(batch_records) + 1) * max_len <= batch_size
+            else:
+                to_append = len(batch_records) < batch_size
+            if to_append:
+                batch_records.append(record)
+            else:
+                yield self._pad_batch_records(batch_records)
+                batch_records, max_len = [record], len(record.token_ids)
+
+        if batch_records:
+            yield self._pad_batch_records(batch_records)
+
+    def get_num_examples(self, input_file):
+        #        examples = self._read_tsv(input_file)
+        #        return len(examples)
+        return self.num_examples
+
+    def data_generator(
+        self, input_file, batch_size, epoch, dev_count=1, trainer_id=0, trainer_num=1, shuffle=True, phase=None
+    ):
+
+        if phase == "train":
+            #            examples = examples[trainer_id: (len(examples) //trainer_num) * trainer_num : trainer_num]
+            self.num_examples_per_node = self.total_num // trainer_num
+            self.num_examples = self.num_examples_per_node * trainer_num
+            examples = self._read_tsv(
+                input_file, trainer_id=trainer_id, trainer_num=trainer_num, num_examples=self.num_examples_per_node
+            )
+            log.info("apply sharding %d/%d" % (trainer_id, trainer_num))
+        else:
+            examples = self._read_tsv(input_file)
+
+        def wrapper():
+            all_dev_batches = []
+            for epoch_index in range(epoch):
+                if phase == "train":
+                    self.current_example = 0
+                    self.current_epoch = epoch_index
+                if shuffle:
+                    np.random.shuffle(examples)
+
+                for batch_data in self._prepare_batch_data(examples, batch_size, phase=phase):
+                    if len(all_dev_batches) < dev_count:
+                        all_dev_batches.append(batch_data)
+                    if len(all_dev_batches) == dev_count:
+                        for batch in all_dev_batches:
+                            yield batch
+                        all_dev_batches = []
+
+        def f():
+            try:
+                for i in wrapper():
+                    yield i
+            except Exception:
+                import traceback
+
+                traceback.print_exc()
+
+        return f
+
+
+class ClassifyReader(BaseReader):
+    def _read_tsv(self, input_file, quotechar=None, trainer_id=0, trainer_num=1, num_examples=0):
+        """Reads a tab separated value file."""
+        with open(input_file, "r", encoding="utf8") as f:
+            reader = csv_reader(f, trainer_id=trainer_id, trainer_num=trainer_num)
+            #            headers = next(reader)
+            headers = "query\ttitle\tpara\tlabel".split("\t")
+            text_indices = [index for index, h in enumerate(headers) if h != "label"]
+            Example = namedtuple("Example", headers)
+
+            examples = []
+            for cnt, line in enumerate(reader):
+                if num_examples != 0 and cnt == num_examples:
+                    break
+                for index, text in enumerate(line):
+                    if index in text_indices:
+                        if self.for_cn:
+                            line[index] = text.replace(" ", "")
+                        else:
+                            line[index] = text
+                example = Example(*line)
+                examples.append(example)
+            return examples
+
+    def _pad_batch_records(self, batch_records):
+        batch_token_ids = [record.token_ids for record in batch_records]
+        batch_text_type_ids = [record.text_type_ids for record in batch_records]
+        batch_position_ids = [record.position_ids for record in batch_records]
+
+        if not self.is_inference:
+            batch_labels = [record.label_id for record in batch_records]
+            batch_labels = np.array(batch_labels).astype("int64").reshape([-1, 1])
+
+            if batch_records[0].qid:
+                batch_qids = [record.qid for record in batch_records]
+                batch_qids = np.array(batch_qids).astype("int64").reshape([-1, 1])
+            else:
+                batch_qids = np.array([]).astype("int64").reshape([-1, 1])
+
+        # padding
+        padded_token_ids, input_mask = pad_batch_data(batch_token_ids, pad_idx=self.pad_id, return_input_mask=True)
+        padded_text_type_ids = pad_batch_data(batch_text_type_ids, pad_idx=self.pad_id)
+        padded_position_ids = pad_batch_data(batch_position_ids, pad_idx=self.pad_id)
+        padded_task_ids = np.ones_like(padded_token_ids, dtype="int64") * self.task_id
+
+        return_list = [padded_token_ids, padded_text_type_ids, padded_position_ids, padded_task_ids, input_mask]
+        if not self.is_inference:
+            return_list += [batch_labels, batch_qids]
+
+        return return_list

applications/document_intelligence/doc_vqa/Rerank/src/tokenization.py

@@ -0,0 +1,400 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+import collections
+import unicodedata
+from io import open
+
+
+def convert_to_unicode(text):
+    """Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, bytes):
+        return text.decode("utf-8", "ignore")
+    else:
+        raise ValueError("Unsupported string type: %s" % (type(text)))
+
+
+def printable_text(text):
+    """Returns text encoded in a way suitable for print or `tf.logging`."""
+    if isinstance(text, str):
+        return text
+    elif isinstance(text, bytes):
+        return text.decode("utf-8", "ignore")
+    else:
+        raise ValueError("Unsupported string type: %s" % (type(text)))
+
+
+def load_vocab(vocab_file):
+    """Loads a vocabulary file into a dictionary."""
+    vocab = collections.OrderedDict()
+    with open(vocab_file, encoding="utf8") as fin:
+        for num, line in enumerate(fin):
+            items = convert_to_unicode(line.strip()).split("\t")
+            if len(items) > 2:
+                break
+            token = items[0]
+            index = items[1] if len(items) == 2 else num
+            token = token.strip()
+            vocab[token] = int(index)
+    return vocab
+
+
+def convert_by_vocab(vocab, items):
+    """Converts a sequence of [tokens|ids] using the vocab."""
+    output = []
+    for item in items:
+        output.append(vocab[item])
+    return output
+
+
+def convert_tokens_to_ids(vocab, tokens):
+    return convert_by_vocab(vocab, tokens)
+
+
+def convert_ids_to_tokens(inv_vocab, ids):
+    return convert_by_vocab(inv_vocab, ids)
+
+
+def whitespace_tokenize(text):
+    """Runs basic whitespace cleaning and splitting on a piece of text."""
+    text = text.strip()
+    if not text:
+        return []
+    tokens = text.split()
+    return tokens
+
+
+class FullTokenizer(object):
+    """Runs end-to-end tokenization."""
+
+    def __init__(self, vocab_file, do_lower_case=True):
+        self.vocab = load_vocab(vocab_file)
+        self.inv_vocab = {v: k for k, v in self.vocab.items()}
+        self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)
+        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+
+    def tokenize(self, text):
+        split_tokens = []
+        for token in self.basic_tokenizer.tokenize(text):
+            for sub_token in self.wordpiece_tokenizer.tokenize(token):
+                split_tokens.append(sub_token)
+
+        return split_tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return convert_by_vocab(self.vocab, tokens)
+
+    def convert_ids_to_tokens(self, ids):
+        return convert_by_vocab(self.inv_vocab, ids)
+
+
+class CharTokenizer(object):
+    """Runs end-to-end tokenization."""
+
+    def __init__(self, vocab_file, do_lower_case=True):
+        self.vocab = load_vocab(vocab_file)
+        self.inv_vocab = {v: k for k, v in self.vocab.items()}
+        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+
+    def tokenize(self, text):
+        split_tokens = []
+        for token in text.lower().split(" "):
+            for sub_token in self.wordpiece_tokenizer.tokenize(token):
+                split_tokens.append(sub_token)
+
+        return split_tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return convert_by_vocab(self.vocab, tokens)
+
+    def convert_ids_to_tokens(self, ids):
+        return convert_by_vocab(self.inv_vocab, ids)
+
+
+class BasicTokenizer(object):
+    """Runs basic tokenization (punctuation splitting, lower casing, etc.)."""
+
+    def __init__(self, do_lower_case=True):
+        """Constructs a BasicTokenizer.
+
+        Args:
+            do_lower_case: Whether to lower case the input.
+        """
+        self.do_lower_case = do_lower_case
+
+    def tokenize(self, text):
+        """Tokenizes a piece of text."""
+        text = convert_to_unicode(text)
+        text = self._clean_text(text)
+
+        # This was added on November 1st, 2018 for the multilingual and Chinese
+        # models. This is also applied to the English models now, but it doesn't
+        # matter since the English models were not trained on any Chinese data
+        # and generally don't have any Chinese data in them (there are Chinese
+        # characters in the vocabulary because Wikipedia does have some Chinese
+        # words in the English Wikipedia.).
+        text = self._tokenize_chinese_chars(text)
+
+        orig_tokens = whitespace_tokenize(text)
+        split_tokens = []
+        for token in orig_tokens:
+            if self.do_lower_case:
+                token = token.lower()
+                token = self._run_strip_accents(token)
+            split_tokens.extend(self._run_split_on_punc(token))
+
+        output_tokens = whitespace_tokenize(" ".join(split_tokens))
+        return output_tokens
+
+    def _run_strip_accents(self, text):
+        """Strips accents from a piece of text."""
+        text = unicodedata.normalize("NFD", text)
+        output = []
+        for char in text:
+            cat = unicodedata.category(char)
+            if cat == "Mn":
+                continue
+            output.append(char)
+        return "".join(output)
+
+    def _run_split_on_punc(self, text):
+        """Splits punctuation on a piece of text."""
+        chars = list(text)
+        i = 0
+        start_new_word = True
+        output = []
+        while i < len(chars):
+            char = chars[i]
+            if _is_punctuation(char):
+                output.append([char])
+                start_new_word = True
+            else:
+                if start_new_word:
+                    output.append([])
+                start_new_word = False
+                output[-1].append(char)
+            i += 1
+
+        return ["".join(x) for x in output]
+
+    def _tokenize_chinese_chars(self, text):
+        """Adds whitespace around any CJK character."""
+        output = []
+        for char in text:
+            cp = ord(char)
+            if self._is_chinese_char(cp):
+                output.append(" ")
+                output.append(char)
+                output.append(" ")
+            else:
+                output.append(char)
+        return "".join(output)
+
+    def _is_chinese_char(self, cp):
+        """Checks whether CP is the codepoint of a CJK character."""
+        # This defines a "chinese character" as anything in the CJK Unicode block:
+        #     https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+        #
+        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+        # despite its name. The modern Korean Hangul alphabet is a different block,
+        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+        # space-separated words, so they are not treated specially and handled
+        # like the all of the other languages.
+        if (
+            (cp >= 0x4E00 and cp <= 0x9FFF)
+            or (cp >= 0x3400 and cp <= 0x4DBF)  #
+            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
+            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
+            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
+            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
+            or (cp >= 0xF900 and cp <= 0xFAFF)
+            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
+        ):  #
+            return True
+
+        return False
+
+    def _clean_text(self, text):
+        """Performs invalid character removal and whitespace cleanup on text."""
+        output = []
+        for char in text:
+            cp = ord(char)
+            if cp == 0 or cp == 0xFFFD or _is_control(char):
+                continue
+            if _is_whitespace(char):
+                output.append(" ")
+            else:
+                output.append(char)
+        return "".join(output)
+
+
+class WordpieceTokenizer(object):
+    """Runs WordPiece tokenization."""
+
+    def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100):
+        self.vocab = vocab
+        self.unk_token = unk_token
+        self.max_input_chars_per_word = max_input_chars_per_word
+
+    def tokenize(self, text):
+        """Tokenizes a piece of text into its word pieces.
+
+        This uses a greedy longest-match-first algorithm to perform tokenization
+        using the given vocabulary.
+
+        For example:
+            input = "unaffable"
+            output = ["un", "##aff", "##able"]
+
+        Args:
+            text: A single token or whitespace separated tokens. This should have
+                already been passed through `BasicTokenizer.
+
+        Returns:
+            A list of wordpiece tokens.
+        """
+
+        text = convert_to_unicode(text)
+
+        output_tokens = []
+        for token in whitespace_tokenize(text):
+            chars = list(token)
+            if len(chars) > self.max_input_chars_per_word:
+                output_tokens.append(self.unk_token)
+                continue
+
+            is_bad = False
+            start = 0
+            sub_tokens = []
+            while start < len(chars):
+                end = len(chars)
+                cur_substr = None
+                while start < end:
+                    substr = "".join(chars[start:end])
+                    if start > 0:
+                        substr = "##" + substr
+                    if substr in self.vocab:
+                        cur_substr = substr
+                        break
+                    end -= 1
+                if cur_substr is None:
+                    is_bad = True
+                    break
+                sub_tokens.append(cur_substr)
+                start = end
+
+            if is_bad:
+                output_tokens.append(self.unk_token)
+            else:
+                output_tokens.extend(sub_tokens)
+        return output_tokens
+
+
+def _is_whitespace(char):
+    """Checks whether `chars` is a whitespace character."""
+    # \t, \n, and \r are technically control characters but we treat them
+    # as whitespace since they are generally considered as such.
+    if char == " " or char == "\t" or char == "\n" or char == "\r":
+        return True
+    cat = unicodedata.category(char)
+    if cat == "Zs":
+        return True
+    return False
+
+
+def _is_control(char):
+    """Checks whether `chars` is a control character."""
+    # These are technically control characters but we count them as whitespace
+    # characters.
+    if char == "\t" or char == "\n" or char == "\r":
+        return False
+    cat = unicodedata.category(char)
+    if cat.startswith("C"):
+        return True
+    return False
+
+
+def _is_punctuation(char):
+    """Checks whether `chars` is a punctuation character."""
+    cp = ord(char)
+    # We treat all non-letter/number ASCII as punctuation.
+    # Characters such as "^", "$", and "`" are not in the Unicode
+    # Punctuation class but we treat them as punctuation anyways, for
+    # consistency.
+    if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):
+        return True
+    cat = unicodedata.category(char)
+    if cat.startswith("P"):
+        return True
+    return False
+
+
+def tokenize_chinese_chars(text):
+    """Adds whitespace around any CJK character."""
+
+    def _is_chinese_char(cp):
+        """Checks whether CP is the codepoint of a CJK character."""
+        # This defines a "chinese character" as anything in the CJK Unicode block:
+        #     https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+        #
+        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+        # despite its name. The modern Korean Hangul alphabet is a different block,
+        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+        # space-separated words, so they are not treated specially and handled
+        # like the all of the other languages.
+        if (
+            (cp >= 0x4E00 and cp <= 0x9FFF)
+            or (cp >= 0x3400 and cp <= 0x4DBF)  #
+            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
+            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
+            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
+            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
+            or (cp >= 0xF900 and cp <= 0xFAFF)
+            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
+        ):  #
+            return True
+
+        return False
+
+    def _is_whitespace(c):
+        if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
+            return True
+        return False
+
+    output = []
+    buff = ""
+    for char in text:
+        cp = ord(char)
+        if _is_chinese_char(cp) or _is_whitespace(char):
+            if buff != "":
+                output.append(buff)
+                buff = ""
+            output.append(char)
+        else:
+            buff += char
+
+    if buff != "":
+        output.append(buff)
+
+    return output

applications/document_intelligence/doc_vqa/Rerank/src/train_ce.py

@@ -0,0 +1,354 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Finetuning on classification tasks."""
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import logging
+import multiprocessing
+import os
+import time
+import warnings
+
+# NOTE(paddle-dev): All of these flags should be
+# set before `import paddle`. Otherwise, it would
+# not take any effect.
+os.environ["FLAGS_eager_delete_tensor_gb"] = "0"  # enable gc
+
+import paddle  # noqa: E402
+import paddle.fluid as fluid  # noqa: E402
+
+if hasattr(paddle, "enable_static"):
+    paddle.enable_static()
+import paddle.fluid.incubate.fleet.base.role_maker as role_maker  # noqa: E402
+import reader_ce as reader_ce  # noqa: E402
+from cross_encoder import create_model, evaluate, predict  # noqa: E402
+from finetune_args import parser  # noqa: E402
+from model.ernie import ErnieConfig  # noqa: E402
+from optimization import optimization  # noqa: E402
+from paddle.fluid.incubate.fleet.collective import (  # noqa: E402
+    DistributedStrategy,
+    fleet,
+)
+from src.utils.args import check_cuda, prepare_logger, print_arguments  # noqa: E402
+from src.utils.init import init_checkpoint, init_pretraining_params  # noqa: E402
+
+warnings.filterwarnings("ignore")
+args = parser.parse_args()
+log = logging.getLogger()
+
+
+def main(args):
+    ernie_config = ErnieConfig(args.ernie_config_path)
+    ernie_config.print_config()
+
+    if args.use_cuda:
+        dev_list = fluid.cuda_places()
+        place = dev_list[0]
+        dev_count = len(dev_list)
+    else:
+        place = fluid.CPUPlace()
+        dev_count = int(os.environ.get("CPU_NUM", multiprocessing.cpu_count()))
+    exe = fluid.Executor(place)
+
+    reader = reader_ce.ClassifyReader(
+        vocab_path=args.vocab_path,
+        label_map_config=args.label_map_config,
+        max_seq_len=args.max_seq_len,
+        total_num=args.train_data_size,
+        do_lower_case=args.do_lower_case,
+        in_tokens=args.in_tokens,
+        random_seed=args.random_seed,
+        tokenizer=args.tokenizer,
+        for_cn=args.for_cn,
+        task_id=args.task_id,
+    )
+
+    if not (args.do_train or args.do_val or args.do_test):
+        raise ValueError("For args `do_train`, `do_val` and `do_test`, at " "least one of them must be True.")
+
+    if args.do_test:
+        assert args.test_save is not None
+    startup_prog = fluid.Program()
+    if args.random_seed is not None:
+        startup_prog.random_seed = args.random_seed
+
+    if args.predict_batch_size is None:
+        args.predict_batch_size = args.batch_size
+
+    if args.do_train:
+        role = role_maker.PaddleCloudRoleMaker(is_collective=True)
+        fleet.init(role)
+        dev_count = fleet.worker_num()
+
+        train_data_generator = reader.data_generator(
+            input_file=args.train_set,
+            batch_size=args.batch_size,
+            epoch=args.epoch,
+            dev_count=1,
+            trainer_id=fleet.worker_index(),
+            trainer_num=fleet.worker_num(),
+            shuffle=True,
+            phase="train",
+        )
+
+        num_train_examples = reader.get_num_examples(args.train_set)
+
+        if args.in_tokens:
+            max_train_steps = args.epoch * num_train_examples // (args.batch_size // args.max_seq_len) // dev_count
+        else:
+            max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count
+
+        warmup_steps = int(max_train_steps * args.warmup_proportion)
+        log.info("Device count: %d" % dev_count)
+        log.info("Num train examples: %d" % num_train_examples)
+        log.info("Max train steps: %d" % max_train_steps)
+        log.info("Num warmup steps: %d" % warmup_steps)
+
+        train_program = fluid.Program()
+
+        # use fleet api
+        exec_strategy = fluid.ExecutionStrategy()
+        if args.use_fast_executor:
+            exec_strategy.use_experimental_executor = True
+        exec_strategy.num_threads = dev_count
+        if args.is_distributed:
+            exec_strategy.num_threads = 3
+
+        exec_strategy.num_iteration_per_drop_scope = args.num_iteration_per_drop_scope
+
+        dist_strategy = DistributedStrategy()
+        dist_strategy.exec_strategy = exec_strategy
+        dist_strategy.nccl_comm_num = 1
+        if args.is_distributed:
+            dist_strategy.nccl_comm_num = 2
+        dist_strategy.use_hierarchical_allreduce = True
+
+        if args.use_mix_precision:
+            dist_strategy.use_amp = True
+
+        with fluid.program_guard(train_program, startup_prog):
+            with fluid.unique_name.guard():
+                train_pyreader, graph_vars = create_model(
+                    args, pyreader_name="train_reader", ernie_config=ernie_config
+                )
+                scheduled_lr = optimization(
+                    loss=graph_vars["loss"],
+                    warmup_steps=warmup_steps,
+                    num_train_steps=max_train_steps,
+                    learning_rate=args.learning_rate,
+                    train_program=train_program,
+                    startup_prog=startup_prog,
+                    weight_decay=args.weight_decay,
+                    scheduler=args.lr_scheduler,
+                    use_dynamic_loss_scaling=args.use_dynamic_loss_scaling,
+                    incr_every_n_steps=args.incr_every_n_steps,
+                    decr_every_n_nan_or_inf=args.decr_every_n_nan_or_inf,
+                    incr_ratio=args.incr_ratio,
+                    decr_ratio=args.decr_ratio,
+                    dist_strategy=dist_strategy,
+                )
+
+        if args.verbose:
+            if args.in_tokens:
+                lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
+                    program=train_program, batch_size=args.batch_size // args.max_seq_len
+                )
+            else:
+                lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
+                    program=train_program, batch_size=args.batch_size
+                )
+            log.info("Theoretical memory usage in training: %.3f - %.3f %s" % (lower_mem, upper_mem, unit))
+
+    if args.do_val or args.do_test:
+        test_prog = fluid.Program()
+        with fluid.program_guard(test_prog, startup_prog):
+            with fluid.unique_name.guard():
+                test_pyreader, graph_vars = create_model(
+                    args, pyreader_name="test_reader", ernie_config=ernie_config, is_prediction=True
+                )
+
+        test_prog = test_prog.clone(for_test=True)
+
+    train_program = fleet.main_program
+
+    exe = fluid.Executor(place)
+    exe.run(startup_prog)
+
+    if args.do_train:
+        if args.init_checkpoint and args.init_pretraining_params:
+            log.warning(
+                "WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
+                "both are set! Only arg 'init_checkpoint' is made valid."
+            )
+        if args.init_checkpoint:
+            init_checkpoint(exe, args.init_checkpoint, main_program=startup_prog)
+        elif args.init_pretraining_params:
+            init_pretraining_params(exe, args.init_pretraining_params, main_program=startup_prog)
+    elif args.do_val or args.do_test:
+        if not args.init_checkpoint:
+            raise ValueError("args 'init_checkpoint' should be set if" "only doing validation or testing!")
+        init_checkpoint(exe, args.init_checkpoint, main_program=startup_prog)
+
+    if args.do_train:
+        train_exe = exe
+        train_pyreader.decorate_tensor_provider(train_data_generator)
+    else:
+        train_exe = None
+
+    test_exe = exe
+
+    current_epoch = 0
+    steps = 0
+    if args.do_train:
+        train_pyreader.start()
+        if warmup_steps > 0:
+            graph_vars["learning_rate"] = scheduled_lr
+
+        ce_info = []
+        time_begin = time.time()
+        last_epoch = 0
+        while True:
+            try:
+                steps += 1
+
+                if fleet.worker_index() != 0:
+                    train_exe.run(fetch_list=[], program=train_program)
+                    continue
+
+                if steps % args.skip_steps != 0:
+                    train_exe.run(fetch_list=[], program=train_program)
+
+                else:
+                    outputs = evaluate(
+                        train_exe, train_program, train_pyreader, graph_vars, "train", metric=args.metric
+                    )
+
+                    if args.verbose:
+                        verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size()
+                        verbose += "learning rate: %f" % (
+                            outputs["learning_rate"] if warmup_steps > 0 else args.learning_rate
+                        )
+                        log.info(verbose)
+
+                    current_example, current_epoch = reader.get_train_progress()
+                    time_end = time.time()
+                    used_time = time_end - time_begin
+
+                    log.info(
+                        "epoch: %d, progress: %d/%d, step: %d, ave loss: %f, "
+                        "ave acc: %f, speed: %f steps/s"
+                        % (
+                            current_epoch,
+                            current_example * dev_count,
+                            num_train_examples,
+                            steps,
+                            outputs["loss"],
+                            outputs["accuracy"],
+                            args.skip_steps / used_time,
+                        )
+                    )
+                    ce_info.append([outputs["loss"], outputs["accuracy"], used_time])
+
+                    time_begin = time.time()
+
+                if steps % args.save_steps == 0:
+                    save_path = os.path.join(args.checkpoints, "step_" + str(steps))
+                    fluid.io.save_persistables(exe, save_path, fleet._origin_program)
+
+                if steps % args.validation_steps == 0:
+                    # evaluate dev set
+                    if args.do_val:
+                        evaluate_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, current_epoch, steps)
+
+                    if args.do_test:
+                        predict_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, current_epoch, steps)
+
+                if last_epoch != current_epoch:
+                    last_epoch = current_epoch
+
+            except fluid.core.EOFException:
+                save_path = os.path.join(args.checkpoints, "step_" + str(steps))
+                fluid.io.save_persistables(exe, save_path, fleet._origin_program)
+                train_pyreader.reset()
+                break
+
+    # final eval on dev set
+    if args.do_val:
+        evaluate_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, current_epoch, steps)
+
+    # final eval on test set
+    if args.do_test:
+        predict_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars)
+
+    # final eval on diagnostic, hack for glue-ax
+    if args.diagnostic:
+        test_pyreader.decorate_tensor_provider(
+            reader.data_generator(args.diagnostic, batch_size=args.batch_size, epoch=1, dev_count=1, shuffle=False)
+        )
+
+        log.info("Final diagnostic")
+        qids, preds, probs = predict(test_exe, test_prog, test_pyreader, graph_vars)
+        assert len(qids) == len(preds), "{} v.s. {}".format(len(qids), len(preds))
+        with open(args.diagnostic_save, "w") as f:
+            for id, s, p in zip(qids, preds, probs):
+                f.write("{}\t{}\t{}\n".format(id, s, p))
+
+        log.info("Done final diagnostic, saving to {}".format(args.diagnostic_save))
+
+
+def evaluate_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, epoch, steps):
+    # evaluate dev set
+    for ds in args.dev_set.split(","):
+        test_pyreader.decorate_tensor_provider(
+            reader.data_generator(ds, batch_size=args.predict_batch_size, epoch=1, dev_count=1, shuffle=False)
+        )
+        log.info("validation result of dataset {}:".format(ds))
+        evaluate_info = evaluate(exe, test_prog, test_pyreader, graph_vars, "dev", metric=args.metric)
+        log.info(evaluate_info + ", file: {}, epoch: {}, steps: {}".format(ds, epoch, steps))
+
+
+def predict_wrapper(args, reader, exe, test_prog, test_pyreader, graph_vars, epoch=None, steps=None):
+    test_sets = args.test_set.split(",")
+    save_dirs = args.test_save.split(",")
+    assert len(test_sets) == len(save_dirs)
+
+    for test_f, save_f in zip(test_sets, save_dirs):
+        test_pyreader.decorate_tensor_provider(
+            reader.data_generator(test_f, batch_size=args.predict_batch_size, epoch=1, dev_count=1, shuffle=False)
+        )
+
+        if epoch is not None or steps is not None:
+            save_path = save_f + "." + str(epoch) + "." + str(steps)
+        else:
+            save_path = save_f
+        log.info("testing {}, save to {}".format(test_f, save_path))
+        qids, preds, probs = predict(exe, test_prog, test_pyreader, graph_vars)
+
+        save_dir = os.path.dirname(save_path)
+        if not os.path.exists(save_dir):
+            os.makedirs(save_dir)
+        else:
+            log.warning("save dir exists: %s, will skip saving" % save_dir)
+
+        with open(save_path, "w") as f:
+            for p in probs:
+                f.write("{}\n".format(p[1]))
+
+
+if __name__ == "__main__":
+    prepare_logger(log)
+    print_arguments(args)
+    check_cuda(args.use_cuda)
+    main(args)

applications/document_intelligence/doc_vqa/Rerank/src/utils/args.py

@@ -0,0 +1,71 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Arguments for configuration."""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import logging
+import os
+import sys
+
+import paddle.fluid as fluid
+import six
+
+from paddlenlp.trainer.argparser import strtobool
+
+log = logging.getLogger(__name__)
+
+
+def prepare_logger(logger, debug=False, save_to_file=None):
+    formatter = logging.Formatter(fmt="[%(levelname)s] %(asctime)s [%(filename)12s:%(lineno)5d]:\t%(message)s")
+    console_hdl = logging.StreamHandler()
+    console_hdl.setFormatter(formatter)
+    logger.addHandler(console_hdl)
+    if save_to_file is not None and not os.path.exists(save_to_file):
+        file_hdl = logging.FileHandler(save_to_file)
+        file_hdl.setFormatter(formatter)
+        logger.addHandler(file_hdl)
+    logger.setLevel(logging.DEBUG)
+    logger.propagate = False
+
+
+class ArgumentGroup(object):
+    def __init__(self, parser, title, des):
+        self._group = parser.add_argument_group(title=title, description=des)
+
+    def add_arg(self, name, type, default, help, positional_arg=False, **kwargs):
+        prefix = "" if positional_arg else "--"
+        type = strtobool if type == bool else type
+        self._group.add_argument(
+            prefix + name, default=default, type=type, help=help + " Default: %(default)s.", **kwargs
+        )
+
+
+def print_arguments(args):
+    log.info("-----------  Configuration Arguments -----------")
+    for arg, value in sorted(six.iteritems(vars(args))):
+        log.info("%s: %s" % (arg, value))
+    log.info("------------------------------------------------")
+
+
+def check_cuda(
+    use_cuda,
+    err="\nYou can not set use_cuda = True in the model because you are using paddlepaddle-cpu.\n \
+    Please: 1. Install paddlepaddle-gpu to run your models on GPU or 2. Set use_cuda = False to run models on CPU.\n",
+):
+    try:
+        if use_cuda is True and fluid.is_compiled_with_cuda() is False:
+            log.error(err)
+            sys.exit(1)
+    except Exception:
+        pass

applications/document_intelligence/doc_vqa/Rerank/src/utils/init.py

@@ -0,0 +1,52 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+import paddle.fluid as fluid
+
+log = logging.getLogger(__name__)
+
+
+def init_checkpoint(exe, init_checkpoint_path, main_program):
+    assert os.path.exists(init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path
+
+    def existed_persitables(var):
+        if not fluid.io.is_persistable(var):
+            return False
+        if not os.path.exists(os.path.join(init_checkpoint_path, var.name)):
+            print("Var not exists: [%s]\t%s" % (var.name, os.path.join(init_checkpoint_path, var.name)))
+        # else:
+        #    print ("Var exists: [%s]" % (var.name))
+        return os.path.exists(os.path.join(init_checkpoint_path, var.name))
+
+    fluid.io.load_vars(exe, init_checkpoint_path, main_program=main_program, predicate=existed_persitables)
+    log.info("Load model from {}".format(init_checkpoint_path))
+
+
+def init_pretraining_params(exe, pretraining_params_path, main_program):
+    assert os.path.exists(pretraining_params_path), "[%s] cann't be found." % pretraining_params_path
+
+    def existed_params(var):
+        if not isinstance(var, fluid.framework.Parameter):
+            return False
+        if not os.path.exists(os.path.join(pretraining_params_path, var.name)):
+            print("Var not exists: [%s]\t%s" % (var.name, os.path.join(pretraining_params_path, var.name)))
+        # else:
+        #    print ("Var exists: [%s]" % (var.name))
+        return os.path.exists(os.path.join(pretraining_params_path, var.name))
+
+    fluid.io.load_vars(exe, pretraining_params_path, main_program=main_program, predicate=existed_params)
+    log.info("Load pretraining parameters from {}.".format(pretraining_params_path))

applications/document_intelligence/doc_vqa/run_test.sh

@@ -0,0 +1,34 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export CUDA_VISIBLE_DEVICES=0
+
+QUESTION=$1
+
+# Question: NFC咋开门
+
+if [ $# != 1 ];then
+    echo "USAGE: sh script/run_cross_encoder_test.sh \$QUESTION"
+    exit 1
+fi
+
+# compute scores for QUESTION and OCR parsing results  with Rerank module
+cd Rerank
+bash run_test.sh ${QUESTION}
+cd ..
+
+# extraction answer for QUESTION from the top1 of rank
+cd Extraction
+bash run_test.sh ${QUESTION}
+cd ..

applications/document_intelligence/docprompt/README.md

@@ -0,0 +1 @@
+[ERNIE-Layout](../../../model_zoo/ernie-layout)

applications/information_extraction/README.md

@@ -0,0 +1,171 @@
+简体中文 | [English](README_en.md)
+
+# 信息抽取应用
+
+**目录**
+- [1. 信息抽取应用简介](#1)
+- [2. 技术特色](#2)
+  - [2.1 信息抽取方案全覆盖](#21)
+  - [2.2 强大的训练基座](#22)
+  - [2.3 产业级全流程方案](#23)
+  - [2.4 效果展示](#24)
+- [3. 快速开始](#快速开始)
+  - [3.1 Taskflow开箱即用](#31)
+  - [3.2 文本信息抽取](#32)
+  - [3.3 文档信息抽取](#33)
+
+<a name="1"></a>
+
+## 1. 信息抽取应用简介
+
+信息抽取应用针对信息抽取一系列高频场景开源了产业级解决方案，**具备多领域、多任务、跨模态的能力**，打通**数据标注-模型训练-模型调优-预测部署全流程**，可快速实现信息抽取产品落地。
+
+信息抽取通俗地说就是从给定的文本/图片等输入数据中抽取出结构化信息的过程。在信息抽取的落地过程中通常面临领域多变、任务多样、数据稀缺等许多挑战。针对信息抽取领域的难点和痛点，PaddleNLP信息抽取应用**基于UIE统一建模的思想**，提供了信息抽取产业级应用方案，**除支持纯文本场景实体、关系、事件、观点等不同任务抽取外，还支持文档/图片/表格的端到端信息抽取**。该应用**不限定行业领域和抽取目标**，可实现从产品原型研发、业务POC阶段到业务落地、迭代阶段的无缝衔接，助力开发者实现特定领域抽取场景的快速适配与落地。
+
+**信息抽取应用亮点：**
+
+- **覆盖场景全面🎓：** 覆盖信息抽取各类主流任务，面向纯文本和文档场景，支持多语言，满足开发者多样信息抽取落地需求。
+- **效果领先🏃：** 以在纯文本、多模态上均有突出效果的UIE系列模型作为训练基座，提供多种尺寸的预训练模型满足不同需求，具有广泛成熟的实践应用性。
+- **简单易用⚡：** 通过Taskflow实现三行代码可实现无标注数据的情况下进行快速调用，一行命令即可开启信息抽取训练，轻松完成部署上线，降低信息抽取技术落地门槛。
+- **高效调优✊：** 开发者无需机器学习背景知识，即可轻松上手数据标注及模型训练流程。
+
+<a name="2"></a>
+
+## 2. 技术特色
+
+<a name="21"></a>
+
+### 2.1 信息抽取方案全覆盖
+
+多模型选择，满足精度、速度，适配不同信息抽取使用场景。
+
+|                           模型名称                           | 使用场景                                                   | 支持任务                                             |
+| :----------------------------------------------------------: | :--------------------------------------------------------- | :--------------------------------------------------- |
+| `uie-base`<br />`uie-medium`<br />`uie-mini`<br />`uie-micro`<br />`uie-nano` | 面向**纯文本**场景的**抽取式**模型，支持**中文**         | 具备实体、关系、事件、评论观点等通用信息抽取能力 |
+|                       `uie-base-en`                          | 面向**纯文本**场景的**抽取式**模型，支持**英文**         | 具备实体、关系、事件、评论观点等通用信息抽取能力 |
+|                     `uie-m-base`<br />`uie-m-large`          | 面向**纯文本**场景的**抽取式**模型，支持**中英**         | 具备实体、关系、事件、评论观点等通用信息抽取能力 |
+|                      <b>`uie-x-base`</b>                     | 面向**纯文本**和**文档**场景的**抽取式**模型，支持**中英** | 支持纯文本场景的全部功能，还支持文档/图片/表格的端到端信息抽取 |
+
+<a name="22"></a>
+
+### 2.2 强大的训练基座
+
+信息抽取应用使用ERNIE 3.0轻量级模型作为预训练模型，同时在大量信息抽取数据上进行了二次预训练，从而让模型适配固定prompt。
+
+- 中文文本数据集实验效果
+
+我们在互联网、医疗、金融三大垂类文本自建测试集上进行了实验：
+
+<table>
+<tr><th row_span='2'><th colspan='2'>金融<th colspan='2'>医疗<th colspan='2'>互联网
+<tr><td><th>0-shot<th>5-shot<th>0-shot<th>5-shot<th>0-shot<th>5-shot
+<tr><td>uie-base (12L768H)<td>46.43<td>70.92<td><b>71.83</b><td>85.72<td>78.33<td>81.86
+<tr><td>uie-medium (6L768H)<td>41.11<td>64.53<td>65.40<td>75.72<td>78.32<td>79.68
+<tr><td>uie-mini (6L384H)<td>37.04<td>64.65<td>60.50<td>78.36<td>72.09<td>76.38
+<tr><td>uie-micro (4L384H)<td>37.53<td>62.11<td>57.04<td>75.92<td>66.00<td>70.22
+<tr><td>uie-nano (4L312H)<td>38.94<td>66.83<td>48.29<td>76.74<td>62.86<td>72.35
+<tr><td>uie-m-large (24L1024H)<td><b>49.35</b><td><b>74.55</b><td>70.50<td><b>92.66</b><td>78.49<td><b>83.02</b>
+<tr><td>uie-m-base (12L768H)<td>38.46<td>74.31<td>63.37<td>87.32<td>76.27<td>80.13
+<tr><td>🧾 🎓<b>uie-x-base (12L768H)</b><td>48.84<td>73.87<td>65.60<td>88.81<td><b>79.36</b><td>81.65
+</table>
+
+0-shot表示无训练数据直接通过```paddlenlp.Taskflow```进行预测，5-shot表示每个类别包含5条标注数据进行模型微调。**实验表明UIE在垂类场景可以通过少量数据（few-shot）进一步提升效果**。
+
+- 多模态数据集实验效果
+
+我们在通用、金融、医疗三大场景自建多模态测试集上对UIE-X的零样本效果进行了实验：
+
+<table>
+<tr><th ><th>通用<th>金融<th colspan='2'>医疗
+<tr><td>🧾 🎓<b>uie-x-base (12L768H)</b><td>65.03<td>73.51<td>84.24
+</table>
+
+通用测试集包含了不同领域的复杂样本，抽取难度最大。
+
+<a name="23"></a>
+
+### 2.3 产业级全流程方案
+
+**调研阶段**
+
+- 该阶段目标需求开放且缺少数据积累。我们提供Taskflow三行代码极简调用的方式，无需标注数据即可在业务场景上快速验证效果。
+  - [文本抽取 Taskflow使用指南](./taskflow_text.md)
+  - [文档抽取 Taskflow使用指南](./taskflow_doc.md)
+
+**数据准备阶段**
+
+- 我们推荐在实际的业务场景中定制自己的信息抽取模型。我们提供了不同抽取场景的Label Studio标注解决方案，可基于该方案实现从数据标注到训练数据构造的无缝衔接，大大降低了数据标注、模型定制的时间成本。
+  - [文本抽取标注指南](./label_studio_text.md)
+  - [文档抽取标注指南](./label_studio_doc.md)。
+
+**模型微调及封闭域蒸馏**
+
+- 基于UIE优秀的小样本微调能力，实现低成本模型定制适配。同时提供封闭域蒸馏的加速方案，解决抽取速度慢的问题。
+  - [文本信息抽取全流程示例](./text/README.md)
+  - [文档信息抽取全流程示例](./document/README.md)
+
+**模型部署**
+
+- 提供HTTP部署方案，快速实现定制模型的部署上线。
+  - [文本抽取HTTP部署指南](./text/deploy/simple_serving/README.md)
+  - [文档抽取HTTP部署指南](./document/deploy/simple_serving/README.md)
+
+<a name="24"></a>
+
+### 2.4 效果展示
+
+- 🧾 通过[Huggingface网页](https://huggingface.co/spaces/PaddlePaddle/UIE-X)体验UIE-X功能：
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/207856955-a01cd5dd-fd5c-48ae-b8fd-c69512a88845.png height=500 width=900 hspace='10'/>
+</div>
+
+- UIE-X端到端文档抽取产业应用示例
+
+  - 报关单
+
+    <div align="center">
+        <img src=https://user-images.githubusercontent.com/40840292/205879840-239ada90-1692-40e4-a17f-c5e963fdd204.png height=800 width=500 />
+    </div>
+
+  - Delivery Note（需微调）
+
+    <div align="center">
+        <img src=https://user-images.githubusercontent.com/40840292/205922422-f2615050-83cb-4bf5-8887-461f5633e85c.png height=250 width=700 />
+    </div>
+
+  - 增值税发票（需微调）
+
+    <div align="center">
+        <img src=https://user-images.githubusercontent.com/40840292/206084942-44ba477c-9244-4ce2-bbb5-ba430c9b926e.png height=550 width=700 />
+    </div>
+
+  - 表单（需微调）
+
+    <div align="center">
+        <img src=https://user-images.githubusercontent.com/40840292/207856330-7aa0d158-47e0-477f-a88f-e23a040504a3.png height=400 width=700 />
+    </div>
+
+<a name="3"></a>
+
+## 3. 快速开始
+
+<a name="31"></a>
+
+### 3.1 Taskflow开箱即用
+
+- 通过Taskflow实现开箱即用
+  👉 [文本抽取 Taskflow使用指南](./taskflow_text.md)
+  👉 [文档抽取 Taskflow使用指南](./taskflow_doc.md)
+
+<a name="32"></a>
+
+### 3.2 文本信息抽取
+
+- 快速开启文本信息抽取 👉 [文本信息抽取指南](./text/README.md)
+
+<a name="33"></a>
+
+### 3.3 文档信息抽取
+
+- 快速开启文档信息抽取 👉 [文档信息抽取指南](./document/README.md)

applications/information_extraction/document/README.md

@@ -0,0 +1,301 @@
+简体中文 | [English](README_en.md)
+
+# 文档信息抽取
+
+**目录**
+- [1. 文档信息抽取应用](#1)
+- [2. 快速开始](#2)
+  - [2.1 代码结构](#代码结构)
+  - [2.2 数据标注](#数据标注)
+  - [2.3 模型微调](#模型微调)
+  - [2.4 模型评估](#模型评估)
+  - [2.5 定制模型一键预测](#定制模型一键预测)
+  - [2.6 实验指标](#实验指标)
+
+<a name="1"></a>
+
+## 1. 文档信息抽取应用
+
+本项目提供基于UIE微调的文档抽取端到端应用方案，打通**数据标注-模型训练-模型调优-预测部署全流程**，可快速实现文档信息抽取产品落地。
+
+信息抽取通俗地说就是从给定的文本/图片等输入数据中抽取出结构化信息的过程。在信息抽取的落地过程中通常面临领域多变、任务多样、数据稀缺等许多挑战。针对信息抽取领域的难点和痛点，PaddleNLP信息抽取应用UIE统一建模的思想，提供了文档信息抽取产业级应用方案，支持**文档/图片/表格和纯文本场景下实体、关系、事件、观点等不同任务信息抽取**。该应用**不限定行业领域和抽取目标**，可实现从产品原型研发、业务POC阶段到业务落地、迭代阶段的无缝衔接，助力开发者实现特定领域抽取场景的快速适配与落地。
+
+**文档信息抽取应用亮点：**
+
+- **覆盖场景全面🎓：** 覆盖文档信息抽取各类主流任务，支持多语言，满足开发者多样信息抽取落地需求。
+- **效果领先🏃：** 以在多模态信息抽取上有突出效果的模型UIE-X作为训练基座，具有广泛成熟的实践应用性。
+- **简单易用⚡：** 通过Taskflow实现三行代码可实现无标注数据的情况下进行快速调用，一行命令即可开启信息抽取训练，轻松完成部署上线，降低信息抽取技术落地门槛。
+- **高效调优✊：** 开发者无需机器学习背景知识，即可轻松上手数据标注及模型训练流程。
+
+<a name="2"></a>
+
+## 2. 快速开始
+
+对于简单的抽取目标可以直接使用```paddlenlp.Taskflow```实现零样本（zero-shot）抽取，对于细分场景我们推荐使用定制功能（标注少量数据进行模型微调）以进一步提升效果。
+
+<a name="代码结构"></a>
+
+### 2.1 代码结构
+
+```shell
+.
+├── deploy            # 部署目录
+│ └── simple_serving  # 基于PaddleNLP SimpleServing 服务化部署
+├── utils.py          # 数据处理工具
+├── finetune.py       # 模型微调、压缩脚本
+├── evaluate.py       # 模型评估脚本
+└── README.md
+```
+
+<a name="数据标注"></a>
+
+### 2.2 数据标注
+我们推荐使用 [Label Studio](https://labelstud.io/) 进行文档信息抽取数据标注，本项目打通了从数据标注到训练的通道，也即Label Studio导出数据可以通过 [label_studio.py](../label_studio.py) 脚本轻松将数据转换为输入模型时需要的形式，实现无缝衔接。标注方法的详细介绍请参考 [Label Studio数据标注指南](../label_studio_doc.md)。
+
+这里我们提供预先标注好的`增值税发票数据集`的文件，可以运行下面的命令行下载数据集，我们将展示如何使用数据转化脚本生成训练/验证/测试集文件，并使用UIE-X模型进行微调。
+
+下载增值税发票数据集：
+```shell
+wget https://paddlenlp.bj.bcebos.com/datasets/tax.tar.gz
+tar -zxvf tax.tar.gz
+mv tax data
+rm tax.tar.gz
+```
+
+生成训练/验证集文件：
+```shell
+python ../label_studio.py \
+    --label_studio_file ./data/label_studio.json \
+    --save_dir ./data \
+    --splits 0.8 0.2 0 \
+    --task_type ext
+```
+
+生成训练/验证集文件，可以使用PP-Structure的布局分析优化OCR结果的排序：
+```shell
+python ../label_studio.py \
+    --label_studio_file ./data/label_studio.json \
+    --save_dir ./data \
+    --splits 0.8 0.2 0\
+    --task_type ext \
+    --layout_analysis True
+```
+
+更多不同类型任务（含实体抽取、关系抽取、文档分类等）的标注规则及参数说明，请参考[Label Studio数据标注指南](../label_studio_doc.md)。
+
+<a name="模型微调"></a>
+
+### 2.3 模型微调
+
+推荐使用 [Trainer API ](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/docs/trainer.md) 对模型进行微调。只需输入模型、数据集等就可以使用 Trainer API 高效快速地进行预训练、微调和模型压缩等任务，可以一键启动多卡训练、混合精度训练、梯度累积、断点重启、日志显示等功能，Trainer API 还针对训练过程的通用训练配置做了封装，比如：优化器、学习率调度等。
+
+使用下面的命令，使用 `uie-x-base` 作为预训练模型进行模型微调，将微调后的模型保存至`./checkpoint/model_best`：
+
+单卡启动：
+
+```shell
+python finetune.py  \
+    --device gpu \
+    --logging_steps 5 \
+    --save_steps 25 \
+    --eval_steps 25 \
+    --seed 42 \
+    --model_name_or_path uie-x-base \
+    --output_dir ./checkpoint/model_best \
+    --train_path data/train.txt \
+    --dev_path data/dev.txt  \
+    --max_seq_len 512  \
+    --per_device_train_batch_size  8 \
+    --per_device_eval_batch_size 8 \
+    --num_train_epochs 10 \
+    --learning_rate 1e-5 \
+    --do_train \
+    --do_eval \
+    --do_export \
+    --export_model_dir ./checkpoint/model_best \
+    --overwrite_output_dir \
+    --disable_tqdm True \
+    --metric_for_best_model eval_f1 \
+    --load_best_model_at_end  True \
+    --save_total_limit 1
+```
+
+如果在GPU环境中使用，可以指定gpus参数进行多卡训练：
+
+```shell
+python -u -m paddle.distributed.launch --gpus "0" finetune.py \
+    --device gpu \
+    --logging_steps 5 \
+    --save_steps 25 \
+    --eval_steps 25 \
+    --seed 42 \
+    --model_name_or_path uie-x-base \
+    --output_dir ./checkpoint/model_best \
+    --train_path data/train.txt \
+    --dev_path data/dev.txt  \
+    --max_seq_len 512  \
+    --per_device_train_batch_size  8 \
+    --per_device_eval_batch_size 8 \
+    --num_train_epochs 10 \
+    --learning_rate 1e-5 \
+    --do_train \
+    --do_eval \
+    --do_export \
+    --export_model_dir ./checkpoint/model_best \
+    --overwrite_output_dir \
+    --disable_tqdm True \
+    --metric_for_best_model eval_f1 \
+    --load_best_model_at_end  True \
+    --save_total_limit 1
+```
+
+该示例代码中由于设置了参数 `--do_eval`，因此在训练完会自动进行评估。
+
+可配置参数说明：
+* `device`: 训练设备，可选择 'cpu'、'gpu'、'npu' 其中的一种；默认为 GPU 训练。
+* `logging_steps`: 训练过程中日志打印的间隔 steps 数，默认10。
+* `save_steps`: 训练过程中保存模型 checkpoint 的间隔 steps 数，默认100。
+* `eval_steps`: 训练过程中保存模型 checkpoint 的间隔 steps 数，默认100。
+* `seed`：全局随机种子，默认为 42。
+* `model_name_or_path`：进行 few shot 训练使用的预训练模型。默认为 "uie-x-base"。
+* `output_dir`：必须，模型训练或压缩后保存的模型目录；默认为 `None` 。
+* `train_path`：训练集路径；默认为 `None` 。
+* `dev_path`：开发集路径；默认为 `None` 。
+* `max_seq_len`：文本最大切分长度，输入超过最大长度时会对输入文本进行自动切分，默认为512。
+* `per_device_train_batch_size`:用于训练的每个 GPU 核心/NPU 核心/CPU 的batch大小，默认为8。
+* `per_device_eval_batch_size`:用于评估的每个 GPU 核心/NPU 核心/CPU 的batch大小，默认为8。
+* `num_train_epochs`: 训练轮次，使用早停法时可以选择 100；默认为10。
+* `learning_rate`：训练最大学习率，UIE-X 推荐设置为 1e-5；默认值为3e-5。
+* `label_names`：训练数据标签label的名称，UIE-X 设置为'start_positions' 'end_positions'；默认值为None。
+* `do_train`:是否进行微调训练，设置该参数表示进行微调训练，默认不设置。
+* `do_eval`:是否进行评估，设置该参数表示进行评估，默认不设置。
+* `do_export`:是否进行导出，设置该参数表示进行静态图导出，默认不设置。
+* `export_model_dir`:静态图导出地址，默认为None。
+* `overwrite_output_dir`： 如果 `True`，覆盖输出目录的内容。如果 `output_dir` 指向检查点目录，则使用它继续训练。
+* `disable_tqdm`： 是否使用tqdm进度条。
+* `metric_for_best_model`：最优模型指标,UIE-X 推荐设置为 `eval_f1`，默认为None。
+* `load_best_model_at_end`：训练结束后是否加载最优模型，通常与`metric_for_best_model`配合使用，默认为False。
+* `save_total_limit`：如果设置次参数，将限制checkpoint的总数。删除旧的checkpoints `输出目录`，默认为None。
+
+<a name="模型评估"></a>
+
+### 2.4 模型评估
+
+```shell
+python evaluate.py \
+    --device "gpu" \
+    --model_path ./checkpoint/model_best \
+    --test_path ./data/dev.txt \
+    --output_dir ./checkpoint/model_best \
+    --label_names 'start_positions' 'end_positions'\
+    --max_seq_len 512 \
+    --per_device_eval_batch_size 16
+```
+评估方式说明：采用单阶段评价的方式，即关系抽取、事件抽取等需要分阶段预测的任务对每一阶段的预测结果进行分别评价。验证/测试集默认会利用同一层级的所有标签来构造出全部负例。
+
+可开启`debug`模式对每个正例类别分别进行评估，该模式仅用于模型调试：
+
+```shell
+python evaluate.py \
+    --device "gpu" \
+    --model_path ./checkpoint/model_best \
+    --test_path ./data/dev.txt \
+    --output_dir ./checkpoint/model_best \
+    --label_names 'start_positions' 'end_positions'\
+    --max_seq_len 512 \
+    --per_device_eval_batch_size 16 \
+    --debug True
+```
+
+输出结果：
+```text
+[2022-11-14 09:41:18,424] [    INFO] - ***** Running Evaluation *****
+[2022-11-14 09:41:18,424] [    INFO] -   Num examples = 160
+[2022-11-14 09:41:18,424] [    INFO] -   Pre device batch size = 4
+[2022-11-14 09:41:18,424] [    INFO] -   Total Batch size = 4
+[2022-11-14 09:41:18,424] [    INFO] -   Total prediction steps = 40
+[2022-11-14 09:41:26,451] [    INFO] - -----Evaluate model-------
+[2022-11-14 09:41:26,451] [    INFO] - Class Name: ALL CLASSES
+[2022-11-14 09:41:26,451] [    INFO] - Evaluation Precision: 0.94521 | Recall: 0.88462 | F1: 0.91391
+[2022-11-14 09:41:26,451] [    INFO] - -----------------------------
+[2022-11-14 09:41:26,452] [    INFO] - ***** Running Evaluation *****
+[2022-11-14 09:41:26,452] [    INFO] -   Num examples = 8
+[2022-11-14 09:41:26,452] [    INFO] -   Pre device batch size = 4
+[2022-11-14 09:41:26,452] [    INFO] -   Total Batch size = 4
+[2022-11-14 09:41:26,452] [    INFO] -   Total prediction steps = 2
+[2022-11-14 09:41:26,692] [    INFO] - Class Name: 开票日期
+[2022-11-14 09:41:26,692] [    INFO] - Evaluation Precision: 1.00000 | Recall: 1.00000 | F1: 1.00000
+[2022-11-14 09:41:26,692] [    INFO] - -----------------------------
+[2022-11-14 09:41:26,693] [    INFO] - ***** Running Evaluation *****
+[2022-11-14 09:41:26,693] [    INFO] -   Num examples = 8
+[2022-11-14 09:41:26,693] [    INFO] -   Pre device batch size = 4
+[2022-11-14 09:41:26,693] [    INFO] -   Total Batch size = 4
+[2022-11-14 09:41:26,693] [    INFO] -   Total prediction steps = 2
+[2022-11-14 09:41:26,952] [    INFO] - Class Name: 名称
+[2022-11-14 09:41:26,952] [    INFO] - Evaluation Precision: 0.87500 | Recall: 0.87500 | F1: 0.87500
+[2022-11-14 09:41:26,952] [    INFO] - -----------------------------
+...
+```
+
+可配置参数：
+* `device`: 评估设备，可选择 'cpu'、'gpu'、'npu' 其中的一种；默认为 GPU 评估。
+* `model_path`: 进行评估的模型文件夹路径，路径下需包含模型权重文件`model_state.pdparams`及配置文件`model_config.json`。
+* `test_path`: 进行评估的测试集文件。
+* `label_names`：训练数据标签label的名称，UIE-X 设置为'start_positions' 'end_positions'；默认值为None。
+* `batch_size`: 批处理大小，请结合机器情况进行调整，默认为16。
+* `max_seq_len`: 文本最大切分长度，输入超过最大长度时会对输入文本进行自动切分，默认为512。
+* `per_device_eval_batch_size`:用于评估的每个 GPU 核心/NPU 核心/CPU 的batch大小，默认为8。
+* `debug`: 是否开启debug模式对每个正例类别分别进行评估，该模式仅用于模型调试，默认关闭。
+* `schema_lang`: 选择schema的语言，可选有`ch`和`en`。默认为`ch`，英文数据集请选择`en`。
+
+<a name="定制模型一键预测"></a>
+
+### 2.5 定制模型一键预测
+
+`paddlenlp.Taskflow`装载定制模型，通过`task_path`指定模型权重文件的路径，路径下需要包含训练好的模型权重文件`model_state.pdparams`。
+
+```python
+from pprint import pprint
+from paddlenlp import Taskflow
+from paddlenlp.utils.doc_parser import DocParser
+
+schema = ['开票日期', '名称', '纳税人识别号', '开户行及账号', '金额', '价税合计', 'No', '税率', '地址、电话', '税额']
+my_ie = Taskflow("information_extraction", model="uie-x-base", schema=schema, task_path='./checkpoint/model_best', precision='fp16')
+```
+
+我们可以根据设置的`schema`，对指定的`doc_path`文档进行信息抽取并进行可视化：
+
+```python
+doc_path = "./data/images/b199.jpg"
+results = my_ie({"doc": doc_path})
+pprint(results)
+
+# 结果可视化
+DocParser.write_image_with_results(
+    doc_path,
+    result=results[0],
+    save_path="./image_show.png")
+```
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/206084942-44ba477c-9244-4ce2-bbb5-ba430c9b926e.png height=550 width=700 />
+</div>
+
+<a name="实验指标"></a>
+
+### 2.6 实验指标
+
+我们在自标注的增值税数据集上进行实验：
+
+
+  |  |  Precision  | Recall | F1 Score |
+  | :---: | :--------: | :--------: | :--------: |
+  | 0-shot| 0.44898 | 0.56410  | 0.50000 |
+  | 5-shot| 0.9000 | 0.9231 | 0.9114 |
+  | 10-shot| 0.9125 | 0.93590 |  0.9241 |
+  | 20-shot| 0.9737 | 0.9487 | 0.9610 |
+  | 30-shot|  0.9744  | 0.9744  | 0.9744 |
+  | 30-shot+PP-Structure| 1.0  | 0.9625 |  0.9809 |
+
+
+n-shot表示训练集包含n张标注图片数据进行模型微调，实验表明UIE-X可以通过少量数据（few-shot）和PP-Structure的布局分析进一步提升结果。

applications/information_extraction/document/deploy/simple_serving/README.md

@@ -0,0 +1,57 @@
+# 基于PaddleNLP SimpleServing 的服务化部署
+
+## 目录
+- [环境准备](#环境准备)
+- [Server服务启动](#Server服务启动)
+- [Client请求启动](#Client请求启动)
+- [服务化自定义参数](#服务化自定义参数)
+
+## 环境准备
+使用有SimpleServing功能的PaddleNLP版本(或者最新的develop版本)
+
+```shell
+pip install paddlenlp >= 2.4.4
+```
+
+
+## Server服务启动
+
+```bash
+paddlenlp server server:app --workers 1 --host 0.0.0.0 --port 8189
+```
+
+## Client请求启动
+
+```bash
+python client.py
+```
+
+## 服务化自定义参数
+
+### Server 自定义参数
+#### schema替换
+```python
+# Default schema
+schema = ['开票日期', '名称', '纳税人识别号', '开户行及账号', '金额', '价税合计', 'No', '税率', '地址、电话', '税额']
+```
+
+#### 设置模型路径
+```
+# Default task_path
+uie = Taskflow('information_extraction', task_path='../../checkpoint/model_best/', schema=schema)
+```
+
+#### 多卡服务化预测
+PaddleNLP SimpleServing 支持多卡负载均衡预测，主要在服务化注册的时候，注册两个Taskflow的task即可，下面是示例代码
+```
+uie1 = Taskflow('information_extraction', task_path='../../checkpoint/model_best/', schema=schema, device_id=0)
+uie2 = Taskflow('information_extraction', task_path='../../checkpoint/model_best/', schema=schema, device_id=1)
+service.register_taskflow('uie', [uie1, uie2])
+```
+
+### Client 自定义参数
+
+```python
+# Changed to image paths you wanted
+image_paths = ['../../data/images/b1.jpg']
+```

applications/information_extraction/label_studio.py

@@ -0,0 +1,139 @@
+# coding=utf-8
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+import random
+import time
+from decimal import Decimal
+
+import numpy as np
+import paddle
+
+from paddlenlp.trainer.argparser import strtobool
+from paddlenlp.utils.log import logger
+from paddlenlp.utils.tools import DataConverter
+
+
+def set_seed(seed):
+    paddle.seed(seed)
+    random.seed(seed)
+    np.random.seed(seed)
+
+
+def do_convert():
+    set_seed(args.seed)
+
+    tic_time = time.time()
+    if not os.path.exists(args.label_studio_file):
+        raise ValueError("Please input the correct path of label studio file.")
+
+    if not os.path.exists(args.save_dir):
+        os.makedirs(args.save_dir)
+
+    if len(args.splits) != 0 and len(args.splits) != 3:
+        raise ValueError("Only []/ len(splits)==3 accepted for splits.")
+
+    def _check_sum(splits):
+        return Decimal(str(splits[0])) + Decimal(str(splits[1])) + Decimal(str(splits[2])) == Decimal("1")
+
+    if len(args.splits) == 3 and not _check_sum(args.splits):
+        raise ValueError("Please set correct splits, sum of elements in splits should be equal to 1.")
+
+    with open(args.label_studio_file, "r", encoding="utf-8") as f:
+        raw_examples = json.loads(f.read())
+
+    if args.is_shuffle:
+        indexes = np.random.permutation(len(raw_examples))
+        index_list = indexes.tolist()
+        raw_examples = [raw_examples[i] for i in indexes]
+
+    i1, i2, _ = args.splits
+    p1 = int(len(raw_examples) * i1)
+    p2 = int(len(raw_examples) * (i1 + i2))
+
+    train_ids = index_list[:p1]
+    dev_ids = index_list[p1:p2]
+    test_ids = index_list[p2:]
+
+    with open(os.path.join(args.save_dir, "sample_index.json"), "w") as fp:
+        maps = {"train_ids": train_ids, "dev_ids": dev_ids, "test_ids": test_ids}
+        fp.write(json.dumps(maps))
+
+    if raw_examples[0]["data"].get("image"):
+        anno_type = "image"
+    else:
+        anno_type = "text"
+
+    data_converter = DataConverter(
+        args.label_studio_file,
+        negative_ratio=args.negative_ratio,
+        prompt_prefix=args.prompt_prefix,
+        options=args.options,
+        separator=args.separator,
+        layout_analysis=args.layout_analysis,
+        schema_lang=args.schema_lang,
+        ocr_lang=args.ocr_lang,
+        anno_type=anno_type,
+    )
+
+    if args.task_type == "ext":
+        train_examples = data_converter.convert_ext_examples(raw_examples[:p1])
+        dev_examples = data_converter.convert_ext_examples(raw_examples[p1:p2], is_train=False)
+        test_examples = data_converter.convert_ext_examples(raw_examples[p2:], is_train=False)
+    else:
+        train_examples = data_converter.convert_cls_examples(raw_examples[:p1])
+        dev_examples = data_converter.convert_cls_examples(raw_examples[p1:p2])
+        test_examples = data_converter.convert_cls_examples(raw_examples[p2:])
+
+    def _save_examples(save_dir, file_name, examples):
+        count = 0
+        save_path = os.path.join(save_dir, file_name)
+        with open(save_path, "w", encoding="utf-8") as f:
+            for example in examples:
+                f.write(json.dumps(example, ensure_ascii=False) + "\n")
+                count += 1
+        logger.info("Save %d examples to %s." % (count, save_path))
+
+    _save_examples(args.save_dir, "train.txt", train_examples)
+    _save_examples(args.save_dir, "dev.txt", dev_examples)
+    _save_examples(args.save_dir, "test.txt", test_examples)
+
+    logger.info("Finished! It takes %.2f seconds" % (time.time() - tic_time))
+
+
+if __name__ == "__main__":
+    # yapf: disable
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--label_studio_file", default="./data/label_studio.json", type=str, help="The annotation file exported from label studio platform.")
+    parser.add_argument("--save_dir", default="./data", type=str, help="The path of data that you wanna save.")
+    parser.add_argument("--negative_ratio", default=5, type=int, help="Used only for the extraction task, the ratio of positive and negative samples, number of negtive samples = negative_ratio * number of positive samples")
+    parser.add_argument("--splits", default=[0.8, 0.1, 0.1], type=float, nargs="*", help="The ratio of samples in datasets. [0.6, 0.2, 0.2] means 60% samples used for training, 20% for evaluation and 20% for test.")
+    parser.add_argument("--task_type", choices=['ext', 'cls'], default="ext", type=str, help="Select task type, ext for the extraction task and cls for the classification task, defaults to ext.")
+    parser.add_argument("--options", default=["正向", "负向"], type=str, nargs="+", help="Used only for the classification task, the options for classification")
+    parser.add_argument("--prompt_prefix", default="情感倾向", type=str, help="Used only for the classification task, the prompt prefix for classification")
+    parser.add_argument("--is_shuffle", default="True", type=strtobool, help="Whether to shuffle the labeled dataset, defaults to True.")
+    parser.add_argument("--layout_analysis", default=False, type=bool, help="Enable layout analysis to optimize the order of OCR result.")
+    parser.add_argument("--seed", type=int, default=1000, help="Random seed for initialization")
+    parser.add_argument("--separator", type=str, default='##', help="Used only for entity/aspect-level classification task, separator for entity label and classification label")
+    parser.add_argument("--schema_lang", choices=["ch", "en"], default="ch", help="Select the language type for schema.")
+    parser.add_argument("--ocr_lang", choices=["ch", "en"], default="ch", help="Select the language type for OCR.")
+
+    args = parser.parse_args()
+    # yapf: enable
+
+    do_convert()

applications/information_extraction/label_studio_doc.md

@@ -1,4 +1,4 @@
-# 文档抽取任务 Label Studio 使用指南
+# 文档抽取任务Label Studio使用指南
 
  **目录**
 
@@ -21,7 +21,7 @@
 - label-studio == 1.6.0
 - paddleocr >= 2.6.0.1
 
-在终端(terminal)使用 pip 安装 label-studio：
+在终端(terminal)使用pip安装label-studio：
 
 ```shell
 pip install label-studio==1.6.0
@@ -32,7 +32,7 @@ pip install label-studio==1.6.0
 label-studio start
 ```
 
-在浏览器打开[http://localhost:8080/](http://127.0.0.1:8080/)，输入用户名和密码登录，开始使用 label-studio 进行标注。
+在浏览器打开[http://localhost:8080/](http://127.0.0.1:8080/)，输入用户名和密码登录，开始使用label-studio进行标注。
 
 
 <a name="2"></a>
@@ -63,19 +63,19 @@ label-studio start
     <img src=https://user-images.githubusercontent.com/40840292/199729973-53a994d8-da71-4ab9-84f5-83297e19a7a1.png height=400 width=1200 />
 </div>
 
-- 添加标签(也可跳过后续在 Setting/Labeling Interface 中添加)
+- 添加标签(也可跳过后续在Setting/Labeling Interface中添加)
 
 <div align="center">
     <img src=https://user-images.githubusercontent.com/40840292/199450930-4c0cd189-6085-465a-aca0-6ba6f52a0c0d.png height=600 width=1200 />
 </div>
 
-图中展示了 Span 实体类型标签的构建，其他类型标签的构建可参考[2.3标签构建](#23)
+图中展示了Span实体类型标签的构建，其他类型标签的构建可参考[2.3标签构建](#23)
 
 <a name="22"></a>
 
 #### 2.2 数据上传
 
-先从本地或 HTTP 链接上传图片，然后选择导入本项目。
+先从本地或HTTP链接上传图片，然后选择导入本项目。
 
 <div align="center">
     <img src=https://user-images.githubusercontent.com/40840292/199452007-2d45f7ba-c631-46b4-b21f-729a2ed652e9.png height=270 width=1200 />
@@ -85,20 +85,20 @@ label-studio start
 
 #### 2.3 标签构建
 
-- Span 实体类型标签
+- Span实体类型标签
 
 <div align="center">
     <img src=https://user-images.githubusercontent.com/40840292/199456432-ce601ab0-7d6c-458f-ac46-8839dbc4d013.png height=500 width=1200 />
 </div>
 
 
-- Relation 关系类型标签
+- Relation关系类型标签
 
 <div align="center">
     <img src=https://user-images.githubusercontent.com/40840292/199877621-f60e00c7-81ae-42e1-b498-8ebc5b5bd0fd.png height=650 width=1200 />
 </div>
 
-Relation XML 模板：
+Relation XML模板：
 
 ```xml
   <Relations>
@@ -127,7 +127,7 @@ Relation XML 模板：
             <img src=https://user-images.githubusercontent.com/40840292/199879427-82806ffc-dc60-4ec7-bda5-e16419ee9d15.png height=650 width=800 />
         </div>
 
-    - 该标注示例对应的 schema 为：
+    - 该标注示例对应的schema为：
 
         ```text
         schema = ['开票日期', '名称', '纳税人识别号', '地址、电话', '开户行及账号', '金额', '税额', '价税合计', 'No', '税率']
@@ -168,7 +168,7 @@ Relation XML 模板：
         </div>
 
 
-    - 该标注示例对应的 schema 为：
+    - 该标注示例对应的schema为：
 
         ```text
         schema = {
@@ -188,7 +188,7 @@ Relation XML 模板：
             <img src=https://user-images.githubusercontent.com/40840292/199879238-b8b41d4a-7e77-47cd-8def-2fc8ba89442f.png height=650 width=800 />
         </div>
 
-    - 该标注示例对应的 schema 为：
+    - 该标注示例对应的schema为：
 
         ```text
         schema = '文档类别[发票，报关单]'
@@ -199,7 +199,7 @@ Relation XML 模板：
 
 #### 2.5 数据导出
 
-勾选已标注图片 ID，选择导出的文件类型为``JSON``，导出数据：
+勾选已标注图片ID，选择导出的文件类型为``JSON``，导出数据：
 
 <div align="center">
     <img src=https://user-images.githubusercontent.com/40840292/199890897-b33ede99-97d8-4d44-877a-2518a87f8b67.png height=200 width=1200 />
@@ -209,7 +209,7 @@ Relation XML 模板：
 
 #### 2.6 数据转换
 
-将导出的文件重命名为``label_studio.json``后，放入``./document/data``目录下，并将对应的标注图片放入``./document/data/images``目录下（图片的文件名需与上传到 label studio 时的命名一致）。通过[label_studio.py](./label_studio.py)脚本可转为 UIE 的数据格式。
+将导出的文件重命名为``label_studio.json``后，放入``./document/data``目录下，并将对应的标注图片放入``./document/data/images``目录下（图片的文件名需与上传到label studio时的命名一致）。通过[label_studio.py](./label_studio.py)脚本可转为UIE的数据格式。
 
 - 路径示例
 
@@ -247,25 +247,25 @@ python label_studio.py \
 
 #### 2.7 更多配置
 
-- ``label_studio_file``: 从 label studio 导出的数据标注文件。
+- ``label_studio_file``: 从label studio导出的数据标注文件。
 - ``save_dir``: 训练数据的保存目录，默认存储在``data``目录下。
 - ``negative_ratio``: 最大负例比例，该参数只对抽取类型任务有效，适当构造负例可提升模型效果。负例数量和实际的标签数量有关，最大负例数量 = negative_ratio * 正例数量。该参数只对训练集有效，默认为5。为了保证评估指标的准确性，验证集和测试集默认构造全负例。
 - ``splits``: 划分数据集时训练集、验证集所占的比例。默认为[0.8, 0.1, 0.1]表示按照``8:1:1``的比例将数据划分为训练集、验证集和测试集。
 - ``task_type``: 选择任务类型，可选有抽取和分类两种类型的任务。
 - ``options``: 指定分类任务的类别标签，该参数只对分类类型任务有效。默认为["正向", "负向"]。
-- ``prompt_prefix``: 声明分类任务的 prompt 前缀信息，该参数只对分类类型任务有效。默认为"情感倾向"。
-- ``is_shuffle``: 是否对数据集进行随机打散，默认为 True。
+- ``prompt_prefix``: 声明分类任务的prompt前缀信息，该参数只对分类类型任务有效。默认为"情感倾向"。
+- ``is_shuffle``: 是否对数据集进行随机打散，默认为True。
 - ``seed``: 随机种子，默认为1000.
 - ``separator``: 实体类别/评价维度与分类标签的分隔符，该参数只对实体/评价维度分类任务有效。默认为"##"。
-- ``schema_lang``：选择 schema 的语言，将会应该训练数据 prompt 的构造方式，可选有`ch`和`en`。默认为`ch`。
-- ``ocr_lang``：选择 OCR 的语言，可选有`ch`和`en`。默认为`ch`。
-- ``layout_analysis``：是否使用 PPStructure 对文档进行布局分析，该参数只对文档类型标注任务有效。默认为 False。
+- ``schema_lang``：选择schema的语言，将会应该训练数据prompt的构造方式，可选有`ch`和`en`。默认为`ch`。
+- ``ocr_lang``：选择OCR的语言，可选有`ch`和`en`。默认为`ch`。
+- ``layout_analysis``：是否使用PPStructure对文档进行布局分析，该参数只对文档类型标注任务有效。默认为False。
 
 备注：
 - 默认情况下 [label_studio.py](./label_studio.py) 脚本会按照比例将数据划分为 train/dev/test 数据集
 - 每次执行 [label_studio.py](./label_studio.py) 脚本，将会覆盖已有的同名数据文件
 - 在模型训练阶段我们推荐构造一些负例以提升模型效果，在数据转换阶段我们内置了这一功能。可通过`negative_ratio`控制自动构造的负样本比例；负样本数量 = negative_ratio * 正样本数量。
-- 对于从 label_studio 导出的文件，默认文件中的每条数据都是经过人工正确标注的。
+- 对于从label_studio导出的文件，默认文件中的每条数据都是经过人工正确标注的。
 
 
 ## References

applications/information_extraction/label_studio_text.md

@@ -0,0 +1,287 @@
+# 文本抽取任务Label Studio使用指南
+
+ **目录**
+
+- [1. 安装](#1)
+- [2. 文本抽取任务标注](#2)
+    - [2.1 项目创建](#21)
+    - [2.2 数据上传](#22)
+    - [2.3 标签构建](#23)
+    - [2.4 任务标注](#24)
+    - [2.5 数据导出](#25)
+    - [2.6 数据转换](#26)
+    - [2.7 更多配置](#27)
+
+<a name="1"></a>
+
+## 1. 安装
+**以下标注示例用到的环境配置：**
+
+- Python 3.8+
+- label-studio == 1.6.0
+- paddleocr >= 2.6.0.1
+
+在终端(terminal)使用pip安装label-studio：
+
+```shell
+pip install label-studio==1.6.0
+```
+
+安装完成后，运行以下命令行：
+```shell
+label-studio start
+```
+
+在浏览器打开[http://localhost:8080/](http://127.0.0.1:8080/)，输入用户名和密码登录，开始使用label-studio进行标注。
+
+<a name="2"></a>
+
+## 2. 文本抽取任务标注
+
+<a name="21"></a>
+
+#### 2.1 项目创建
+
+点击创建（Create）开始创建一个新的项目，填写项目名称、描述，然后选择``Object Detection with Bounding Boxes``。
+
+- 填写项目名称、描述
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199661377-d9664165-61aa-4462-927d-225118b8535b.png height=230 width=1200 />
+</div>
+
+- **命名实体识别、关系抽取、事件抽取、实体/评价维度分类**任务选择``Relation Extraction`。
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199661638-48a870eb-a1df-4db5-82b9-bc8e985f5190.png height=350 width=1200 />
+</div>
+
+- **文本分类、句子级情感倾向分类**任务选择``Text Classification``。
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/212617773-34534e68-4544-4b24-8f39-ae7f9573d397.png height=420 width=1200 />
+</div>
+
+- 添加标签(也可跳过后续在Setting/Labeling Interface中配置)
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199662737-ed996a2c-7a24-4077-8a36-239c4bfb0a16.png height=380 width=1200 />
+</div>
+
+图中展示了实体类型标签的构建，其他类型标签的构建可参考[2.3标签构建](#23)
+
+<a name="22"></a>
+
+#### 2.2 数据上传
+
+先从本地上传txt格式文件，选择``List of tasks``，然后选择导入本项目。
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199667670-1b8f6755-b41f-41c4-8afc-06bb051690b6.png height=210 width=1200 />
+</div>
+
+<a name="23"></a>
+
+#### 2.3 标签构建
+
+- Span类型标签
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199667941-04e300c5-3cd7-4b8e-aaf5-561415414891.png height=480 width=1200 />
+</div>
+
+- Relation类型标签
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199725229-f5e998bf-367c-4449-b83a-c799f1e3de00.png height=620 width=1200 />
+</div>
+
+Relation XML模板：
+
+```xml
+  <Relations>
+    <Relation value="歌手"/>
+    <Relation value="发行时间"/>
+    <Relation value="所属专辑"/>
+  </Relations>
+```
+
+- 分类类别标签
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199724082-ee82dceb-dab0-496d-a930-a8ecb284d8b2.png height=370 width=1200 />
+</div>
+
+
+<a name="24"></a>
+
+#### 2.4 任务标注
+
+- 实体抽取
+
+标注示例：
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199879957-aeec9d17-d342-4ea0-a840-457b49f6066e.png height=140 width=1000 />
+</div>
+
+该标注示例对应的schema为：
+
+```text
+schema = [
+    '时间',
+    '选手',
+    '赛事名称',
+    '得分'
+]
+```
+
+- 关系抽取
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199879866-03c1ecac-1828-4f35-af70-9ae61701c303.png height=230 width=1200 />
+</div>
+
+对于关系抽取，其P的类型设置十分重要，需要遵循以下原则
+
+“{S}的{P}为{O}”需要能够构成语义合理的短语。比如对于三元组(S, 父子, O)，关系类别为父子是没有问题的。但按照UIE当前关系类型prompt的构造方式，“S的父子为O”这个表达不是很通顺，因此P改成孩子更好，即“S的孩子为O”。**合理的P类型设置，将显著提升零样本效果**。
+
+该标注示例对应的schema为：
+
+```text
+schema = {
+    '作品名': [
+        '歌手',
+        '发行时间',
+        '所属专辑'
+    ]
+}
+```
+
+- 事件抽取
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199879776-75abbade-9bea-44dc-ac36-322fecdc03e0.png height=220 width=1200 />
+</div>
+
+该标注示例对应的schema为：
+
+```text
+schema = {
+    '地震触发词': [
+        '时间',
+        '震级'
+    ]
+}
+```
+
+- 句子级分类
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199879672-c3f286fe-a217-4888-950f-d4ee45b19f5a.png height=210 width=1000 />
+</div>
+
+
+该标注示例对应的schema为：
+
+```text
+schema = '情感倾向[正向，负向]'
+```
+
+- 实体/评价维度分类
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199879586-8c6e4826-a3b0-49e0-9920-98ca062dccff.png height=240 width=1200 />
+</div>
+
+该标注示例对应的schema为：
+
+```text
+schema = {
+    '评价维度': [
+        '观点词',
+        '情感倾向[正向，负向]'
+    ]
+}
+```
+
+<a name="25"></a>
+
+#### 2.5 数据导出
+
+勾选已标注文本ID，选择导出的文件类型为``JSON``，导出数据：
+
+<div align="center">
+    <img src=https://user-images.githubusercontent.com/40840292/199891344-023736e2-6f9d-454b-b72a-dec6689f8436.png height=180 width=1200 />
+</div>
+
+<a name="26"></a>
+
+#### 2.6 数据转换
+
+将导出的文件重命名为``label_studio.json``后，放入``./data``目录下。通过[label_studio.py](./label_studio.py)脚本可转为UIE的数据格式。
+
+- 抽取式任务
+
+```shell
+python label_studio.py \
+    --label_studio_file ./data/label_studio.json \
+    --save_dir ./data \
+    --splits 0.8 0.1 0.1 \
+    --task_type ext
+```
+
+- 句子级分类任务
+
+在数据转换阶段，我们会自动构造用于模型训练的prompt信息。例如句子级情感分类中，prompt为``情感倾向[正向,负向]``，可以通过`prompt_prefix`和`options`参数进行配置。
+
+```shell
+python label_studio.py \
+    --label_studio_file ./data/label_studio.json \
+    --task_type cls \
+    --save_dir ./data \
+    --splits 0.8 0.1 0.1 \
+    --prompt_prefix "情感倾向" \
+    --options "正向" "负向"
+```
+
+- 实体/评价维度分类任务
+
+在数据转换阶段，我们会自动构造用于模型训练的prompt信息。例如评价维度情感分类中，prompt为``XXX的情感倾向[正向,负向]``，可以通过`prompt_prefix`和`options`参数进行声明。
+
+```shell
+python label_studio.py \
+    --label_studio_file ./data/label_studio.json \
+    --task_type ext \
+    --save_dir ./data \
+    --splits 0.8 0.1 0.1 \
+    --prompt_prefix "情感倾向" \
+    --options "正向" "负向" \
+    --separator "##"
+```
+
+<a name="27"></a>
+
+#### 2.7 更多配置
+
+- ``label_studio_file``: 从label studio导出的数据标注文件。
+- ``save_dir``: 训练数据的保存目录，默认存储在``data``目录下。
+- ``negative_ratio``: 最大负例比例，该参数只对抽取类型任务有效，适当构造负例可提升模型效果。负例数量和实际的标签数量有关，最大负例数量 = negative_ratio * 正例数量。该参数只对训练集有效，默认为5。为了保证评估指标的准确性，验证集和测试集默认构造全负例。
+- ``splits``: 划分数据集时训练集、验证集所占的比例。默认为[0.8, 0.1, 0.1]表示按照``8:1:1``的比例将数据划分为训练集、验证集和测试集。
+- ``task_type``: 选择任务类型，可选有抽取和分类两种类型的任务。
+- ``options``: 指定分类任务的类别标签，该参数只对分类类型任务有效。默认为["正向", "负向"]。
+- ``prompt_prefix``: 声明分类任务的prompt前缀信息，该参数只对分类类型任务有效。默认为"情感倾向"。
+- ``is_shuffle``: 是否对数据集进行随机打散，默认为True。
+- ``seed``: 随机种子，默认为1000.
+- ``schema_lang``：选择schema的语言，将会应该训练数据prompt的构造方式，可选有`ch`和`en`。默认为`ch`。
+- ``separator``: 实体类别/评价维度与分类标签的分隔符，该参数只对实体/评价维度分类任务有效。默认为"##"。
+
+备注：
+- 默认情况下 [label_studio.py](./label_studio.py) 脚本会按照比例将数据划分为 train/dev/test 数据集
+- 每次执行 [label_studio.py](./label_studio.py) 脚本，将会覆盖已有的同名数据文件
+- 在模型训练阶段我们推荐构造一些负例以提升模型效果，在数据转换阶段我们内置了这一功能。可通过`negative_ratio`控制自动构造的负样本比例；负样本数量 = negative_ratio * 正样本数量。
+- 对于从label_studio导出的文件，默认文件中的每条数据都是经过人工正确标注的。
+
+
+## References
+- **[Label Studio](https://labelstud.io/)**

applications/information_extraction/taskflow_doc.md

@@ -1,4 +1,4 @@
-# UIE Taskflow 使用指南
+# UIE Taskflow使用指南
 
 **目录**
 - [1. 功能简介](#1)
@@ -21,13 +21,13 @@
 
 ## 2. 文档信息抽取
 
-本章节主要介绍 Taskflow 的文档抽取功能，以下示例图片[下载链接](https://bj.bcebos.com/paddlenlp/taskflow/information_extraction/cases.zip)。
+本章节主要介绍Taskflow的文档抽取功能，以下示例图片[下载链接](https://bj.bcebos.com/paddlenlp/taskflow/information_extraction/cases.zip)。
 
 <a name="21"></a>
 
 #### 2.1 实体抽取
 
-实体抽取，又称命名实体识别（Named Entity Recognition，简称 NER），是指识别文本中具有特定意义的实体。在开放域信息抽取中，抽取的类别没有限制，用户可以自己定义。
+实体抽取，又称命名实体识别（Named Entity Recognition，简称NER），是指识别文本中具有特定意义的实体。在开放域信息抽取中，抽取的类别没有限制，用户可以自己定义。
 
 - 报关单
 
@@ -117,7 +117,7 @@
 
 #### 2.2 关系抽取
 
-关系抽取（Relation Extraction，简称 RE），是指从文本中识别实体并抽取实体之间的语义关系，进而获取三元组信息，即<主体，谓语，客体>。
+关系抽取（Relation Extraction，简称RE），是指从文本中识别实体并抽取实体之间的语义关系，进而获取三元组信息，即<主体，谓语，客体>。
 
 - 表格
 
@@ -139,7 +139,7 @@
 
 - 实体、关系多任务抽取
 
-对文档进行实体+关系抽取，schema 构造如下：
+对文档进行实体+关系抽取，schema构造如下：
 
 ```text
 schema = [
@@ -176,7 +176,7 @@ schema = [
 
 - 输入格式
 
-文档抽取 UIE-X 支持图片路径、http 图片链接、base64的输入形式，支持图片和 PDF 两种文档格式。文本抽取可以通过`text`指定输入文本。
+文档抽取UIE-X支持图片路径、http图片链接、base64的输入形式，支持图片和PDF两种文档格式。文本抽取可以通过`text`指定输入文本。
 
 ```python
 [
@@ -186,9 +186,9 @@ schema = [
 ]
 ```
 
-**NOTE**: 多页 PDF 输入目前只抽取第一页的结果，UIE-X 比较适合单证文档（如票据、单据等）的信息提取，目前还不适合过长或多页的文档。
+**NOTE**: 多页PDF输入目前只抽取第一页的结果，UIE-X比较适合单证文档（如票据、单据等）的信息提取，目前还不适合过长或多页的文档。
 
-- 使用自己的 layout / OCR 作为输入
+- 使用自己的layout / OCR作为输入
 
 ```python
 layout = [
@@ -204,9 +204,9 @@ ie({"doc": doc_path, 'layout': layout})
 
 #### 2.5 使用技巧
 
-- 使用 PP-Structure 版面分析功能
+- 使用PP-Structure版面分析功能
 
-OCR 中识别出来的文字会按照左上到右下进行排序，对于分栏、表格内有多行文本等情况我们推荐使用版面分析功能``layout_analysis=True``以优化文字排序并增强抽取效果。以下例子仅举例版面分析功能的使用场景，实际场景一般需要标注微调。
+OCR中识别出来的文字会按照左上到右下进行排序，对于分栏、表格内有多行文本等情况我们推荐使用版面分析功能``layout_analysis=True``以优化文字排序并增强抽取效果。以下例子仅举例版面分析功能的使用场景，实际场景一般需要标注微调。
 
 <div align="center">
     <img src=https://user-images.githubusercontent.com/40840292/206139057-aedec98f-683c-4648-999d-81ce5ea04a86.png height=250 width=500 hspace='10'/>
@@ -235,7 +235,7 @@ OCR 中识别出来的文字会按照左上到右下进行排序，对于分栏
 
 #### 2.6 结果可视化
 
-- OCR 识别结果可视化：
+- OCR识别结果可视化：
 
 ```python
 >>> from paddlenlp.utils.doc_parser import DocParser
@@ -296,13 +296,14 @@ OCR 中识别出来的文字会按照左上到右下进行排序，对于分栏
 ```
 
 * `schema`：定义任务抽取目标，可参考开箱即用中不同任务的调用示例进行配置。
-* `schema_lang`：设置 schema 的语言，默认为`ch`, 可选有`ch`和`en`。因为中英 schema 的构造有所不同，因此需要指定 schema 的语言。
-* `ocr_lang`：选择 PaddleOCR 的语言，`ch`可在中英混合的图片中使用，`en`在英文图片上的效果更好，默认为`ch`。
+* `schema_lang`：设置schema的语言，默认为`ch`, 可选有`ch`和`en`。因为中英schema的构造有所不同，因此需要指定schema的语言。
+* `ocr_lang`：选择PaddleOCR的语言，`ch`可在中英混合的图片中使用，`en`在英文图片上的效果更好，默认为`ch`。
 * `batch_size`：批处理大小，请结合机器情况进行调整，默认为16。
 * `model`：选择任务使用的模型，默认为`uie-base`，可选有`uie-base`, `uie-medium`, `uie-mini`, `uie-micro`, `uie-nano`和`uie-medical-base`, `uie-base-en`，`uie-x-base`。
-* `layout_analysis`：是否使用 PP-Structure 对文档进行布局分析以优化布局信息的排序，默认为 False。
-* `position_prob`：模型对于 span 的起始位置/终止位置的结果概率在0~1之间，返回结果去掉小于这个阈值的结果，默认为0.5，span 的最终概率输出为起始位置概率和终止位置概率的乘积。
-* `precision`：选择模型精度，默认为`fp32`，可选有`fp16`和`fp32`。`fp16`推理速度更快，支持 GPU 和 NPU 硬件环境。如果选择`fp16`，在 GPU 硬件环境下，请先确保机器正确安装 NVIDIA 相关驱动和基础软件，**确保 CUDA>=11.2，cuDNN>=8.1.1**，初次使用需按照提示安装相关依赖。其次，需要确保 GPU 设备的 CUDA 计算能力（CUDA Compute Capability）大于7.0，典型的设备包括 V100、T4、A10、A100、GTX 20系列和30系列显卡等。更多关于 CUDA Compute Capability 和精度支持情况请参考 NVIDIA 文档：[GPU 硬件与支持精度对照表](https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-840-ea/support-matrix/index.html#hardware-precision-matrix)。
+* `layout_analysis`：是否使用PP-Structure对文档进行布局分析以优化布局信息的排序，默认为False。
+* `position_prob`：模型对于span的起始位置/终止位置的结果概率在0~1之间，返回结果去掉小于这个阈值的结果，默认为0.5，span的最终概率输出为起始位置概率和终止位置概率的乘积。
+* `precision`：选择模型精度，默认为`fp32`，可选有`fp16`和`fp32`。`fp16`推理速度更快，支持GPU和NPU硬件环境。如果选择`fp16`，在GPU硬件环境下，请先确保机器正确安装NVIDIA相关驱动和基础软件，**确保CUDA>=11.2，cuDNN>=8.1.1**，初次使用需按照提示安装相关依赖。其次，需要确保GPU设备的CUDA计算能力（CUDA Compute Capability）大于7.0，典型的设备包括V100、T4、A10、A100、GTX 20系列和30系列显卡等。更多关于CUDA Compute Capability和精度支持情况请参考NVIDIA文档：[GPU硬件与支持精度对照表](https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-840-ea/support-matrix/index.html#hardware-precision-matrix)。
+* `use_fast`: 使用C++实现的高性能分词算子FastTokenizer进行文本预处理加速。需要通过`pip install fast-tokenizer-python`安装FastTokenizer库后方可使用。默认为`False`。更多使用说明可参考[FastTokenizer文档](../../fast_tokenizer)。
 
 ## References
 - **[PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR)**

applications/information_extraction/taskflow_doc_en.md

@@ -286,7 +286,8 @@ The text recognized in OCR will be sorted from top left to bottom right. For cas
                   model='uie-x-base',
                   layout_analysis=False,
                   position_prob=0.5,
-                  precision='fp32')
+                  precision='fp32',
+                  use_fast=False)
 ```
 
 * `schema`: Define the task extraction target, which can be configured by referring to the calling examples of different tasks in the out-of-the-box.
@@ -297,6 +298,7 @@ The text recognized in OCR will be sorted from top left to bottom right. For cas
 * `layout_analysis`: Whether to use PP-Structure to analyze the layout of the document to optimize the sorting of layout information, the default is False.
 * `position_prob`: The result probability of the model for the start position/end position of the span is between 0 and 1, and the returned result removes the results less than this threshold, the default is 0.5, and the final probability output of the span is the start position probability and end position The product of the position probabilities.
 * `precision`: select the model precision, the default is `fp32`, optional `fp16` and `fp32`. `fp16` inference is faster, support GPU and NPU hardware. If you choose `fp16` and GPU hardware, please ensure that the machine is correctly installed with NVIDIA-related drivers and basic software. **Ensure that CUDA>=11.2, cuDNN>=8.1.1**. For the first time use, you need to follow the prompts to install the relevant dependencies. Secondly, it is necessary to ensure that the CUDA Compute Capability of the GPU device is greater than 7.0. Typical devices include V100, T4, A10, A100, GTX 20 series and 30 series graphics cards, etc. For more information about CUDA Compute Capability and precision support, please refer to NVIDIA documentation: [GPU Hardware and Supported Precision Comparison Table](https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-840-ea/support-matrix/index.html#hardware-precision-matrix).
+* `use_fast`: Use the high-performance word segmentation operator FastTokenizer implemented in C++ to accelerate text preprocessing. The FastTokenizer library needs to be installed through `pip install fast-tokenizer-python` before it can be used. Defaults to `False`. For more usage instructions, please refer to [FastTokenizer Documentation](../../fast_tokenizer).
 
 ## References
 - **[PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR)**

applications/information_extraction/taskflow_text.md

@@ -1,4 +1,4 @@
-# 文本抽取任务 UIE Taskflow 使用指南
+# 文本抽取任务UIE Taskflow使用指南
 
 **目录**
 - [1. 功能简介](#1)
@@ -23,7 +23,7 @@
 
 ## 2. 应用示例
 
-UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现开箱即用的行业示例：
+UIE不限定行业领域和抽取目标，以下是一些通过Taskflow实现开箱即用的行业示例：
 
 - 医疗场景-专病结构化
 
@@ -53,9 +53,9 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
 
 #### 3.1 实体抽取
 
-  实体抽取，又称命名实体识别（Named Entity Recognition，简称 NER），是指识别文本中具有特定意义的实体。在开放域信息抽取中，抽取的类别没有限制，用户可以自己定义。
+  实体抽取，又称命名实体识别（Named Entity Recognition，简称NER），是指识别文本中具有特定意义的实体。在开放域信息抽取中，抽取的类别没有限制，用户可以自己定义。
 
-  - 例如抽取的目标实体类型是"时间"、"选手"和"赛事名称", schema 构造如下：
+  - 例如抽取的目标实体类型是"时间"、"选手"和"赛事名称", schema构造如下：
 
     ```text
     ['时间', '选手', '赛事名称']
@@ -84,7 +84,7 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
               'text': '谷爱凌'}]}]
     ```
 
-  - 例如抽取的目标实体类型是"肿瘤的大小"、"肿瘤的个数"、"肝癌级别"和"脉管内癌栓分级", schema 构造如下：
+  - 例如抽取的目标实体类型是"肿瘤的大小"、"肿瘤的个数"、"肝癌级别"和"脉管内癌栓分级", schema构造如下：
 
     ```text
     ['肿瘤的大小', '肿瘤的个数', '肝癌级别', '脉管内癌栓分级']
@@ -116,7 +116,7 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
                   'text': 'M0级'}]}]
     ```
 
-  - 例如抽取的目标实体类型是"person"和"organization"，schema 构造如下：
+  - 例如抽取的目标实体类型是"person"和"organization"，schema构造如下：
 
     ```text
     ['person', 'organization']
@@ -144,9 +144,9 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
 
 #### 3.2 关系抽取
 
-  关系抽取（Relation Extraction，简称 RE），是指从文本中识别实体并抽取实体之间的语义关系，进而获取三元组信息，即<主体，谓语，客体>。
+  关系抽取（Relation Extraction，简称RE），是指从文本中识别实体并抽取实体之间的语义关系，进而获取三元组信息，即<主体，谓语，客体>。
 
-  - 例如以"竞赛名称"作为抽取主体，抽取关系类型为"主办方"、"承办方"和"已举办次数", schema 构造如下：
+  - 例如以"竞赛名称"作为抽取主体，抽取关系类型为"主办方"、"承办方"和"已举办次数", schema构造如下：
 
     ```text
     {
@@ -194,7 +194,7 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
                 'text': '2022语言与智能技术竞赛'}]}]
     ```
 
-  - 例如以"person"作为抽取主体，抽取关系类型为"Company"和"Position", schema 构造如下：
+  - 例如以"person"作为抽取主体，抽取关系类型为"Company"和"Position", schema构造如下：
 
     ```text
     {
@@ -229,9 +229,9 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
 
 #### 3.3 事件抽取
 
-  事件抽取 (Event Extraction, 简称 EE)，是指从自然语言文本中抽取预定义的事件触发词(Trigger)和事件论元(Argument)，组合为相应的事件结构化信息。
+  事件抽取 (Event Extraction, 简称EE)，是指从自然语言文本中抽取预定义的事件触发词(Trigger)和事件论元(Argument)，组合为相应的事件结构化信息。
 
-  - 例如抽取的目标是"地震"事件的"地震强度"、"时间"、"震中位置"和"震源深度"这些信息，schema 构造如下：
+  - 例如抽取的目标是"地震"事件的"地震强度"、"时间"、"震中位置"和"震源深度"这些信息，schema构造如下：
 
     ```text
     {
@@ -244,7 +244,7 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
     }
     ```
 
-    触发词的格式统一为`触发词`或``XX 触发词`，`XX`表示具体事件类型，上例中的事件类型是`地震`，则对应触发词为`地震触发词`。
+    触发词的格式统一为`触发词`或``XX触发词`，`XX`表示具体事件类型，上例中的事件类型是`地震`，则对应触发词为`地震触发词`。
 
     调用示例：
 
@@ -263,7 +263,7 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
 
   评论观点抽取，是指抽取文本中包含的评价维度、观点词。
 
-  - 例如抽取的目标是文本中包含的评价维度及其对应的观点词和情感倾向，schema 构造如下：
+  - 例如抽取的目标是文本中包含的评价维度及其对应的观点词和情感倾向，schema构造如下：
 
     ```text
     {
@@ -302,7 +302,7 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
                 'text': '店面'}]}]
     ```
 
-  - 英文模型 schema 构造如下：
+  - 英文模型schema构造如下：
 
     ```text
     {
@@ -335,7 +335,7 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
 
 #### 3.5 情感分类
 
-  - 句子级情感倾向分类，即判断句子的情感倾向是“正向”还是“负向”，schema 构造如下：
+  - 句子级情感倾向分类，即判断句子的情感倾向是“正向”还是“负向”，schema构造如下：
 
     ```text
     '情感倾向[正向，负向]'
@@ -350,7 +350,7 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
     [{'情感倾向[正向，负向]': [{'text': '正向', 'probability': 0.9988661643929895}]}]
     ```
 
-    英文模型 schema 构造如下：
+    英文模型schema构造如下：
 
     ```text
     'Sentiment classification [negative, positive]'
@@ -369,7 +369,7 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
 
 #### 3.6 跨任务抽取
 
-  - 例如在法律场景同时对文本进行实体抽取和关系抽取，schema 可按照如下方式进行构造：
+  - 例如在法律场景同时对文本进行实体抽取和关系抽取，schema可按照如下方式进行构造：
 
     ```text
     [
@@ -489,12 +489,14 @@ UIE 不限定行业领域和抽取目标，以下是一些通过 Taskflow 实现
                   batch_size=16,
                   model='uie-base',
                   position_prob=0.5,
-                  precision='fp32')
+                  precision='fp32',
+                  use_fast=False)
 ```
 
 * `schema`：定义任务抽取目标，可参考开箱即用中不同任务的调用示例进行配置。
-* `schema_lang`：设置 schema 的语言，默认为`ch`, 可选有`ch`和`en`。因为中英 schema 的构造有所不同，因此需要指定 schema 的语言。该参数只对`uie-x-base`,`uie-m-base`和`uie-m-large`模型有效。
+* `schema_lang`：设置schema的语言，默认为`ch`, 可选有`ch`和`en`。因为中英schema的构造有所不同，因此需要指定schema的语言。该参数只对`uie-x-base`,`uie-m-base`和`uie-m-large`模型有效。
 * `batch_size`：批处理大小，请结合机器情况进行调整，默认为16。
 * `model`：选择任务使用的模型，默认为`uie-base`，可选有`uie-base`, `uie-medium`, `uie-mini`, `uie-micro`, `uie-nano`和`uie-medical-base`, `uie-base-en`，`uie-x-base`。
-* `position_prob`：模型对于 span 的起始位置/终止位置的结果概率在0~1之间，返回结果去掉小于这个阈值的结果，默认为0.5，span 的最终概率输出为起始位置概率和终止位置概率的乘积。
-* `precision`：选择模型精度，默认为`fp32`，可选有`fp16`和`fp32`。`fp16`推理速度更快，支持 GPU 和 NPU 硬件环境。如果选择`fp16`，在 GPU 硬件环境下，请先确保机器正确安装 NVIDIA 相关驱动和基础软件，**确保 CUDA>=11.2，cuDNN>=8.1.1**，初次使用需按照提示安装相关依赖。其次，需要确保 GPU 设备的 CUDA 计算能力（CUDA Compute Capability）大于7.0，典型的设备包括 V100、T4、A10、A100、GTX 20系列和30系列显卡等。更多关于 CUDA Compute Capability 和精度支持情况请参考 NVIDIA 文档：[GPU 硬件与支持精度对照表](https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-840-ea/support-matrix/index.html#hardware-precision-matrix)。
+* `position_prob`：模型对于span的起始位置/终止位置的结果概率在0~1之间，返回结果去掉小于这个阈值的结果，默认为0.5，span的最终概率输出为起始位置概率和终止位置概率的乘积。
+* `precision`：选择模型精度，默认为`fp32`，可选有`fp16`和`fp32`。`fp16`推理速度更快，支持GPU和NPU硬件环境。如果选择`fp16`，在GPU硬件环境下，请先确保机器正确安装NVIDIA相关驱动和基础软件，**确保CUDA>=11.2，cuDNN>=8.1.1**，初次使用需按照提示安装相关依赖。其次，需要确保GPU设备的CUDA计算能力（CUDA Compute Capability）大于7.0，典型的设备包括V100、T4、A10、A100、GTX 20系列和30系列显卡等。更多关于CUDA Compute Capability和精度支持情况请参考NVIDIA文档：[GPU硬件与支持精度对照表](https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-840-ea/support-matrix/index.html#hardware-precision-matrix)。
+* `use_fast`: 使用C++实现的高性能分词算子FastTokenizer进行文本预处理加速。需要通过`pip install fast-tokenizer-python`安装FastTokenizer库后方可使用。默认为`False`。更多使用说明可参考[FastTokenizer文档](../../fast_tokenizer)。

applications/information_extraction/taskflow_text_en.md

@@ -299,7 +299,8 @@ UIE does not limit industry fields and extraction targets. The following are som
                   batch_size=16,
                   model='uie-base',
                   position_prob=0.5,
-                  precision='fp32')
+                  precision='fp32',
+                  use_fast=False)
 ```
 
 * `schema`: Define the task extraction target, which can be configured by referring to the calling examples of different tasks in the out-of-the-box.
@@ -308,3 +309,4 @@ UIE does not limit industry fields and extraction targets. The following are som
 * `model`: select the model used by the task, the default is `uie-base`, optional `uie-base`, `uie-medium`, `uie-mini`, `uie-micro`, `uie-nano` and `uie-medical-base`, `uie-base-en`, `uie-x-base`.
 * `position_prob`: The result probability of the model for the start position/end position of the span is between 0 and 1, and the returned result removes the results less than this threshold, the default is 0.5, and the final probability output of the span is the start position probability and end position The product of the position probabilities.
 * `precision`: select the model precision, the default is `fp32`, optional `fp16` and `fp32`. `fp16` inference is faster, support GPU and NPU hardware. If you choose `fp16` and GPU hardware, please ensure that the machine is correctly installed with NVIDIA-related drivers and basic software. **Ensure that CUDA>=11.2, cuDNN>=8.1.1**. For the first time use, you need to follow the prompts to install the relevant dependencies. Secondly, it is necessary to ensure that the CUDA Compute Capability of the GPU device is greater than 7.0. Typical devices include V100, T4, A10, A100, GTX 20 series and 30 series graphics cards, etc. For more information about CUDA Compute Capability and precision support, please refer to NVIDIA documentation: [GPU Hardware and Supported Precision Comparison Table](https://docs.nvidia.com/deeplearning/tensorrt/archives/tensorrt-840-ea/support-matrix/index.html#hardware-precision-matrix).
+* `use_fast`: Use the high-performance word segmentation operator FastTokenizer implemented in C++ to accelerate text preprocessing. The FastTokenizer library needs to be installed through `pip install fast-tokenizer-python` before it can be used. Defaults to `False`. For more usage instructions, please refer to [FastTokenizer Documentation](../../fast_tokenizer).

applications/information_extraction/text/README.md

@@ -0,0 +1,289 @@
+简体中文 | [English](README_en.md)
+
+# 文本信息抽取
+
+**目录**
+- [1. 文本信息抽取应用](#1)
+- [2. 快速开始](#2)
+  - [2.1 代码结构](#代码结构)
+  - [2.2 数据标注](#数据标注)
+  - [2.3 模型微调](#模型微调)
+  - [2.4 模型评估](#模型评估)
+  - [2.5 定制模型一键预测](#定制模型一键预测)
+  - [2.6 实验指标](#实验指标)
+  - [2.7 封闭域蒸馏](#封闭域蒸馏)
+
+<a name="1"></a>
+
+## 1. 文本信息抽取应用
+
+本项目提供基于UIE微调的纯文本抽取端到端应用方案，打通**数据标注-模型训练-模型调优-预测部署全流程**，可快速实现文档信息抽取产品落地。
+
+信息抽取通俗地说就是从给定的文本/图片等输入数据中抽取出结构化信息的过程。在信息抽取的落地过程中通常面临领域多变、任务多样、数据稀缺等许多挑战。针对信息抽取领域的难点和痛点，PaddleNLP信息抽取应用UIE统一建模的思想，提供了文档信息抽取产业级应用方案，支持**文档/图片/表格和纯文本场景下实体、关系、事件、观点等不同任务信息抽取**。该应用**不限定行业领域和抽取目标**，可实现从产品原型研发、业务POC阶段到业务落地、迭代阶段的无缝衔接，助力开发者实现特定领域抽取场景的快速适配与落地。
+
+**文本信息抽取应用亮点：**
+
+- **覆盖场景全面🎓：** 覆盖文本信息抽取各类主流任务，支持多语言，满足开发者多样信息抽取落地需求。
+- **效果领先🏃：** 以在纯文本具有突出效果的UIE系列模型作为训练基座，提供多种尺寸的预训练模型满足不同需求，具有广泛成熟的实践应用性。
+- **简单易用⚡：** 通过Taskflow实现三行代码可实现无标注数据的情况下进行快速调用，一行命令即可开启信息抽取训练，轻松完成部署上线，降低信息抽取技术落地门槛。
+- **高效调优✊：** 开发者无需机器学习背景知识，即可轻松上手数据标注及模型训练流程。
+
+<a name="2"></a>
+
+## 2. 快速开始
+
+对于简单的抽取目标可以直接使用```paddlenlp.Taskflow```实现零样本（zero-shot）抽取，对于细分场景我们推荐使用定制功能（标注少量数据进行模型微调）以进一步提升效果。
+
+<a name="代码结构"></a>
+
+### 2.1 代码结构
+
+```shell
+.
+├── utils.py          # 数据处理工具
+├── finetune.py       # 模型微调、压缩脚本
+├── evaluate.py       # 模型评估脚本
+└── README.md
+```
+
+<a name="数据标注"></a>
+
+### 2.2 数据标注
+
+我们推荐使用 [Label Studio](https://labelstud.io/) 进行文本信息抽取数据标注，本项目打通了从数据标注到训练的通道，也即Label Studio导出数据可以通过 [label_studio.py](../label_studio.py) 脚本轻松将数据转换为输入模型时需要的形式，实现无缝衔接。标注方法的详细介绍请参考 [Label Studio数据标注指南](../label_studio_text.md)。
+
+这里我们提供预先标注好的`军事关系抽取数据集`的文件，可以运行下面的命令行下载数据集，我们将展示如何使用数据转化脚本生成训练/验证/测试集文件，并使用UIE模型进行微调。
+
+下载军事关系抽取数据集：
+
+```shell
+wget https://bj.bcebos.com/paddlenlp/datasets/military.tar.gz
+tar -xvf military.tar.gz
+mv military data
+rm military.tar.gz
+```
+
+生成训练/验证集文件：
+```shell
+python ../label_studio.py \
+    --label_studio_file ./data/label_studio.json \
+    --save_dir ./data \
+    --splits 0.76 0.24 0 \
+    --negative_ratio 3 \
+    --task_type ext
+```
+
+更多不同类型任务（含实体抽取、关系抽取、文档分类等）的标注规则及参数说明，请参考[Label Studio数据标注指南](../label_studio_text.md)。
+
+
+<a name="模型微调"></a>
+
+### 2.3 模型微调
+
+推荐使用 [Trainer API ](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/docs/trainer.md) 对模型进行微调。只需输入模型、数据集等就可以使用 Trainer API 高效快速地进行预训练、微调和模型压缩等任务，可以一键启动多卡训练、混合精度训练、梯度累积、断点重启、日志显示等功能，Trainer API 还针对训练过程的通用训练配置做了封装，比如：优化器、学习率调度等。
+
+使用下面的命令，使用 `uie-base` 作为预训练模型进行模型微调，将微调后的模型保存至`$finetuned_model`：
+
+单卡启动：
+
+```shell
+python finetune.py  \
+    --device gpu \
+    --logging_steps 10 \
+    --save_steps 100 \
+    --eval_steps 100 \
+    --seed 1000 \
+    --model_name_or_path uie-base \
+    --output_dir ./checkpoint/model_best \
+    --train_path data/train.txt \
+    --dev_path data/dev.txt  \
+    --max_seq_len 512  \
+    --per_device_train_batch_size  16 \
+    --per_device_eval_batch_size 16 \
+    --num_train_epochs 20 \
+    --learning_rate 1e-5 \
+    --do_train \
+    --do_eval \
+    --do_export \
+    --export_model_dir ./checkpoint/model_best \
+    --overwrite_output_dir \
+    --disable_tqdm True \
+    --metric_for_best_model eval_f1 \
+    --load_best_model_at_end  True \
+    --save_total_limit 1
+```
+
+如果在GPU环境中使用，可以指定gpus参数进行多卡训练：
+
+```shell
+python -u -m paddle.distributed.launch --gpus "0,1" finetune.py \
+    --device gpu \
+    --logging_steps 10 \
+    --save_steps 100 \
+    --eval_steps 100 \
+    --seed 1000 \
+    --model_name_or_path uie-base \
+    --output_dir ./checkpoint/model_best \
+    --train_path data/train.txt \
+    --dev_path data/dev.txt  \
+    --max_seq_len 512  \
+    --per_device_train_batch_size  8 \
+    --per_device_eval_batch_size 8 \
+    --num_train_epochs 20 \
+    --learning_rate 1e-5 \
+    --do_train \
+    --do_eval \
+    --do_export \
+    --export_model_dir ./checkpoint/model_best \
+    --overwrite_output_dir \
+    --disable_tqdm True \
+    --metric_for_best_model eval_f1 \
+    --load_best_model_at_end  True \
+    --save_total_limit 1
+```
+
+该示例代码中由于设置了参数 `--do_eval`，因此在训练完会自动进行评估。
+
+可配置参数说明：
+* `device`: 训练设备，可选择 'cpu'、'gpu'、'npu' 其中的一种；默认为 GPU 训练。
+* `logging_steps`: 训练过程中日志打印的间隔 steps 数，默认10。
+* `save_steps`: 训练过程中保存模型 checkpoint 的间隔 steps 数，默认100。
+* `eval_steps`: 训练过程中保存模型 checkpoint 的间隔 steps 数，默认100。
+* `seed`：全局随机种子，默认为 42。
+* `model_name_or_path`：进行 few shot 训练使用的预训练模型。默认为 "uie-x-base"。
+* `output_dir`：必须，模型训练或压缩后保存的模型目录；默认为 `None` 。
+* `train_path`：训练集路径；默认为 `None` 。
+* `dev_path`：开发集路径；默认为 `None` 。
+* `max_seq_len`：文本最大切分长度，输入超过最大长度时会对输入文本进行自动切分，默认为512。
+* `per_device_train_batch_size`:用于训练的每个 GPU 核心/CPU 的batch大小，默认为8。
+* `per_device_eval_batch_size`:用于评估的每个 GPU 核心/CPU 的batch大小，默认为8。
+* `num_train_epochs`: 训练轮次，使用早停法时可以选择 100；默认为10。
+* `learning_rate`：训练最大学习率，UIE-X 推荐设置为 1e-5；默认值为3e-5。
+* `label_names`：训练数据标签label的名称，UIE-X 设置为'start_positions' 'end_positions'；默认值为None。
+* `do_train`:是否进行微调训练，设置该参数表示进行微调训练，默认不设置。
+* `do_eval`:是否进行评估，设置该参数表示进行评估，默认不设置。
+* `do_export`:是否进行导出，设置该参数表示进行静态图导出，默认不设置。
+* `export_model_dir`:静态图导出地址，默认为None。
+* `overwrite_output_dir`： 如果 `True`，覆盖输出目录的内容。如果 `output_dir` 指向检查点目录，则使用它继续训练。
+* `disable_tqdm`： 是否使用tqdm进度条。
+* `metric_for_best_model`：最优模型指标,UIE-X 推荐设置为 `eval_f1`，默认为None。
+* `load_best_model_at_end`：训练结束后是否加载最优模型，通常与`metric_for_best_model`配合使用，默认为False。
+* `save_total_limit`：如果设置次参数，将限制checkpoint的总数。删除旧的checkpoints `输出目录`，默认为None。
+
+<a name="模型评估"></a>
+
+### 2.4 模型评估
+
+通过运行以下命令进行模型评估：
+
+```shell
+python evaluate.py \
+    --model_path ./checkpoint/model_best \
+    --test_path ./data/dev.txt \
+    --device gpu \
+    --batch_size 16 \
+    --max_seq_len 512
+```
+
+通过运行以下命令对 UIE-M 进行模型评估：
+
+```
+python evaluate.py \
+    --model_path ./checkpoint/model_best \
+    --test_path ./data/dev.txt \
+    --batch_size 16 \
+    --device gpu \
+    --max_seq_len 512 \
+    --multilingual
+```
+
+评估方式说明：采用单阶段评价的方式，即关系抽取、事件抽取等需要分阶段预测的任务对每一阶段的预测结果进行分别评价。验证/测试集默认会利用同一层级的所有标签来构造出全部负例。
+
+可开启`debug`模式对每个正例类别分别进行评估，该模式仅用于模型调试：
+
+```shell
+python evaluate.py \
+    --model_path ./checkpoint/model_best \
+    --test_path ./data/dev.txt \
+    --debug
+```
+
+输出打印示例：
+
+```text
+[2022-11-21 12:48:41,794] [    INFO] - -----------------------------
+[2022-11-21 12:48:41,795] [    INFO] - Class Name: 武器名称
+[2022-11-21 12:48:41,795] [    INFO] - Evaluation Precision: 0.96667 | Recall: 0.96667 | F1: 0.96667
+[2022-11-21 12:48:44,093] [    INFO] - -----------------------------
+[2022-11-21 12:48:44,094] [    INFO] - Class Name: X的产国
+[2022-11-21 12:48:44,094] [    INFO] - Evaluation Precision: 1.00000 | Recall: 0.99275 | F1: 0.99636
+[2022-11-21 12:48:46,474] [    INFO] - -----------------------------
+[2022-11-21 12:48:46,475] [    INFO] - Class Name: X的研发单位
+[2022-11-21 12:48:46,475] [    INFO] - Evaluation Precision: 0.77519 | Recall: 0.64935 | F1: 0.70671
+[2022-11-21 12:48:48,800] [    INFO] - -----------------------------
+[2022-11-21 12:48:48,801] [    INFO] - Class Name: X的类型
+[2022-11-21 12:48:48,801] [    INFO] - Evaluation Precision: 1.00000 | Recall: 1.00000 | F1: 1.00000
+```
+
+可配置参数说明：
+
+- `device`: 评估设备，可选择 'cpu'、'gpu'、'npu' 其中的一种；默认为 GPU 评估。
+- `model_path`: 进行评估的模型文件夹路径，路径下需包含模型权重文件`model_state.pdparams`及配置文件`model_config.json`。
+- `test_path`: 进行评估的测试集文件。
+- `batch_size`: 批处理大小，请结合机器情况进行调整，默认为16。
+- `max_seq_len`: 文本最大切分长度，输入超过最大长度时会对输入文本进行自动切分，默认为512。
+- `debug`: 是否开启debug模式对每个正例类别分别进行评估，该模式仅用于模型调试，默认关闭。
+- `multilingual`: 是否是跨语言模型，默认关闭。
+- `schema_lang`: 选择schema的语言，可选有`ch`和`en`。默认为`ch`，英文数据集请选择`en`。
+
+<a name="定制模型一键预测"></a>
+
+### 2.5 定制模型一键预测
+
+`paddlenlp.Taskflow`装载定制模型，通过`task_path`指定模型权重文件的路径，路径下需要包含训练好的模型权重文件`model_state.pdparams`。
+
+```python
+>>> from pprint import pprint
+>>> from paddlenlp import Taskflow
+
+>>> schema = {"武器名称": ["产国", "类型", "研发单位"]}
+# 设定抽取目标和定制化模型权重路径
+>>> my_ie = Taskflow("information_extraction", schema=schema, task_path='./checkpoint/model_best')
+>>> pprint(my_ie("威尔哥（Virgo）减速炸弹是由瑞典FFV军械公司专门为瑞典皇家空军的攻击机实施低空高速轰炸而研制，1956年开始研制，1963年进入服役，装备于A32“矛盾”、A35“龙”、和AJ134“雷”攻击机，主要用于攻击登陆艇、停放的飞机、高炮、野战火炮、轻型防护装甲车辆以及有生力量。"))
+[{'武器名称': [{'end': 14,
+            'probability': 0.9998632702221926,
+            'relations': {'产国': [{'end': 18,
+                                  'probability': 0.9998815094394331,
+                                  'start': 16,
+                                  'text': '瑞典'}],
+                          '研发单位': [{'end': 25,
+                                    'probability': 0.9995875123178521,
+                                    'start': 18,
+                                    'text': 'FFV军械公司'}],
+                          '类型': [{'end': 14,
+                                  'probability': 0.999877336059086,
+                                  'start': 12,
+                                  'text': '炸弹'}]},
+            'start': 0,
+            'text': '威尔哥（Virgo）减速炸弹'}]}]
+```
+
+<a name="实验指标"></a>
+
+### 2.6 实验指标
+
+军事关系抽取数据集实验指标：
+
+  |  |  Precision  | Recall | F1 Score |
+  | :---: | :--------: | :--------: | :--------: |
+  | 0-shot | 0.64634| 0.53535 | 0.58564 |
+  | 5-shot | 0.89474 | 0.85000 | 0.87179 |
+  | 10-shot | 0.92793 | 0.85833 | 0.89177 |
+  | full-set | 0.93103 | 0.90000 | 0.91525 |
+
+
+<a name="封闭域蒸馏"></a>
+
+### 2.7 封闭域蒸馏
+
+在一些工业应用场景中对性能的要求较高，模型若不能有效压缩则无法实际应用。因此，我们基于数据蒸馏技术构建了UIE Slim数据蒸馏系统。其原理是通过数据作为桥梁，将UIE模型的知识迁移到封闭域信息抽取小模型，以达到精度损失较小的情况下却能达到大幅度预测速度提升的效果。详细介绍请参考[UIE Slim 数据蒸馏](./data_distill/README.md)

applications/information_extraction/text/data_distill/README.md

@@ -0,0 +1,156 @@
+# UIE Slim 数据蒸馏
+
+在UIE强大的抽取能力背后，同样需要较大的算力支持计算。在一些工业应用场景中对性能的要求较高，若不能有效压缩则无法实际应用。因此，我们基于数据蒸馏技术构建了UIE Slim数据蒸馏系统。其原理是通过数据作为桥梁，将UIE模型的知识迁移到封闭域信息抽取小模型，以达到精度损失较小的情况下却能达到大幅度预测速度提升的效果。
+
+#### UIE数据蒸馏三步
+
+- **Step 1**: 使用UIE模型对标注数据进行finetune，得到Teacher Model。
+
+- **Step 2**: 用户提供大规模无标注数据，需与标注数据同源。使用Taskflow UIE对无监督数据进行预测。
+
+- **Step 3**: 使用标注数据以及步骤2得到的合成数据训练出封闭域Student Model。
+
+## UIE Finetune
+
+参考[UIE关系抽取微调](../README.md)完成模型微调，得到``../checkpoint/model_best``。
+
+## 离线蒸馏
+
+#### 通过训练好的UIE定制模型预测无监督数据的标签
+
+```shell
+python data_distill.py \
+    --data_path ../data \
+    --save_dir student_data \
+    --task_type relation_extraction \
+    --synthetic_ratio 10 \
+    --model_path ../checkpoint/model_best
+```
+
+**NOTE**：schema需要根据标注数据在`data_distill.py`中进行配置，且schema需要包含标注数据中的所有标签类型。
+
+可配置参数说明：
+
+- `data_path`: 标注数据（`doccano_ext.json`）及无监督文本（`unlabeled_data.txt`）路径。
+- `model_path`: 训练好的UIE定制模型路径。
+- `save_dir`: 学生模型训练数据保存路径。
+- `synthetic_ratio`: 控制合成数据的比例。最大合成数据数量=synthetic_ratio*标注数据数量。
+- `platform`: 标注数据的所使用的标注平台，可选有`doccano`，`label_studio`，默认为`label_studio`。
+- `task_type`: 选择任务类型，可选有`entity_extraction`，`relation_extraction`，`event_extraction`和`opinion_extraction`。因为是封闭域抽取，不同任务的后处理逻辑不同，因此需指定任务类型。
+- `seed`: 随机种子，默认为1000。
+
+#### 老师模型评估
+
+UIE微调阶段针对UIE训练格式数据评估模型效果（该评估方式非端到端评估，非关系抽取或事件抽取的标准评估方式），可通过以下评估脚本进行端到端评估。
+
+```shell
+python evaluate_teacher.py \
+    --task_type relation_extraction \
+    --test_path ./student_data/dev_data.json \
+    --label_maps_path ./student_data/label_maps.json \
+    --model_path ../checkpoint/model_best
+```
+
+可配置参数说明：
+
+- `model_path`: 训练好的UIE定制模型路径。
+- `test_path`: 测试数据集路径。
+- `label_maps_path`: 学生模型标签字典。
+- `batch_size`: 批处理大小，默认为8。
+- `max_seq_len`: 最大文本长度，默认为256。
+- `task_type`: 选择任务类型，可选有`entity_extraction`，`relation_extraction`，`event_extraction`和`opinion_extraction`。因为是封闭域信息抽取的评估，需指定任务类型。
+
+
+#### 学生模型训练
+
+```shell
+python train.py \
+    --task_type relation_extraction \
+    --train_path student_data/train_data.json \
+    --dev_path student_data/dev_data.json \
+    --label_maps_path student_data/label_maps.json \
+    --num_epochs 50 \
+    --encoder ernie-3.0-mini-zh
+```
+
+可配置参数说明：
+
+- `train_path`: 训练集文件路径。
+- `dev_path`: 验证集文件路径。
+- `batch_size`: 批处理大小，默认为16。
+- `learning_rate`: 学习率，默认为3e-5。
+- `save_dir`: 模型存储路径，默认为`./checkpoint`。
+- `max_seq_len`: 最大文本长度，默认为256。
+- `weight_decay`: 表示AdamW优化器中使用的 weight_decay 的系数。
+- `warmup_proportion`: 学习率warmup策略的比例，如果0.1，则学习率会在前10%训练step的过程中从0慢慢增长到learning_rate, 而后再缓慢衰减，默认为0.0。
+- `num_epochs`: 训练轮数，默认为100。
+- `seed`: 随机种子，默认为1000。
+- `encoder`: 选择学生模型的模型底座，默认为`ernie-3.0-mini-zh`。
+- `task_type`: 选择任务类型，可选有`entity_extraction`，`relation_extraction`，`event_extraction`和`opinion_extraction`。因为是封闭域信息抽取，需指定任务类型。
+- `logging_steps`: 日志打印的间隔steps数，默认10。
+- `eval_steps`: evaluate的间隔steps数，默认200。
+- `device`: 选用什么设备进行训练，可选cpu或gpu。
+- `init_from_ckpt`: 可选，模型参数路径，热启动模型训练；默认为None。
+
+#### 学生模型评估
+
+```shell
+python evaluate.py \
+    --model_path ./checkpoint/model_best \
+    --test_path student_data/dev_data.json \
+    --task_type relation_extraction \
+    --label_maps_path student_data/label_maps.json \
+    --encoder ernie-3.0-mini-zh
+```
+
+可配置参数说明：
+
+- `model_path`: 训练好的UIE定制模型路径。
+- `test_path`: 测试数据集路径。
+- `label_maps_path`: 学生模型标签字典。
+- `batch_size`: 批处理大小，默认为8。
+- `max_seq_len`: 最大文本长度，默认为256。
+- `encoder`: 选择学生模型的模型底座，默认为`ernie-3.0-mini-zh`。
+- `task_type`: 选择任务类型，可选有`entity_extraction`，`relation_extraction`，`event_extraction`和`opinion_extraction`。因为是封闭域信息抽取的评估，需指定任务类型。
+
+## Taskflow部署学生模型
+
+- 通过Taskflow一键部署封闭域信息抽取模型，`task_path`为学生模型路径。
+
+```python
+>>> from pprint import pprint
+>>> from paddlenlp import Taskflow
+
+>>> my_ie = Taskflow("information_extraction", model="uie-data-distill-gp", task_path="checkpoint/model_best/") # Schema is fixed in closed-domain information extraction
+>>> pprint(my_ie("威尔哥（Virgo）减速炸弹是由瑞典FFV军械公司专门为瑞典皇家空军的攻击机实施低空高速轰炸而研制，1956年开始研制，1963年进入服役，装备于A32“矛盾”、A35“龙”、和AJ134“雷”攻击机，主要用于攻击登陆艇、停放的飞机、高炮、野战火炮、轻型防护装甲车辆以及有生力量。"))
+[{'武器名称': [{'end': 14,
+            'probability': 0.9976037,
+            'relations': {'产国': [{'end': 18,
+                                  'probability': 0.9988706,
+                                  'relations': {},
+                                  'start': 16,
+                                  'text': '瑞典'}],
+                          '研发单位': [{'end': 25,
+                                    'probability': 0.9978277,
+                                    'relations': {},
+                                    'start': 18,
+                                    'text': 'FFV军械公司'}],
+                          '类型': [{'end': 14,
+                                  'probability': 0.99837446,
+                                  'relations': {},
+                                  'start': 12,
+                                  'text': '炸弹'}]},
+            'start': 0,
+            'text': '威尔哥（Virgo）减速炸弹'}]}]
+```
+
+
+# References
+
+- **[GlobalPointer](https://kexue.fm/search/globalpointer/)**
+
+- **[GPLinker](https://kexue.fm/archives/8888)**
+
+- **[JunnYu/GPLinker_pytorch](https://github.com/JunnYu/GPLinker_pytorch)**
+
+- **[CBLUE](https://github.com/CBLUEbenchmark/CBLUE)**

applications/information_extraction/text/data_distill/deploy/simple_serving/README.md

@@ -0,0 +1,58 @@
+# 基于PaddleNLP SimpleServing 的服务化部署
+
+## 目录
+- [环境准备](#环境准备)
+- [Server服务启动](#Server服务启动)
+- [Client请求启动](#Client请求启动)
+- [服务化自定义参数](#服务化自定义参数)
+
+## 环境准备
+使用有SimpleServing功能的PaddleNLP版本(或者最新的develop版本)
+
+```shell
+pip install paddlenlp >= 2.4.4
+```
+
+
+## Server服务启动
+
+```bash
+paddlenlp server server:app --workers 1 --host 0.0.0.0 --port 8189
+```
+
+## Client请求启动
+
+```bash
+python client.py
+```
+
+## 服务化自定义参数
+
+### Server 自定义参数
+#### schema替换
+```python
+# Default schema
+schema = {"武器名称": ["产国", "类型", "研发单位"]}
+```
+
+#### 设置模型路径
+```
+# Default task_path
+uie = Taskflow('information_extraction', model='uie-data-distill-gp', task_path='../../checkpoint/model_best/', schema=schema)
+```
+
+#### 多卡服务化预测
+PaddleNLP SimpleServing 支持多卡负载均衡预测，主要在服务化注册的时候，注册两个Taskflow的task即可，下面是示例代码
+```
+uie1 = Taskflow('information_extraction', model='uie-data-distill-gp', task_path='../../checkpoint/model_best/', schema=schema, device_id=0)
+uie2 = Taskflow('information_extraction', model='uie-data-distill-gp', task_path='../../checkpoint/model_best/', schema=schema, device_id=1)
+service.register_taskflow('uie', [uie1, uie2])
+```
+
+### Client 自定义参数
+
+```python
+# Changed to input texts you wanted
+texts = ['威尔哥（Virgo）减速炸弹是由瑞典FFV军械公司专门为瑞典皇家空军的攻击机实施低空高速轰炸而研制，1956年开始研制，1963年进入服役，装备于A32“矛盾”、A35“龙”、和AJ134“雷”攻击机，主要用于攻击登陆艇、停放的飞机、高炮、野战火炮、轻型防护装甲车辆以及有生力量。']
+
+```

applications/information_extraction/text/deploy/simple_serving/README.md

@@ -0,0 +1,58 @@
+# 基于PaddleNLP SimpleServing 的服务化部署
+
+## 目录
+- [环境准备](#环境准备)
+- [Server服务启动](#Server服务启动)
+- [Client请求启动](#Client请求启动)
+- [服务化自定义参数](#服务化自定义参数)
+
+## 环境准备
+使用有SimpleServing功能的PaddleNLP版本(或者最新的develop版本)
+
+```shell
+pip install paddlenlp >= 2.4.4
+```
+
+
+## Server服务启动
+
+```bash
+paddlenlp server server:app --workers 1 --host 0.0.0.0 --port 8189
+```
+
+## Client请求启动
+
+```bash
+python client.py
+```
+
+## 服务化自定义参数
+
+### Server 自定义参数
+#### schema替换
+```python
+# Default schema
+schema = {"武器名称": ["产国", "类型", "研发单位"]}
+```
+
+#### 设置模型路径
+```
+# Default task_path
+uie = Taskflow('information_extraction', task_path='../../checkpoint/model_best/', schema=schema)
+```
+
+#### 多卡服务化预测
+PaddleNLP SimpleServing 支持多卡负载均衡预测，主要在服务化注册的时候，注册两个Taskflow的task即可，下面是示例代码
+```
+uie1 = Taskflow('information_extraction', task_path='../../checkpoint/model_best/', schema=schema, device_id=0)
+uie2 = Taskflow('information_extraction', task_path='../../checkpoint/model_best/', schema=schema, device_id=1)
+service.register_taskflow('uie', [uie1, uie2])
+```
+
+### Client 自定义参数
+
+```python
+# Changed to input texts you wanted
+texts = ['威尔哥（Virgo）减速炸弹是由瑞典FFV军械公司专门为瑞典皇家空军的攻击机实施低空高速轰炸而研制，1956年开始研制，1963年进入服役，装备于A32“矛盾”、A35“龙”、和AJ134“雷”攻击机，主要用于攻击登陆>艇、停放的飞机、高炮、野战火炮、轻型防护装甲车辆以及有生力量。']
+
+```