Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
dca8425045 | ||
|
|
ffbc5d05ca |
134
.copyright.hook
134
.copyright.hook
@@ -1,134 +0,0 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
import datetime
|
||||
|
||||
COPYRIGHT = '''Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.'''
|
||||
|
||||
def _generate_copyright(comment_mark):
|
||||
copyright=COPYRIGHT.split(os.linesep)
|
||||
header = copyright[0].rstrip()
|
||||
|
||||
p = re.search('(\d{4})', header).group(0)
|
||||
now = datetime.datetime.now()
|
||||
|
||||
header = header.replace(p,str(now.year))
|
||||
|
||||
ans=[comment_mark + " " + header + os.linesep]
|
||||
for idx, line in enumerate(copyright[1:]):
|
||||
ans.append(comment_mark + " " + line.rstrip() + os.linesep)
|
||||
|
||||
return ans
|
||||
|
||||
def _get_comment_mark(path):
|
||||
lang_type=re.compile(r"\.(py|sh)$")
|
||||
if lang_type.search(path) is not None:
|
||||
return "#"
|
||||
|
||||
lang_type=re.compile(r"\.(h|c|hpp|cc|cpp|cu|go|cuh|proto)$")
|
||||
if lang_type.search(path) is not None:
|
||||
return "//"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
RE_ENCODE = re.compile(r"^[ \t\v]*#.*?coding[:=]", re.IGNORECASE)
|
||||
RE_COPYRIGHT = re.compile(r".*Copyright( \(c\))* \d{4}", re.IGNORECASE)
|
||||
RE_SHEBANG = re.compile(r"^[ \t\v]*#[ \t]?\!")
|
||||
|
||||
def _check_copyright(path):
|
||||
head=[]
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
head = [next(f) for x in range(4)]
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
for idx, line in enumerate(head):
|
||||
if RE_COPYRIGHT.search(line) is not None:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def generate_copyright(path, comment_mark):
|
||||
original_contents = io.open(path, encoding="utf-8").readlines()
|
||||
head = original_contents[0:4]
|
||||
|
||||
insert_line_no=0
|
||||
for i, line in enumerate(head):
|
||||
if RE_ENCODE.search(line) or RE_SHEBANG.search(line):
|
||||
insert_line_no=i+1
|
||||
|
||||
copyright = _generate_copyright(comment_mark)
|
||||
if insert_line_no == 0:
|
||||
new_contents = copyright
|
||||
if len(original_contents) > 0 and len(original_contents[0].strip()) != 0:
|
||||
new_contents.append(os.linesep)
|
||||
new_contents.extend(original_contents)
|
||||
else:
|
||||
new_contents=original_contents[0:insert_line_no]
|
||||
new_contents.append(os.linesep)
|
||||
new_contents.extend(copyright)
|
||||
if len(original_contents) > insert_line_no and len(original_contents[insert_line_no].strip()) != 0:
|
||||
new_contents.append(os.linesep)
|
||||
new_contents.extend(original_contents[insert_line_no:])
|
||||
new_contents="".join(new_contents)
|
||||
|
||||
with io.open(path, 'w') as output_file:
|
||||
output_file.write(new_contents)
|
||||
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Checker for copyright declaration.')
|
||||
parser.add_argument('filenames', nargs='*', help='Filenames to check')
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
retv = 0
|
||||
for path in args.filenames:
|
||||
comment_mark = _get_comment_mark(path)
|
||||
if comment_mark is None:
|
||||
print("warning:Unsupported file", path, file=sys.stderr)
|
||||
continue
|
||||
|
||||
if _check_copyright(path):
|
||||
continue
|
||||
|
||||
generate_copyright(path, comment_mark)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
exit(main())
|
||||
7
.flake8
7
.flake8
@@ -1,7 +0,0 @@
|
||||
[flake8]
|
||||
ignore = E203, E402, E501, E731, E741, W503, W605, E722
|
||||
max-line-length = 119
|
||||
|
||||
# E402: module level import not at top of file
|
||||
per-file-ignores =
|
||||
__init__.py:F401,F403,E402
|
||||
89
.github/CODE_OF_CONDUCT.md
vendored
89
.github/CODE_OF_CONDUCT.md
vendored
@@ -1,89 +0,0 @@
|
||||
**简体中文**🀄 | [English🌎](./CODE_OF_CONDUCT_en.md)
|
||||
|
||||
# 贡献者公约
|
||||
|
||||
## 我们的承诺
|
||||
|
||||
身为社区成员、贡献者和领袖，我们承诺使社区参与者不受骚扰，无论其年龄、体型、可见或不可见的缺陷、族裔、性征、性别认同和表达、经验水平、教育程度、社会与经济地位、国籍、相貌、种族、种姓、肤色、宗教信仰、性倾向或性取向如何。
|
||||
|
||||
我们承诺以有助于建立开放、友善、多样化、包容、健康社区的方式行事和互动。
|
||||
|
||||
## 我们的准则
|
||||
|
||||
有助于为我们的社区创造积极环境的行为例子包括但不限于：
|
||||
|
||||
* 表现出对他人的同情和善意
|
||||
* 尊重不同的主张、观点和感受
|
||||
* 提出和大方接受建设性意见
|
||||
* 承担责任并向受我们错误影响的人道歉
|
||||
* 注重社区共同诉求，而非个人得失
|
||||
|
||||
不当行为例子包括：
|
||||
|
||||
* 使用情色化的语言或图像，及性引诱或挑逗
|
||||
* 嘲弄、侮辱或诋毁性评论，以及人身或政治攻击
|
||||
* 公开或私下的骚扰行为
|
||||
* 未经他人明确许可，公布他人的私人信息，如物理或电子邮件地址
|
||||
* 其他有理由认定为违反职业操守的不当行为
|
||||
|
||||
## 责任和权力
|
||||
|
||||
社区领袖有责任解释和落实我们所认可的行为准则，并妥善公正地对他们认为不当、威胁、冒犯或有害的任何行为采取纠正措施。
|
||||
|
||||
社区领导有权力和责任删除、编辑或拒绝或拒绝与本行为准则不相符的评论（comment）、提交（commits）、代码、维基（wiki）编辑、议题（issues）或其他贡献，并在适当时机知采取措施的理由。
|
||||
|
||||
## 适用范围
|
||||
|
||||
本行为准则适用于所有社区场合，也适用于在公共场所代表社区时的个人。
|
||||
|
||||
代表社区的情形包括使用官方电子邮件地址、通过官方社交媒体帐户发帖或在线上或线下活动中担任指定代表。
|
||||
|
||||
## 监督
|
||||
|
||||
辱骂、骚扰或其他不可接受的行为可通过 paddlenlp@baidu.com 向负责监督的社区领袖报告。
|
||||
所有投诉都将得到及时和公平的审查和调查。
|
||||
|
||||
所有社区领袖都有义务尊重任何事件报告者的隐私和安全。
|
||||
|
||||
## 处理方针
|
||||
|
||||
社区领袖将遵循下列社区处理方针来明确他们所认定违反本行为准则的行为的处理方式：
|
||||
|
||||
### 1. 纠正
|
||||
|
||||
**社区影响**：使用不恰当的语言或其他在社区中被认定为不符合职业道德或不受欢迎的行为。
|
||||
|
||||
**处理意见**：由社区领袖发出非公开的书面警告，明确说明违规行为的性质，并解释举止如何不妥。或将要求公开道歉。
|
||||
|
||||
### 2. 警告
|
||||
|
||||
**社区影响**：单个或一系列违规行为。
|
||||
|
||||
**处理意见**：警告并对连续性行为进行处理。在指定时间内，不得与相关人员互动，包括主动与行为准则执行者互动。这包括避免在社区场所和外部渠道中的互动。违反这些条款可能会导致临时或永久封禁。
|
||||
|
||||
### 3. 临时封禁
|
||||
|
||||
**社区影响**: 严重违反社区准则，包括持续的不当行为。
|
||||
|
||||
**处理意见**: 在指定时间内，暂时禁止与社区进行任何形式的互动或公开交流。在此期间，不得与相关人员进行公开或私下互动，包括主动与行为准则执行者互动。违反这些条款可能会导致永久封禁。
|
||||
|
||||
### 4. 永久封禁
|
||||
|
||||
**社区影响**：行为模式表现出违反社区准则，包括持续的不当行为、骚扰个人或攻击或贬低某个类别的个体。
|
||||
|
||||
**处理意见**：永久禁止在社区内进行任何形式的公开互动。
|
||||
|
||||
## 参见
|
||||
|
||||
本行为准则改编自 [Contributor Covenant][homepage] 2.1 版, 参见 [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]。
|
||||
|
||||
社区处理方针灵感来源于 [Mozilla's code of conduct enforcement ladder][Mozilla CoC]。
|
||||
|
||||
有关本行为准则的常见问题的答案，参见 [https://www.contributor-covenant.org/faq][FAQ]。
|
||||
其他语言翻译参见 [https://www.contributor-covenant.org/translations][translations]。
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
|
||||
[Mozilla CoC]: https://github.com/mozilla/diversity
|
||||
[FAQ]: https://www.contributor-covenant.org/faq
|
||||
[translations]: https://www.contributor-covenant.org/translations
|
||||
134
.github/CODE_OF_CONDUCT_en.md
vendored
134
.github/CODE_OF_CONDUCT_en.md
vendored
@@ -1,134 +0,0 @@
|
||||
[简体中文🀄](./CODE_OF_CONDUCT.md) | **English**🌎
|
||||
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our
|
||||
community a harassment-free experience for everyone, regardless of age, body
|
||||
size, visible or invisible disability, ethnicity, sex characteristics, gender
|
||||
identity and expression, level of experience, education, socio-economic status,
|
||||
nationality, personal appearance, race, caste, color, religion, or sexual
|
||||
identity and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming,
|
||||
diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to a positive environment for our
|
||||
community include:
|
||||
|
||||
* Demonstrating empathy and kindness toward other people
|
||||
* Being respectful of differing opinions, viewpoints, and experiences
|
||||
* Giving and gracefully accepting constructive feedback
|
||||
* Accepting responsibility and apologizing to those affected by our mistakes,
|
||||
and learning from the experience
|
||||
* Focusing on what is best not just for us as individuals, but for the overall
|
||||
community
|
||||
|
||||
Examples of unacceptable behavior include:
|
||||
|
||||
* The use of sexualized language or imagery, and sexual attention or advances of
|
||||
any kind
|
||||
* Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or email address,
|
||||
without their explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of
|
||||
acceptable behavior and will take appropriate and fair corrective action in
|
||||
response to any behavior that they deem inappropriate, threatening, offensive,
|
||||
or harmful.
|
||||
|
||||
Community leaders have the right and responsibility to remove, edit, or reject
|
||||
comments, commits, code, wiki edits, issues, and other contributions that are
|
||||
not aligned to this Code of Conduct, and will communicate reasons for moderation
|
||||
decisions when appropriate.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when
|
||||
an individual is officially representing the community in public spaces.
|
||||
Examples of representing our community include using an official e-mail address,
|
||||
posting via an official social media account, or acting as an appointed
|
||||
representative at an online or offline event.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement at
|
||||
paddlenlp@baidu.com.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
reporter of any incident.
|
||||
|
||||
## Enforcement Guidelines
|
||||
|
||||
Community leaders will follow these Community Impact Guidelines in determining
|
||||
the consequences for any action they deem in violation of this Code of Conduct:
|
||||
|
||||
### 1. Correction
|
||||
|
||||
**Community Impact**: Use of inappropriate language or other behavior deemed
|
||||
unprofessional or unwelcome in the community.
|
||||
|
||||
**Consequence**: A private, written warning from community leaders, providing
|
||||
clarity around the nature of the violation and an explanation of why the
|
||||
behavior was inappropriate. A public apology may be requested.
|
||||
|
||||
### 2. Warning
|
||||
|
||||
**Community Impact**: A violation through a single incident or series of
|
||||
actions.
|
||||
|
||||
**Consequence**: A warning with consequences for continued behavior. No
|
||||
interaction with the people involved, including unsolicited interaction with
|
||||
those enforcing the Code of Conduct, for a specified period of time. This
|
||||
includes avoiding interactions in community spaces as well as external channels
|
||||
like social media. Violating these terms may lead to a temporary or permanent
|
||||
ban.
|
||||
|
||||
### 3. Temporary Ban
|
||||
|
||||
**Community Impact**: A serious violation of community standards, including
|
||||
sustained inappropriate behavior.
|
||||
|
||||
**Consequence**: A temporary ban from any sort of interaction or public
|
||||
communication with the community for a specified period of time. No public or
|
||||
private interaction with the people involved, including unsolicited interaction
|
||||
with those enforcing the Code of Conduct, is allowed during this period.
|
||||
Violating these terms may lead to a permanent ban.
|
||||
|
||||
### 4. Permanent Ban
|
||||
|
||||
**Community Impact**: Demonstrating a pattern of violation of community
|
||||
standards, including sustained inappropriate behavior, harassment of an
|
||||
individual, or aggression toward or disparagement of classes of individuals.
|
||||
|
||||
**Consequence**: A permanent ban from any sort of public interaction within the
|
||||
community.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
||||
version 2.1, available at
|
||||
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
|
||||
|
||||
Community Impact Guidelines were inspired by
|
||||
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
|
||||
|
||||
For answers to common questions about this code of conduct, see the FAQ at
|
||||
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
|
||||
[https://www.contributor-covenant.org/translations][translations].
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
|
||||
[Mozilla CoC]: https://github.com/mozilla/diversity
|
||||
[FAQ]: https://www.contributor-covenant.org/faq
|
||||
[translations]: https://www.contributor-covenant.org/translations
|
||||
154
.github/CONTRIBUTING_en.md
vendored
154
.github/CONTRIBUTING_en.md
vendored
@@ -1,154 +0,0 @@
|
||||
[简体中文🀄](../CONTRIBUTING.md) | **English**🌎
|
||||
|
||||
# Contributing to PaddleNLP
|
||||
|
||||
We highly welcome and value your contributions to `PaddleNLP`. The first step to start your contribution is to sign the [PaddlePaddle Contributor License Agreement](https://cla-assistant.io/PaddlePaddle/PaddleNLP).
|
||||
|
||||
This document explains our workflow and work style:
|
||||
|
||||
## Finding out what to work on Workflow
|
||||
|
||||
## Development Workflow
|
||||
|
||||
PaddleNLP uses the [Git branching model](http://nvie.com/posts/a-successful-git-branching-model/). The following steps guide usual contributions.
|
||||
|
||||
#### 1. Fork
|
||||
|
||||
Our development community has been growing fastly; it doesn't make sense for everyone to write into the official repo. So, please file Pull Requests from your fork. To make a fork, just head over to the GitHub page and click the ["Fork" button](https://help.github.com/articles/fork-a-repo/).
|
||||
|
||||
#### 2. Clone
|
||||
|
||||
To make a copy of your fork to your local computers, please run
|
||||
|
||||
```bash
|
||||
git clone https://github.com/<your-github-account>/PaddleNLP
|
||||
cd PaddleNLP
|
||||
```
|
||||
|
||||
#### 3. Create the local feature branch
|
||||
|
||||
For daily works like adding a new feature or fixing a bug, please open your feature branch before coding:
|
||||
|
||||
```bash
|
||||
git checkout -b my-cool-feature
|
||||
```
|
||||
|
||||
#### 4. Set up the development environment
|
||||
|
||||
Before you start coding, you need to setup the development environment. We highly recommend doing all your development in a virtual environment such as
|
||||
[venv](https://docs.python.org/3/library/venv.html) or [conda](https://docs.conda.io/en/latest/). After you setup and activated your virtual environment,
|
||||
run the following command:
|
||||
|
||||
```bash
|
||||
make install
|
||||
```
|
||||
|
||||
This will setup all the dependencies of `PaddleNLP` as well as the [`pre-commit`](http://pre-commit.com/) tool.
|
||||
|
||||
If you are working on the `examples` or `applications` module and require importing from `PaddleNLP`, make sure you install `PaddleNLP` in editable mode.
|
||||
If `PaddleNLP` is already installed in the virtual environment, remove it with `pip uninstall paddlenlp` before reinstalling it in editable mode with
|
||||
`pip install -e .`
|
||||
|
||||
#### 5. Develop
|
||||
|
||||
As you develop your new exciting feature, keep in mind that it should be covered by unit tests. All of our unit tests can be found under the `tests` directory.
|
||||
You can either modify existing unit test to cover the new feature, or create a new test from scratch.
|
||||
As you finish up the your code, you should make sure the test suite passes. You can run the tests impacted by your changes like this:
|
||||
|
||||
```bash
|
||||
pytest tests/<test_to_run>.py
|
||||
```
|
||||
|
||||
#### 6. Commit
|
||||
|
||||
We utilizes [`pre-commit`](http://pre-commit.com/) (with [black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/) and
|
||||
[flake8](https://flake8.pycqa.org/en/latest/) under the hood) to check the style of code and documentation in every commit. When you run run `git commit`, you will see
|
||||
something like the following:
|
||||
|
||||
```
|
||||
➜ (my-virtual-env) git commit -m "commiting my cool feature"
|
||||
black....................................................................Passed
|
||||
isort....................................................................Passed
|
||||
flake8...................................................................Passed
|
||||
check for merge conflicts................................................Passed
|
||||
check for broken symlinks............................(no files to check)Skipped
|
||||
detect private key.......................................................Passed
|
||||
fix end of files.....................................(no files to check)Skipped
|
||||
trim trailing whitespace.............................(no files to check)Skipped
|
||||
CRLF end-lines checker...............................(no files to check)Skipped
|
||||
CRLF end-lines remover...............................(no files to check)Skipped
|
||||
No-tabs checker......................................(no files to check)Skipped
|
||||
Tabs remover.........................................(no files to check)Skipped
|
||||
copyright_checker........................................................Passed
|
||||
```
|
||||
|
||||
But most of the time things don't go so smoothly. When your code or documentation doesn't meet the standard, the `pre-commit` check will fail.
|
||||
```
|
||||
➜ (my-virtual-env) git commit -m "commiting my cool feature"
|
||||
black....................................................................Passed
|
||||
isort....................................................................Failed
|
||||
- hook id: isort
|
||||
- files were modified by this hook
|
||||
|
||||
Fixing examples/information_extraction/waybill_ie/run_ernie_crf.py
|
||||
|
||||
flake8...................................................................Passed
|
||||
check for merge conflicts................................................Passed
|
||||
check for broken symlinks............................(no files to check)Skipped
|
||||
detect private key.......................................................Passed
|
||||
fix end of files.....................................(no files to check)Skipped
|
||||
trim trailing whitespace.............................(no files to check)Skipped
|
||||
CRLF end-lines checker...............................(no files to check)Skipped
|
||||
CRLF end-lines remover...............................(no files to check)Skipped
|
||||
No-tabs checker......................................(no files to check)Skipped
|
||||
Tabs remover.........................................(no files to check)Skipped
|
||||
copyright_checker........................................................Passed
|
||||
```
|
||||
|
||||
But **don't panic**!
|
||||
Our tooling will fix most of the style errors automatically. Some errors will need to be addressed manually. Fortunately, the error messages are straight forward and
|
||||
the errors are usually simple to fix. After addressing the errors, you can run `git add <files>` and `git commit` again, which will trigger `pre-commit` again.
|
||||
Once the `pre-commit` checks pass, you are ready to push the code.
|
||||
|
||||
[Google][http://google.com/] or [StackOverflow](https://stackoverflow.com/) are great tools to help you understand the code style errors.
|
||||
Don't worry if you still can't figure it out. You can commit with `git commit -m "style error" --no-verify` and we are happy to help you once you create a Pull Request.
|
||||
|
||||
#### 7. Keep pulling
|
||||
|
||||
An experienced Git user pulls from the official repo often -- daily or even hourly, so they notice conflicts with others work early, and it's easier to resolve smaller conflicts.
|
||||
|
||||
```bash
|
||||
git remote add upstream https://github.com/PaddlePaddle/PaddleNLP
|
||||
git pull upstream develop
|
||||
```
|
||||
|
||||
#### 8. Push and file a pull request
|
||||
|
||||
You can "push" your local work into your forked repo:
|
||||
|
||||
```bash
|
||||
git push origin my-cool-stuff
|
||||
```
|
||||
|
||||
The push allows you to create a pull request, requesting owners of this [official repo](https://github.com/PaddlePaddle/PaddleNLP) to pull your change into the official one.
|
||||
|
||||
To create a pull request, please follow [these steps](https://help.github.com/articles/creating-a-pull-request/).
|
||||
|
||||
#### 9. Delete local and remote branches
|
||||
|
||||
To keep your local workspace and your fork clean, you might want to remove merged branches:
|
||||
|
||||
```bash
|
||||
git push origin my-cool-stuff
|
||||
git checkout develop
|
||||
git pull upstream develop
|
||||
git branch -d my-cool-stuff
|
||||
```
|
||||
|
||||
## Code Review
|
||||
|
||||
- Please feel free to ping your reviewers by @-mentioning the in the Pull Request. Please do this after your pull request passes the CI.
|
||||
|
||||
- Please answer reviewers' every comment. If you are to follow the comment, please write "Done"; Otherwise, please start a discussion under the comment.
|
||||
|
||||
- If you don't want your reviewers to get overwhelmed by email notifications, you might reply their comments by [in a batch](https://help.github.com/articles/reviewing-proposed-changes-in-a-pull-request/).
|
||||
15
.github/ISSUE_TEMPLATE/----.md
vendored
Normal file
15
.github/ISSUE_TEMPLATE/----.md
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
---
|
||||
name: 问题反馈
|
||||
about: PaddleNLP问题反馈
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
欢迎您反馈PaddleNLP使用问题，非常感谢您对PaddleNLP的贡献！
|
||||
在留下您的问题时，辛苦您同步提供如下信息：
|
||||
- 版本、环境信息
|
||||
1）PaddleNLP和PaddlePaddle版本：请提供您的PaddleNLP和PaddlePaddle版本号，例如PaddleNLP 2.0.4，PaddlePaddle2.1.1
|
||||
2）系统环境：请您描述系统类型，例如Linux/Windows/MacOS/，python版本
|
||||
- 复现信息：如为报错，请给出复现环境、复现步骤
|
||||
23
.github/ISSUE_TEMPLATE/ask-question.yml
vendored
23
.github/ISSUE_TEMPLATE/ask-question.yml
vendored
@@ -1,23 +0,0 @@
|
||||
name: 🐛 Ask Question
|
||||
description: 请描述您使用PaddleNLP时遇到的问题
|
||||
title: "[Question]: "
|
||||
labels:
|
||||
- question
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
#### 你可以在这里提出一个使用/咨询问题，提问之前请确保：
|
||||
|
||||
- 1）已经百度/谷歌搜索过你的问题，但是没有找到解答；
|
||||
|
||||
- 2）已经在官网查询过[API文档](https://www.paddlepaddle.org.cn/documentation/docs/zh/api/index_cn.html)与[FAQ](https://www.paddlepaddle.org.cn/documentation/docs/zh/faq/index_cn.html)，但是没有找到解答；
|
||||
|
||||
- 3）已经在[历史issue](https://github.com/PaddlePaddle/Paddle/issues)中搜索过，没有找到同类issue或issue未被解答。
|
||||
|
||||
- type: textarea
|
||||
id: question
|
||||
attributes:
|
||||
label: 请提出你的问题
|
||||
validations:
|
||||
required: true
|
||||
45
.github/ISSUE_TEMPLATE/bug-report.yml
vendored
45
.github/ISSUE_TEMPLATE/bug-report.yml
vendored
@@ -1,45 +0,0 @@
|
||||
name: 🐛 Bug Report
|
||||
description: PaddleNLP问题反馈
|
||||
title: "[Bug]: "
|
||||
labels: bug
|
||||
body:
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: 软件环境
|
||||
description: |
|
||||
请使用以下命令给出您本地Paddle相关包信息
|
||||
```sh
|
||||
pip list | grep paddle
|
||||
|
||||
```
|
||||
value: |
|
||||
- paddlepaddle:
|
||||
- paddlepaddle-gpu:
|
||||
- paddlenlp:
|
||||
render: Markdown
|
||||
validations:
|
||||
required: true
|
||||
- type: checkboxes
|
||||
id: dumplicated-problem
|
||||
attributes:
|
||||
label: 重复问题
|
||||
description: 是否已在issues中搜索相关问题
|
||||
options:
|
||||
- label: I have searched the existing issues
|
||||
required: true
|
||||
- type: textarea
|
||||
id: descripton
|
||||
attributes:
|
||||
label: 错误描述
|
||||
description: 给出错误详细描述，以便能够更好的追踪相关问题
|
||||
render: Markdown
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: mvp-code
|
||||
attributes:
|
||||
label: 稳定复现步骤 & 代码
|
||||
description: 请给出稳定复现该问题的步骤 & 代码，以便相关人员能够快速定位到具体问题。
|
||||
validations:
|
||||
required: true
|
||||
32
.github/ISSUE_TEMPLATE/docs-report.yml
vendored
32
.github/ISSUE_TEMPLATE/docs-report.yml
vendored
@@ -1,32 +0,0 @@
|
||||
name: 🐛 Docs Report
|
||||
description: PaddleNLP文档反馈
|
||||
title: "[Docs]: "
|
||||
labels:
|
||||
- documentation
|
||||
|
||||
body:
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: 软件环境
|
||||
description: |
|
||||
请使用以下命令给出您本地Paddle相关包信息
|
||||
```sh
|
||||
pip list | grep paddle
|
||||
|
||||
```
|
||||
value: |
|
||||
- paddlepaddle:
|
||||
- paddlepaddle-gpu:
|
||||
- paddlenlp:
|
||||
render: Markdown
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: 详细描述
|
||||
description: 请详细描述您想要反馈的具体问题
|
||||
render: Markdown
|
||||
validations:
|
||||
required: true
|
||||
30
.github/ISSUE_TEMPLATE/feature-request.yml
vendored
30
.github/ISSUE_TEMPLATE/feature-request.yml
vendored
@@ -1,30 +0,0 @@
|
||||
name: "\U0001F680 Feature request"
|
||||
description: 请详细描述您所需功能
|
||||
labels: [ "feature" ]
|
||||
body:
|
||||
- type: textarea
|
||||
id: feature-request
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Feature request
|
||||
description: |
|
||||
对特性提案的清晰而简明的描述。如果论文和代码存在，请提供链接。
|
||||
|
||||
- type: textarea
|
||||
id: motivation
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Motivation
|
||||
description: |
|
||||
请概述这项建议的动机。您的特性要求与问题有关吗?
|
||||
|
||||
- type: textarea
|
||||
id: contribution
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Your contribution
|
||||
description: |
|
||||
Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md)
|
||||
28
.github/ISSUE_TEMPLATE/new-model.yaml
vendored
28
.github/ISSUE_TEMPLATE/new-model.yaml
vendored
@@ -1,28 +0,0 @@
|
||||
name: "\U0001F31F 添加新模型"
|
||||
description: 请为新模型提交一份说明
|
||||
labels: [ "New model" ]
|
||||
|
||||
body:
|
||||
- type: textarea
|
||||
id: description-request
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: 简要描述
|
||||
description: |
|
||||
请简要描述模型的类型、解决的问题等。
|
||||
|
||||
- type: checkboxes
|
||||
id: information-tasks
|
||||
attributes:
|
||||
label: 是否已开源
|
||||
options:
|
||||
- label: 已开源
|
||||
- label: 未开源
|
||||
|
||||
- type: textarea
|
||||
id: additional-info
|
||||
attributes:
|
||||
label: 模型详细信息
|
||||
description: |
|
||||
请给出新模型相关信息，如论文地址、现存代码地址等。
|
||||
23
.github/ISSUE_TEMPLATE/others.yml
vendored
23
.github/ISSUE_TEMPLATE/others.yml
vendored
@@ -1,23 +0,0 @@
|
||||
name: 🧩 其他 Others
|
||||
description: 提出其他问题。
|
||||
labels: [others]
|
||||
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
#### 你可以在这里提出任何前面几类模板不适用的问题，包括但不限于：优化性建议、框架使用体验反馈、版本兼容性问题、报错信息不清楚等。
|
||||
|
||||
- type: textarea
|
||||
id: others
|
||||
attributes:
|
||||
label: 问题描述
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
感谢你的贡献 🎉！
|
||||
|
||||
|
||||
14
.github/PULL_REQUEST_TEMPLATE.md
vendored
14
.github/PULL_REQUEST_TEMPLATE.md
vendored
@@ -1,18 +1,4 @@
|
||||
<!-- Demo: https://github.com/PaddlePaddle/PaddleNLP/pull/26 -->
|
||||
#### Before submitting
|
||||
|
||||
- [ ] Lint code. If there are lint issues, please format the code first.
|
||||
|
||||
```shell
|
||||
# Install and register `pre-commit` in the project folder
|
||||
pip install pre-commit && pre-commit install
|
||||
|
||||
# Process previous code files separately
|
||||
pre-commit run --file XXXX.py
|
||||
```
|
||||
|
||||
- [ ] Add test cases into `tests` folder. If there are codecov issues, please add tests cases first.
|
||||
|
||||
### PR types
|
||||
<!-- One of [ New features | Bug fixes | Function optimization | Performance optimization | Breaking changes | Others ] -->
|
||||
|
||||
|
||||
41
.github/actions/check-bypass/action.yml
vendored
41
.github/actions/check-bypass/action.yml
vendored
@@ -1,41 +0,0 @@
|
||||
name: "Check bypass"
|
||||
description: "A custom action to encapsulate PFCCLab/ci-bypass"
|
||||
inputs:
|
||||
github-token:
|
||||
description: "GitHub token"
|
||||
required: true
|
||||
workflow-name:
|
||||
description: "Workflow name"
|
||||
required: true
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether the workflow can be skipped."
|
||||
value: ${{ steps.check-bypass.outputs.can-skip }}
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- id: check-bypass
|
||||
name: Check Bypass
|
||||
env:
|
||||
CI_TEAM_MEMBERS: '["tianshuo78520a", "swgu98", "risemeup1", "XieYunshen","luotao1","From00"]'
|
||||
uses: PFCCLab/ci-bypass@v1
|
||||
with:
|
||||
github-token: ${{ inputs.github-token }}
|
||||
non-pull-request-event-strategy: 'never-skipped'
|
||||
type: 'composite'
|
||||
composite-rule: |
|
||||
{
|
||||
"any": [
|
||||
{
|
||||
"type": "labeled",
|
||||
"label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
},
|
||||
{
|
||||
"type": "commented",
|
||||
"comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
}
|
||||
]
|
||||
}
|
||||
30
.github/actions/rerun-workflow/action.yml
vendored
30
.github/actions/rerun-workflow/action.yml
vendored
@@ -1,30 +0,0 @@
|
||||
name: 'Rerun Workflow'
|
||||
description: 'Re-run GitHub Actions workflow for a given Pull Request'
|
||||
inputs:
|
||||
GITHUB_TOKEN:
|
||||
description: 'GitHub token with repo scope'
|
||||
required: true
|
||||
OWNER:
|
||||
description: 'Repository owner'
|
||||
required: true
|
||||
REPO:
|
||||
description: 'Repository name'
|
||||
required: true
|
||||
PR_ID:
|
||||
description: 'Pull Request ID'
|
||||
required: true
|
||||
JOB_NAME:
|
||||
description: 'Job name to rerun'
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- run: bash ./.github/actions/rerun-workflow/rerun.sh
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }}
|
||||
OWNER: ${{ inputs.OWNER }}
|
||||
REPO: ${{ inputs.REPO }}
|
||||
PR_ID: ${{ inputs.PR_ID }}
|
||||
JOB_NAME: ${{ inputs.JOB_NAME }}
|
||||
77
.github/actions/rerun-workflow/rerun.sh
vendored
77
.github/actions/rerun-workflow/rerun.sh
vendored
@@ -1,77 +0,0 @@
|
||||
# Copyright (c) 2025 PaddleNLP Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
|
||||
COMMIT_SHA=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/pulls/$PR_ID" | jq -r '.head.sha')
|
||||
|
||||
echo "Commit SHA: $COMMIT_SHA"
|
||||
|
||||
response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/actions/runs?head_sha=$COMMIT_SHA&per_page=100")
|
||||
|
||||
echo "Response: $response"
|
||||
|
||||
run_ids=$(echo "$response" | jq -r '.workflow_runs[].id')
|
||||
|
||||
if [ -n "$run_ids" ]; then
|
||||
echo "Found run_ids for commit $COMMIT_SHA: $run_ids"
|
||||
|
||||
for run_id in $run_ids; do
|
||||
if [ "$JOB_NAME" = "all-failed" ]; then
|
||||
echo "Rerunning all failed jobs for run_id: $run_id"
|
||||
|
||||
rerun_response=$(curl -X POST -s -w "%{http_code}" -o /dev/null \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: Bearer $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/rerun-failed-jobs")
|
||||
if [ "$rerun_response" -eq 201 ]; then
|
||||
echo "Successfully requested rerun for all blocked jobs in run_id: $run_id"
|
||||
else
|
||||
echo "Failed to request rerun for run_id: $run_id with status code $rerun_response"
|
||||
fi
|
||||
|
||||
else
|
||||
jobs_response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/jobs")
|
||||
|
||||
echo "Jobs Response for run_id $run_id: $jobs_response"
|
||||
|
||||
# if [[ "$JOB_NAME" == *"bypass"* ]]; then
|
||||
block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
|
||||
'.jobs[] | select(.name == $job_name) | .id')
|
||||
# else
|
||||
# block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
|
||||
# '.jobs[] | select(.name == $job_name and .conclusion != "success") | .id')
|
||||
# fi
|
||||
|
||||
if [ -n "$block_jobs" ]; then
|
||||
echo "Found block jobs for run_id $run_id: $block_jobs"
|
||||
|
||||
for job_id in $block_jobs; do
|
||||
echo "Rerunning job_id: $job_id"
|
||||
curl -X POST -H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: token $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/actions/jobs/$job_id/rerun"
|
||||
done
|
||||
else
|
||||
echo "No block jobs found for run_id $run_id with name $JOB_NAME."
|
||||
fi
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "No matching workflow runs found for commit $COMMIT_SHA."
|
||||
exit 1
|
||||
fi
|
||||
13
.github/codecov.yml
vendored
13
.github/codecov.yml
vendored
@@ -1,13 +0,0 @@
|
||||
codecov:
|
||||
notify:
|
||||
wait_for_ci: false
|
||||
|
||||
coverage:
|
||||
status:
|
||||
project:
|
||||
default:
|
||||
target: 58% # overall project Coverage
|
||||
threshold: 1% # Allow the coverage to drop by 1%, and posting a success status.
|
||||
patch:
|
||||
default:
|
||||
target: 80% # lines adjusted Coverage < 80% CI will fail
|
||||
47
.github/workflows/PaddleFormers-PR-Check.yml
vendored
47
.github/workflows/PaddleFormers-PR-Check.yml
vendored
@@ -1,47 +0,0 @@
|
||||
name: PaddleFormers PR Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
|
||||
env:
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
token: ${{ vars.ACTION_GITHUB_TOKEN }}
|
||||
|
||||
jobs:
|
||||
check-approvers:
|
||||
name: Check approval
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Update paddle
|
||||
run: |
|
||||
wget -q --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleNLP/PaddleNLP-develop.tar.gz --no-check-certificate
|
||||
tar zxf PaddleNLP-develop.tar.gz --strip-components=1 >/dev/null
|
||||
rm -rf PaddleNLP-develop.tar.gz >/dev/null
|
||||
git fetch origin pull/${PR_ID}/head
|
||||
git checkout -b origin_pr FETCH_HEAD
|
||||
git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
|
||||
git fetch upstream
|
||||
git checkout -b test_pr upstream/${BRANCH}
|
||||
git merge --no-edit origin_pr
|
||||
git log --pretty=oneline -10
|
||||
|
||||
- name: Check bypass
|
||||
id: check-bypass
|
||||
uses: ./.github/actions/check-bypass
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
workflow-name: approval
|
||||
|
||||
- name: Display Required Approvers
|
||||
if: steps.check-bypass.outputs.can-skip != 'true'
|
||||
run: |
|
||||
cd scripts/ci_approval
|
||||
bash run_ci_approval_cherry-pick.sh
|
||||
47
.github/workflows/approval.yml
vendored
47
.github/workflows/approval.yml
vendored
@@ -1,47 +0,0 @@
|
||||
name: Approval
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
|
||||
env:
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
token: ${{ vars.ACTION_GITHUB_TOKEN }}
|
||||
|
||||
jobs:
|
||||
check-approvers:
|
||||
name: Check approval
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Update paddle
|
||||
run: |
|
||||
wget -q --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleNLP/PaddleNLP-develop.tar.gz --no-check-certificate
|
||||
tar zxf PaddleNLP-develop.tar.gz --strip-components=1 >/dev/null
|
||||
rm -rf PaddleNLP-develop.tar.gz >/dev/null
|
||||
git fetch origin pull/${PR_ID}/head
|
||||
git checkout -b origin_pr FETCH_HEAD
|
||||
git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
|
||||
git fetch upstream
|
||||
git checkout -b test_pr upstream/${BRANCH}
|
||||
git merge --no-edit origin_pr
|
||||
git log --pretty=oneline -10
|
||||
|
||||
- name: Check bypass
|
||||
id: check-bypass
|
||||
uses: ./.github/actions/check-bypass
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
workflow-name: approval
|
||||
|
||||
- name: Display Required Approvers
|
||||
if: steps.check-bypass.outputs.can-skip != 'true'
|
||||
run: |
|
||||
cd scripts/ci_approval
|
||||
bash -x run_ci_approval.sh
|
||||
52
.github/workflows/check-bypass.yml
vendored
52
.github/workflows/check-bypass.yml
vendored
@@ -1,52 +0,0 @@
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
workflow-name:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
github-token:
|
||||
required: true
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether the workflow can be skipped."
|
||||
value: ${{ jobs.check-bypass.outputs.can-skip }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
CI_TEAM_MEMBERS: '["tianshuo78520a", "swgu98", "risemeup1" , "XieYunshen","luotao1","From00"]'
|
||||
outputs:
|
||||
can-skip: ${{ steps.check-bypass.outputs.can-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
|
||||
- id: check-bypass
|
||||
name: Check Bypass
|
||||
uses: PFCCLab/ci-bypass@v1
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
non-pull-request-event-strategy: 'never-skipped'
|
||||
type: 'composite'
|
||||
composite-rule: |
|
||||
{
|
||||
"any": [
|
||||
{
|
||||
"type": "labeled",
|
||||
"label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
},
|
||||
{
|
||||
"type": "commented",
|
||||
"comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
}
|
||||
]
|
||||
}
|
||||
113
.github/workflows/lint.yml
vendored
113
.github/workflows/lint.yml
vendored
@@ -1,113 +0,0 @@
|
||||
name: Codestyle Check
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
TASK: PaddleNLP-CI-Lint-${{ github.event.pull_request.number }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: 'lint'
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
Lint:
|
||||
name: Lint
|
||||
needs: check-bypass
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
runs-on: [self-hosted, ernie-cpu]
|
||||
steps:
|
||||
- name: Run Container
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
python_version: "3.10"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image="iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu20.04-gcc12.2-cuda12.3-cudnn9.0-nccl2.20.3.1-openmpi4.1.5-latest"
|
||||
docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
|
||||
-v $work_dir/../../..:$work_dir/../../.. \
|
||||
-v $work_dir:/workspace \
|
||||
-v /home/.cache/pip:/home/.cache/pip \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e no_proxy \
|
||||
-e python_version \
|
||||
-w /workspace ${docker_image}
|
||||
|
||||
- name: Download Code
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading PaddleNLP.tar"
|
||||
wget -q --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
|
||||
echo "Extracting PaddleNLP.tar"
|
||||
tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
|
||||
source $work_dir/../../../proxy
|
||||
cd PaddleNLP
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git pull
|
||||
git submodule update --init --recursive --force
|
||||
if [ -n "${PR_ID}" ]; then
|
||||
git fetch origin pull/${PR_ID}/head
|
||||
git checkout -b PR_${PR_ID} FETCH_HEAD
|
||||
git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
|
||||
git fetch upstream ${BRANCH}
|
||||
git merge ${BRANCH} --no-edit
|
||||
git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
|
||||
else
|
||||
echo "Not in a pull_request event. Skipping PR-specific operations."
|
||||
fi
|
||||
git log --pretty=oneline -10
|
||||
|
||||
if ! git show-ref --quiet refs/heads/develop; then \
|
||||
echo "local develop branch is missing, creating local develop branch that tracks remote develop branch"
|
||||
git fetch origin develop
|
||||
git branch develop --track origin/develop
|
||||
else
|
||||
echo "local develop branch exist, skipping"
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Setup Environment
|
||||
run: |
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
unlink /usr/local/bin/python
|
||||
ln -sf $(which python${python_version}) /usr/local/bin/python
|
||||
set -e
|
||||
python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
python -m pip config set global.cache-dir "/home/.cache/pip"
|
||||
source $work_dir/../../../proxy
|
||||
python -m pip install --upgrade pip
|
||||
cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
|
||||
make install
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
set -e
|
||||
source $work_dir/../../../proxy
|
||||
cd /workspace/PaddleNLP
|
||||
make lint
|
||||
'
|
||||
|
||||
- name: Terminate And Delete the Container
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f $container_name 2>/dev/null || true
|
||||
207
.github/workflows/llm.yml
vendored
207
.github/workflows/llm.yml
vendored
@@ -1,207 +0,0 @@
|
||||
# Disabled LLM CI workflow - manually enabled when needed
|
||||
name: LLM CI (DISABLED)
|
||||
|
||||
#on:
|
||||
# pull_request:
|
||||
# types: [opened, synchronize, reopened]
|
||||
# branches: [develop]
|
||||
# schedule:
|
||||
# - cron: "2 0 * * *"
|
||||
# workflow_call:
|
||||
# inputs:
|
||||
# run_downstream:
|
||||
# required: true
|
||||
# type: string
|
||||
# image_name:
|
||||
# required: true
|
||||
# type: string
|
||||
|
||||
#concurrency:
|
||||
# group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
# cancel-in-progress: true
|
||||
|
||||
#env:
|
||||
# PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddlenlp-CI-${{ github.event.pull_request.number }}-llm
|
||||
ci_scripts: /workspace/PaddleNLP/scripts/regression
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: llm-ci
|
||||
no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
|
||||
PPNLP_HOME: /ssd1/paddlenlp
|
||||
HF_DATASETS_CACHE: /ssd1/paddlenlp/huggingface/datasets
|
||||
TRANSFORMERS_CACHE: /ssd1/paddlenlp/huggingface
|
||||
CCACHE_DIR: /home/data/gzcfs/.ccache/gpubox
|
||||
RUN_DOWNSTREAM: ${{ inputs.run_downstream }}
|
||||
|
||||
#defaults:
|
||||
# run:
|
||||
# shell: bash
|
||||
|
||||
#jobs:
|
||||
# llm-ci:
|
||||
# name: llm-ci (DISABLED)
|
||||
runs-on: [self-hosted, ernie-8gpu]
|
||||
steps:
|
||||
- name: Determine Image Name
|
||||
env:
|
||||
IMAGE_NAME: ${{ inputs.image_name }}
|
||||
run: |
|
||||
if [[ -n "${IMAGE_NAME}" ]]; then
|
||||
echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-paddlenlp-latest" >> "$GITHUB_ENV"
|
||||
fi
|
||||
|
||||
- name: Run Container
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
FLAGS_dynamic_static_unified_comm: "True"
|
||||
python_version: "3.10"
|
||||
paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> "$GITHUB_ENV"
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping..."
|
||||
else
|
||||
docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
|
||||
-v $work_dir/../../..:$work_dir/../../.. \
|
||||
-v $work_dir:/workspace \
|
||||
-v /home/.cache/pip:/home/.cache/pip \
|
||||
-v /ssd1/paddlenlp:/ssd1/paddlenlp \
|
||||
-v /home/data/gzcfs/.ccache/gpubox:/home/data/gzcfs/.ccache/gpubox \
|
||||
-e BRANCH \
|
||||
-e AGILE_COMPILE_BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e paddle_whl \
|
||||
-e HF_ENDPOINT \
|
||||
-e STUDIO_GIT_HOST \
|
||||
-e PPNLP_HOME \
|
||||
-e HF_DATASETS_CACHE \
|
||||
-e TRANSFORMERS_CACHE \
|
||||
-e CACHE_DIR \
|
||||
-e FLAGS_dynamic_static_unified_comm \
|
||||
-e python_version \
|
||||
-w /workspace --runtime=nvidia $IMAGE_NAME
|
||||
fi
|
||||
|
||||
- name: Download Code
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping.."
|
||||
else
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading PaddleNLP.tar.gz"
|
||||
wget -q --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
|
||||
echo "Extracting PaddleNLP.tar.gz"
|
||||
tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
|
||||
source $work_dir/../../../proxy
|
||||
cd PaddleNLP
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git pull
|
||||
git submodule update --init --recursive --force
|
||||
if [ -n "${PR_ID}" ]; then
|
||||
git fetch origin pull/${PR_ID}/head
|
||||
git checkout -b PR_${PR_ID} FETCH_HEAD
|
||||
git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
|
||||
git fetch upstream ${BRANCH}
|
||||
git merge ${BRANCH} --no-edit
|
||||
git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
|
||||
else
|
||||
echo "Not in a pull_request event. Skipping PR-specific operations."
|
||||
fi
|
||||
git log --pretty=oneline -10
|
||||
'
|
||||
fi
|
||||
|
||||
- name: Skip For Bug
|
||||
run: |
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping..."
|
||||
else
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
cd /workspace/PaddleNLP
|
||||
git revert f2477c07272d04244cd3287d1f21c70482a4a85f --no-edit # 套件PR#10413引入bug-待修复
|
||||
git revert d74c950e15a35b7a100d1688c89318195cc83bca --no-edit # 框架升级后配合修改，当前因框架bug固定commit适配
|
||||
'
|
||||
fi
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping..."
|
||||
else
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
ldconfig
|
||||
unlink /usr/bin/python3
|
||||
ln -sf $(which python${python_version}) /usr/bin/python3
|
||||
pip config set global.cache-dir "/home/.cache/pip"
|
||||
set -e
|
||||
source $work_dir/../../../proxy
|
||||
cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
|
||||
echo "Paddle PR#73283 PR#74484 import bug, set paddle_commit=8ae742 to skip "
|
||||
export paddle_whl=https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/8ae7423e99b2ea96e410968a0ebb3f1795e37205/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
timeout 2h bash scripts/regression/run_ci.sh python${python_version} ${paddle_whl}
|
||||
'
|
||||
fi
|
||||
|
||||
- name: Upload Allure-reports & Logs
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/../../..
|
||||
bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
|
||||
allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
|
||||
run: |
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping..."
|
||||
else
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
unset http_proxy && unset https_proxy
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
if [ ! -f "${{ env.allure_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
|
||||
unzip -q ${{ env.home_path }}/allure-2.19.0.zip -d ${{ env.home_path }}/
|
||||
fi
|
||||
if [[ "${{ env.RUN_DOWNSTREAM }}" == "" && -n "${PR_ID}" ]]; then
|
||||
bos_prefix="${PR_ID}/${COMMIT_ID}"
|
||||
elif [[ "${{ env.RUN_DOWNSTREAM }}" == "true" && -n "${PR_ID}" ]]; then
|
||||
bos_prefix="${PR_ID}/${COMMIT_ID}/test_build"
|
||||
else
|
||||
bos_prefix="schedule/$(date +%Y%m%d)"
|
||||
fi
|
||||
cd /workspace/PaddleNLP/model_logs
|
||||
for FILE in /workspace/PaddleNLP/model_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleNLP/llm/${bos_prefix}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/llm/${bos_prefix}/logs/$file"
|
||||
done
|
||||
cd /workspace/PaddleNLP/
|
||||
${{ env.allure_file }} generate result -o report
|
||||
tar -czf products.tar.gz report model_logs
|
||||
python ${{ env.bos_file }} products.tar.gz paddle-github-action/PR/PaddleNLP/llm/${bos_prefix}/logs
|
||||
echo "products: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/llm/${bos_prefix}/logs/products.tar.gz"
|
||||
'
|
||||
fi
|
||||
|
||||
- name: Terminate And Delete the Container
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f ${{ env.container_name }} 2>/dev/null || true
|
||||
30
.github/workflows/pipelines.yml
vendored
30
.github/workflows/pipelines.yml
vendored
@@ -1,30 +0,0 @@
|
||||
name: Pipelines-Test
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'slm/pipelines/*'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'slm/pipelines/*'
|
||||
|
||||
|
||||
jobs:
|
||||
Pipelines-Test:
|
||||
name: Pipelines-Test
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.10'
|
||||
cache: 'pip' # caching pip dependencies
|
||||
- name: Install dependencies
|
||||
working-directory: ./slm/pipelines
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
make install
|
||||
pip install -r tests/requirements.txt
|
||||
- name: run the command
|
||||
working-directory: ./slm/pipelines
|
||||
run: make test
|
||||
58
.github/workflows/release.yml
vendored
58
.github/workflows/release.yml
vendored
@@ -1,58 +0,0 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
package:
|
||||
description: 'Package to be deployed'
|
||||
required: true
|
||||
default: 'paddlenlp'
|
||||
type: choice
|
||||
options:
|
||||
- paddlenlp
|
||||
- paddle-pipelines
|
||||
- ppdiffusers
|
||||
|
||||
jobs:
|
||||
Release:
|
||||
name: Release
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.10'
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install setuptools wheel twine
|
||||
- name: Should Deploy
|
||||
id: should-deploy
|
||||
env:
|
||||
PACKAGE: ${{ inputs.package }}
|
||||
run: python scripts/should_deploy.py --name $PACKAGE >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Check Branch
|
||||
id: check-branch
|
||||
run: |
|
||||
if [[ ${{ github.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "match=true" >> $GITHUB_OUTPUT
|
||||
fi # See: https://stackoverflow.com/a/58869470/1123955
|
||||
|
||||
- name: Is A Publish Branch
|
||||
if: steps.check-branch.outputs.match == 'true' && steps.should-deploy.outputs.should_deploy == 'true'
|
||||
env:
|
||||
TWINE_USERNAME: __token__
|
||||
TWINE_PASSWORD: ${{ secrets.paddlenlp }}
|
||||
PACKAGE: ${{ inputs.package }}
|
||||
PADDLENLP_STABLE_VERSION: "true"
|
||||
run: |
|
||||
make deploy-$PACKAGE
|
||||
|
||||
- name: Should Not Deploy To Pypi Server
|
||||
if: steps.should-deploy.outputs.should_deploy != 'true'
|
||||
run: echo 'should not deploy pypi server'
|
||||
|
||||
- name: Is Not A Publish Branch
|
||||
if: steps.check-branch.outputs.match != 'true'
|
||||
run: echo 'Not A Publish Branch'
|
||||
73
.github/workflows/rerun.yml
vendored
73
.github/workflows/rerun.yml
vendored
@@ -1,73 +0,0 @@
|
||||
name: Re-run
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
re-run:
|
||||
if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/re-run') && github.event.comment.user.login == github.event.issue.user.login }}
|
||||
runs-on: [self-hosted, ernie-cpu]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Rerun all failed jobs
|
||||
if: ${{ contains(github.event.comment.body, 'all-failed') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'all-failed'
|
||||
|
||||
- name: Rerun LLM
|
||||
if: ${{ contains(github.event.comment.body, 'LLM') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'LLM CI / llm-ci'
|
||||
|
||||
- name: Unittest GPU
|
||||
if: ${{ contains(github.event.comment.body, 'Unittest GPU') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Unittest GPU CI / unittest-gpu-ci'
|
||||
|
||||
- name: Rerun Unittest CPU
|
||||
if: ${{ contains(github.event.comment.body, 'Unittest CPU') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Unittest CPU CI / unit-cpu-ci'
|
||||
|
||||
- name: Rerun Distribute (V100)
|
||||
if: ${{ contains(github.event.comment.body, 'Distribute (V100)') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Distribute CI (V100)/ distribute-v100-ci'
|
||||
|
||||
- name: Rerun Codestyle Check
|
||||
if: ${{ contains(github.event.comment.body, 'Codestyle Check') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Codestyle Check / Lint'
|
||||
35
.github/workflows/stale.yml
vendored
35
.github/workflows/stale.yml
vendored
@@ -1,35 +0,0 @@
|
||||
name: Stale
|
||||
|
||||
on:
|
||||
# Allow manual run via GitHub web or CLI
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Run daily at midnight UTC
|
||||
- cron: 0 0 * * *
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- uses: actions/stale@v6.0.1
|
||||
with:
|
||||
days-before-issue-stale: 60
|
||||
days-before-issue-close: 14
|
||||
stale-issue-label: "stale"
|
||||
stale-issue-message: "This issue is stale because it has been open for 60 days with no activity. 当前issue 60天内无活动，被标记为stale。"
|
||||
close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale. 当前issue 被标记为stale已有14天，即将关闭。"
|
||||
exempt-issue-labels: 'triage,keep'
|
||||
days-before-pr-stale: 60
|
||||
days-before-pr-close: -1
|
||||
stale-pr-label: "stale"
|
||||
stale-pr-message: "This Pull Request is stale because it has been open for 60 days with no activity. 当前Pull Request 60天内无活动，被标记为stale。"
|
||||
operations-per-run: 400
|
||||
120
.github/workflows/unittest-cpu.yml
vendored
120
.github/workflows/unittest-cpu.yml
vendored
@@ -1,120 +0,0 @@
|
||||
# Disabled unittest CPU workflow - manually enabled when needed
|
||||
name: Unittest CPU CI (DISABLED)
|
||||
|
||||
#on: [pull_request]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
TASK: PaddleNLP-CI-${{ github.event.pull_request.number }}-unittest-cpu
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
|
||||
PPNLP_HOME: /home/disk1/cache
|
||||
HF_DATASETS_CACHE: /home/disk1/cache/huggingface/datasets
|
||||
TRANSFORMERS_CACHE: /home/disk1/cache/huggingface
|
||||
|
||||
#jobs:
|
||||
# unittest-cpu-ci:
|
||||
# name: unittest-cpu-ci (DISABLED)
|
||||
runs-on: [self-hosted, ernie-cpu]
|
||||
permissions:
|
||||
pull-requests: write
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Run Container
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
python_version: "3.10"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image="iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu20.04-gcc12.2-cuda12.3-cudnn9.0-nccl2.20.3.1-openmpi4.1.5-latest"
|
||||
docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
|
||||
-v $work_dir/../../..:$work_dir/../../.. \
|
||||
-v $work_dir:/workspace \
|
||||
-v /home/.cache/pip:/home/.cache/pip \
|
||||
-v /home/disk1/cache:/home/disk1/cache \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e no_proxy \
|
||||
-e python_version \
|
||||
-e HF_ENDPOINT \
|
||||
-e STUDIO_GIT_HOST \
|
||||
-e PPNLP_HOME \
|
||||
-e HF_DATASETS_CACHE \
|
||||
-e TRANSFORMERS_CACHE \
|
||||
-w /workspace ${docker_image}
|
||||
|
||||
- name: Download Code
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading PaddleNLP.tar"
|
||||
wget -q --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
|
||||
echo "Extracting PaddleNLP.tar"
|
||||
tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
|
||||
source $work_dir/../../../proxy
|
||||
cd PaddleNLP
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git pull
|
||||
git submodule update --init --recursive --force
|
||||
if [ -n "${PR_ID}" ]; then
|
||||
git fetch origin pull/${PR_ID}/head
|
||||
git checkout -b PR_${PR_ID} FETCH_HEAD
|
||||
git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
|
||||
git fetch upstream ${BRANCH}
|
||||
git merge ${BRANCH} --no-edit
|
||||
git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
|
||||
else
|
||||
echo "Not in a pull_request event. Skipping PR-specific operations."
|
||||
fi
|
||||
git log --pretty=oneline -10
|
||||
'
|
||||
|
||||
- name: Setup Environment
|
||||
run: |
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
unlink /usr/bin/python3
|
||||
ln -sf $(which python${python_version}) /usr/bin/python3
|
||||
set -e
|
||||
python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
python -m pip config set global.cache-dir "/home/.cache/pip"
|
||||
source $work_dir/../../../proxy
|
||||
python -m pip install --upgrade pip
|
||||
cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
|
||||
pip install -r tests/requirements.txt
|
||||
make install
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
source $work_dir/../../../proxy
|
||||
cd /workspace/PaddleNLP
|
||||
set -e
|
||||
make test
|
||||
'
|
||||
|
||||
# - name: Upload Coverage To Codecov
|
||||
# uses: codecov/codecov-action@v3
|
||||
# env:
|
||||
# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
# with:
|
||||
# token: ${{ secrets.CODECOV_TOKEN }}
|
||||
# files: ./PaddleNLP/coverage.xml
|
||||
|
||||
- name: Terminate And Delete the Container
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f $container_name 2>/dev/null || true
|
||||
204
.github/workflows/unittest-gpu.yml
vendored
204
.github/workflows/unittest-gpu.yml
vendored
@@ -1,204 +0,0 @@
|
||||
# Disabled unittest GPU workflow - manually enabled when needed
|
||||
name: Unittest GPU CI (DISABLED)
|
||||
|
||||
#on:
|
||||
# pull_request:
|
||||
# types: [opened, synchronize, reopened]
|
||||
# branches: [develop]
|
||||
# schedule:
|
||||
# - cron: "3 0 * * *"
|
||||
# workflow_call:
|
||||
# inputs:
|
||||
# run_downstream:
|
||||
# required: true
|
||||
# type: string
|
||||
# image_name:
|
||||
# required: true
|
||||
# type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddlenlp-CI-${{ github.event.pull_request.number }}-unit-gpu
|
||||
ci_scripts: /workspace/PaddleNLP/scripts/unit_test
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: unittest-gpu-ci
|
||||
no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
HF_ENDPOINT: https://hf-mirror.com
|
||||
STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
|
||||
PPNLP_HOME: /ssd1/paddlenlp
|
||||
HF_DATASETS_CACHE: /ssd1/paddlenlp/huggingface/datasets
|
||||
TRANSFORMERS_CACHE: /ssd1/paddlenlp/huggingface
|
||||
CCACHE_DIR: /home/data/gzcfs/.ccache/gpubox
|
||||
RUN_DOWNSTREAM: ${{ inputs.run_downstream }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
#jobs:
|
||||
# unittest-gpu-ci:
|
||||
# name: unittest-gpu-ci (DISABLED)
|
||||
runs-on: [self-hosted, ernie-8gpu]
|
||||
steps:
|
||||
- name: Determine Image Name
|
||||
env:
|
||||
IMAGE_NAME: ${{ inputs.image_name }}
|
||||
run: |
|
||||
if [[ -n "${IMAGE_NAME}" ]]; then
|
||||
echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "IMAGE_NAME=iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu18.04-gcc8.2-cuda11.8-cudnn8.6-nccl2.15.5-paddlenlp-latest" >> "$GITHUB_ENV"
|
||||
fi
|
||||
|
||||
- name: Run Container
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
FLAGS_dynamic_static_unified_comm: "True"
|
||||
python_version: "3.10"
|
||||
paddle_whl: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-GpuSome-LinuxCentos-Gcc82-Cuda118-Cudnn86-Trt85-Py310-CINN-Compile/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> "$GITHUB_ENV"
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping..."
|
||||
else
|
||||
docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
|
||||
-v $work_dir/../../..:$work_dir/../../.. \
|
||||
-v $work_dir:/workspace \
|
||||
-v /home/.cache/pip:/home/.cache/pip \
|
||||
-v /ssd1/paddlenlp:/ssd1/paddlenlp \
|
||||
-v /home/data/gzcfs/.ccache/gpubox:/home/data/gzcfs/.ccache/gpubox \
|
||||
-e BRANCH \
|
||||
-e AGILE_COMPILE_BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e paddle_whl \
|
||||
-e HF_ENDPOINT \
|
||||
-e STUDIO_GIT_HOST \
|
||||
-e PPNLP_HOME \
|
||||
-e HF_DATASETS_CACHE \
|
||||
-e TRANSFORMERS_CACHE \
|
||||
-e CACHE_DIR \
|
||||
-e FLAGS_dynamic_static_unified_comm \
|
||||
-e python_version \
|
||||
-w /workspace --runtime=nvidia $IMAGE_NAME
|
||||
fi
|
||||
|
||||
- name: Download Code
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping.."
|
||||
else
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading PaddleNLP.tar.gz"
|
||||
wget -q --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
|
||||
echo "Extracting PaddleNLP.tar.gz"
|
||||
tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
|
||||
source $work_dir/../../../proxy
|
||||
cd PaddleNLP
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git pull
|
||||
git submodule update --init --recursive --force
|
||||
if [ -n "${PR_ID}" ]; then
|
||||
git fetch origin pull/${PR_ID}/head
|
||||
git checkout -b PR_${PR_ID} FETCH_HEAD
|
||||
git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git
|
||||
git fetch upstream ${BRANCH}
|
||||
git merge ${BRANCH} --no-edit
|
||||
git diff --numstat ${BRANCH} -- | awk "{print \$NF}"
|
||||
else
|
||||
echo "Not in a pull_request event. Skipping PR-specific operations."
|
||||
fi
|
||||
git log --pretty=oneline -10
|
||||
'
|
||||
fi
|
||||
|
||||
- name: Skip For Bug
|
||||
run: |
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping..."
|
||||
else
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
cd /workspace/PaddleNLP
|
||||
echo "no skip for bug"
|
||||
'
|
||||
fi
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping..."
|
||||
else
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
ldconfig
|
||||
unlink /usr/bin/python3
|
||||
ln -sf $(which python${python_version}) /usr/bin/python3
|
||||
pip config set global.cache-dir "/home/.cache/pip"
|
||||
source $work_dir/../../../proxy
|
||||
set -e
|
||||
cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD
|
||||
timeout 50m bash scripts/unit_test/ci_unit.sh ${paddle_whl}
|
||||
'
|
||||
fi
|
||||
|
||||
- name: Upload Allure-reports & Logs
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/../../..
|
||||
bos_file: ${{ github.workspace }}/../../../bos/BosClient.py
|
||||
allure_file: ${{ github.workspace }}/../../../allure-2.19.0/bin/allure
|
||||
run: |
|
||||
if [[ "$RUN_DOWNSTREAM" == "false" ]]; then
|
||||
echo "Not in a pull_request or test_build event. Skipping..."
|
||||
else
|
||||
docker exec -t $container_name /bin/bash -c '
|
||||
unset http_proxy && unset https_proxy
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
if [ ! -f "${{ env.allure_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/allure-2.19.0.zip https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip --no-check-certificate
|
||||
unzip -q ${{ env.home_path }}/allure-2.19.0.zip -d ${{ env.home_path }}/
|
||||
fi
|
||||
if [[ "${{ env.RUN_DOWNSTREAM }}" == "" && -n "${PR_ID}" ]]; then
|
||||
bos_prefix="${PR_ID}/${COMMIT_ID}"
|
||||
elif [[ "${{ env.RUN_DOWNSTREAM }}" == "true" && -n "${PR_ID}" ]]; then
|
||||
bos_prefix="${PR_ID}/${COMMIT_ID}/test_build"
|
||||
else
|
||||
bos_prefix="schedule/$(date +%Y%m%d)"
|
||||
fi
|
||||
cd /workspace/PaddleNLP/unittest_logs
|
||||
for FILE in /workspace/PaddleNLP/unittest_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs/$file"
|
||||
done
|
||||
cd /workspace/PaddleNLP/
|
||||
${{ env.allure_file }} generate result -o report
|
||||
tar -czf products.tar.gz report unittest_logs
|
||||
python ${{ env.bos_file }} products.tar.gz paddle-github-action/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs
|
||||
echo "report: https://paddle-github-action.bj.bcebos.com/PR/PaddleNLP/unittest-gpu/${bos_prefix}/logs/products.tar.gz"
|
||||
'
|
||||
fi
|
||||
|
||||
- name: Terminate And Delete the Container
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f $container_name 2>/dev/null || true
|
||||
18
.gitignore
vendored
18
.gitignore
vendored
@@ -123,21 +123,3 @@ FETCH_HEAD
|
||||
|
||||
# vscode
|
||||
.vscode
|
||||
./ppdiffusers/ppdiffusers/version.py
|
||||
|
||||
# third party
|
||||
csrc/third_party/
|
||||
dataset/
|
||||
output/
|
||||
|
||||
# gen codes
|
||||
autogen/
|
||||
|
||||
# cutlass kernel
|
||||
!csrc/gpu/cutlass_kernels/gemm/collective/builders
|
||||
|
||||
|
||||
#fp8
|
||||
ops/csrc/fp8/deep_gemm/include/cutlass
|
||||
ops/csrc/fp8/deep_gemm/include/cute
|
||||
.ccls-cache
|
||||
6
.gitmodules
vendored
6
.gitmodules
vendored
@@ -1,6 +0,0 @@
|
||||
[submodule "csrc/third_party/cutlass"]
|
||||
path = csrc/third_party/cutlass
|
||||
url = https://github.com/NVIDIA/cutlass.git
|
||||
[submodule "csrc/third_party/nlohmann_json"]
|
||||
path = csrc/third_party/nlohmann_json
|
||||
url = https://github.com/nlohmann/json.git
|
||||
@@ -1,22 +1,11 @@
|
||||
exclude: 'slm/model_zoo/gpt-3;csrc/third_party'
|
||||
repos:
|
||||
# For Python files
|
||||
- repo: https://github.com/psf/black.git
|
||||
rev: 22.8.0
|
||||
- repo: https://github.com/PaddlePaddle/mirrors-yapf.git
|
||||
sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37
|
||||
hooks:
|
||||
- id: black
|
||||
files: \.(py|pyi)$
|
||||
additional_dependencies: [toml]
|
||||
- repo: https://github.com/PyCQA/isort
|
||||
rev: 5.11.5
|
||||
hooks:
|
||||
- id: isort
|
||||
- repo: https://github.com/PyCQA/flake8
|
||||
rev: 4.0.1
|
||||
hooks:
|
||||
- id: flake8
|
||||
- id: yapf
|
||||
files: \.py$
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.1.0
|
||||
sha: a11d9314b22d8f8c7556443875b731ef05965464
|
||||
hooks:
|
||||
- id: check-merge-conflict
|
||||
- id: check-symlinks
|
||||
@@ -27,7 +16,7 @@ repos:
|
||||
- id: trailing-whitespace
|
||||
files: \.md$
|
||||
- repo: https://github.com/Lucas-C/pre-commit-hooks
|
||||
rev: v1.1.14
|
||||
sha: v1.0.1
|
||||
hooks:
|
||||
- id: forbid-crlf
|
||||
files: \.md$
|
||||
@@ -37,37 +26,3 @@ repos:
|
||||
files: \.md$
|
||||
- id: remove-tabs
|
||||
files: \.md$
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: copyright_checker
|
||||
name: copyright_checker
|
||||
entry: python .copyright.hook
|
||||
language: system
|
||||
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|xpu|kps|py|sh)$
|
||||
# For Markdown files
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: add-spaces-between-chinese-and-english
|
||||
name: Add spaces between Chinese and English characters
|
||||
entry: python scripts/codestyle/check_spaces.py
|
||||
language: python
|
||||
files: \.(md|markdown)$
|
||||
pass_filenames: true
|
||||
# For dead links
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: check-dead-links
|
||||
name: Check dead links
|
||||
entry: python scripts/codestyle/check_dead_links.py
|
||||
language: python
|
||||
files: \.(md|markdown|rst)$
|
||||
pass_filenames: true
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: clang-format
|
||||
name: clang-format
|
||||
description: Format files with ClangFormat.
|
||||
entry: bash ./tools/codestyle/clang_format.sh -i
|
||||
language: system
|
||||
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$
|
||||
|
||||
@@ -4,10 +4,6 @@
|
||||
|
||||
# Required
|
||||
version: 2
|
||||
build:
|
||||
os: "ubuntu-20.04"
|
||||
tools:
|
||||
python: "3.10"
|
||||
|
||||
submodules:
|
||||
include: all
|
||||
@@ -15,7 +11,7 @@ submodules:
|
||||
|
||||
# Build documentation in the docs/ directory with Sphinx
|
||||
sphinx:
|
||||
configuration: docs/zh/conf.py
|
||||
configuration: docs/conf.py
|
||||
|
||||
# Optionally build your docs in additional formats such as PDF
|
||||
#formats:
|
||||
@@ -23,5 +19,10 @@ sphinx:
|
||||
|
||||
# Optionally set the version of Python and requirements required to build your docs
|
||||
python:
|
||||
version: 3.8
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
- requirements: requirements.txt
|
||||
- method: setuptools
|
||||
path: .
|
||||
system_packages: true
|
||||
3
.style.yapf
Normal file
3
.style.yapf
Normal file
@@ -0,0 +1,3 @@
|
||||
[style]
|
||||
based_on_style = pep8
|
||||
column_limit = 80
|
||||
158
CONTRIBUTING.md
158
CONTRIBUTING.md
@@ -1,158 +0,0 @@
|
||||
**简体中文**🀄 | [English🌎](.github/CONTRIBUTING_en.md)
|
||||
|
||||
# Contributing to PaddleNLP
|
||||
|
||||
我们非常欢迎并希望您对`PaddleNLP`做出开源贡献。在您开始提交您的贡献之前，请先行签署[PaddlePaddle 贡献者许可协议](https://cla-assistant.io/PaddlePaddle/PaddleNLP)。
|
||||
本文接下来将介绍我们的开发与贡献流程：
|
||||
|
||||
## 贡献方式
|
||||
|
||||
我们欢迎不同的向`PaddleNLP`做出贡献的方式，例如：
|
||||
|
||||
- 修复已知的 Issue
|
||||
- 提交新的 Issue，例如提出功能需求或者 bug 报告
|
||||
- 实现新的模型结构
|
||||
|
||||
如果您不知道从哪里开始，请查看 Issues 板块中的`Good First Issue`标签。它为您提供一个对初学者友好的已知 Issue 列表，可以降低贡献的门槛，帮助您开始为开源做出贡献。您只需在您想处理的 Issue 中告知我们您想负责此 Issue 即可。
|
||||
|
||||
## 开发流程
|
||||
|
||||
PaddleNLP 使用 [Git 分支模型](http://nvie.com/posts/a-successful-git-branching-model/)。对于常见的开源贡献，我们有以下的贡献流程：
|
||||
|
||||
### 1. Fork
|
||||
|
||||
因为 PaddleNLP 的开发社区一直在发展，如果每位贡献者都直接向官方 Repo 提交 commit 将会难以管理。因此，请从您的分支中提交 Pull Requests。建议您通过 GitHub 的[“Fork”按钮](https://help.github.com/articles/fork-a-repo/)来创建您的 Fork 分支。
|
||||
|
||||
### 2. Clone
|
||||
|
||||
请运行一下命令将您的分支 clone 到本地
|
||||
|
||||
```bash
|
||||
git clone https://github.com/<your-github-account>/PaddleNLP
|
||||
cd PaddleNLP
|
||||
```
|
||||
|
||||
### 3. 创建本地开发分支
|
||||
|
||||
对于添加新功能或修复错误等日常工作，请在开发前创建您的本地开发分支：
|
||||
|
||||
```bash
|
||||
git checkout -b my-cool-feature
|
||||
```
|
||||
|
||||
### 4. 配置开发环境
|
||||
|
||||
在开始编码之前，您需要设置开发环境。我们强烈建议您在虚拟环境中进行所有开发，例如[venv](https://docs.python.org/3/library/venv.html)或[conda](https://docs.conda.io/en/latest/)。
|
||||
请您设置并激活虚拟环境后，运行以下命令：
|
||||
|
||||
```bash
|
||||
make install
|
||||
```
|
||||
|
||||
这将设置 `PaddleNLP` 的所有依赖以及 [`pre-commit`](http://pre-commit.com/) 工具。
|
||||
|
||||
如果您需要开发 `examples` 或 `applications` 模块并加载 `PaddleNLP`，请确保以可编辑模式（`-e`）安装 `PaddleNLP`。
|
||||
如果在虚拟环境中已经安装 `PaddleNLP` ，请使用 `pip uninstall paddlenlp` 将其删除，然后以可编辑模式重新安装它
|
||||
`pip install -e .`
|
||||
|
||||
### 5. 开发
|
||||
|
||||
当您开发时，请确保您新增的代码会被单元测试所覆盖。我们所有的单元测试都可以在 `tests` 目录下找到。
|
||||
您可以修改现有单元测试以覆盖新功能，也可以从头开始创建新测试。
|
||||
当您完成代码时，您应该确保相关的单元测试可以通过。您可以像这样运行受更改影响的测试：
|
||||
|
||||
```bash
|
||||
pytest tests/<test_to_run>.py
|
||||
```
|
||||
|
||||
### 6. Commit
|
||||
|
||||
我们使用 [`pre-commit`](http://pre-commit.com/)工具（包括[black](https://black.readthedocs.io/en/stable/)、[isort](https:/ /pycqa.github.io/isort/) 和
|
||||
[flake8](https://flake8.pycqa.org/en/latest/)）来检查每次提交中的代码和文档的风格。当你运行 `git commit` 时，你会看到
|
||||
类似于以下内容：
|
||||
|
||||
```text
|
||||
➜ (my-virtual-env) git commit -m "committing my cool feature"
|
||||
black....................................................................Passed
|
||||
isort....................................................................Passed
|
||||
flake8...................................................................Passed
|
||||
check for merge conflicts................................................Passed
|
||||
check for broken symlinks............................(no files to check)Skipped
|
||||
detect private key.......................................................Passed
|
||||
fix end of files.....................................(no files to check)Skipped
|
||||
trim trailing whitespace.............................(no files to check)Skipped
|
||||
CRLF end-lines checker...............................(no files to check)Skipped
|
||||
CRLF end-lines remover...............................(no files to check)Skipped
|
||||
No-tabs checker......................................(no files to check)Skipped
|
||||
Tabs remover.........................................(no files to check)Skipped
|
||||
copyright_checker........................................................Passed
|
||||
```
|
||||
|
||||
但大多数时候事情并没有那么顺利。当您的代码或文档不符合标准时，`pre-commit` 检查将失败。
|
||||
|
||||
```text
|
||||
➜ (my-virtual-env) git commit -m "committing my cool feature"
|
||||
black....................................................................Passed
|
||||
isort....................................................................Failed
|
||||
- hook id: isort
|
||||
- files were modified by this hook
|
||||
|
||||
Fixing examples/information_extraction/waybill_ie/run_ernie_crf.py
|
||||
|
||||
flake8...................................................................Passed
|
||||
check for merge conflicts................................................Passed
|
||||
check for broken symlinks............................(no files to check)Skipped
|
||||
detect private key.......................................................Passed
|
||||
fix end of files.....................................(no files to check)Skipped
|
||||
trim trailing whitespace.............................(no files to check)Skipped
|
||||
CRLF end-lines checker...............................(no files to check)Skipped
|
||||
CRLF end-lines remover...............................(no files to check)Skipped
|
||||
No-tabs checker......................................(no files to check)Skipped
|
||||
Tabs remover.........................................(no files to check)Skipped
|
||||
copyright_checker........................................................Passed
|
||||
```
|
||||
|
||||
我们的工具将自动修复大部分样式错误，但是有些错误需要手动解决。幸运的是，错误信息一般通俗易懂，很容易修复。
|
||||
解决错误后，您可以再次运行 `git add <files>` 和 `git commit`，这将再次触发 pre-commit 。
|
||||
一旦 pre-commit 检查通过，您就可以推送代码了。
|
||||
|
||||
[Google](https://google.com/) 或 [StackOverflow](https://stackoverflow.com/) 是帮助您了解代码风格错误的好工具。
|
||||
如果您仍然无法弄清楚，请不要担心。您可以使用 `git commit -m "style error" --no-verify` 提交，我们很乐意在您创建 Pull Request 后帮助您。
|
||||
|
||||
### 7. git pull 与代码冲突
|
||||
|
||||
有经验的 Git 用户经常从官方 Repo 中 git pull。因为这样子他们会及早注意到与其他人的代码冲突，并且让代码冲突更容易解决
|
||||
|
||||
```bash
|
||||
git remote add upstream https://github.com/PaddlePaddle/PaddleNLP
|
||||
git pull upstream develop
|
||||
```
|
||||
|
||||
### 8. git push 与提交 Pull Request
|
||||
|
||||
您可以将您的本地开发分支中的工作 push 到您的 fork 的分支中：
|
||||
|
||||
```bash
|
||||
git push origin my-cool-stuff
|
||||
```
|
||||
|
||||
git push 之后，您可以提交 Pull Request，请求[官方 repo](https://github.com/PaddlePaddle/PaddleNLP) 采纳您的开发工作。请您依照[这些步骤](https://help.github.com/articles/creating-a-pull-request/)创建 Pull Request。
|
||||
|
||||
### 9. 删除已经合入的本地和远程分支
|
||||
|
||||
为了保持您本地的工作区和 fork 分支的干净整洁，建议您在 Pull Request 合入之后删除本地的残余分支：
|
||||
|
||||
```bash
|
||||
git push origin my-cool-stuff
|
||||
git checkout develop
|
||||
git pull upstream develop
|
||||
git branch -d my-cool-stuff
|
||||
```
|
||||
|
||||
## 代码 Review
|
||||
|
||||
- 在您的 Pull Request 能够顺利通过本地测试以及 CI 的情况下，您可以在 Pull Request 中 @ 相关的 Reviewer，提醒他们尽快对您的 Pull Request 进行 Review。
|
||||
|
||||
- 请处理 Reviewer 的每一条评论。如果您已按照评论修改，请回复“完成”；否则，可以在评论下展开讨论。
|
||||
|
||||
- 如果您不希望您的 Reviewer 被电子邮件通知淹没，您可以[批量回复](https://help.github.com/articles/reviewing-proposed-changes-in-a-pull-request/)。
|
||||
79
Makefile
79
Makefile
@@ -1,79 +0,0 @@
|
||||
# Makefile for PaddleNLP
|
||||
#
|
||||
# GitHb: https://github.com/PaddlePaddle/PaddleNLP
|
||||
# Author: Paddle Team https://github.com/PaddlePaddle
|
||||
#
|
||||
|
||||
.PHONY: all
|
||||
all : lint test
|
||||
check_dirs := applications examples model_zoo paddlenlp pipelines ppdiffusers scripts tests
|
||||
# # # # # # # # # # # # # # # Format Block # # # # # # # # # # # # # # #
|
||||
|
||||
format:
|
||||
pre-commit run isort
|
||||
pre-commit run black
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
||||
# # # # # # # # # # # # # # # Lint Block # # # # # # # # # # # # # # #
|
||||
|
||||
.PHONY: lint
|
||||
lint:
|
||||
$(eval modified_py_files := $(shell python scripts/get_modified_files.py $(check_dirs)))
|
||||
@if test -n "$(modified_py_files)"; then \
|
||||
echo ${modified_py_files}; \
|
||||
pre-commit run --files ${modified_py_files}; \
|
||||
else \
|
||||
echo "No library .py files were modified"; \
|
||||
fi
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
||||
# # # # # # # # # # # # # # # Test Block # # # # # # # # # # # # # # #
|
||||
|
||||
.PHONY: test
|
||||
test: unit-test
|
||||
|
||||
unit-test:
|
||||
PYTHONPATH=$(shell pwd) pytest -v \
|
||||
-n auto \
|
||||
--retries 1 --retry-delay 1 \
|
||||
--durations 20 \
|
||||
--cov paddlenlp \
|
||||
--cov-report xml:coverage.xml
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
||||
.PHONY: install
|
||||
install:
|
||||
pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/
|
||||
pip install -r requirements-dev.txt
|
||||
pip install -r requirements.txt
|
||||
pip install -r paddlenlp/experimental/autonlp/requirements.txt
|
||||
pre-commit install
|
||||
|
||||
|
||||
.PHONY: deploy-ppdiffusers
|
||||
deploy-ppdiffusers:
|
||||
cd ppdiffusers && make install && make
|
||||
|
||||
.PHONY: deploy-paddle-pipelines
|
||||
deploy-paddle-pipelines:
|
||||
cd pipelines && make install && make
|
||||
|
||||
.PHONY: deploy-paddlenlp
|
||||
deploy-paddlenlp:
|
||||
# install related package
|
||||
make install
|
||||
# build
|
||||
python3 setup.py sdist bdist_wheel
|
||||
# upload
|
||||
twine upload --skip-existing dist/*
|
||||
|
||||
.PHONY: regression-all
|
||||
release:
|
||||
bash ./scripts/regression/run_release.sh 0 0,1 all
|
||||
|
||||
.PHONY: regression-key
|
||||
key:
|
||||
bash ./scripts/regression/run_release.sh 0 0,1 p0
|
||||
483
README.md
483
README.md
@@ -1,286 +1,295 @@
|
||||
**简体中文**🀄 | [English🌎](./README_en.md)
|
||||
简体中文 | [English](./README_en.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/1371212/175816733-8ec25eb0-9af3-4380-9218-27c154518258.png" align="middle" width="500" />
|
||||
<img src="./docs/imgs/paddlenlp.png" align="middle" width="500" />
|
||||
</p>
|
||||
|
||||
------------------------------------------------------------------------------------------
|
||||
[](https://pypi.org/project/paddlenlp/)
|
||||
[](https://pypi.org/project/paddlenlp/)
|
||||
[](https://pepy.tech/project/paddlenlp)
|
||||
|
||||
|
||||
|
||||
<p align="center">
|
||||
<a href="https://paddlenlp.readthedocs.io/en/latest/?badge=latest"><img src="https://readthedocs.org/projects/paddlenlp/badge/?version=latest">
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/PaddleNLP?color=ffa"></a>
|
||||
<a href=""><img src="https://img.shields.io/badge/python-3.7+-aff.svg"></a>
|
||||
<a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/graphs/contributors"><img src="https://img.shields.io/github/contributors/PaddlePaddle/PaddleNLP?color=9ea"></a>
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/commits"><img src="https://img.shields.io/github/commit-activity/m/PaddlePaddle/PaddleNLP?color=3af"></a>
|
||||
<a href="https://pypi.org/project/paddlenlp/"><img src="https://img.shields.io/pypi/dm/paddlenlp?color=9cf"></a>
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/PaddleNLP?color=9cc"></a>
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/PaddleNLP?color=ccf"></a>
|
||||
<a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
|
||||
</p>
|
||||
## News <img src="./docs/imgs/news_icon.png" width="40"/>
|
||||
|
||||
<h4 align="center">
|
||||
<a href=#特性> 特性 </a> |
|
||||
<a href=#模型支持> 模型支持 </a> |
|
||||
<a href=#安装> 安装 </a> |
|
||||
<a href=#快速开始> 快速开始 </a> |
|
||||
<a href=#社区交流> 社区交流 </a>
|
||||
</h4>
|
||||
* [2022-03-21] PaddleNLP**一键预测工具**[Taskflow](./docs/model_zoo/taskflow.md)全新升级！🚀欢迎体验更丰富的功能、更便捷的使用方式；新推出适合不同场景的中文分词、命名实体识别模式！
|
||||
* [2021-12-28] PaddleNLP新发**语义检索、问答、评论观点抽取和情感倾向分析** [产业化案例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/applications)，🚀快速搭建系统！配套视频课程[直通车](https://aistudio.baidu.com/aistudio/course/introduce/24902)！
|
||||
|
||||
**PaddleNLP**是一款基于飞桨深度学习框架的大语言模型(LLM)开发套件，支持在多种硬件上进行高效的大模型训练、无损压缩以及高性能推理。PaddleNLP 具备**简单易用**和**性能极致**的特点，致力于助力开发者实现高效的大模型产业级应用。
|
||||
## 简介
|
||||
|
||||
<a href="https://trendshift.io/repositories/2246" target="_blank"><img src="https://trendshift.io/api/badge/repositories/2246" alt="PaddlePaddle%2FPaddleNLP | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
PaddleNLP是飞桨自然语言处理开发库，具备**易用的文本领域API**，**多场景的应用示例**、和**高性能分布式训练**三大特点，旨在提升开发者在文本领域的开发效率，并提供丰富的NLP应用示例。
|
||||
|
||||
## News 📢
|
||||
- **易用的文本领域API**
|
||||
- 提供丰富的产业级预置任务能力[Taskflow](./docs/model_zoo/taskflow.md)和全流程的文本领域API：支持丰富中文数据集加载的[Dataset API](https://paddlenlp.readthedocs.io/zh/latest/data_prepare/dataset_list.html)；灵活高效地完成数据预处理的[Data API](https://paddlenlp.readthedocs.io/zh/latest/source/paddlenlp.data.html)；提供100+预训练模型的[Transformers API](./docs/model_zoo/transformers.rst)等，可大幅提升NLP任务建模的效率。
|
||||
- **多场景的应用示例**
|
||||
- 覆盖从学术到产业级的NLP[应用示例](#多场景的应用示例)，涵盖NLP基础技术、NLP系统应用以及相关拓展应用。全面基于飞桨核心框架2.0全新API体系开发，为开发者提供飞桨文本领域的最佳实践。
|
||||
- **高性能分布式训练**
|
||||
- 基于飞桨核心框架领先的自动混合精度优化策略，结合分布式Fleet API，支持4D混合并行策略，可高效地完成大规模预训练模型训练。
|
||||
|
||||
* **2025.04.29 PaddleNLP 现已支持 Qwen3 系列模型**: Qwen3 系列模型支持持两种思考模式，预训练约 36 万亿个 token、119 种语言和方言。包括六个 Dense 模型, Qwen3-32B、Qwen3-14B、Qwen3-8B、Qwen3-4B、Qwen3-1.7B 和 Qwen3-0.6B。两个 MoE 模型的权重：Qwen3-235B-A22B，Qwen3-30B-A3B。
|
||||
## 社区交流
|
||||
|
||||
* **2025.03.12 [PaddleNLP v3.0 Beta4](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta4)**：全面支持 DeepSeek V3/R1/R1-Distill, 及 QwQ-32B 等热门思考模型。**DeepSeek V3/R1完整版支持 FP8、INT8、4-bit 量化推理，MTP 投机解码**。单机 FP8推理输出超**1000 tokens/s**; 4-bit 推理输出超**2100 tokens/s**! 发布新版推理部署镜像，热门模型[一键部署](https://paddlenlp.readthedocs.io/zh/latest/llm/server/docs/general_model_inference.html)。推理部署[使用文档](https://paddlenlp.readthedocs.io/zh/latest/llm/docs/predict/index.html)全面更新，体验全面提升！自研下一代通用信息抽取模型 PP-UIE [全新发布](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/application/information_extraction)，支持8K 长度信息抽取。新增大模型 Embedding 训练，支持 INF-CL 超大 batch size 训练。新增[MergeKit](https://paddlenlp.readthedocs.io/zh/latest/llm/docs/mergekit.html)模型融合工具，缓解对齐代价。低资源训练全面优化，16G 小显存可以流畅训练。
|
||||
微信扫描下方二维码加入官方交流群，与各行各业开发者充分交流，期待您的加入⬇️
|
||||
|
||||
<div align="center">
|
||||
<img src="https://raw.githubusercontent.com/PaddlePaddle/PaddleNLP/release/2.2/docs/imgs/wechat.png" width="188" height="188" />
|
||||
</div>
|
||||
|
||||
* **2025.02.10 PaddleNLP 现已支持 DeepSeek-R1系列模型，[在线使用](https://aistudio.baidu.com/projectdetail/8775758)**：依托全新的 PaddleNLP 3.0套件，DeepSeek-R1系列模型现已全面支持。凭借数据并行、数据分组切分并行、模型并行、流水线并行以及专家并行等一系列先进的分布式训练能力，结合 Paddle 框架独有的列稀疏注意力掩码表示技术——FlashMask 方法，DeepSeek-R1系列模型在训练过程中显著降低了显存消耗，同时取得了卓越的训练性能提升。
|
||||
|
||||
<details><summary> <b>点击展开</b> </summary><div>
|
||||
|
||||
* **2025.03.17 《DeepSeek-R1满血版单机部署实测》** 🔥🔥🔥 飞桨框架3.0大模型推理部署全面升级，支持多款主流大模型，DeepSeek-R1满血版实现单机部署，吞吐提升一倍！欢迎广大用户开箱体验～现已开启有奖活动：完成 DeepSeek-R1-MTP 单机部署任务、提交高质量测评 blog，即可实时赢取奖金！💰💰💰
|
||||
报名[地址](https://www.wjx.top/vm/OlzzmbG.aspx#)， 活动详情：https://github.com/PaddlePaddle/PaddleNLP/issues/10166 ， 参考文档：https://github.com/PaddlePaddle/PaddleNLP/issues/10157 。
|
||||
|
||||
* **2025.03.06 PaddleNLP 现已支持 Qwen/QwQ-32B 模型**: 其模型参数仅有 32B，但其数学推理、编程能力和通用能力可与具备 671B 参数（其中 37B 被激活）的 DeepSeek-R1 媲美。借助 PaddleNLP 3.0套件，现可实现多种并行策略[微调训练](./llm/README.md)、[高性能推理、低比特量化](./llm/docs/predict/qwen.md)和[服务化部署](./llm/server/README.md)。
|
||||
|
||||
* **2025.02.20 🔥🔥《PP-UIE 信息抽取智能引擎全新升级》** 强化零样本学习能力，支持极少甚至零标注数据实现高效冷启动与迁移学习，显著降低数据标注成本；具备处理长文本能力，支持 8192 个 Token 长度文档信息抽取，实现跨段落识别关键信息，形成完整理解；提供完整可定制化的训练和推理全流程，训练效率相较于 LLama-Factory 实现了1.8倍的提升。
|
||||
2月26日（周三）19：00为您深度解析全新 PP-UIE 技术方案及在部署方面的功能、优势与技巧。报名链接：https://www.wjx.top/vm/mBKC6pb.aspx?udsid=606418
|
||||
|
||||
* **2024.12.16 [PaddleNLP v3.0 Beta3](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta3)**：大模型功能全新升级，新增了 Llama-3.2、DeepSeekV2模型，升级了 TokenizerFast，快速分词，重构了 SFTTrainer，一键开启 SFT 训练。此外，PaddleNLP 还支持了优化器状态的卸载和重载功能，实现了精细化的重新计算，训练性能提升7%。在 Unified Checkpoint 方面，进一步优化了异步保存逻辑，新增 Checkpoint 压缩功能，可节省78.5%存储空间。
|
||||
最后，在大模型推理方面，升级 Append Attention，支持了 FP8量化，支持投机解码。
|
||||
|
||||
* **2024.12.13 📚《飞桨大模型套件 Unified Checkpoint 技术》**，加速模型存储95%，节省空间78%。支持全分布式策略调整自适应转换，提升模型训练的灵活性与可扩展性。训练-压缩-推理统一存储协议，无需手动转换提升全流程体验。Checkpoint 无损压缩结合异步保存，实现秒级存储并降低模型存储成本。适用于智能制造、指挥交通、医疗健康、金融服务等产业实际场景。12月24日（周二）19：00直播为您详细解读该技术如何优化大模型训练流程。报名链接：https://www.wjx.top/vm/huZkHn9.aspx?udsid=787976
|
||||
|
||||
* **2024.11.28 📚《FlashRAG-Paddle | 基于 PaddleNLP 的高效开发与评测 RAG 框架》**，为文本更快更好构建准确嵌入表示、加速推理生成速度。PaddleNLP 支持超大 Batch 嵌入表示学习与多硬件高性能推理，涵盖 INT8/INT4量化技术及多种高效注意力机制优化与 TensorCore 深度优化。内置全环节算子融合技术，使得 FlashRAG 推理性能相比 transformers 动态图提升70%以上，结合检索增强知识输出结果更加准确，带来敏捷高效的使用体验。直播时间：12月3日（周二）19：00。报名链接：https://www.wjx.top/vm/eaBa1vA.aspx?udsid=682361
|
||||
|
||||
* **2024.08.08 📚《飞桨产业级大语言模型开发利器 PaddleNLP 3.0 重磅发布》**，训压推全流程贯通，主流模型全覆盖。大模型自动并行，千亿模型训推全流程开箱即用。提供产业级高性能精调与对齐解决方案，压缩推理领先，多硬件适配。覆盖产业级智能助手、内容创作、知识问答、关键信息抽取等应用场景。直播时间：8月22日（周四）19：00。报名链接：https://www.wjx.top/vm/Y2f7FFY.aspx?udsid=143844
|
||||
|
||||
* **2024.06.27 [PaddleNLP v3.0 Beta](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta0)**：拥抱大模型，体验全升级。统一大模型套件，实现国产计算芯片全流程接入；全面支持飞桨4D 并行配置、高效精调策略、高效对齐算法、高性能推理等大模型产业级应用流程；自研极致收敛的 RsLoRA+算法、自动扩缩容存储机制 Unified Checkpoint 和通用化支持的 FastFFN、FusedQKV 助力大模型训推；主流模型持续支持更新，提供高效解决方案。
|
||||
|
||||
* **2024.04.24 [PaddleNLP v2.8](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.8.0)**：自研极致收敛的 RsLoRA+算法，大幅提升 PEFT 训练收敛速度以及训练效果；引入高性能生成加速到 RLHF PPO 算法，打破 PPO 训练中生成速度瓶颈，PPO 训练性能大幅领先。通用化支持 FastFFN、FusedQKV 等多个大模型训练性能优化方式，大模型训练更快、更稳定。
|
||||
</div></details>
|
||||
|
||||
## 特性
|
||||
|
||||
### <a href=#多硬件训推一体> 🔧 多硬件训推一体 </a>
|
||||
|
||||
支持英伟达 GPU、昆仑 XPU、昇腾 NPU、燧原 GCU 和海光 DCU 等多个硬件的大模型和自然语言理解模型训练和推理，套件接口支持硬件快速切换，大幅降低硬件切换研发成本。
|
||||
当前支持的自然语言理解模型：[多硬件自然语言理解模型列表](./docs/zh/model_zoo/model_list_multy_device.md)
|
||||
|
||||
### <a href=#高效易用的预训练> 🚀 高效易用的预训练 </a>
|
||||
|
||||
支持纯数据并行策略、分组参数切片的数据并行策略、张量模型并行策略和流水线模型并行策略的4D 高性能训练，Trainer 支持分布式策略配置化，降低复杂分布式组合带来的使用成本；
|
||||
[Unified Checkpoint 大模型存储工具](./llm/docs/unified_checkpoint.md)可以使得训练断点支持机器资源动态扩缩容恢复。此外，异步保存，模型存储可加速95%，Checkpoint 压缩，可节省78.5%存储空间。
|
||||
|
||||
### <a href=#高效精调> 🤗 高效精调 </a>
|
||||
|
||||
精调算法深度结合零填充数据流和 [FlashMask](./llm/docs/flashmask.md) 高性能算子，降低训练无效数据填充和计算，大幅提升精调训练吞吐。
|
||||
|
||||
### <a href=#无损压缩和高性能推理> 🎛️ 无损压缩和高性能推理 </a>
|
||||
|
||||
大模型套件高性能推理模块内置动态插入和全环节算子融合策略，极大加快并行推理速度。底层实现细节封装化，实现开箱即用的高性能并行推理能力。
|
||||
|
||||
## 文档
|
||||
更多详细文档, 请访问 [PaddleNLP Documentation](https://paddlenlp.readthedocs.io/).
|
||||
|
||||
------------------------------------------------------------------------------------------
|
||||
|
||||
## 模型支持
|
||||
|
||||
* 模型参数已支持 LLaMA 系列、Baichuan 系列、Bloom 系列、ChatGLM 系列、Gemma 系列、Mistral 系列、OPT 系列和 Qwen 系列，详细列表👉【LLM】模型参数支持列表如下：
|
||||
|
||||
| 模型系列 | 模型名称 |
|
||||
|:-------------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| [PP-UIE](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/application/information_extraction) | paddlenlp/PP-UIE-0.5B, paddlenlp/PP-UIE-1.5B, paddlenlp/PP-UIE-7B, paddlenlp/PP-UIE-14B |
|
||||
| [LLaMA](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama) | facebook/llama-7b, facebook/llama-13b, facebook/llama-30b, facebook/llama-65b |
|
||||
| [Llama2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama) | meta-llama/Llama-2-7b, meta-llama/Llama-2-7b-chat, meta-llama/Llama-2-13b, meta-llama/Llama-2-13b-chat, meta-llama/Llama-2-70b, meta-llama/Llama-2-70b-chat |
|
||||
| [Llama3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama) | meta-llama/Meta-Llama-3-8B, meta-llama/Meta-Llama-3-8B-Instruct, meta-llama/Meta-Llama-3-70B, meta-llama/Meta-Llama-3-70B-Instruct |
|
||||
| [Llama3.1](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama) | meta-llama/Meta-Llama-3.1-8B, meta-llama/Meta-Llama-3.1-8B-Instruct, meta-llama/Meta-Llama-3.1-70B, meta-llama/Meta-Llama-3.1-70B-Instruct, meta-llama/Meta-Llama-3.1-405B, meta-llama/Meta-Llama-3.1-405B-Instruct, meta-llama/Llama-Guard-3-8B |
|
||||
| [Llama3.2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama) | meta-llama/Llama-3.2-1B, meta-llama/Llama-3.2-1B-Instruct, meta-llama/Llama-3.2-3B, meta-llama/Llama-3.2-3B-Instruct, meta-llama/Llama-Guard-3-1B |
|
||||
| [Llama3.3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/llama) | meta-llama/Llama-3.3-70B-Instruct |
|
||||
| [Baichuan](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/baichuan) | baichuan-inc/Baichuan-7B, baichuan-inc/Baichuan-13B-Base, baichuan-inc/Baichuan-13B-Chat |
|
||||
| [Baichuan2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/baichuan) | baichuan-inc/Baichuan2-7B-Base, baichuan-inc/Baichuan2-7B-Chat, baichuan-inc/Baichuan2-13B-Base, baichuan-inc/Baichuan2-13B-Chat |
|
||||
| [Bloom](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/bloom) | bigscience/bloom-560m, bigscience/bloom-560m-bf16, bigscience/bloom-1b1, bigscience/bloom-3b, bigscience/bloom-7b1, bigscience/bloomz-560m, bigscience/bloomz-1b1, bigscience/bloomz-3b, bigscience/bloomz-7b1-mt, bigscience/bloomz-7b1-p3, bigscience/bloomz-7b1, bellegroup/belle-7b-2m |
|
||||
| [ChatGLM](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/chatglm/) | THUDM/chatglm-6b, THUDM/chatglm-6b-v1.1 |
|
||||
| [ChatGLM2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/chatglm2) | THUDM/chatglm2-6b |
|
||||
| [ChatGLM3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/chatglm2) | THUDM/chatglm3-6b |
|
||||
| [DeepSeekV2](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/config/deepseek-v2) | deepseek-ai/DeepSeek-V2, deepseek-ai/DeepSeek-V2-Chat, deepseek-ai/DeepSeek-V2-Lite, deepseek-ai/DeepSeek-V2-Lite-Chat, deepseek-ai/DeepSeek-Coder-V2-Base, deepseek-ai/DeepSeek-Coder-V2-Instruct, deepseek-ai/DeepSeek-Coder-V2-Lite-Base, deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct |
|
||||
| [DeepSeekV3](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/config/deepseek-v2) | deepseek-ai/DeepSeek-V3, deepseek-ai/DeepSeek-V3-Base |
|
||||
| [DeepSeek-R1](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/config/deepseek-v2) | deepseek-ai/DeepSeek-R1, deepseek-ai/DeepSeek-R1-Zero, deepseek-ai/DeepSeek-R1-Distill-Llama-70B, deepseek-ai/DeepSeek-R1-Distill-Llama-8B, deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, deepseek-ai/DeepSeek-R1-Distill-Qwen-7B |
|
||||
| [Gemma](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/gemma) | google/gemma-7b, google/gemma-7b-it, google/gemma-2b, google/gemma-2b-it |
|
||||
| [Mistral](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/mistral) | mistralai/Mistral-7B-Instruct-v0.3, mistralai/Mistral-7B-v0.1 |
|
||||
| [Mixtral](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/mixtral) | mistralai/Mixtral-8x7B-Instruct-v0.1 |
|
||||
| [OPT](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/opt) | facebook/opt-125m, facebook/opt-350m, facebook/opt-1.3b, facebook/opt-2.7b, facebook/opt-6.7b, facebook/opt-13b, facebook/opt-30b, facebook/opt-66b, facebook/opt-iml-1.3b, opt-iml-max-1.3b |
|
||||
| [Qwen](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/) | qwen/qwen-7b, qwen/qwen-7b-chat, qwen/qwen-14b, qwen/qwen-14b-chat, qwen/qwen-72b, qwen/qwen-72b-chat, |
|
||||
| [Qwen1.5](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/) | Qwen/Qwen1.5-0.5B, Qwen/Qwen1.5-0.5B-Chat, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-1.8B-Chat, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-4B-Chat, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-7B-Chat, Qwen/Qwen1.5-14B, Qwen/Qwen1.5-14B-Chat, Qwen/Qwen1.5-32B, Qwen/Qwen1.5-32B-Chat, Qwen/Qwen1.5-72B, Qwen/Qwen1.5-72B-Chat, Qwen/Qwen1.5-110B, Qwen/Qwen1.5-110B-Chat, Qwen/Qwen1.5-MoE-A2.7B, Qwen/Qwen1.5-MoE-A2.7B-Chat |
|
||||
| [Qwen2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/) | Qwen/Qwen2-0.5B, Qwen/Qwen2-0.5B-Instruct, Qwen/Qwen2-1.5B, Qwen/Qwen2-1.5B-Instruct, Qwen/Qwen2-7B, Qwen/Qwen2-7B-Instruct, Qwen/Qwen2-72B, Qwen/Qwen2-72B-Instruct, Qwen/Qwen2-57B-A14B, Qwen/Qwen2-57B-A14B-Instruct |
|
||||
| [Qwen2-Math](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/) | Qwen/Qwen2-Math-1.5B, Qwen/Qwen2-Math-1.5B-Instruct, Qwen/Qwen2-Math-7B, Qwen/Qwen2-Math-7B-Instruct, Qwen/Qwen2-Math-72B, Qwen/Qwen2-Math-72B-Instruct, Qwen/Qwen2-Math-RM-72B |
|
||||
| [Qwen2.5](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/) | Qwen/Qwen2.5-0.5B, Qwen/Qwen2.5-0.5B-Instruct, Qwen/Qwen2.5-1.5B, Qwen/Qwen2.5-1.5B-Instruct, Qwen/Qwen2.5-3B, Qwen/Qwen2.5-3B-Instruct, Qwen/Qwen2.5-7B, Qwen/Qwen2.5-7B-Instruct, Qwen/Qwen2.5-7B-Instruct-1M, Qwen/Qwen2.5-14B, Qwen/Qwen2.5-14B-Instruct, Qwen/Qwen2.5-14B-Instruct-1M, Qwen/Qwen2.5-32B, Qwen/Qwen2.5-32B-Instruct, Qwen/Qwen2.5-72B, Qwen/Qwen2.5-72B-Instruct |
|
||||
| [Qwen2.5-Math](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/) | Qwen/Qwen2.5-Math-1.5B, Qwen/Qwen2.5-Math-1.5B-Instruct, Qwen/Qwen2.5-Math-7B, Qwen/Qwen2.5-Math-7B-Instruct, Qwen/Qwen2.5-Math-72B, Qwen/Qwen2.5-Math-72B-Instruct, Qwen/Qwen2.5-Math-RM-72B |
|
||||
| [Qwen2.5-Coder](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/) | Qwen/Qwen2.5-Coder-1.5B, Qwen/Qwen2.5-Coder-1.5B-Instruct, Qwen/Qwen2.5-Coder-7B, Qwen/Qwen2.5-Coder-7B-Instruct |
|
||||
| [Qwen3](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/) | Qwen/Qwen3-0.6B, Qwen/Qwen3-1.7B, Qwen/Qwen3-4B, Qwen/Qwen3-8B, Qwen/Qwen3-14B, Qwen/Qwen3-32B, Qwen/Qwen3-30B-A3B, Qwen/Qwen3-235B-A22B, Qwen/Qwen3-0.6B-Base, Qwen/Qwen3-1.7B-Base, Qwen/Qwen3-4B-Base, Qwen/Qwen3-8B-Base, Qwen/Qwen3-14B-Base, Qwen/Qwen3-30B-A3B-Base |
|
||||
| [QwQ](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/qwen/) | Qwen/QwQ-32B, Qwen/QwQ-32B-Preview |
|
||||
| [Yuan2](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm/config/yuan/) | IEITYuan/Yuan2-2B, IEITYuan/Yuan2-51B, IEITYuan/Yuan2-102B |
|
||||
|
||||
* 4D 并行和算子优化已支持 LLaMA 系列、Baichuan 系列、Bloom 系列、ChatGLM 系列、Gemma 系列、Mistral 系列、OPT 系列和 Qwen 系列，【LLM】模型4D 并行和算子支持列表如下：
|
||||
|
||||
| 模型名称/并行能力支持 | 数据并行 | 张量模型并行 | | 参数分片并行 | | | 流水线并行 |
|
||||
|:---------------------:|:--------:|:------------:|:--------:|:------------:|:------:|:------:|:----------:|
|
||||
| | | 基础能力 | 序列并行 | stage1 | stage2 | stage3 | |
|
||||
| Llama | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| Qwen | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| Qwen1.5 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| Qwen2 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| Mixtral(moe) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 |
|
||||
| Mistral | ✅ | ✅ | 🚧 | ✅ | ✅ | ✅ | 🚧 |
|
||||
| Baichuan | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| Baichuan2 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| ChatGLM | ✅ | ✅ | 🚧 | ✅ | ✅ | ✅ | 🚧 |
|
||||
| ChatGLM2 | ✅ | 🚧 | 🚧 | ✅ | ✅ | ✅ | 🚧 |
|
||||
| ChatGLM3 | ✅ | 🚧 | 🚧 | ✅ | ✅ | ✅ | 🚧 |
|
||||
| Bloom | ✅ | ✅ | 🚧 | ✅ | ✅ | ✅ | 🚧 |
|
||||
| GPT-2/GPT-3 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| OPT | ✅ | ✅ | 🚧 | ✅ | ✅ | ✅ | 🚧 |
|
||||
| Gemma | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| Yuan2 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 |
|
||||
|
||||
* 大模型预训练、精调（包含 SFT、PEFT 技术）、对齐、量化已支持 LLaMA 系列、Baichuan 系列、Bloom 系列、ChatGLM 系列、Mistral 系列、OPT 系列和 Qwen 系列，【LLM】模型预训练、精调、对齐、量化支持列表如下：
|
||||
|
||||
|
||||
| Model | Pretrain | SFT | LoRA | FlashMask | Prefix Tuning | DPO/SimPO/ORPO/KTO | RLHF | Mergekit | Quantization |
|
||||
|--------------------------------------------|:--------:|:---:|:----:|:---------:|:-------------:|:------------------:|:----:|:--------:|:------------:|
|
||||
| [Llama](./llm/config/llama) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| [Qwen](./llm/config/qwen) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 |
|
||||
| [Mixtral](./llm/config/mixtral) | ✅ | ✅ | ✅ | 🚧 | 🚧 | ✅ | 🚧 | ✅ | 🚧 |
|
||||
| [Mistral](./llm/config/mistral) | ✅ | ✅ | ✅ | 🚧 | ✅ | ✅ | 🚧 | ✅ | 🚧 |
|
||||
| [Baichuan/Baichuan2](./llm/config/llama) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 | ✅ | ✅ |
|
||||
| [ChatGLM-6B](./llm/config/chatglm) | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 | 🚧 | ✅ | ✅ |
|
||||
| [ChatGLM2/ChatGLM3](./llm/config/chatglm2) | ✅ | ✅ | ✅ | 🚧 | ✅ | ✅ | 🚧 | ✅ | ✅ |
|
||||
| [Bloom](./llm/config/bloom) | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 | 🚧 | ✅ | ✅ |
|
||||
| [GPT-3](./llm/config/gpt-3) | ✅ | ✅ | 🚧 | 🚧 | 🚧 | 🚧 | 🚧 | ✅ | 🚧 |
|
||||
| [OPT](./llm/config/opt) | ✅ | ✅ | ✅ | 🚧 | 🚧 | 🚧 | 🚧 | ✅ | 🚧 |
|
||||
| [Gemma](./llm/config/gemma) | ✅ | ✅ | ✅ | 🚧 | 🚧 | ✅ | 🚧 | ✅ | 🚧 |
|
||||
| [Yuan](./llm/config/yuan) | ✅ | ✅ | ✅ | 🚧 | 🚧 | ✅ | 🚧 | ✅ | 🚧 |
|
||||
* [大模型推理](./llm/docs/predict/inference.md)已支持 LLaMA 系列、Qwen 系列、DeepSeek 系列、Mistral 系列、ChatGLM 系列、Bloom 系列和 Baichuan 系列，支持 Weight Only INT8及 INT4推理，支持 WAC（权重、激活、Cache KV）进行 INT8、FP8量化的推理，【LLM】模型推理支持列表如下：
|
||||
|
||||
| 模型名称/量化类型支持 | FP16/BF16 | WINT8 | WINT4 | INT8-A8W8 | FP8-A8W8 | INT8-A8W8C8 |
|
||||
|:------------------------------------------:|:---------:|:-----:|:-----:|:---------:|:--------:|:-----------:|
|
||||
| [LLaMA](./llm/docs/predict/llama.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| [Qwen](./llm/docs/predict/qwen.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| [DeepSeek](./llm/docs/predict/deepseek.md) | ✅ | ✅ | ✅ | 🚧 | ✅ | 🚧 |
|
||||
| [Qwen-Moe](./llm/docs/predict/qwen.md) | ✅ | ✅ | ✅ | 🚧 | 🚧 | 🚧 |
|
||||
| [Mixtral](./llm/docs/predict/mixtral.md) | ✅ | ✅ | ✅ | 🚧 | 🚧 | 🚧 |
|
||||
| ChatGLM | ✅ | ✅ | ✅ | 🚧 | 🚧 | 🚧 |
|
||||
| Bloom | ✅ | ✅ | ✅ | 🚧 | 🚧 | 🚧 |
|
||||
| BaiChuan | ✅ | ✅ | ✅ | ✅ | ✅ | 🚧 |
|
||||
|
||||
## 安装
|
||||
|
||||
### 环境依赖
|
||||
|
||||
* python >= 3.8
|
||||
* paddlepaddle >= 3.0.0rc1
|
||||
- python >= 3.6
|
||||
- paddlepaddle >= 2.2
|
||||
|
||||
如果您尚未安装 PaddlePaddle，请参考 [飞桨官网](https://www.paddlepaddle.org.cn/) 进行安装。
|
||||
|
||||
### pip 安装
|
||||
### pip安装
|
||||
|
||||
```shell
|
||||
pip install --upgrade paddlenlp==3.0.0b4
|
||||
pip install --upgrade paddlenlp
|
||||
```
|
||||
|
||||
或者可通过以下命令安装最新 develop 分支代码：
|
||||
更多关于PaddlePaddle和PaddleNLP安装的详细教程请查看[Installation](./docs/get_started/installation.rst)。
|
||||
|
||||
```shell
|
||||
pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html
|
||||
```
|
||||
## 易用的文本领域API
|
||||
|
||||
更多关于 PaddlePaddle 和 PaddleNLP 安装的详细教程请查看[Installation](./docs/zh/get_started/installation.rst)。
|
||||
### Taskflow：开箱即用的产业级NLP能力
|
||||
|
||||
------------------------------------------------------------------------------------------
|
||||
Taskflow旨在提供**开箱即用**的NLP预置任务能力，覆盖自然语言理解与生成两大场景，提供**产业级的效果**与**极致的预测性能**。
|
||||
|
||||
## 快速开始
|
||||
|
||||
|
||||
### 大模型文本生成
|
||||
更多使用方法请参考[Taskflow文档](./docs/model_zoo/taskflow.md)。
|
||||
|
||||
PaddleNLP 提供了方便易用的 Auto API，能够快速的加载模型和 Tokenizer。这里以使用 `Qwen/Qwen2-0.5B` 模型做文本生成为例：
|
||||
### Transformers API: 强大的预训练模型生态底座
|
||||
|
||||
覆盖**30**个网络结构和**100**余个预训练模型参数，既包括百度自研的预训练模型如ERNIE系列, PLATO, SKEP等，也涵盖业界主流的中文预训练模型如BERT，GPT，XLNet，BART等。使用AutoModel可以下载不同网络结构的预训练模型。欢迎开发者加入贡献更多预训练模型！🤗
|
||||
|
||||
```python
|
||||
from paddlenlp.transformers import AutoTokenizer, AutoModelForCausalLM
|
||||
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B")
|
||||
# if using CPU, please change float16 to float32
|
||||
model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B", dtype="float16")
|
||||
input_features = tokenizer("你好！请自我介绍一下。", return_tensors="pd")
|
||||
outputs = model.generate(**input_features, max_new_tokens=128)
|
||||
print(tokenizer.batch_decode(outputs[0], skip_special_tokens=True))
|
||||
# ['我是一个AI语言模型，我可以回答各种问题，包括但不限于：天气、新闻、历史、文化、科学、教育、娱乐等。请问您有什么需要了解的吗？']
|
||||
from paddlenlp.transformers import *
|
||||
|
||||
ernie = AutoModel.from_pretrained('ernie-1.0')
|
||||
ernie_gram = AutoModel.from_pretrained('ernie-gram-zh')
|
||||
bert = AutoModel.from_pretrained('bert-wwm-chinese')
|
||||
albert = AutoModel.from_pretrained('albert-chinese-tiny')
|
||||
roberta = AutoModel.from_pretrained('roberta-wwm-ext')
|
||||
electra = AutoModel.from_pretrained('chinese-electra-small')
|
||||
gpt = AutoModelForPretraining.from_pretrained('gpt-cpm-large-cn')
|
||||
```
|
||||
|
||||
### 大模型预训练
|
||||
|
||||
```shell
|
||||
git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # 如已clone或下载PaddleNLP可跳过
|
||||
mkdir -p llm/data && cd llm/data
|
||||
wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
|
||||
wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
|
||||
cd .. # change folder to PaddleNLP/llm
|
||||
# 如需使用use_fused_rms_norm=true，需要前往slm/model_zoo/gpt-3/external_ops安装fused_ln
|
||||
python -u run_pretrain.py ./config/qwen/pretrain_argument_0p5b.json
|
||||
```
|
||||
|
||||
### 大模型 SFT 精调
|
||||
|
||||
```shell
|
||||
git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # 如已clone或下载PaddleNLP可跳过
|
||||
mkdir -p llm/data && cd llm/data
|
||||
wget https://bj.bcebos.com/paddlenlp/datasets/examples/AdvertiseGen.tar.gz && tar -zxvf AdvertiseGen.tar.gz
|
||||
cd .. # change folder to PaddleNLP/llm
|
||||
python -u run_finetune.py ./config/qwen/sft_argument_0p5b.json
|
||||
```
|
||||
|
||||
更多大模型全流程步骤，请参考[飞桨大模型套件](./llm)介绍。
|
||||
另外我们还提供了快速微调方式, 无需 clone 源代码：
|
||||
对预训练模型应用范式如语义表示、文本分类、句对匹配、序列标注、问答等，提供统一的API体验。
|
||||
|
||||
```python
|
||||
from paddlenlp.trl import SFTConfig, SFTTrainer
|
||||
from datasets import load_dataset
|
||||
import paddle
|
||||
from paddlenlp.transformers import *
|
||||
|
||||
dataset = load_dataset("ZHUI/alpaca_demo", split="train")
|
||||
tokenizer = AutoTokenizer.from_pretrained('ernie-1.0')
|
||||
text = tokenizer('自然语言处理')
|
||||
|
||||
training_args = SFTConfig(output_dir="Qwen/Qwen2.5-0.5B-SFT", device="gpu")
|
||||
trainer = SFTTrainer(
|
||||
args=training_args,
|
||||
model="Qwen/Qwen2.5-0.5B-Instruct",
|
||||
train_dataset=dataset,
|
||||
)
|
||||
trainer.train()
|
||||
# 语义表示
|
||||
model = AutoModel.from_pretrained('ernie-1.0')
|
||||
sequence_output, pooled_output = model(input_ids=paddle.to_tensor([text['input_ids']]))
|
||||
# 文本分类 & 句对匹配
|
||||
model = AutoModelForSequenceClassification.from_pretrained('ernie-1.0')
|
||||
# 序列标注
|
||||
model = AutoModelForTokenClassification.from_pretrained('ernie-1.0')
|
||||
# 问答
|
||||
model = AutoModelForQuestionAnswering.from_pretrained('ernie-1.0')
|
||||
```
|
||||
|
||||
更多 PaddleNLP 内容可参考：
|
||||
请参考[Transformer API文档](./docs/model_zoo/transformers.rst)查看目前支持的预训练模型结构、参数和详细用法。
|
||||
|
||||
* [精选模型库](./slm/model_zoo)，包含优质预训练模型的端到端全流程使用。
|
||||
* [多场景示例](./slm/examples)，了解如何使用 PaddleNLP 解决 NLP 多种技术问题，包含基础技术、系统应用与拓展应用。
|
||||
* [交互式教程](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)，在🆓免费算力平台 AI Studio 上快速学习 PaddleNLP。
|
||||
### Datasets API: 丰富的中文数据集
|
||||
|
||||
------------------------------------------------------------------------------------------
|
||||
Dataset API提供便捷、高效的数据集加载功能；内置[千言数据集](https://www.luge.ai/)，提供丰富的面向自然语言理解与生成场景的中文数据集，为NLP研究人员提供一站式的科研体验。
|
||||
|
||||
## 社区交流
|
||||
```python
|
||||
from paddlenlp.datasets import load_dataset
|
||||
|
||||
* 微信扫描二维码并填写问卷，即可加入交流群与众多社区开发者以及官方团队深度交流.
|
||||
train_ds, dev_ds, test_ds = load_dataset("chnsenticorp", splits=["train", "dev", "test"])
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/user-attachments/assets/3a58cc9f-69c7-4ccb-b6f5-73e966b8051a" width="150" height="150" />
|
||||
</div>
|
||||
train_ds, dev_ds = load_dataset("lcqmc", splits=["train", "dev"])
|
||||
```
|
||||
|
||||
可参考[Dataset文档](https://paddlenlp.readthedocs.io/zh/latest/data_prepare/dataset_list.html) 查看更多数据集。
|
||||
|
||||
### Embedding API: 一键加载预训练词向量
|
||||
|
||||
```python
|
||||
from paddlenlp.embeddings import TokenEmbedding
|
||||
|
||||
wordemb = TokenEmbedding("w2v.baidu_encyclopedia.target.word-word.dim300")
|
||||
print(wordemb.cosine_sim("国王", "王后"))
|
||||
>>> 0.63395125
|
||||
wordemb.cosine_sim("艺术", "火车")
|
||||
>>> 0.14792643
|
||||
```
|
||||
|
||||
内置50+中文词向量，覆盖多种领域语料、如百科、新闻、微博等。更多使用方法请参考[Embedding文档](./docs/model_zoo/embeddings.md)。
|
||||
|
||||
### 更多API使用文档
|
||||
|
||||
- [Data API](./docs/data.md): 提供便捷高效的文本数据处理功能
|
||||
- [Metrics API](./docs/metrics.md): 提供NLP任务的评估指标，与飞桨高层API兼容。
|
||||
|
||||
更多的API示例与使用说明请查阅[PaddleNLP官方文档](https://paddlenlp.readthedocs.io/)
|
||||
|
||||
## 多场景的应用示例
|
||||
|
||||
PaddleNLP提供了多粒度、多场景的NLP应用示例，面向动态图模式和全新的API体系开发，更加简单易懂。
|
||||
涵盖了[NLP基础技术](#nlp-基础技术)、[NLP系统应用](#nlp-系统应用)以及文本相关的[NLP拓展应用](#拓展应用)、与知识库结合的[文本知识关联](./examples/text_to_knowledge)、与图结合的[文本图学习](./examples/text_graph/)等。
|
||||
|
||||
### NLP 基础技术
|
||||
|
||||
| 任务 | 简介 |
|
||||
| :------------ | ---- |
|
||||
| [词向量](./examples/word_embedding/) | 利用`TokenEmbedding API`展示如何快速计算词之间语义距离和词的特征提取。 |
|
||||
| [词法分析](./examples/lexical_analysis/) | 基于BiGRU-CRF模型实现了分词、词性标注和命名实体识的联合训练任务。 |
|
||||
| [语言模型](./examples/language_model/) | 覆盖了经典的LSTM类语言模型，和Transformer类的预训练语言模型，如BERT, ERNIE, GPT, XLNet, BART，ELECTRA，BigBird等。 |
|
||||
| [语义解析](./examples/text_to_sql/):star: | 语义解析Text-to-SQL任务是让机器自动让自然语言问题转换数据库可操作的SQL查询语句，是实现基于数据库自动问答的核心模块。|
|
||||
|
||||
#### 句法依存分析 (Dependency Parsing)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :----- | ------ |
|
||||
| [Deep Biaffine Parsing](./examples/dependency_parsing/) | 实现经典的Deep Biaffine Parsing句法分析模型，并提供高质量的中文预训练模型可供直接使用。|
|
||||
|
||||
#### 文本分类 (Text Classification)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :----- | ------ |
|
||||
| [RNN/CNN/GRU/LSTM](./examples/text_classification/rnn) | 实现了经典的RNN, CNN, GRU, LSTM等经典文本分类结构。|
|
||||
| [BiLSTM-Attention](./examples/text_classification/rnn) | 基于BiLSTM网络结构引入注意力机制提升文本分类效果。 |
|
||||
| [BERT/ERNIE](./examples/text_classification/pretrained_models) | 提供基于预训练模型的文本分类任务实现，包含训练、预测和推理部署的全流程应用。 |
|
||||
|
||||
#### 文本匹配 (Text Matching)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :--------------- | ---------- |
|
||||
| [SimCSE](./examples/text_matching/simcse/):star2: | 基于论文[SimCSE: Simple Contrastive Learning of Sentence Embeddings](https://arxiv.org/abs/2104.08821)实现无监督语义匹配模型，无需标注数据仅利用无监督数据也能训练效果出众的语义匹配模型。|
|
||||
| [ERNIE-Gram w/ R-Drop](./examples/text_matching/question_matching/) | 提供基于ERNIE-Gram预训练模型结合R-Drop策略的问题匹配任在千言数据集上的基线代码。|
|
||||
| [SimNet](./examples/text_matching/simnet/) | 百度自研的语义匹配框架，使用BOW、CNN、GRNN等核心网络作为表示层，在百度内搜索、推荐等多个应用场景得到广泛易用。|
|
||||
| [ERNIE](./examples/text_matching/ernie_matching/) | 基于ERNIE使用LCQMC数据完成中文句对匹配任务，提供了Pointwise和Pairwise两种类型学习方式。 |
|
||||
| [Sentence-BERT](./examples/text_matching/sentence_transformers/) | 提供基于Siamese双塔结构的文本匹配模型[Sentence-BERT](https://arxiv.org/abs/1908.1008)实现，可用于获取文本的向量化表示。 |
|
||||
| [SimBERT](./examples/text_matching/simbert/) | 提供[SimBERT](https://github.com/ZhuiyiTechnology/simbert)模型实现，用于获取文本的向量化表示。|
|
||||
|
||||
#### 文本生成 (Text Generation)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :------------ | ---------- |
|
||||
| [Seq2Seq](./examples/text_generation/couplet) | 实现了经典的Seq2Seq with Attention的网络结构，并提供在自动对联的文本生成应用示例。 |
|
||||
| [VAE-Seq2Seq](./examples/text_generation/vae-seq2seq) | 在Seq2Seq框架基础上，加入VAE结构以实现更加多样化的文本生成。|
|
||||
| [ERNIE-GEN](./examples/text_generation/ernie-gen) | [ERNIE-GEN](https://arxiv.org/abs/2001.11314)是百度NLP提出的基于多流(multi-flow)机制生成完整语义片段的预训练模型，基于该模型实现了提供了智能写诗的应用示例。|
|
||||
|
||||
#### 文本纠错 (Text Correction)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :------------ | ---------- |
|
||||
| [ERNIE-CSC](./examples/text_correction/ernie-csc) :star2: | [ERNIE-CSC](https://aclanthology.org/2021.findings-acl.198.pdf)是基于ERNIE预训练模型融合了拼音特征的端到端中文拼写纠错模型，在SIGHAN数据集上取得SOTA的效果。|
|
||||
|
||||
#### 语义索引 (Semantic Indexing)
|
||||
|
||||
提供一套完整的语义索引开发流程，并提供了In-Batch Negative和Hardest Negatives两种策略，开发者可基于该示例实现一个轻量级的语义索引系统，更多信息请查看[语义索引应用示例](./examples/semantic_indexing/)。
|
||||
|
||||
#### 信息抽取 (Information Extraction)
|
||||
|
||||
| 任务 | 简介 |
|
||||
| :--------------- | ---- |
|
||||
| [DuEE](./examples/information_extraction/DuEE/) | 基于[DuEE](https://link.springer.com/chapter/10.1007/978-3-030-60457-8_44)数据集，使用预训练模型的方式提供句子级和篇章级的事件抽取示例。 |
|
||||
| [DuIE](./examples/information_extraction/DuIE/) | 基于[DuIE](http://tcci.ccf.org.cn/conference/2019/papers/EV10.pdf)数据集，使用预训练模型的方式提供关系抽取示例。 |
|
||||
| [快递单信息抽取](./examples/information_extraction/waybill_ie/) | 提供BiLSTM+CRF和预训练模型两种方式完成真实的快递单信息抽取案例。 |
|
||||
|
||||
### NLP 系统应用
|
||||
|
||||
#### 情感分析 (Sentiment Analysis)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :--------- | ---------- |
|
||||
| [SKEP](./examples/sentiment_analysis/skep/):star2: | [SKEP](https://arxiv.org/abs/2005.05635)是百度提出的基于情感知识增强的预训练算法，利用无监督挖掘的海量情感知识构建预训练目标，让模型更好理解情感语义，可为各类情感分析任务提供统一且强大的情感语义表示。 |
|
||||
|
||||
#### 阅读理解 (Machine Reading Comprehension)
|
||||
|
||||
| 任务 | 简介 |
|
||||
| :------------------- | ---- |
|
||||
| [SQuAD](./examples/machine_reading_comprehension/SQuAD/) | 提供预训练模型在SQuAD 2.0数据集上微调的应用示例。 |
|
||||
| [DuReader-yesno](./examples/machine_reading_comprehension/DuReader-yesno/) | 提供预训练模型在**千言数据集DuReader-yesno**上微调的应用示例。 |
|
||||
| [DuReader-robust](./examples/machine_reading_comprehension/DuReader-robust/) | 提供预训练模型在**千言数据集DuReader-robust**上微调的应用示例。 |
|
||||
|
||||
#### 文本翻译 (Text Translation)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :--------------- | ------- |
|
||||
| [Seq2Seq-Attn](./examples/machine_translation/seq2seq) | 提供了[Effective Approaches to Attention-based Neural Machine Translation](https://arxiv.org/abs/1508.04025v5)基于注意力机制改进的Seq2Seq经典神经网络机器翻译模型实现。|
|
||||
| [Transformer](./examples/machine_translation/transformer) | 提供了基于[Attention Is All You Need](https://arxiv.org/abs/1706.03762)论文的Transformer机器翻译实现，包含了完整的训练到推理部署的全流程实现。|
|
||||
|
||||
#### 同传翻译 (Simultaneous Translation)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :---------- | ------- |
|
||||
| [STACL](./examples/simultaneous_translation/stacl) :star:| [STACL](https://www.aclweb.org/anthology/P19-1289/)是百度自研的基于Prefix-to-Prefix框架的同传翻译模型，结合Wait-k策略可以在保持较高的翻译质量的同时实现任意字级别的翻译延迟，并提供了轻量级同声传译系统搭建教程。|
|
||||
|
||||
#### 对话系统 (Dialogue System)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :---------------- | ------|
|
||||
| [PLATO-2](./examples/dialogue/plato-2) | [PLATO-2](https://arxiv.org/abs/2006.16779)是百度自研领先的基于课程学习两阶段方式训练的开放域对话预训练模型。|
|
||||
| [PLATO-mini](./examples/dialogue/unified_transformer):star2: | 基于6层UnifiedTransformer预训练结构，结合海量中文对话语料数据预训练的轻量级**中文**闲聊对话模型。|
|
||||
|
||||
### 拓展应用
|
||||
|
||||
#### 文本知识关联 (Text to Knowledge)
|
||||
|
||||
:star2:[**解语**](./examples/text_to_knowledge/)是由百度知识图谱部开发的文本知识关联框架，覆盖中文全词类的知识库和知识标注工具，能够帮助开发者面对更加多元的应用场景，方便地融合自有知识体系，显著提升中文文本解析和挖掘效果，还可以便捷地利用知识增强机器学习模型效果。
|
||||
|
||||
* [TermTree: 中文全词类的知识库](./examples/text_to_knowledge/termtree):star2:
|
||||
* [WordTag: 中文词类知识标注工具](./examples/text_to_knowledge/wordtag):star2:
|
||||
* [NPTag: 中文名词短语标注工具](./examples/text_to_knowledge/nptag):star2:
|
||||
|
||||
#### 文本图学习 (Text Graph Learning)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :------------ | ------- |
|
||||
| [ERNIESage](./examples/text_graph/erniesage)| 基于[飞桨PGL](https://github.com/PaddlePaddle/PGL)图学习框架结合PaddleNLP Transformer API实现的文本与图结构融合的模型。|
|
||||
|
||||
#### 模型压缩 (Model Compression)
|
||||
|
||||
| 模型 | 简介 |
|
||||
| :--------------------------------------------------------- | ------------------------------------------------------------ |
|
||||
| [MiniLMv2](examples/model_compression/minilmv2) | 基于[MiniLMv2: Multi-Head Self-Attention Relation Distillation for Compressing Pretrained Transformers](https://arxiv.org/abs/2012.15828)论文策略的实现，是一种通用蒸馏方法。本实例以`bert-base-chinese`为教师模型，利用中文数据进行了通用蒸馏。 |
|
||||
| [TinyBERT](./examples/model_compression/tinybert) | 基于论文[TinyBERT: Distilling BERT for Natural Language Understanding](https://arxiv.org/abs/1909.10351)的实现，提供了通用蒸馏和下游任务蒸馏的脚本。本实例利用开源模型`tinybert-6l-768d-v2`初始化，在GLUE的7个数据集上进行下游任务的蒸馏，最终模型参数量缩小1/2，预测速度提升2倍，同时保证模型精度几乎无损，其中精度可达教师模型`bert-base-uncased`的 98.90%。 |
|
||||
| [OFA-BERT](./examples/model_compression/ofa/) | 基于PaddleSlim Once-For-ALL(OFA)策略对BERT在GLUE任务的下游模型进行压缩，在精度无损的情况下可减少33%参数量，达到模型小型化的提速的效果。 |
|
||||
| [Distill-LSTM](./examples/model_compression/distill_lstm/) | 基于[Distilling Task-Specific Knowledge from BERT into Simple Neural Networks](https://arxiv.org/abs/1903.12136)论文策略的实现，将BERT中英文分类的下游模型知识通过蒸馏的方式迁移至LSTM的小模型结构中，取得比LSTM单独训练更好的效果。 |
|
||||
| [PP-MiniLM](examples/model_compression/pp-minilm) :star2: | 基于 PaddleSlim 通过模型蒸馏、剪裁、量化等级联模型压缩技术发布中文特色小模型 PP-MiniLM(6L768H) 及压缩方案，保证模型精度的同时模型推理速度达 BERT-base 的4.2倍，参数量相比减少52%，模型精度在中文语言理解评测基准 CLUE 高0.32。 |
|
||||
|
||||
#### 小样本学习 (Few-Shot Learning)
|
||||
|
||||
| 算法 | 简介 |
|
||||
| :--------------- | ------- |
|
||||
| [PET](./examples/few_shot/pet/) | 基于[Exploiting Cloze Questions for Few Shot Text Classification and Natural Language Inference](https://arxiv.org/abs/2001.07676) 论文策略实现, 基于人工知识设计 Prompt, 将下游目标任务转换为完形填空任务来充分挖掘预训练模型中的知识, 显著提升模型效果。|
|
||||
| [P-Tuning](./examples/few_shot/p-tuning/) |基于[GPT Understands, Too](https://arxiv.org/pdf/2103.10385.pdf) 论文策略实现, 首次提出连续可学习的模板参数，在全参数空间对模板进行连续优化，大幅提升模型稳定性和模型效果。|
|
||||
| [EFL](./examples/few_shot/efl/) | 基于[Entailment as Few-Shot Learner](https://arxiv.org/abs/2104.14690) 论文策略实现，将下游目标任务转换为蕴含任务降低模型预测空间，显著提升模型效果。|
|
||||
|
||||
## 交互式Notebook教程
|
||||
|
||||
- [使用Seq2Vec模块进行句子情感分类](https://aistudio.baidu.com/aistudio/projectdetail/1283423)
|
||||
- [如何通过预训练模型Fine-tune下游任务](https://aistudio.baidu.com/aistudio/projectdetail/1294333)
|
||||
- [使用BiGRU-CRF模型完成快递单信息抽取](https://aistudio.baidu.com/aistudio/projectdetail/1317771)
|
||||
- [使用预训练模型ERNIE优化快递单信息抽取](https://aistudio.baidu.com/aistudio/projectdetail/1329361)
|
||||
- [使用Seq2Seq模型完成自动对联](https://aistudio.baidu.com/aistudio/projectdetail/1321118)
|
||||
- [使用预训练模型ERNIE-GEN实现智能写诗](https://aistudio.baidu.com/aistudio/projectdetail/1339888)
|
||||
- [使用TCN网络完成新冠疫情病例数预测](https://aistudio.baidu.com/aistudio/projectdetail/1290873)
|
||||
|
||||
更多教程参见[PaddleNLP on AI Studio](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)。
|
||||
|
||||
|
||||
## 版本更新
|
||||
|
||||
更多版本更新说明请查看[ChangeLog](./docs/changelog.md)
|
||||
|
||||
## Citation
|
||||
|
||||
如果 PaddleNLP 对您的研究有帮助，欢迎引用
|
||||
|
||||
```bibtex
|
||||
如果PaddleNLP对您的研究有帮助，欢迎引用
|
||||
```
|
||||
@misc{=paddlenlp,
|
||||
title={PaddleNLP: An Easy-to-use and High Performance NLP Library},
|
||||
author={PaddleNLP Contributors},
|
||||
@@ -291,8 +300,8 @@ trainer.train()
|
||||
|
||||
## Acknowledge
|
||||
|
||||
我们借鉴了 Hugging Face 的[Transformers](https://github.com/huggingface/transformers)🤗关于预训练模型使用的优秀设计，在此对 Hugging Face 作者及其开源社区表示感谢。
|
||||
我们借鉴了Hugging Face的[Transformers](https://github.com/huggingface/transformers)🤗关于预训练模型使用的优秀设计，在此对Hugging Face作者及其开源社区表示感谢。
|
||||
|
||||
## License
|
||||
|
||||
PaddleNLP 遵循[Apache-2.0开源协议](./LICENSE)。
|
||||
PaddleNLP遵循[Apache-2.0开源协议](./LICENSE)。
|
||||
|
||||
303
README_en.md
303
README_en.md
@@ -1,139 +1,43 @@
|
||||
[简体中文🀄](./README.md) | **English🌎**
|
||||
|
||||
|
||||
English | [简体中文](./README.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="https://user-images.githubusercontent.com/1371212/175816733-8ec25eb0-9af3-4380-9218-27c154518258.png" align="middle" width="500" />
|
||||
<img src="./docs/imgs/paddlenlp.png" align="middle" width="500" />
|
||||
</p>
|
||||
|
||||
------------------------------------------------------------------------------------------
|
||||
[](https://pypi.org/project/paddlenlp/)
|
||||
[](https://pypi.org/project/paddlenlp/)
|
||||
[](https://pepy.tech/project/paddlenlp)
|
||||
|
||||
|
||||
|
||||
<p align="center">
|
||||
<a href="https://paddlenlp.readthedocs.io/en/latest/?badge=latest"><img src="https://readthedocs.org/projects/paddlenlp/badge/?version=latest">
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/releases"><img src="https://img.shields.io/github/v/release/PaddlePaddle/PaddleNLP?color=ffa"></a>
|
||||
<a href=""><img src="https://img.shields.io/badge/python-3.7+-aff.svg"></a>
|
||||
<a href=""><img src="https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg"></a>
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/graphs/contributors"><img src="https://img.shields.io/github/contributors/PaddlePaddle/PaddleNLP?color=9ea"></a>
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/commits"><img src="https://img.shields.io/github/commit-activity/m/PaddlePaddle/PaddleNLP?color=3af"></a>
|
||||
<a href="https://pypi.org/project/paddlenlp/"><img src="https://img.shields.io/pypi/dm/paddlenlp?color=9cf"></a>
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/issues"><img src="https://img.shields.io/github/issues/PaddlePaddle/PaddleNLP?color=9cc"></a>
|
||||
<a href="https://github.com/PaddlePaddle/PaddleNLP/stargazers"><img src="https://img.shields.io/github/stars/PaddlePaddle/PaddleNLP?color=ccf"></a>
|
||||
<a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache%202-dfd.svg"></a>
|
||||
</p>
|
||||
## News <img src="./docs/imgs/news_icon.png" width="40"/>
|
||||
|
||||
<h4 align="center">
|
||||
<a href=#Features> Features </a> |
|
||||
<a href=#Support-Models> Supported Models </a> |
|
||||
<a href=#Installation> Installation </a> |
|
||||
<a href=#Quick-start> Quick Start </a> |
|
||||
<a href=#community> Community </a>
|
||||
</h4>
|
||||
* [2021-12-12] PaddleNLP v2.2 has been officially relealsed! :tada: For more information please refer to [Release Note](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.2.0).
|
||||
* [2021-12-12] *End-to-end Question Answering Toolkit**🚀[RocketQA](https://github.com/PaddlePaddle/RocketQA) has been released！:tada:
|
||||
|
||||
**PaddleNLP** is a Large Language Model (LLM) development suite based on the PaddlePaddle deep learning framework, supporting efficient large model training, lossless compression, and high-performance inference on various hardware devices. With its **simplicity** and **ultimate performance**, PaddleNLP is dedicated to helping developers achieve efficient industrial applications of large models.
|
||||
## Introduction
|
||||
|
||||
## News 📢
|
||||
**PaddleNLP** is an **easy-to-use** and **high performance** NLP library with **awesome** pre-trained Transformer models, supporting wide-range of NLP tasks from research to industrial applications.
|
||||
|
||||
* **2024.06.27 [PaddleNLP v3.0 Beta](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v3.0.0-beta0)**：Embrace large models and experience a complete upgrade. With a unified large model suite, we achieve full-process access to domestically produced computing chips. We fully support industrial-level application processes for large models, such as PaddlePaddle's 4D parallel configuration, efficient fine-tuning strategies, efficient alignment algorithms, and high-performance reasoning. Our developed RsLoRA+ algorithm, full checkpoint storage mechanism Unified Checkpoint, and generalized support for FastFNN and FusedQKV all contribute to the training and inference of large models. We continuously support updates to mainstream models for providing efficient solutions.
|
||||
|
||||
* **2024.04.24 [PaddleNLP v2.8](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.8.0)**：Our self-developed RsLoRA+ algorithm with extreme convergence significantly improves the convergence speed and training effectiveness of PEFT training. By introducing high-performance generation acceleration into the RLHF PPO algorithm, we have broken through the generation speed bottleneck in PPO training, achieving a significant lead in PPO training performance. We generally support multiple large model training performance optimization methods such as FastFFN and FusedQKV, making large model training faster and more stable.
|
||||
* **Easy-to-Use API**
|
||||
- The API is fully integrated with PaddlePaddle 2.0 high-level API system. It minimizes the number of user actions required for common use cases like data loading, text pre-processing, awesome transfomer models, and fast inference, which enables developer to deal with text problems more productively.
|
||||
|
||||
## Features
|
||||
* **Wide-range NLP Task Support**
|
||||
- PaddleNLP support NLP task from research to industrial applications, including Lexical Analysis, Text Classification, Text Matching, Text Generation, Information Extraction, Machine Translation, General Dialogue and Question Answering etc.
|
||||
|
||||
### <a href=#Integrated training and inference on multiple hardware platforms> 🔧 Integrated training and inference on multiple hardware platforms </a>
|
||||
|
||||
Our development suit supports large model training and inference on multiple hardware platforms, including NVIDIA GPUs, Kunlun XPUs, Ascend NPUs, Enflame GCUs, and Hygon DCUs. The toolkit's interface allows for quick hardware switching, significantly reducing research and development costs associated with hardware transitions.
|
||||
|
||||
### <a href=Efficient and easy-to-use pre-training> 🚀 Efficient and easy-to-use pre-training </a>
|
||||
|
||||
We support 4D high-performance training with data parallelism, sharding parallelism, tensor parallelism, and pipeline parallelism. The Trainer supports configurable distributed strategies, reducing the cost associated with complex distributed combinations. The Unified Checkpoint large model storage format supports dynamic scaling of model parameter distribution during training, thereby reducing the migration cost caused by hardware switching.
|
||||
|
||||
### <a href=#Efficient fine-tuning> 🤗 Efficient fine-tuning </a>
|
||||
|
||||
The fine-tuning algorithms are deeply integrated with zero-padding data streams and high-performance FlashMask operators, reducing invalid data padding and computation during training, and significantly improving the throughput of fine-tuning training.
|
||||
|
||||
### <a href=#Lossless compression and high-performance inference> 🎛️ Lossless compression and high-performance inference </a>
|
||||
|
||||
The high-performance inference module of the large model toolkit incorporates dynamic insertion and operator fusion strategies throughout the entire process, greatly accelerating parallel inference speed. The underlying implementation details are encapsulated, enabling out-of-the-box high-performance parallel inference capabilities.
|
||||
|
||||
## Documentation
|
||||
For detailed documentation, visit the [PaddleNLP Documentation](https://paddlenlp.readthedocs.io/).
|
||||
|
||||
------------------------------------------------------------------------------------------
|
||||
|
||||
## Support Models
|
||||
|
||||
Detailed list 👉 [Supported Model List](https://github.com/PaddlePaddle/PaddleNLP/issues/8663)
|
||||
|
||||
## Installation
|
||||
|
||||
### Prerequisites
|
||||
|
||||
* python >= 3.8
|
||||
* paddlepaddle >= 3.0.0b0
|
||||
|
||||
### Pip Installation
|
||||
|
||||
```shell
|
||||
pip install --upgrade paddlenlp==3.0.0b3
|
||||
```
|
||||
|
||||
or you can install the latest develop branch code with the following command:
|
||||
|
||||
```shell
|
||||
pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html
|
||||
```
|
||||
|
||||
More information about PaddlePaddle installation please refer to [PaddlePaddle's Website](https://www.paddlepaddle.org.cn).
|
||||
|
||||
------------------------------------------------------------------------------------------
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Text generation with large language model
|
||||
|
||||
PaddleNLP provides a convenient and easy-to-use Auto API, which can quickly load models and Tokenizers. Here, we use the `Qwen/Qwen2-0.5B` large model as an example for text generation:
|
||||
|
||||
```python
|
||||
>>> from paddlenlp.transformers import AutoTokenizer, AutoModelForCausalLM
|
||||
>>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B")
|
||||
>>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B", dtype="float16")
|
||||
>>> input_features = tokenizer("你好！请自我介绍一下。", return_tensors="pd")
|
||||
>>> outputs = model.generate(**input_features, max_length=128)
|
||||
>>> print(tokenizer.batch_decode(outputs[0], skip_special_tokens=True))
|
||||
['我是一个AI语言模型，我可以回答各种问题，包括但不限于：天气、新闻、历史、文化、科学、教育、娱乐等。请问您有什么需要了解的吗？']
|
||||
```
|
||||
|
||||
### Pre-training for large language model
|
||||
|
||||
```shell
|
||||
git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # if cloned or downloaded, can skip this step
|
||||
mkdir -p llm/data && cd llm/data
|
||||
wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.bin
|
||||
wget https://bj.bcebos.com/paddlenlp/models/transformers/llama/data/llama_openwebtext_100k.idx
|
||||
cd .. # change folder to PaddleNLP/llm
|
||||
python -u -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" run_pretrain.py ./config/llama/pretrain_argument.json
|
||||
```
|
||||
|
||||
### SFT finetuning forlarge language model
|
||||
|
||||
```shell
|
||||
git clone https://github.com/PaddlePaddle/PaddleNLP.git && cd PaddleNLP # if cloned or downloaded, can skip this step
|
||||
mkdir -p llm/data && cd llm/data
|
||||
wget https://bj.bcebos.com/paddlenlp/datasets/examples/AdvertiseGen.tar.gz && tar -zxvf AdvertiseGen.tar.gz
|
||||
cd .. # change folder to PaddleNLP/llm
|
||||
python -u -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" run_finetune.py ./config/llama/sft_argument.json
|
||||
```
|
||||
|
||||
For more steps in the entire large model process, please refer to the[Large Model Full-Process Suite](./llm).
|
||||
|
||||
For more PaddleNLP content, please refer to:
|
||||
|
||||
* [Model Library](./slm/model_zoo)，which includes end-to-end usage of high-quality pre-trained models.
|
||||
* [Multi-scenario Examples](./slm/examples)，to understand how to use PaddleNLP to solve various NLP technical problems, including basic techniques, system applications, and extended applications.
|
||||
* [Interactive Tutorial](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)，to quickly learn PaddleNLP on the free computing platform AI Studio.
|
||||
|
||||
------------------------------------------------------------------------------------------
|
||||
* **High Performance Distributed Training**
|
||||
- We provide an industrial level training pipeline for super large-scale Transformer model based on **Auto Mixed Precision** and Fleet distributed training API by PaddlePaddle, which can support customized model pre-training efficiently.
|
||||
|
||||
## Community
|
||||
|
||||
### Special Interest Group (SIG)
|
||||
|
||||
Welcome to join [PaddleNLP SIG](https://iwenjuan.baidu.com/?code=bkypg8) for contribution, eg. Dataset, Models and Toolkit.
|
||||
|
||||
### Slack
|
||||
|
||||
To connect with other users and contributors, welcome to join our [Slack channel](https://paddlenlp.slack.com/).
|
||||
@@ -143,14 +47,163 @@ To connect with other users and contributors, welcome to join our [Slack channel
|
||||
Scan the QR code below with your Wechat⬇️. You can access to official technical exchange group. Look forward to your participation.
|
||||
|
||||
<div align="center">
|
||||
<img src="https://github.com/user-attachments/assets/3a58cc9f-69c7-4ccb-b6f5-73e966b8051a" width="150" height="150" />
|
||||
<img src="https://raw.githubusercontent.com/PaddlePaddle/PaddleNLP/release/2.2/docs/imgs/wechat.png" width="188" height="188" />
|
||||
</div>
|
||||
|
||||
|
||||
## Installation
|
||||
|
||||
### Prerequisites
|
||||
|
||||
* python >= 3.6
|
||||
* paddlepaddle >= 2.2
|
||||
|
||||
More information about PaddlePaddle installation please refer to [PaddlePaddle's Website](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/conda/linux-conda.html).
|
||||
|
||||
### Python pip Installation
|
||||
|
||||
```
|
||||
pip install --upgrade paddlenlp
|
||||
```
|
||||
|
||||
## Easy-to-use API
|
||||
|
||||
### Taskflow：Off-the-shelf Industial NLP Pre-built Task
|
||||
|
||||
Taskflow aims to provide **off-the-shelf** NLP pre-built task covering NLU and NLG scenario, in the meanwhile with extreamly fast infernece satisfying industrial applications.
|
||||
|
||||
|
||||
|
||||
For more usage please refer to [Taskflow Docs](./docs/model_zoo/taskflow.md)
|
||||
|
||||
### Transformer API: Awesome Pre-trained Model Ecosystem
|
||||
|
||||
We provide **30** network architectures and over **100** pretrained models. Not only includes all the SOTA model like ERNIE, PLATO and SKEP released by Baidu, but also integrates most of the high quality Chinese pretrained model developed by other organizations. Use AutoModel to download pretrained mdoels of different architecture. We welcome all developers to contribute your Transformer models to PaddleNLP! 🤗
|
||||
|
||||
```python
|
||||
from paddlenlp.transformers import *
|
||||
|
||||
ernie = AutoModel.from_pretrained('ernie-1.0')
|
||||
ernie_gram = AutoModel.from_pretrained('ernie-gram-zh')
|
||||
bert = AutoModel.from_pretrained('bert-wwm-chinese')
|
||||
albert = AutoModel.from_pretrained('albert-chinese-tiny')
|
||||
roberta = AutoModel.from_pretrained('roberta-wwm-ext')
|
||||
electra = AutoModel.from_pretrained('chinese-electra-small')
|
||||
gpt = AutoModelForPretraining.from_pretrained('gpt-cpm-large-cn')
|
||||
```
|
||||
|
||||
PaddleNLP also provides unified API experience for NLP task like semantic representation, text classification, sentence matching, sequence labeling, question answering, etc.
|
||||
|
||||
```python
|
||||
import paddle
|
||||
from paddlenlp.transformers import *
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained('ernie-1.0')
|
||||
text = tokenizer('natural language understanding')
|
||||
|
||||
# Semantic Representation
|
||||
model = AutoModel.from_pretrained('ernie-1.0')
|
||||
sequence_output, pooled_output = model(input_ids=paddle.to_tensor([text['input_ids']]))
|
||||
# Text Classificaiton and Matching
|
||||
model = AutoModelForSequenceClassification.from_pretrained('ernie-1.0')
|
||||
# Sequence Labeling
|
||||
model = AutoModelForTokenClassification.from_pretrained('ernie-1.0')
|
||||
# Question Answering
|
||||
model = AutoModelForQuestionAnswering.from_pretrained('ernie-1.0')
|
||||
```
|
||||
|
||||
For more pretrained model usage, please refer to [Transformer API](./docs/model_zoo/transformers.rst)
|
||||
|
||||
### Dataset API: Abundant Dataset Integration and Quick Loading
|
||||
|
||||
```python
|
||||
from paddlenlp.datasets import load_dataset
|
||||
|
||||
train_ds, dev_ds, test_ds = load_dataset("chnsenticorp", splits=["train", "dev", "test"])
|
||||
```
|
||||
|
||||
For more dataset API usage please refer to [Dataset API](./docs/datasets.md).
|
||||
|
||||
### Embedding API: Quick Loading for Word Embedding
|
||||
|
||||
```python
|
||||
from paddlenlp.embeddings import TokenEmbedding
|
||||
|
||||
wordemb = TokenEmbedding("fasttext.wiki-news.target.word-word.dim300.en")
|
||||
wordemb.cosine_sim("king", "queen")
|
||||
>>> 0.77053076
|
||||
wordemb.cosine_sim("apple", "rail")
|
||||
>>> 0.29207364
|
||||
```
|
||||
|
||||
For more `TokenEmbedding` usage, please refer to [Embedding API](./docs/model_zoo/embeddings.md)
|
||||
|
||||
### More API Usage
|
||||
|
||||
- [Transformer API](./docs/model_zoo/transformers.rst)
|
||||
- [Data API](./docs/data.md)
|
||||
- [Dataset API](./docs/datasets.md)
|
||||
- [Embedding API](./docs/model_zoo/embeddings.md)
|
||||
- [Metrics API](./docs/metrics.md)
|
||||
|
||||
Please find more API Reference from our [readthedocs](https://paddlenlp.readthedocs.io/).
|
||||
|
||||
## Wide-range NLP Task Support
|
||||
|
||||
PaddleNLP provides rich application examples covering mainstream NLP task to help developers accelerate problem solving.
|
||||
|
||||
### NLP Basic Technique
|
||||
|
||||
- [Word Embedding](./examples/word_embedding/)
|
||||
- [Lexical Analysis](./examples/lexical_analysis/)
|
||||
- [Dependency Parsing](./examples/dependency_parsing/)
|
||||
- [Language Model](./examples/language_model/)
|
||||
- [Semantic Parsing (Text to SQL)](./examples/text_to_sql):star:
|
||||
- [Text Classification](./examples/text_classification/)
|
||||
- [Text Matching](./examples/text_matching/)
|
||||
- [Text Generation](./examples/text_generation/)
|
||||
- [Text Correction](./examples/text_correction/):star:
|
||||
- [Semantic Indexing](./examples/semantic_indexing/)
|
||||
- [Information Extraction](./examples/information_extraction/)
|
||||
|
||||
### NLP System
|
||||
|
||||
- [Sentiment Analysis](./examples/sentiment_analysis/):star2:
|
||||
- [General Dialogue System](./examples/dialogue/)
|
||||
- [Machine Translation](./examples/machine_translation/)
|
||||
- [Simultaneous Translation](././examples/simultaneous_translation/)
|
||||
- [Machine Reading Comprehension](./examples/machine_reading_comprehension/)
|
||||
|
||||
### NLP Extented Applications
|
||||
|
||||
- [Few-shot Learning](./examples/few_shot/):star2:
|
||||
- [Text Knowledge Mining](./examples/text_to_knowledge/):star2:
|
||||
- [Model Compression](./examples/model_compression/)
|
||||
- [Text Graph Learning](./examples/text_graph/erniesage/)
|
||||
- [Time Series Prediction](./examples/time_series/)
|
||||
|
||||
## Tutorials
|
||||
|
||||
Please refer to our official AI Studio account for more interactive tutorials: [PaddleNLP on AI Studio](https://aistudio.baidu.com/aistudio/personalcenter/thirdview/574995)
|
||||
|
||||
* [What's Seq2Vec?](https://aistudio.baidu.com/aistudio/projectdetail/1283423) shows how to use simple API to finish LSTM model and solve sentiment analysis task.
|
||||
|
||||
* [Sentiment Analysis with ERNIE](https://aistudio.baidu.com/aistudio/projectdetail/1294333) shows how to exploit the pretrained ERNIE to solve sentiment analysis problem.
|
||||
|
||||
* [Waybill Information Extraction with BiGRU-CRF Model](https://aistudio.baidu.com/aistudio/projectdetail/1317771) shows how to make use of Bi-GRU plus CRF to finish information extraction task.
|
||||
|
||||
* [Waybill Information Extraction with ERNIE](https://aistudio.baidu.com/aistudio/projectdetail/1329361) shows how to use ERNIE, the Chinese pre-trained model improve information extraction performance.
|
||||
|
||||
* [Use TCN Model to predict COVID-19 confirmed cases](https://aistudio.baidu.com/aistudio/projectdetail/1290873)
|
||||
|
||||
## ChangeLog
|
||||
|
||||
For more details about our release, please refer to [ChangeLog](./docs/changelog.md)
|
||||
|
||||
## Citation
|
||||
|
||||
If you find PaddleNLP useful in your research, please consider citing
|
||||
|
||||
```bibtext
|
||||
If you find PaddleNLP useful in your research, please consider cite
|
||||
```
|
||||
@misc{=paddlenlp,
|
||||
title={PaddleNLP: An Easy-to-use and High Performance NLP Library},
|
||||
author={PaddleNLP Contributors},
|
||||
@@ -161,7 +214,7 @@ If you find PaddleNLP useful in your research, please consider citing
|
||||
|
||||
## Acknowledge
|
||||
|
||||
We have borrowed from Hugging Face's [Transformers](https://github.com/huggingface/transformers)🤗 excellent design on pretrained models usage, and we would like to express our gratitude to the authors of Hugging Face and its open source community.
|
||||
We have borrowed from Hugging Face's [Transformer](https://github.com/huggingface/transformers)🤗 excellent design on pretrained models usage, and we would like to express our gratitude to the authors of Hugging Face and its open source community.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
287
applications/neural_search/README.md
Normal file
287
applications/neural_search/README.md
Normal file
@@ -0,0 +1,287 @@
|
||||
# 手把手搭建一个语义检索系统
|
||||
|
||||
## 1. 场景概述
|
||||
|
||||
检索系统存在于我们日常使用的很多产品中，比如商品搜索系统、学术文献检索系等等，本方案提供了检索系统完整实现。限定场景是用户通过输入检索词 Query，快速在海量数据中查找相似文档。
|
||||
|
||||
所谓语义检索（也称基于向量的检索），是指检索系统不再拘泥于用户 Query 字面本身，而是能精准捕捉到用户 Query 后面的真正意图并以此来搜索，从而更准确地向用户返回最符合的结果。通过使用最先进的语义索引模型找到文本的向量表示，在高维向量空间中对它们进行索引，并度量查询向量与索引文档的相似程度，从而解决了关键词索引带来的缺陷。
|
||||
|
||||
例如下面两组文本 Pair，如果基于关键词去计算相似度，两组的相似度是相同的。而从实际语义上看，第一组相似度高于第二组。
|
||||
|
||||
```
|
||||
车头如何放置车牌 前牌照怎么装
|
||||
车头如何放置车牌 后牌照怎么装
|
||||
```
|
||||
|
||||
语义检索系统的关键就在于，采用语义而非关键词方式进行召回，达到更精准、更广泛得召回相似结果的目的。
|
||||
|
||||
## 2. 产品功能介绍
|
||||
|
||||
通常检索业务的数据都比较庞大，都会分为召回（索引）、排序两个环节。召回阶段主要是从至少千万级别的候选集合里面，筛选出相关的文档，这样候选集合的数目就会大大降低，在之后的排序阶段就可以使用一些复杂的模型做精细化或者个性化的排序。一般采用多路召回策略（例如关键词召回、热点召回、语义召回结合等），多路召回结果聚合后，经过统一的打分以后选出最优的 TopK 的结果。
|
||||
|
||||
### 2.1 系统特色
|
||||
|
||||
+ 低门槛
|
||||
+ 手把手搭建起检索系统
|
||||
+ 无需标注数据也能构建检索系统
|
||||
+ 提供 训练、预测、ANN 引擎一站式能力
|
||||
|
||||
+ 效果好
|
||||
+ 针对多种数据场景的专业方案
|
||||
+ 仅有无监督数据: SimCSE
|
||||
+ 仅有有监督数据: InBatchNegative
|
||||
+ 兼具无监督数据 和 有监督数据：融合模型
|
||||
+ 进一步优化方案: 面向领域的预训练 Domain-adaptive Pretraining
|
||||
+ 性能快
|
||||
+ 基于 Paddle Inference 快速抽取向量
|
||||
+ 基于 Milvus 快速查询和高性能建库
|
||||
|
||||
### 2.2 功能架构
|
||||
|
||||
索引环节有两类方法：基于字面的关键词索引；语义索引。语义索引能够较好地表征语义信息，解决字面不相似但语义相似的情形。本系统给出的是语义索引方案，实际业务中可融合其他方案使用。下面就详细介绍整个方案的架构和功能。
|
||||
|
||||
#### 2.2.1 整体介绍
|
||||
|
||||
|
||||
|
||||
|
||||
以上是nerual_search的系统流程图，其中左侧为召回环节，核心是语义向量抽取模块；右侧是排序环节，核心是排序模型。图中红色虚线框表示在线计算，黑色虚线框表示离线批量处理。下面我们分别介绍召回中的语义向量抽取模块，以及排序模型。
|
||||
|
||||
|
||||
#### 2.2.2 召回模块
|
||||
|
||||
召回模块需要从千万量级数据中快速召回候选数据。首先需要抽取语料库中文本的 Embedding，然后借助向量搜索引擎实现高效 ANN，从而实现候选集召回。
|
||||
|
||||
我们针对不同的数据情况推出三种语义索引方案，如下图所示，您可以参照此方案，快速建立语义索引：
|
||||
|
||||
| ⭐️ 无监督数据 | ⭐️ 有监督数据 | **召回方案** |
|
||||
| ------------ | ------------ | ------------ |
|
||||
| 多 | 无 | SimCSE |
|
||||
| 无 | 多 | In-batch Negatives|
|
||||
| 有 | 有 | SimCSE+ In-batch Negatives |
|
||||
|
||||
最基本的情况是只有无监督数据，我们推荐您使用 SimCSE 进行无监督训练；另一种方案是只有有监督数据，我们推荐您使用 In-batch Negatives 的方法进行有监督训练。
|
||||
|
||||
如果想进一步提升模型效果：还可以使用大规模业务数据，对预训练模型进行 Domain-adaptive Pretraining，训练完以后得到预训练模型，再进行无监督的 SimCSE。
|
||||
|
||||
此外，如果您同时拥有监督数据和无监督数据，我们推荐将两种方案结合使用，这样能训练出更加强大的语义索引模型。
|
||||
|
||||
#### 2.2.3 排序模块
|
||||
|
||||
排序模块基于前沿的预训练模型 ERNIE-Gram，训练 Pair-wise 语义匹配模型。召回模型负责从海量（千万级）候选文本中快速（毫秒级）筛选出与 Query 相关性较高的 TopK Doc，排序模型会在召回模型筛选出的 TopK Doc 结果基础之上针对每一个 (Query, Doc) Pair 对进行两两匹配计算相关性，排序效果更精准。
|
||||
|
||||
## 3. 文献检索实践
|
||||
|
||||
### 3.1 技术方案和评估指标
|
||||
|
||||
#### 3.1.1 技术方案
|
||||
|
||||
**语义索引**：由于我们既有无监督数据，又有有监督数据，所以结合 SimCSE 和 In-batch Negatives 方案，并采取 Domain-adaptive Pretraining 优化模型效果。
|
||||
|
||||
首先是利用 ERNIE 1.0 模型进行 Domain-adaptive Pretraining，在得到的预训练模型基础上，进行无监督的 SimCSE 训练，最后利用 In-batch Negatives 方法进行微调，得到最终的语义索引模型，把建库的文本放入模型中抽取特征向量，然后把抽取后的向量放到语义索引引擎 milvus 中，利用 milvus 就可以很方便得实现召回了。
|
||||
|
||||
**排序**：使用 ERNIE-Gram 的单塔结构对召回后的数据精排序。
|
||||
|
||||
#### 3.1.2 评估指标
|
||||
|
||||
**模型效果指标**
|
||||
* 在语义索引召回阶段使用的指标是 Recall@K，表示的是预测的前topK（从最后的按得分排序的召回列表中返回前K个结果）结果和语料库中真实的前 K 个相关结果的重叠率，衡量的是检索系统的查全率。
|
||||
|
||||
* 在排序阶段使用的指标为AUC，AUC反映的是分类器对样本的排序能力，如果完全随机得对样本分类，那么AUC应该接近0.5。分类器越可能把真正的正样本排在前面，AUC越大，分类性能越好。
|
||||
|
||||
**性能指标**
|
||||
* 基于 Paddle Inference 快速抽取向量
|
||||
|
||||
* 建库性能和 ANN 查询性能快
|
||||
|
||||
### 3.2 数据说明
|
||||
|
||||
数据集来源于某文献检索系统，既有大量无监督数据，又有有监督数据。
|
||||
|
||||
（1）采用文献的 query, title,keywords,abstract 四个字段内容，构建无标签数据集进行 Domain-adaptive Pretraining；
|
||||
|
||||
（2）采用文献的 query,title,keywords 三个字段内容，构造无标签数据集，进行无监督召回训练SimCSE；
|
||||
|
||||
（3）使用文献的的query, title, keywords，构造带正标签的数据集，不包含负标签样本，基于 In-batch Negatives 策略进行训练；
|
||||
|
||||
（4）在排序阶段，使用点击（作为正样本）和展现未点击（作为负样本）数据构造排序阶段的训练集，使用ERNIE-Gram模型进行精排训练。
|
||||
|
||||
| 阶段 |模型 | 训练集 | 评估集（用于评估模型效果） | 召回库 |测试集 |
|
||||
| ------------ | ------------ |------------ | ------------ | ------------ | ------------ |
|
||||
| 召回 | Domain-adaptive Pretraining | 2kw | - | - | - |
|
||||
| 召回 | 无监督预训练 - SimCSE | 798w | 20000 | 300000| 1000 |
|
||||
| 召回 | 有监督训练 - In-batch Negatives | 3998 | 20000 |300000 | 1000 |
|
||||
| 排序 | 有监督训练 - ERNIE-Gram单塔 Pairwise| 1973538 | 57811 | - | 1000 |
|
||||
|
||||
我们将除 Domain-adaptive Pretraining 之外的其他数据集全部开源，下载地址：
|
||||
|
||||
- [literature_search_data](https://bj.bcebos.com/v1/paddlenlp/data/literature_search_data.zip)
|
||||
|
||||
```
|
||||
├── milvus # milvus建库数据集
|
||||
├── milvus_data.csv. # 构建召回库的数据（模拟实际业务线上的语料库，实际语料库远大于这里的规模），用于直观演示相关文献召回效果
|
||||
├── recall # 召回阶段数据集
|
||||
├── train_unsupervised.csv # 无监督训练集，用于训练 SimCSE
|
||||
├── train.csv # 有监督训练集，用于训练 In-batch Negative
|
||||
├── dev.csv # 召回阶段验证集，用于评估召回模型的效果，SimCSE 和 In-batch Negative 共用
|
||||
├── corpus.csv # 构建召回库的数据（模拟实际业务线上的语料库，实际语料库远大于这里的规模），用于评估召回阶段模型效果，SimCSE 和 In-batch Negative 共用
|
||||
├── test.csv # 召回阶段测试数据，预测文本之间的相似度，SimCSE 和 In-batch Negative 共用
|
||||
├── sort # 排序阶段数据集
|
||||
├── train_pairwise.csv # 排序训练集
|
||||
├── dev_pairwise.csv # 排序验证集
|
||||
└── test_pairwise.csv # 排序测试集
|
||||
```
|
||||
|
||||
### 3.3 运行环境和安装说明
|
||||
|
||||
|
||||
（1）运行环境
|
||||
|
||||
本实验采用了以下的运行环境进行，详细说明如下，用户也可以在自己 GPU 硬件环境进行：
|
||||
|
||||
a. 软件环境：
|
||||
|
||||
|
||||
- python >= 3.6
|
||||
- paddlenlp >= 2.2.1
|
||||
- paddlepaddle-gpu >=2.2
|
||||
- CUDA Version: 10.2
|
||||
- NVIDIA Driver Version: 440.64.00
|
||||
- Ubuntu 16.04.6 LTS (Docker)
|
||||
|
||||
|
||||
b. 硬件环境：
|
||||
|
||||
|
||||
- NVIDIA Tesla V100 16GB x4卡
|
||||
- Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz
|
||||
|
||||
|
||||
c. 依赖安装:
|
||||
|
||||
```
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
## 4. 动手实践——搭建自己的检索系统
|
||||
|
||||
这里展示了能够从头至尾跑通的完整代码，您使用自己的业务数据，照着跑，能搭建出一个给定 Query，返回 topK 相关文档的小型检索系统。您可以参照我们给出的效果和性能数据来检查自己的运行过程是否正确。
|
||||
|
||||
### 4.1 召回阶段
|
||||
|
||||
**召回模型训练**
|
||||
|
||||
这里采用 Domain-adaptive Pretraining + SimCSE + In-batch Negatives 方案：
|
||||
|
||||
第一步：无监督训练 Domain-adaptive Pretraining
|
||||
|
||||
训练用时 16hour55min，可参考：[ERNIE 1.0](./recall/domain_adaptive_pretraining/)
|
||||
|
||||
第二步：无监督训练 SimCSE
|
||||
|
||||
训练用时 16hour53min，可参考：[SimCSE](./recall/simcse/)
|
||||
|
||||
第三步：有监督训练
|
||||
|
||||
几分钟内训练完成，可参考 [In-batch Negatives](./recall/in_batch_negative/)
|
||||
|
||||
|
||||
此外，我们进行了多组实践，用来对比说明召回阶段各方案的效果：
|
||||
|
||||
| 模型 | Recall@1 | Recall@5 |Recall@10 |Recall@20 |Recall@50 |策略简要说明|
|
||||
| ------------ | ------------ | ------------ |--------- |--------- |--------- |--------- |
|
||||
| 有监督训练 Baseline | 30.077| 43.513| 48.633 | 53.448 |59.632| 标准 pair-wise 训练范式，通过随机采样产生负样本|
|
||||
| 有监督训练 In-batch Negatives | 51.301 | 65.309| 69.878| 73.996|78.881| In-batch Negatives 有监督训练|
|
||||
| 无监督训练 SimCSE | 42.374 | 57.505| 62.641| 67.09|72.331| SimCSE 无监督训练|
|
||||
| 无监督 + 有监督训练 SimCSE + In-batch Negatives | 55.976 | 71.849| 76.363| 80.49|84.809| SimCSE无监督训练，In-batch Negatives 有监督训练|
|
||||
| Domain-adaptive Pretraining + SimCSE | 51.031 | 66.648| 71.338 | 75.676 |80.144| ERNIE 预训练，SimCSE 无监督训练|
|
||||
| Domain-adaptive Pretraining + SimCSE + In-batch Negatives| **58.248** | **75.099**| **79.813**| **83.801**|**87.733**| ERNIE 预训练，SimCSE 无监督训训练，In-batch Negatives 有监督训练|
|
||||
|
||||
从上述表格可以看出，首先利用ERNIE 1.0 做 Domain-adaptive Pretraining ，然后把训练好的模型加载到 SimCSE 上进行无监督训练，最后利用 In-batch Negatives 在有监督数据上进行训练能够获得最佳的性能。[模型下载](https://paddlenlp.bj.bcebos.com/models/inbatch_model_best.zip)，模型的使用方式参考[In-batch Negatives](./recall/in_batch_negative/) 。
|
||||
|
||||
**召回系统搭建**
|
||||
|
||||
召回系统使用索引引擎 Milvus，可参考 [milvus_system](./recall/milvus/)。
|
||||
我们展示一下系统的效果，输入的文本如下：
|
||||
|
||||
```
|
||||
中西方语言与文化的差异
|
||||
|
||||
```
|
||||
下面是召回的部分结果，第一个是召回的title，第二个数字是计算的相似度距离
|
||||
|
||||
```
|
||||
跨文化中的文化习俗对翻译的影响翻译,跨文化,文化习俗 0.615584135055542
|
||||
试论翻译过程中的文化差异与语言空缺翻译过程,文化差异,语言空缺,文化对比 0.6155391931533813
|
||||
中英文化差异及习语翻译习语,文化差异,翻译 0.6153547763824463
|
||||
英语中的中国文化元素英语,中国文化,语言交流 0.6151996850967407
|
||||
跨文化交际中的文化误读研究文化误读,影响,中华文化,西方文明 0.6137217283248901
|
||||
在语言学习中了解中法文化差异文化差异,对话交际,语言 0.6134252548217773
|
||||
从翻译视角看文化差异影响下的中式英语的应对策略文化差异;中式英语现;汉英翻译;动态对等理论 0.6127341389656067
|
||||
归化与异化在跨文化传播中的动态平衡归化,异化,翻译策略,跨文化传播,文化外译 0.6127211451530457
|
||||
浅谈中西言语交际行为中的文化差异交际用语,文化差异,中国,西方 0.6125463843345642
|
||||
翻译中的文化因素--异化与归化文化翻译,文化因素,异化与归化 0.6111845970153809
|
||||
历史与文化差异对翻译影响的分析研究历史与文化差异,法汉翻译,翻译方法 0.6107486486434937
|
||||
从中、韩、美看跨文化交际中的东西方文化差异跨文化交际,东西方,文化差异 0.6091923713684082
|
||||
试论文化差异对翻译工作的影响文化差异,翻译工作,影响 0.6084284782409668
|
||||
从归化与异化看翻译中的文化冲突现象翻译,文化冲突,归化与异化,跨文化交际 0.6063553690910339
|
||||
中西方问候语的文化差异问候语,文化差异,文化背景 0.6054259538650513
|
||||
中英思维方式的差异对翻译的影响中英文化的差异,中英思维方式的差异,翻译 0.6026732921600342
|
||||
略论中西方语言文字的特性与差异语言,会意,确意,特性,差异 0.6009351015090942
|
||||
......
|
||||
|
||||
```
|
||||
|
||||
|
||||
### 4.2 排序阶段
|
||||
|
||||
排序阶段使用的模型是 ERNIE-Gram，用时 20h，可参考：
|
||||
|
||||
[ernie_matching](./ranking/ernie_matching/)
|
||||
|
||||
排序阶段的效果评估：
|
||||
|
||||
| 模型 | AUC |
|
||||
| ------------ | ------------ |
|
||||
| Baseline: In-batch Negatives | 0.582 |
|
||||
| ERNIE-Gram | 0.801 |
|
||||
|
||||
同样输入文本：
|
||||
|
||||
```
|
||||
中西方语言与文化的差异
|
||||
```
|
||||
排序阶段的结果展示如下，第一个是 Title ，第二个数字是计算的概率，显然经排序阶段筛选的文档与 Query 更相关：
|
||||
|
||||
```
|
||||
中西方文化差异以及语言体现中西方文化,差异,语言体现 0.999848484992981
|
||||
论中西方语言与文化差异的历史渊源中西方语言,中西方文化,差异,历史渊源 0.9998375177383423
|
||||
从日常生活比较中西方语言与文化的差异中西方,语言,文化,比较 0.9985846281051636
|
||||
试论中西方语言文化教育的差异比较与融合中西方,语言文化教育,差异 0.9972485899925232
|
||||
中西方文化差异对英语学习的影响中西方文化,差异,英语,学习 0.9831035137176514
|
||||
跨文化视域下的中西文化差异研究跨文化,中西,文化差异 0.9781349897384644
|
||||
中西方文化差异对跨文化交际的影响分析文化差异,跨文化交际,影响 0.9735479354858398
|
||||
探析跨文化交际中的中西方语言差异跨文化交际,中西方,语言差异 0.9668175578117371
|
||||
中西方文化差异解读中英文差异表达中西文化,差异表达,跨文化交际 0.9629314541816711
|
||||
中西方文化差异对英语翻译的影响中西方文化差异,英语翻译,翻译策略,影响 0.9538986086845398
|
||||
论跨文化交际中的中西方文化冲突跨文化交际,冲突,文化差异,交际策略,全球化 0.9493677616119385
|
||||
中西方文化差异对英汉翻译的影响中西方文化,文化差异,英汉翻译,影响 0.9430705904960632
|
||||
中西方文化差异与翻译中西方,文化差异,翻译影响,策略方法,译者素质 0.9401137828826904
|
||||
外语教学中的中西文化差异外语教学,文化,差异 0.9397934675216675
|
||||
浅析西语国家和中国的文化差异-以西班牙为例跨文化交际,西语国家,文化差异 0.9373322129249573
|
||||
中英文化差异在语言应用中的体现中英文化,汉语言,语言应用,语言差异 0.9359155297279358
|
||||
....
|
||||
```
|
||||
|
||||
|
||||
## Reference
|
||||
|
||||
[1] Tianyu Gao, Xingcheng Yao, Danqi Chen: [SimCSE: Simple Contrastive Learning of Sentence Embeddings](https://arxiv.org/abs/2104.08821). EMNLP (1) 2021: 6894-6910
|
||||
|
||||
[2] Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, Wen-tau Yih, [Dense Passage Retrieval for Open-Domain Question Answering](https://arxiv.org/abs/2004.04906). Preprint 2020.
|
||||
|
||||
[3] Dongling Xiao, Yu-Kun Li, Han Zhang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang:
|
||||
[ERNIE-Gram: Pre-Training with Explicitly N-Gram Masked Language Modeling for Natural Language Understanding](https://arxiv.org/abs/2010.12148). NAACL-HLT 2021: 1702-1715
|
||||
|
||||
[4] Yu Sun, Shuohuan Wang, Yu-Kun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu:
|
||||
[ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223). CoRR abs/1904.09223 (2019)
|
||||
BIN
applications/neural_search/img/mem.png
Normal file
BIN
applications/neural_search/img/mem.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 643 KiB |
|
Before Width: | Height: | Size: 92 KiB After Width: | Height: | Size: 92 KiB |
282
applications/neural_search/ranking/ernie_matching/README.md
Normal file
282
applications/neural_search/ranking/ernie_matching/README.md
Normal file
@@ -0,0 +1,282 @@
|
||||
|
||||
**目录**
|
||||
|
||||
* [背景介绍](#背景介绍)
|
||||
* [ERNIE-Gram](#ERNIE-Gram)
|
||||
* [1. 技术方案和评估指标](#技术方案)
|
||||
* [2. 环境依赖](#环境依赖)
|
||||
* [3. 代码结构](#代码结构)
|
||||
* [4. 数据准备](#数据准备)
|
||||
* [5. 模型训练](#模型训练)
|
||||
* [6. 评估](#开始评估)
|
||||
* [7. 预测](#预测)
|
||||
* [8. 部署](#部署)
|
||||
|
||||
<a name="背景介绍"></a>
|
||||
|
||||
# 背景介绍
|
||||
|
||||
基于ERNIE-Gram训练Pair-wise模型。Pair-wise 匹配模型适合将文本对相似度作为特征之一输入到上层排序模块进行排序的应用场景。
|
||||
|
||||
|
||||
<a name="ERNIE-Gram"></a>
|
||||
|
||||
# ERNIE-Gram
|
||||
|
||||
<a name="技术方案"></a>
|
||||
|
||||
## 1. 技术方案和评估指标
|
||||
|
||||
### 技术方案
|
||||
|
||||
双塔模型，使用ERNIE-Gram预训练模型，使用margin_ranking_loss训练模型。
|
||||
|
||||
|
||||
### 评估指标
|
||||
|
||||
（1）采用 AUC 指标来评估排序模型的排序效果。
|
||||
|
||||
**效果评估**
|
||||
|
||||
| 模型 | AUC |
|
||||
| ------------ | ------------ |
|
||||
| ERNIE-Gram | 0.801 |
|
||||
|
||||
<a name="环境依赖"></a>
|
||||
|
||||
## 2. 环境依赖和安装说明
|
||||
|
||||
**环境依赖**
|
||||
|
||||
* python >= 3.x
|
||||
* paddlepaddle >= 2.1.3
|
||||
* paddlenlp >= 2.2
|
||||
* pandas >= 0.25.1
|
||||
* scipy >= 1.3.1
|
||||
|
||||
<a name="代码结构"></a>
|
||||
|
||||
## 3. 代码结构
|
||||
|
||||
以下是本项目主要代码结构及说明：
|
||||
|
||||
```
|
||||
ernie_matching/
|
||||
├── deply # 部署
|
||||
└── python
|
||||
├── deploy.sh # 预测部署bash脚本
|
||||
└── predict.py # python 预测部署示例
|
||||
|—— scripts
|
||||
├── export_model.sh # 动态图参数导出静态图参数的bash文件
|
||||
├── train_pairwise.sh # Pair-wise 单塔匹配模型训练的bash文件
|
||||
├── evaluate.sh # 评估验证文件bash脚本
|
||||
├── predict_pairwise.sh # Pair-wise 单塔匹配模型预测脚本的bash文件
|
||||
├── export_model.py # 动态图参数导出静态图参数脚本
|
||||
├── model.py # Pair-wise 匹配模型组网
|
||||
├── data.py # Pair-wise 训练样本的转换逻辑 、Pair-wise 生成随机负例的逻辑
|
||||
├── train_pairwise.py # Pair-wise 单塔匹配模型训练脚本
|
||||
├── evaluate.py # 评估验证文件
|
||||
├── predict_pairwise.py # Pair-wise 单塔匹配模型预测脚本，输出文本对是相似度
|
||||
|
||||
```
|
||||
|
||||
<a name="数据准备"></a>
|
||||
|
||||
## 4. 数据准备
|
||||
|
||||
### 数据集说明
|
||||
|
||||
样例数据如下:
|
||||
```
|
||||
个人所得税税务筹划 基于新个税视角下的个人所得税纳税筹划分析新个税;个人所得税;纳税筹划 个人所得税工资薪金税务筹划研究个人所得税,工资薪金,税务筹划
|
||||
液压支架底座受力分析 ZY4000/09/19D型液压支架的有限元分析液压支架,有限元分析,两端加载,偏载,扭转 基于ANSYS的液压支架多工况受力分析液压支架,四种工况,仿真分析,ANSYS,应力集中,优化
|
||||
迟发性血管痉挛 西洛他唑治疗动脉瘤性蛛网膜下腔出血后脑血管痉挛的Meta分析西洛他唑,蛛网膜下腔出血,脑血管痉挛,Meta分析 西洛他唑治疗动脉瘤性蛛网膜下腔出血后脑血管痉挛的Meta分析西洛他唑,蛛网膜下腔出血,脑血管痉挛,Meta分析
|
||||
氧化亚硅 复合溶胶-凝胶一锅法制备锂离子电池氧化亚硅/碳复合负极材料氧化亚硅,溶胶-凝胶法,纳米颗粒,负极,锂离子电池 负载型聚酰亚胺-二氧化硅-银杂化膜的制备和表征聚酰亚胺,二氧化硅,银,杂化膜,促进传输
|
||||
```
|
||||
|
||||
|
||||
### 数据集下载
|
||||
|
||||
|
||||
- [literature_search_data](https://bj.bcebos.com/v1/paddlenlp/data/literature_search_data.zip)
|
||||
|
||||
```
|
||||
├── milvus # milvus建库数据集
|
||||
├── milvus_data.csv. # 构建召回库的数据
|
||||
├── recall # 召回（语义索引）数据集
|
||||
├── corpus.csv # 用于测试的召回库
|
||||
├── dev.csv # 召回验证集
|
||||
├── test.csv # 召回测试集
|
||||
├── train.csv # 召回训练集
|
||||
├── train_unsupervised.csv # 无监督训练集
|
||||
├── sort # 排序数据集
|
||||
├── test_pairwise.csv # 排序测试集
|
||||
├── dev_pairwise.csv # 排序验证集
|
||||
└── train_pairwise.csv # 排序训练集
|
||||
|
||||
```
|
||||
|
||||
<a name="模型训练"></a>
|
||||
|
||||
## 5. 模型训练
|
||||
|
||||
**排序模型下载链接：**
|
||||
|
||||
|
||||
|Model|训练参数配置|硬件|MD5|
|
||||
| ------------ | ------------ | ------------ |-----------|
|
||||
|[ERNIE-Gram-Sort](https://bj.bcebos.com/v1/paddlenlp/models/ernie_gram_sort.zip)|<div style="width: 150pt">epoch:3 lr:5E-5 bs:64 max_len:64 </div>|<div style="width: 100pt">4卡 v100-16g</div>|d24ece68b7c3626ce6a24baa58dd297d|
|
||||
|
||||
|
||||
### 训练环境说明
|
||||
|
||||
|
||||
- NVIDIA Driver Version: 440.64.00
|
||||
- Ubuntu 16.04.6 LTS (Docker)
|
||||
- Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz
|
||||
|
||||
|
||||
### 单机单卡训练/单机多卡训练
|
||||
|
||||
这里采用单机多卡方式进行训练，通过如下命令，指定 GPU 0,1,2,3 卡, 基于ERNIE-Gram训练模型，数据量比较大，需要20小时10分钟左右。如果采用单机单卡训练，只需要把`--gpu`参数设置成单卡的卡号即可
|
||||
|
||||
训练的命令如下：
|
||||
|
||||
```
|
||||
python -u -m paddle.distributed.launch --gpus "0,2,3,4" train_pairwise.py \
|
||||
--device gpu \
|
||||
--save_dir ./checkpoints \
|
||||
--batch_size 32 \
|
||||
--learning_rate 2E-5 \
|
||||
--margin 0.1 \
|
||||
--eval_step 100 \
|
||||
--train_file data/train_pairwise.csv \
|
||||
--test_file data/dev_pairwise.csv
|
||||
```
|
||||
也可以运行bash脚本：
|
||||
|
||||
```
|
||||
sh scripts/train_pairwise.sh
|
||||
```
|
||||
|
||||
<a name="评估"></a>
|
||||
|
||||
## 6. 评估
|
||||
|
||||
|
||||
```
|
||||
unset CUDA_VISIBLE_DEVICES
|
||||
python -u -m paddle.distributed.launch --gpus "0" evaluate.py \
|
||||
--device gpu \
|
||||
--batch_size 32 \
|
||||
--learning_rate 2E-5 \
|
||||
--init_from_ckpt "./checkpoints/model_30000/model_state.pdparams" \
|
||||
--test_file data/dev_pairwise.csv
|
||||
```
|
||||
也可以运行bash脚本：
|
||||
|
||||
```
|
||||
sh scripts/evaluate.sh
|
||||
```
|
||||
|
||||
|
||||
成功运行后会输出下面的指标：
|
||||
|
||||
```
|
||||
eval_dev auc:0.796
|
||||
```
|
||||
|
||||
<a name="预测"></a>
|
||||
|
||||
## 7. 预测
|
||||
|
||||
### 准备预测数据
|
||||
|
||||
待预测数据为 tab 分隔的 tsv 文件，每一行为 1 个文本 Pair，和文本pair的语义索引相似度，部分示例如下:
|
||||
|
||||
```
|
||||
中西方语言与文化的差异 第二语言习得的一大障碍就是文化差异。 0.5160342454910278
|
||||
中西方语言与文化的差异 跨文化视角下中国文化对外传播路径琐谈跨文化,中国文化,传播,翻译 0.5145505666732788
|
||||
中西方语言与文化的差异 从中西方民族文化心理的差异看英汉翻译语言,文化,民族文化心理,思维方式,翻译 0.5141439437866211
|
||||
中西方语言与文化的差异 中英文化差异对翻译的影响中英文化,差异,翻译的影响 0.5138794183731079
|
||||
中西方语言与文化的差异 浅谈文化与语言习得文化,语言,文化与语言的关系,文化与语言习得意识,跨文化交际 0.5131710171699524
|
||||
```
|
||||
|
||||
|
||||
|
||||
### 开始预测
|
||||
|
||||
以上述 demo 数据为例，运行如下命令基于我们开源的 ERNIE-Gram模型开始计算文本 Pair 的语义相似度:
|
||||
|
||||
```shell
|
||||
python -u -m paddle.distributed.launch --gpus "0" \
|
||||
predict_pairwise.py \
|
||||
--device gpu \
|
||||
--params_path "./checkpoints/model_30000/model_state.pdparams"\
|
||||
--batch_size 128 \
|
||||
--max_seq_length 64 \
|
||||
--input_file 'sort/test_pairwise.csv'
|
||||
```
|
||||
也可以直接执行下面的命令：
|
||||
|
||||
```
|
||||
sh scripts/predict_pairwise.sh
|
||||
```
|
||||
得到下面的输出，分别是query，title和对应的预测概率：
|
||||
|
||||
```
|
||||
{'query': '中西方语言与文化的差异', 'title': '第二语言习得的一大障碍就是文化差异。', 'pred_prob': 0.85112214}
|
||||
{'query': '中西方语言与文化的差异', 'title': '跨文化视角下中国文化对外传播路径琐谈跨文化,中国文化,传播,翻译', 'pred_prob': 0.78629625}
|
||||
{'query': '中西方语言与文化的差异', 'title': '从中西方民族文化心理的差异看英汉翻译语言,文化,民族文化心理,思维方式,翻译', 'pred_prob': 0.91767526}
|
||||
{'query': '中西方语言与文化的差异', 'title': '中英文化差异对翻译的影响中英文化,差异,翻译的影响', 'pred_prob': 0.8601749}
|
||||
{'query': '中西方语言与文化的差异', 'title': '浅谈文化与语言习得文化,语言,文化与语言的关系,文化与语言习得意识,跨文化交际', 'pred_prob': 0.8944413}
|
||||
```
|
||||
|
||||
<a name="部署"></a>
|
||||
|
||||
## 8. 部署
|
||||
|
||||
### 动转静导出
|
||||
|
||||
首先把动态图模型转换为静态图：
|
||||
|
||||
```
|
||||
python export_model.py --params_path checkpoints/model_30000/model_state.pdparams --output_path=./output
|
||||
```
|
||||
也可以运行下面的bash脚本：
|
||||
|
||||
```
|
||||
sh scripts/export_model.sh
|
||||
```
|
||||
|
||||
### Paddle Inference
|
||||
|
||||
修改预测文件路径：
|
||||
|
||||
```
|
||||
input_file='../../sort/test_pairwise.csv'
|
||||
```
|
||||
|
||||
然后使用PaddleInference
|
||||
|
||||
```
|
||||
python predict.py --model_dir=../../output
|
||||
```
|
||||
也可以运行下面的bash脚本：
|
||||
|
||||
```
|
||||
sh deploy.sh
|
||||
```
|
||||
得到下面的输出，输出的是样本的query，title以及对应的概率：
|
||||
|
||||
```
|
||||
Data: {'query': '中西方语言与文化的差异', 'title': '第二语言习得的一大障碍就是文化差异。'} prob: [0.8511221]
|
||||
Data: {'query': '中西方语言与文化的差异', 'title': '跨文化视角下中国文化对外传播路径琐谈跨文化,中国文化,传播,翻译'} prob: [0.7862964]
|
||||
Data: {'query': '中西方语言与文化的差异', 'title': '从中西方民族文化心理的差异看英汉翻译语言,文化,民族文化心理,思维方式,翻译'} prob: [0.91767514]
|
||||
Data: {'query': '中西方语言与文化的差异', 'title': '中英文化差异对翻译的影响中英文化,差异,翻译的影响'} prob: [0.8601747]
|
||||
Data: {'query': '中西方语言与文化的差异', 'title': '浅谈文化与语言习得文化,语言,文化与语言的关系,文化与语言习得意识,跨文化交际'} prob: [0.8944413]
|
||||
```
|
||||
|
||||
## Reference
|
||||
|
||||
[1] Xiao, Dongling, Yu-Kun Li, Han Zhang, Yu Sun, Hao Tian, Hua Wu, and Haifeng Wang. “ERNIE-Gram: Pre-Training with Explicitly N-Gram Masked Language Modeling for Natural Language Understanding.” ArXiv:2010.12148 [Cs].
|
||||
152
applications/neural_search/ranking/ernie_matching/data.py
Normal file
152
applications/neural_search/ranking/ernie_matching/data.py
Normal file
@@ -0,0 +1,152 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import paddle
|
||||
import numpy as np
|
||||
|
||||
from paddlenlp.datasets import MapDataset
|
||||
|
||||
|
||||
def create_dataloader(dataset,
|
||||
mode='train',
|
||||
batch_size=1,
|
||||
batchify_fn=None,
|
||||
trans_fn=None):
|
||||
if trans_fn:
|
||||
dataset = dataset.map(trans_fn)
|
||||
|
||||
shuffle = True if mode == 'train' else False
|
||||
if mode == 'train':
|
||||
batch_sampler = paddle.io.DistributedBatchSampler(
|
||||
dataset, batch_size=batch_size, shuffle=shuffle)
|
||||
else:
|
||||
batch_sampler = paddle.io.BatchSampler(
|
||||
dataset, batch_size=batch_size, shuffle=shuffle)
|
||||
|
||||
return paddle.io.DataLoader(
|
||||
dataset=dataset,
|
||||
batch_sampler=batch_sampler,
|
||||
collate_fn=batchify_fn,
|
||||
return_list=True)
|
||||
|
||||
|
||||
def read_text_pair(data_path):
|
||||
"""Reads data."""
|
||||
with open(data_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
data = line.rstrip().split("\t")
|
||||
if len(data) != 3:
|
||||
continue
|
||||
yield {'query': data[0], 'title': data[1]}
|
||||
|
||||
|
||||
def convert_pointwise_example(example,
|
||||
tokenizer,
|
||||
max_seq_length=512,
|
||||
is_test=False):
|
||||
|
||||
query, title = example["query"], example["title"]
|
||||
|
||||
encoded_inputs = tokenizer(
|
||||
text=query, text_pair=title, max_seq_len=max_seq_length)
|
||||
|
||||
input_ids = encoded_inputs["input_ids"]
|
||||
token_type_ids = encoded_inputs["token_type_ids"]
|
||||
|
||||
if not is_test:
|
||||
label = np.array([example["label"]], dtype="int64")
|
||||
return input_ids, token_type_ids, label
|
||||
else:
|
||||
return input_ids, token_type_ids
|
||||
|
||||
|
||||
def convert_pairwise_example(example,
|
||||
tokenizer,
|
||||
max_seq_length=512,
|
||||
phase="train"):
|
||||
|
||||
if phase == "train":
|
||||
query, pos_title, neg_title = example["query"], example[
|
||||
"title"], example["neg_title"]
|
||||
|
||||
pos_inputs = tokenizer(
|
||||
text=query, text_pair=pos_title, max_seq_len=max_seq_length)
|
||||
neg_inputs = tokenizer(
|
||||
text=query, text_pair=neg_title, max_seq_len=max_seq_length)
|
||||
|
||||
pos_input_ids = pos_inputs["input_ids"]
|
||||
pos_token_type_ids = pos_inputs["token_type_ids"]
|
||||
neg_input_ids = neg_inputs["input_ids"]
|
||||
neg_token_type_ids = neg_inputs["token_type_ids"]
|
||||
|
||||
return (pos_input_ids, pos_token_type_ids, neg_input_ids,
|
||||
neg_token_type_ids)
|
||||
|
||||
else:
|
||||
query, title = example["query"], example["title"]
|
||||
|
||||
inputs = tokenizer(
|
||||
text=query, text_pair=title, max_seq_len=max_seq_length)
|
||||
|
||||
input_ids = inputs["input_ids"]
|
||||
token_type_ids = inputs["token_type_ids"]
|
||||
if phase == "eval":
|
||||
return input_ids, token_type_ids, example["label"]
|
||||
elif phase == "predict":
|
||||
return input_ids, token_type_ids
|
||||
else:
|
||||
raise ValueError("not supported phase:{}".format(phase))
|
||||
|
||||
|
||||
def gen_pair(dataset, pool_size=100):
|
||||
"""
|
||||
Generate triplet randomly based on dataset
|
||||
|
||||
Args:
|
||||
dataset: A `MapDataset` or `IterDataset` or a tuple of those.
|
||||
Each example is composed of 2 texts: exampe["query"], example["title"]
|
||||
pool_size: the number of example to sample negative example randomly
|
||||
|
||||
Return:
|
||||
dataset: A `MapDataset` or `IterDataset` or a tuple of those.
|
||||
Each example is composed of 2 texts: exampe["query"], example["pos_title"]、example["neg_title"]
|
||||
"""
|
||||
|
||||
if len(dataset) < pool_size:
|
||||
pool_size = len(dataset)
|
||||
|
||||
new_examples = []
|
||||
pool = []
|
||||
tmp_exmaples = []
|
||||
|
||||
for example in dataset:
|
||||
label = example["label"]
|
||||
|
||||
# Filter negative example
|
||||
if label == 0:
|
||||
continue
|
||||
|
||||
tmp_exmaples.append(example)
|
||||
pool.append(example["title"])
|
||||
|
||||
if len(pool) >= pool_size:
|
||||
np.random.shuffle(pool)
|
||||
for idx, example in enumerate(tmp_exmaples):
|
||||
example["neg_title"] = pool[idx]
|
||||
new_examples.append(example)
|
||||
tmp_exmaples = []
|
||||
pool = []
|
||||
else:
|
||||
continue
|
||||
return MapDataset(new_examples)
|
||||
@@ -0,0 +1 @@
|
||||
python deploy/python/predict.py --model_dir ./output
|
||||
@@ -0,0 +1,244 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import paddle
|
||||
import paddlenlp as ppnlp
|
||||
from scipy.special import softmax
|
||||
from scipy.special import expit
|
||||
from paddle import inference
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.datasets import load_dataset
|
||||
from paddlenlp.utils.log import logger
|
||||
import paddle.nn.functional as F
|
||||
import sys
|
||||
|
||||
sys.path.append('.')
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model_dir", type=str, required=True,
|
||||
help="The directory to static model.")
|
||||
|
||||
parser.add_argument("--max_seq_length", default=128, type=int,
|
||||
help="The maximum total input sequence length after tokenization. Sequences "
|
||||
"longer than this will be truncated, sequences shorter will be padded.")
|
||||
parser.add_argument("--batch_size", default=32, type=int,
|
||||
help="Batch size per GPU/CPU for training.")
|
||||
parser.add_argument('--device', choices=['cpu', 'gpu', 'xpu'], default="gpu",
|
||||
help="Select which device to train model, defaults to gpu.")
|
||||
|
||||
parser.add_argument('--use_tensorrt', default=False, type=eval, choices=[True, False],
|
||||
help='Enable to use tensorrt to speed up.')
|
||||
parser.add_argument("--precision", default="fp32", type=str, choices=["fp32", "fp16", "int8"],
|
||||
help='The tensorrt precision.')
|
||||
|
||||
parser.add_argument('--cpu_threads', default=10, type=int,
|
||||
help='Number of threads to predict when using cpu.')
|
||||
parser.add_argument('--enable_mkldnn', default=False, type=eval, choices=[True, False],
|
||||
help='Enable to use mkldnn to speed up when using cpu.')
|
||||
|
||||
parser.add_argument("--benchmark", type=eval, default=False,
|
||||
help="To log some information about environment and running.")
|
||||
parser.add_argument("--save_log_path", type=str, default="./log_output/",
|
||||
help="The file path to save log.")
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def read_text_pair(data_path):
|
||||
"""Reads data."""
|
||||
with open(data_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
data = line.rstrip().split("\t")
|
||||
if len(data) != 3:
|
||||
continue
|
||||
yield {'query': data[0], 'title': data[1]}
|
||||
|
||||
|
||||
def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
|
||||
|
||||
query, title = example["query"], example["title"]
|
||||
|
||||
encoded_inputs = tokenizer(
|
||||
text=query, text_pair=title, max_seq_len=max_seq_length)
|
||||
|
||||
input_ids = encoded_inputs["input_ids"]
|
||||
token_type_ids = encoded_inputs["token_type_ids"]
|
||||
|
||||
if not is_test:
|
||||
label = np.array([example["label"]], dtype="int64")
|
||||
return input_ids, token_type_ids, label
|
||||
else:
|
||||
return input_ids, token_type_ids
|
||||
|
||||
|
||||
class Predictor(object):
|
||||
def __init__(self,
|
||||
model_dir,
|
||||
device="gpu",
|
||||
max_seq_length=128,
|
||||
batch_size=32,
|
||||
use_tensorrt=False,
|
||||
precision="fp32",
|
||||
cpu_threads=10,
|
||||
enable_mkldnn=False):
|
||||
self.max_seq_length = max_seq_length
|
||||
self.batch_size = batch_size
|
||||
|
||||
model_file = model_dir + "/inference.get_pooled_embedding.pdmodel"
|
||||
params_file = model_dir + "/inference.get_pooled_embedding.pdiparams"
|
||||
if not os.path.exists(model_file):
|
||||
raise ValueError("not find model file path {}".format(model_file))
|
||||
if not os.path.exists(params_file):
|
||||
raise ValueError("not find params file path {}".format(params_file))
|
||||
config = paddle.inference.Config(model_file, params_file)
|
||||
|
||||
if device == "gpu":
|
||||
# set GPU configs accordingly
|
||||
# such as intialize the gpu memory, enable tensorrt
|
||||
config.enable_use_gpu(100, 0)
|
||||
precision_map = {
|
||||
"fp16": inference.PrecisionType.Half,
|
||||
"fp32": inference.PrecisionType.Float32,
|
||||
"int8": inference.PrecisionType.Int8
|
||||
}
|
||||
precision_mode = precision_map[precision]
|
||||
|
||||
if args.use_tensorrt:
|
||||
config.enable_tensorrt_engine(
|
||||
max_batch_size=batch_size,
|
||||
min_subgraph_size=30,
|
||||
precision_mode=precision_mode)
|
||||
elif device == "cpu":
|
||||
# set CPU configs accordingly,
|
||||
# such as enable_mkldnn, set_cpu_math_library_num_threads
|
||||
config.disable_gpu()
|
||||
if args.enable_mkldnn:
|
||||
# cache 10 different shapes for mkldnn to avoid memory leak
|
||||
config.set_mkldnn_cache_capacity(10)
|
||||
config.enable_mkldnn()
|
||||
config.set_cpu_math_library_num_threads(args.cpu_threads)
|
||||
elif device == "xpu":
|
||||
# set XPU configs accordingly
|
||||
config.enable_xpu(100)
|
||||
|
||||
config.switch_use_feed_fetch_ops(False)
|
||||
self.predictor = paddle.inference.create_predictor(config)
|
||||
self.input_handles = [
|
||||
self.predictor.get_input_handle(name)
|
||||
for name in self.predictor.get_input_names()
|
||||
]
|
||||
self.output_handle = self.predictor.get_output_handle(
|
||||
self.predictor.get_output_names()[0])
|
||||
|
||||
if args.benchmark:
|
||||
import auto_log
|
||||
pid = os.getpid()
|
||||
self.autolog = auto_log.AutoLogger(
|
||||
model_name="ernie-tiny",
|
||||
model_precision=precision,
|
||||
batch_size=self.batch_size,
|
||||
data_shape="dynamic",
|
||||
save_path=args.save_log_path,
|
||||
inference_config=config,
|
||||
pids=pid,
|
||||
process_name=None,
|
||||
gpu_ids=0,
|
||||
time_keys=[
|
||||
'preprocess_time', 'inference_time', 'postprocess_time'
|
||||
],
|
||||
warmup=0,
|
||||
logger=logger)
|
||||
|
||||
def predict(self, data, tokenizer):
|
||||
"""
|
||||
Predicts the data labels.
|
||||
|
||||
Args:
|
||||
data (obj:`List(str)`): The batch data whose each element is a raw text.
|
||||
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
|
||||
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
|
||||
label_map(obj:`dict`): The label id (key) to label str (value) map.
|
||||
|
||||
Returns:
|
||||
results(obj:`dict`): All the predictions labels.
|
||||
"""
|
||||
if args.benchmark:
|
||||
self.autolog.times.start()
|
||||
|
||||
examples = []
|
||||
for text in data:
|
||||
input_ids, segment_ids = convert_example(
|
||||
text,
|
||||
tokenizer,
|
||||
max_seq_length=self.max_seq_length,
|
||||
is_test=True)
|
||||
examples.append((input_ids, segment_ids))
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # segment
|
||||
): fn(samples)
|
||||
|
||||
if args.benchmark:
|
||||
self.autolog.times.stamp()
|
||||
|
||||
input_ids, segment_ids = batchify_fn(examples)
|
||||
self.input_handles[0].copy_from_cpu(input_ids)
|
||||
self.input_handles[1].copy_from_cpu(segment_ids)
|
||||
self.predictor.run()
|
||||
sim_score = self.output_handle.copy_to_cpu()
|
||||
if args.benchmark:
|
||||
self.autolog.times.stamp()
|
||||
sim_score = expit(sim_score)
|
||||
|
||||
if args.benchmark:
|
||||
self.autolog.times.end(stamp=True)
|
||||
|
||||
return sim_score
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Define predictor to do prediction.
|
||||
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
|
||||
args.batch_size, args.use_tensorrt, args.precision,
|
||||
args.cpu_threads, args.enable_mkldnn)
|
||||
|
||||
tokenizer = ppnlp.transformers.ErnieGramTokenizer.from_pretrained(
|
||||
'ernie-gram-zh')
|
||||
|
||||
# test_ds = load_dataset("lcqmc", splits=["test"])
|
||||
input_file = 'sort/test_pairwise.csv'
|
||||
|
||||
test_ds = load_dataset(read_text_pair, data_path=input_file, lazy=False)
|
||||
|
||||
data = [{'query': d['query'], 'title': d['title']} for d in test_ds]
|
||||
|
||||
batches = [
|
||||
data[idx:idx + args.batch_size]
|
||||
for idx in range(0, len(data), args.batch_size)
|
||||
]
|
||||
|
||||
results = []
|
||||
for batch_data in batches:
|
||||
results.extend(predictor.predict(batch_data, tokenizer))
|
||||
for idx, text in enumerate(data):
|
||||
print('Data: {} \t prob: {}'.format(text, results[idx]))
|
||||
if args.benchmark:
|
||||
predictor.autolog.report()
|
||||
158
applications/neural_search/ranking/ernie_matching/evaluate.py
Normal file
158
applications/neural_search/ranking/ernie_matching/evaluate.py
Normal file
@@ -0,0 +1,158 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from functools import partial
|
||||
import argparse
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.datasets import load_dataset
|
||||
from paddlenlp.transformers import LinearDecayWithWarmup
|
||||
|
||||
from data import create_dataloader, gen_pair
|
||||
from data import convert_pairwise_example as convert_example
|
||||
from model import PairwiseMatching
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--margin", default=0.1, type=float, help="Margin for pos_score and neg_score.")
|
||||
parser.add_argument("--test_file", type=str, required=True, help="The full path of test file")
|
||||
|
||||
parser.add_argument("--max_seq_length", default=128, type=int, help="The maximum total input sequence length after tokenization. "
|
||||
"Sequences longer than this will be truncated, sequences shorter will be padded.")
|
||||
parser.add_argument("--batch_size", default=32, type=int, help="Batch size per GPU/CPU for training.")
|
||||
parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
|
||||
parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
|
||||
parser.add_argument("--epochs", default=3, type=int, help="Total number of training epochs to perform.")
|
||||
parser.add_argument("--eval_step", default=200, type=int, help="Step interval for evaluation.")
|
||||
parser.add_argument('--save_step', default=10000, type=int, help="Step interval for saving checkpoint.")
|
||||
parser.add_argument("--warmup_proportion", default=0.0, type=float, help="Linear warmup proption over the training process.")
|
||||
parser.add_argument("--init_from_ckpt", type=str, default=None, help="The path of checkpoint to be loaded.")
|
||||
parser.add_argument("--seed", type=int, default=1000, help="Random seed for initialization.")
|
||||
parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to train model, defaults to gpu.")
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def set_seed(seed):
|
||||
"""sets random seed"""
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
paddle.seed(seed)
|
||||
|
||||
|
||||
@paddle.no_grad()
|
||||
def evaluate(model, metric, data_loader, phase="dev"):
|
||||
"""
|
||||
Given a dataset, it evals model and computes the metric.
|
||||
|
||||
Args:
|
||||
model(obj:`paddle.nn.Layer`): A model to classify texts.
|
||||
data_loader(obj:`paddle.io.DataLoader`): The dataset loader which generates batches.
|
||||
metric(obj:`paddle.metric.Metric`): The evaluation metric.
|
||||
"""
|
||||
model.eval()
|
||||
metric.reset()
|
||||
|
||||
for idx, batch in enumerate(data_loader):
|
||||
input_ids, token_type_ids, labels = batch
|
||||
|
||||
pos_probs = model.predict(
|
||||
input_ids=input_ids, token_type_ids=token_type_ids)
|
||||
|
||||
neg_probs = 1.0 - pos_probs
|
||||
|
||||
preds = np.concatenate((neg_probs, pos_probs), axis=1)
|
||||
metric.update(preds=preds, labels=labels)
|
||||
|
||||
print("eval_{} auc:{:.3}".format(phase, metric.accumulate()))
|
||||
metric.reset()
|
||||
model.train()
|
||||
|
||||
|
||||
# 构建读取函数，读取原始数据
|
||||
def read(src_path, is_predict=False):
|
||||
data = pd.read_csv(src_path, sep='\t')
|
||||
for index, row in tqdm(data.iterrows()):
|
||||
query = row['query']
|
||||
title = row['title']
|
||||
neg_title = row['neg_title']
|
||||
yield {'query': query, 'title': title, 'neg_title': neg_title}
|
||||
|
||||
|
||||
def read_test(src_path, is_predict=False):
|
||||
data = pd.read_csv(src_path, sep='\t')
|
||||
for index, row in tqdm(data.iterrows()):
|
||||
query = row['query']
|
||||
title = row['title']
|
||||
label = row['label']
|
||||
yield {'query': query, 'title': title, 'label': label}
|
||||
|
||||
|
||||
def do_train():
|
||||
paddle.set_device(args.device)
|
||||
rank = paddle.distributed.get_rank()
|
||||
if paddle.distributed.get_world_size() > 1:
|
||||
paddle.distributed.init_parallel_env()
|
||||
|
||||
set_seed(args.seed)
|
||||
|
||||
dev_ds = load_dataset(read_test, src_path=args.test_file, lazy=False)
|
||||
print(dev_ds[0])
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieGramModel.from_pretrained(
|
||||
'ernie-gram-zh')
|
||||
tokenizer = ppnlp.transformers.ErnieGramTokenizer.from_pretrained(
|
||||
'ernie-gram-zh')
|
||||
|
||||
trans_func_eval = partial(
|
||||
convert_example,
|
||||
tokenizer=tokenizer,
|
||||
max_seq_length=args.max_seq_length,
|
||||
phase="eval")
|
||||
|
||||
batchify_fn_eval = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # pair_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # pair_segment
|
||||
Stack(dtype="int64") # label
|
||||
): [data for data in fn(samples)]
|
||||
|
||||
dev_data_loader = create_dataloader(
|
||||
dev_ds,
|
||||
mode='dev',
|
||||
batch_size=args.batch_size,
|
||||
batchify_fn=batchify_fn_eval,
|
||||
trans_fn=trans_func_eval)
|
||||
|
||||
model = PairwiseMatching(pretrained_model, margin=args.margin)
|
||||
|
||||
if args.init_from_ckpt and os.path.isfile(args.init_from_ckpt):
|
||||
state_dict = paddle.load(args.init_from_ckpt)
|
||||
model.set_dict(state_dict)
|
||||
|
||||
metric = paddle.metric.Auc()
|
||||
evaluate(model, metric, dev_data_loader, "dev")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
do_train()
|
||||
@@ -0,0 +1,63 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
|
||||
from model import PairwiseMatching
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--params_path", type=str, required=True, default='./checkpoint/model_900/model_state.pdparams', help="The path to model parameters to be loaded.")
|
||||
parser.add_argument("--output_path", type=str, default='./output', help="The path of model parameter in static graph to be saved.")
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
if __name__ == "__main__":
|
||||
# If you want to use ernie1.0 model, plesace uncomment the following code
|
||||
# tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
# pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained("ernie-1.0")
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieGramModel.from_pretrained(
|
||||
'ernie-gram-zh')
|
||||
tokenizer = ppnlp.transformers.ErnieGramTokenizer.from_pretrained(
|
||||
'ernie-gram-zh')
|
||||
model = PairwiseMatching(pretrained_model)
|
||||
|
||||
if args.params_path and os.path.isfile(args.params_path):
|
||||
state_dict = paddle.load(args.params_path)
|
||||
model.set_dict(state_dict)
|
||||
print("Loaded parameters from %s" % args.params_path)
|
||||
|
||||
model.eval()
|
||||
|
||||
# Convert to static graph with specific input description
|
||||
model = paddle.jit.to_static(
|
||||
model,
|
||||
input_spec=[
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None], dtype="int64"), # input_ids
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None], dtype="int64") # segment_ids
|
||||
])
|
||||
# Save in static graph model.
|
||||
save_path = os.path.join(args.output_path, "inference")
|
||||
paddle.jit.save(model, save_path)
|
||||
92
applications/neural_search/ranking/ernie_matching/model.py
Normal file
92
applications/neural_search/ranking/ernie_matching/model.py
Normal file
@@ -0,0 +1,92 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import paddle
|
||||
import paddle.nn as nn
|
||||
import paddle.nn.functional as F
|
||||
|
||||
|
||||
class PairwiseMatching(nn.Layer):
|
||||
def __init__(self, pretrained_model, dropout=None, margin=0.1):
|
||||
super().__init__()
|
||||
self.ptm = pretrained_model
|
||||
self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)
|
||||
self.margin = margin
|
||||
|
||||
# hidden_size -> 1, calculate similarity
|
||||
self.similarity = nn.Linear(self.ptm.config["hidden_size"], 1)
|
||||
|
||||
@paddle.jit.to_static(input_spec=[
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64'), paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64')
|
||||
])
|
||||
def get_pooled_embedding(self,
|
||||
input_ids,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
attention_mask=None):
|
||||
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
|
||||
attention_mask)
|
||||
cls_embedding = self.dropout(cls_embedding)
|
||||
sim = self.similarity(cls_embedding)
|
||||
return sim
|
||||
|
||||
def predict(self,
|
||||
input_ids,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
attention_mask=None):
|
||||
|
||||
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
|
||||
attention_mask)
|
||||
|
||||
cls_embedding = self.dropout(cls_embedding)
|
||||
sim_score = self.similarity(cls_embedding)
|
||||
sim_score = F.sigmoid(sim_score)
|
||||
|
||||
return sim_score
|
||||
|
||||
def forward(self,
|
||||
pos_input_ids,
|
||||
neg_input_ids,
|
||||
pos_token_type_ids=None,
|
||||
neg_token_type_ids=None,
|
||||
pos_position_ids=None,
|
||||
neg_position_ids=None,
|
||||
pos_attention_mask=None,
|
||||
neg_attention_mask=None):
|
||||
|
||||
_, pos_cls_embedding = self.ptm(pos_input_ids, pos_token_type_ids,
|
||||
pos_position_ids, pos_attention_mask)
|
||||
|
||||
_, neg_cls_embedding = self.ptm(neg_input_ids, neg_token_type_ids,
|
||||
neg_position_ids, neg_attention_mask)
|
||||
|
||||
pos_embedding = self.dropout(pos_cls_embedding)
|
||||
neg_embedding = self.dropout(neg_cls_embedding)
|
||||
|
||||
pos_sim = self.similarity(pos_embedding)
|
||||
neg_sim = self.similarity(neg_embedding)
|
||||
|
||||
pos_sim = F.sigmoid(pos_sim)
|
||||
neg_sim = F.sigmoid(neg_sim)
|
||||
|
||||
labels = paddle.full(
|
||||
shape=[pos_cls_embedding.shape[0]], fill_value=1.0, dtype='float32')
|
||||
|
||||
loss = F.margin_ranking_loss(
|
||||
pos_sim, neg_sim, labels, margin=self.margin)
|
||||
|
||||
return loss
|
||||
@@ -0,0 +1,129 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from functools import partial
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.datasets import load_dataset
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
|
||||
from data import create_dataloader, read_text_pair
|
||||
from data import convert_pairwise_example as convert_example
|
||||
from model import PairwiseMatching
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input_file", type=str, required=True, help="The full path of input file")
|
||||
parser.add_argument("--params_path", type=str, required=True, help="The path to model parameters to be loaded.")
|
||||
parser.add_argument("--max_seq_length", default=64, type=int, help="The maximum total input sequence length after tokenization. "
|
||||
"Sequences longer than this will be truncated, sequences shorter will be padded.")
|
||||
parser.add_argument("--batch_size", default=32, type=int, help="Batch size per GPU/CPU for training.")
|
||||
parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to train model, defaults to gpu.")
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def predict(model, data_loader):
|
||||
"""
|
||||
Predicts the data labels.
|
||||
|
||||
Args:
|
||||
model (obj:`SemanticIndexBase`): A model to extract text embedding or calculate similarity of text pair.
|
||||
data_loaer (obj:`List(Example)`): The processed data ids of text pair: [query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids]
|
||||
Returns:
|
||||
results(obj:`List`): cosine similarity of text pairs.
|
||||
"""
|
||||
batch_probs = []
|
||||
|
||||
model.eval()
|
||||
|
||||
with paddle.no_grad():
|
||||
for batch_data in data_loader:
|
||||
input_ids, token_type_ids = batch_data
|
||||
|
||||
input_ids = paddle.to_tensor(input_ids)
|
||||
token_type_ids = paddle.to_tensor(token_type_ids)
|
||||
|
||||
batch_prob = model.predict(
|
||||
input_ids=input_ids, token_type_ids=token_type_ids).numpy()
|
||||
|
||||
batch_probs.append(batch_prob)
|
||||
if (len(batch_prob) == 1):
|
||||
batch_probs = np.array(batch_probs)
|
||||
else:
|
||||
batch_probs = np.concatenate(batch_probs, axis=0)
|
||||
|
||||
return batch_probs
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
paddle.set_device(args.device)
|
||||
|
||||
# If you want to use ernie1.0 model, plesace uncomment the following code
|
||||
# tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
# pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained("ernie-1.0")
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieGramModel.from_pretrained(
|
||||
'ernie-gram-zh')
|
||||
tokenizer = ppnlp.transformers.ErnieGramTokenizer.from_pretrained(
|
||||
'ernie-gram-zh')
|
||||
|
||||
trans_func = partial(
|
||||
convert_example,
|
||||
tokenizer=tokenizer,
|
||||
max_seq_length=args.max_seq_length,
|
||||
phase="predict")
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # input_ids
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # segment_ids
|
||||
): [data for data in fn(samples)]
|
||||
|
||||
valid_ds = load_dataset(
|
||||
read_text_pair, data_path=args.input_file, lazy=False)
|
||||
|
||||
valid_data_loader = create_dataloader(
|
||||
valid_ds,
|
||||
mode='predict',
|
||||
batch_size=args.batch_size,
|
||||
batchify_fn=batchify_fn,
|
||||
trans_fn=trans_func)
|
||||
|
||||
model = PairwiseMatching(pretrained_model)
|
||||
|
||||
if args.params_path and os.path.isfile(args.params_path):
|
||||
state_dict = paddle.load(args.params_path)
|
||||
model.set_dict(state_dict)
|
||||
print("Loaded parameters from %s" % args.params_path)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Please set --params_path with correct pretrained model file")
|
||||
|
||||
y_probs = predict(model, valid_data_loader)
|
||||
|
||||
valid_ds = load_dataset(
|
||||
read_text_pair, data_path=args.input_file, lazy=False)
|
||||
|
||||
for idx, prob in enumerate(y_probs):
|
||||
text_pair = valid_ds[idx]
|
||||
text_pair["pred_prob"] = prob[0]
|
||||
print(text_pair)
|
||||
@@ -0,0 +1,16 @@
|
||||
unset CUDA_VISIBLE_DEVICES
|
||||
# gpu
|
||||
python -u -m paddle.distributed.launch --gpus "0" evaluate.py \
|
||||
--device gpu \
|
||||
--batch_size 32 \
|
||||
--learning_rate 2E-5 \
|
||||
--init_from_ckpt "./checkpoints/model_30000/model_state.pdparams" \
|
||||
--test_file sort/dev_pairwise.csv
|
||||
|
||||
# cpu
|
||||
# python evaluate.py \
|
||||
# --device cpu \
|
||||
# --batch_size 32 \
|
||||
# --learning_rate 2E-5 \
|
||||
# --init_from_ckpt "./checkpoints/model_30000/model_state.pdparams" \
|
||||
# --test_file sort/dev_pairwise.csv
|
||||
@@ -0,0 +1 @@
|
||||
python export_model.py --params_path checkpoints/model_30000/model_state.pdparams --output_path=./output
|
||||
@@ -0,0 +1,15 @@
|
||||
# gpu
|
||||
python -u -m paddle.distributed.launch --gpus "0" \
|
||||
predict_pairwise.py \
|
||||
--device gpu \
|
||||
--params_path "./checkpoints/model_30000/model_state.pdparams"\
|
||||
--batch_size 128 \
|
||||
--max_seq_length 64 \
|
||||
--input_file 'sort/test_pairwise.csv'
|
||||
# cpu
|
||||
# python predict_pairwise.py \
|
||||
# --device gpu \
|
||||
# --params_path "./checkpoints/model_30000/model_state.pdparams"\
|
||||
# --batch_size 128 \
|
||||
# --max_seq_length 64 \
|
||||
# --input_file 'sort/test_pairwise.csv'
|
||||
@@ -0,0 +1,21 @@
|
||||
# gpu
|
||||
python -u -m paddle.distributed.launch --gpus "0,2,3,4" train_pairwise.py \
|
||||
--device gpu \
|
||||
--save_dir ./checkpoints \
|
||||
--batch_size 32 \
|
||||
--learning_rate 2E-5 \
|
||||
--margin 0.1 \
|
||||
--eval_step 100 \
|
||||
--train_file sort/train_pairwise.csv \
|
||||
--test_file sort/test_pairwise.csv
|
||||
|
||||
# cpu
|
||||
# python train_pairwise.py \
|
||||
# --device cpu \
|
||||
# --save_dir ./checkpoints \
|
||||
# --batch_size 32 \
|
||||
# --learning_rate 2E-5 \
|
||||
# --margin 0.1 \
|
||||
# --eval_step 100 \
|
||||
# --train_file sort/train_pairwise.csv \
|
||||
# --test_file sort/test_pairwise.csv
|
||||
@@ -0,0 +1,240 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from functools import partial
|
||||
import argparse
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.datasets import load_dataset
|
||||
from paddlenlp.transformers import LinearDecayWithWarmup
|
||||
|
||||
from data import create_dataloader, gen_pair
|
||||
from data import convert_pairwise_example as convert_example
|
||||
from model import PairwiseMatching
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--margin", default=0.2, type=float, help="Margin for pos_score and neg_score.")
|
||||
parser.add_argument("--train_file", type=str, required=True, help="The full path of train file")
|
||||
parser.add_argument("--test_file", type=str, required=True, help="The full path of test file")
|
||||
|
||||
parser.add_argument("--save_dir", default='./checkpoint', type=str, help="The output directory where the model checkpoints will be written.")
|
||||
parser.add_argument("--max_seq_length", default=128, type=int, help="The maximum total input sequence length after tokenization. "
|
||||
"Sequences longer than this will be truncated, sequences shorter will be padded.")
|
||||
parser.add_argument("--batch_size", default=32, type=int, help="Batch size per GPU/CPU for training.")
|
||||
parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
|
||||
parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
|
||||
parser.add_argument("--epochs", default=3, type=int, help="Total number of training epochs to perform.")
|
||||
parser.add_argument("--eval_step", default=200, type=int, help="Step interval for evaluation.")
|
||||
parser.add_argument('--save_step', default=10000, type=int, help="Step interval for saving checkpoint.")
|
||||
parser.add_argument("--warmup_proportion", default=0.0, type=float, help="Linear warmup proption over the training process.")
|
||||
parser.add_argument("--init_from_ckpt", type=str, default=None, help="The path of checkpoint to be loaded.")
|
||||
parser.add_argument("--seed", type=int, default=1000, help="Random seed for initialization.")
|
||||
parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to train model, defaults to gpu.")
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def set_seed(seed):
|
||||
"""sets random seed"""
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
paddle.seed(seed)
|
||||
|
||||
|
||||
@paddle.no_grad()
|
||||
def evaluate(model, metric, data_loader, phase="dev"):
|
||||
"""
|
||||
Given a dataset, it evals model and computes the metric.
|
||||
|
||||
Args:
|
||||
model(obj:`paddle.nn.Layer`): A model to classify texts.
|
||||
data_loader(obj:`paddle.io.DataLoader`): The dataset loader which generates batches.
|
||||
metric(obj:`paddle.metric.Metric`): The evaluation metric.
|
||||
"""
|
||||
model.eval()
|
||||
metric.reset()
|
||||
|
||||
for idx, batch in enumerate(data_loader):
|
||||
input_ids, token_type_ids, labels = batch
|
||||
|
||||
pos_probs = model.predict(
|
||||
input_ids=input_ids, token_type_ids=token_type_ids)
|
||||
|
||||
neg_probs = 1.0 - pos_probs
|
||||
|
||||
preds = np.concatenate((neg_probs, pos_probs), axis=1)
|
||||
metric.update(preds=preds, labels=labels)
|
||||
|
||||
print("eval_{} auc:{:.3}".format(phase, metric.accumulate()))
|
||||
metric.reset()
|
||||
model.train()
|
||||
|
||||
|
||||
# 构建读取函数，读取原始数据
|
||||
def read(src_path, is_predict=False):
|
||||
data = pd.read_csv(src_path, sep='\t')
|
||||
for index, row in tqdm(data.iterrows()):
|
||||
query = row['query']
|
||||
title = row['title']
|
||||
neg_title = row['neg_title']
|
||||
yield {'query': query, 'title': title, 'neg_title': neg_title}
|
||||
|
||||
|
||||
def read_test(src_path, is_predict=False):
|
||||
data = pd.read_csv(src_path, sep='\t')
|
||||
for index, row in tqdm(data.iterrows()):
|
||||
query = row['query']
|
||||
title = row['title']
|
||||
label = row['label']
|
||||
yield {'query': query, 'title': title, 'label': label}
|
||||
|
||||
|
||||
def do_train():
|
||||
paddle.set_device(args.device)
|
||||
rank = paddle.distributed.get_rank()
|
||||
if paddle.distributed.get_world_size() > 1:
|
||||
paddle.distributed.init_parallel_env()
|
||||
|
||||
set_seed(args.seed)
|
||||
|
||||
# train_ds, dev_ds = load_dataset("lcqmc", splits=["train", "dev"])
|
||||
|
||||
train_ds = load_dataset(read, src_path=args.train_file, lazy=False)
|
||||
dev_ds = load_dataset(read_test, src_path=args.test_file, lazy=False)
|
||||
print(train_ds[0])
|
||||
|
||||
# train_ds = gen_pair(train_ds)
|
||||
|
||||
# If you want to use ernie1.0 model, plesace uncomment the following code
|
||||
# pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained('ernie-1.0')
|
||||
# tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieGramModel.from_pretrained(
|
||||
'ernie-gram-zh')
|
||||
tokenizer = ppnlp.transformers.ErnieGramTokenizer.from_pretrained(
|
||||
'ernie-gram-zh')
|
||||
|
||||
trans_func_train = partial(
|
||||
convert_example,
|
||||
tokenizer=tokenizer,
|
||||
max_seq_length=args.max_seq_length)
|
||||
|
||||
trans_func_eval = partial(
|
||||
convert_example,
|
||||
tokenizer=tokenizer,
|
||||
max_seq_length=args.max_seq_length,
|
||||
phase="eval")
|
||||
|
||||
batchify_fn_train = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # pos_pair_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # pos_pair_segment
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # neg_pair_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id) # neg_pair_segment
|
||||
): [data for data in fn(samples)]
|
||||
|
||||
batchify_fn_eval = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # pair_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # pair_segment
|
||||
Stack(dtype="int64") # label
|
||||
): [data for data in fn(samples)]
|
||||
|
||||
train_data_loader = create_dataloader(
|
||||
train_ds,
|
||||
mode='train',
|
||||
batch_size=args.batch_size,
|
||||
batchify_fn=batchify_fn_train,
|
||||
trans_fn=trans_func_train)
|
||||
|
||||
dev_data_loader = create_dataloader(
|
||||
dev_ds,
|
||||
mode='dev',
|
||||
batch_size=args.batch_size,
|
||||
batchify_fn=batchify_fn_eval,
|
||||
trans_fn=trans_func_eval)
|
||||
|
||||
model = PairwiseMatching(pretrained_model, margin=args.margin)
|
||||
|
||||
if args.init_from_ckpt and os.path.isfile(args.init_from_ckpt):
|
||||
state_dict = paddle.load(args.init_from_ckpt)
|
||||
model.set_dict(state_dict)
|
||||
|
||||
num_training_steps = len(train_data_loader) * args.epochs
|
||||
|
||||
lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps,
|
||||
args.warmup_proportion)
|
||||
|
||||
# Generate parameter names needed to perform weight decay.
|
||||
# All bias and LayerNorm parameters are excluded.
|
||||
decay_params = [
|
||||
p.name for n, p in model.named_parameters()
|
||||
if not any(nd in n for nd in ["bias", "norm"])
|
||||
]
|
||||
optimizer = paddle.optimizer.AdamW(
|
||||
learning_rate=lr_scheduler,
|
||||
parameters=model.parameters(),
|
||||
weight_decay=args.weight_decay,
|
||||
apply_decay_param_fun=lambda x: x in decay_params)
|
||||
|
||||
metric = paddle.metric.Auc()
|
||||
|
||||
global_step = 0
|
||||
tic_train = time.time()
|
||||
for epoch in range(1, args.epochs + 1):
|
||||
for step, batch in enumerate(train_data_loader, start=1):
|
||||
pos_input_ids, pos_token_type_ids, neg_input_ids, neg_token_type_ids = batch
|
||||
|
||||
loss = model(
|
||||
pos_input_ids=pos_input_ids,
|
||||
neg_input_ids=neg_input_ids,
|
||||
pos_token_type_ids=pos_token_type_ids,
|
||||
neg_token_type_ids=neg_token_type_ids)
|
||||
|
||||
global_step += 1
|
||||
if global_step % 10 == 0 and rank == 0:
|
||||
print(
|
||||
"global step %d, epoch: %d, batch: %d, loss: %.5f, speed: %.2f step/s"
|
||||
% (global_step, epoch, step, loss,
|
||||
10 / (time.time() - tic_train)))
|
||||
tic_train = time.time()
|
||||
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
lr_scheduler.step()
|
||||
optimizer.clear_grad()
|
||||
|
||||
if global_step % args.eval_step == 0 and rank == 0:
|
||||
evaluate(model, metric, dev_data_loader, "dev")
|
||||
|
||||
if global_step % args.save_step == 0 and rank == 0:
|
||||
save_dir = os.path.join(args.save_dir, "model_%d" % global_step)
|
||||
if not os.path.exists(save_dir):
|
||||
os.makedirs(save_dir)
|
||||
save_param_path = os.path.join(save_dir, 'model_state.pdparams')
|
||||
paddle.save(model.state_dict(), save_param_path)
|
||||
tokenizer.save_pretrained(save_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
do_train()
|
||||
@@ -0,0 +1,187 @@
|
||||
|
||||
**目录**
|
||||
|
||||
* [背景介绍](#背景介绍)
|
||||
* [ERNIE 1.0](#ERNIE1.0)
|
||||
* [1. 技术方案和评估指标](#技术方案)
|
||||
* [2. 环境依赖](#环境依赖)
|
||||
* [3. 代码结构](#代码结构)
|
||||
* [4. 数据准备](#数据准备)
|
||||
* [5. 模型训练](#模型训练)
|
||||
* [6. 模型转换](#模型转换)
|
||||
|
||||
<a name="背景介绍"></a>
|
||||
|
||||
# 背景介绍
|
||||
|
||||
|
||||
ERNIE是百度开创性提出的基于知识增强的持续学习语义理解框架，它将大数据预训练与多源丰富知识相结合，通过持续学习技术，不断吸收海量文本数据中词汇、结构、语义等方面的知识，实现模型效果不断进化。
|
||||
|
||||
ERNIE在情感分析、文本匹配、自然语言推理、词法分析、阅读理解、智能问答等16个公开数据集上全面显著超越世界领先技术，在国际权威的通用语言理解评估基准GLUE上，得分首次突破90分，获得全球第一。
|
||||
相关创新成果也被国际顶级学术会议AAAI、IJCAI收录。
|
||||
同时，ERNIE在工业界得到了大规模应用，如搜索引擎、新闻推荐、广告系统、语音交互、智能客服等。
|
||||
|
||||
本示例采用了全新数据流程，适配了ERNIE预训练任务，具有高效易用，方便快捷的特点。支持动态文本mask，自动断点训练重启等。
|
||||
用户可以根据自己的需求，灵活修改mask方式。具体可以参考`./data_tools/dataset_utils.py`中`create_masked_lm_predictions`函数。
|
||||
用户可以设置`checkpoint_steps`，间隔`checkpoint_steps`数，即保留最新的checkpoint到`model_last`文件夹。重启训练时，程序默认从最新checkpoint重启训练，学习率、数据集都可以恢复到checkpoint时候的状态。
|
||||
|
||||
|
||||
<a name="ERNIE 1.0"></a>
|
||||
|
||||
# ERNIE 1.0
|
||||
|
||||
|
||||
<a name="技术方案"></a>
|
||||
|
||||
## 1. 技术方案和评估指标
|
||||
|
||||
### 技术方案
|
||||
采用ERNIE1.0预训练垂直领域的模型
|
||||
|
||||
|
||||
<a name="环境依赖"></a>
|
||||
|
||||
## 2. 环境依赖和安装说明
|
||||
|
||||
**环境依赖**
|
||||
* python >= 3.6
|
||||
* paddlepaddle >= 2.1.3
|
||||
* paddlenlp >= 2.2
|
||||
* visualdl >=2.2.2
|
||||
* pybind11
|
||||
|
||||
安装命令 `pip install visualdl pybind11`
|
||||
|
||||
<a name="代码结构"></a>
|
||||
|
||||
## 3. 代码结构
|
||||
|
||||
以下是本项目主要代码结构及说明：
|
||||
|
||||
```
|
||||
ERNIE 1.0/
|
||||
|—— scripts
|
||||
|—— run_pretrain_static.sh # 静态图与训练bash脚本
|
||||
├── ernie_static_to_dynamic.py # 静态图转动态图
|
||||
├── run_pretrain_static.py # ernie1.0静态图预训练
|
||||
├── args.py # 预训练的参数配置文件
|
||||
└── data_tools # 预训练数据处理文件目录
|
||||
```
|
||||
|
||||
<a name="数据准备"></a>
|
||||
|
||||
## 4. 数据准备
|
||||
|
||||
数据准备部分请移步[data_tools](./data_tools/)目录，根据文档，创建训练数据。
|
||||
|
||||
## 5. 模型训练
|
||||
|
||||
**领域适应模型下载链接：**
|
||||
|
||||
|Model|训练参数配置|硬件|MD5|
|
||||
| ------------ | ------------ | ------------ |-----------|
|
||||
|[ERNIE 1.0](https://bj.bcebos.com/v1/paddlenlp/models/ernie_pretrain.zip)|<div style="width: 150pt">max_lr:0.0001 min_lr:0.00001 bs:512 max_len:512 </div>|<div style="width: 100pt">4卡 v100-32g</div>|91b0e30c444ab654dd99c0ee354f6a8f|
|
||||
|
||||
### 训练环境说明
|
||||
|
||||
|
||||
- NVIDIA Driver Version: 440.64.00
|
||||
- Ubuntu 16.04.6 LTS (Docker)
|
||||
- Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz
|
||||
|
||||
|
||||
### 单机单卡训练/单机多卡训练
|
||||
|
||||
这里采用单机多卡方式进行训练，通过如下命令，指定 GPU 0,1,2,3 卡, 基于SimCSE训练模型，数据量比较小，几分钟就可以完成。如果采用单机单卡训练，只需要把 `--gpus` 参数设置成单卡的卡号即可
|
||||
|
||||
|
||||
|
||||
### 模型训练
|
||||
|
||||
|
||||
```
|
||||
python -u -m paddle.distributed.launch \
|
||||
--gpus "0,1,2,3" \
|
||||
--log_dir "output/$task_name/log" \
|
||||
run_pretrain_static.py \
|
||||
--model_type "ernie" \
|
||||
--model_name_or_path "ernie-1.0" \
|
||||
--input_dir "./data" \
|
||||
--output_dir "output/$task_name" \
|
||||
--max_seq_len 512 \
|
||||
--micro_batch_size 32 \
|
||||
--global_batch_size 128 \
|
||||
--sharding_degree 1\
|
||||
--dp_degree 4 \
|
||||
--use_sharding false \
|
||||
--use_amp true \
|
||||
--use_recompute false \
|
||||
--max_lr 0.0001 \
|
||||
--min_lr 0.00001 \
|
||||
--max_steps 200000 \
|
||||
--save_steps 100000 \
|
||||
--checkpoint_steps 5000 \
|
||||
--decay_steps 1980000 \
|
||||
--weight_decay 0.01\
|
||||
--warmup_rate 0.01 \
|
||||
--grad_clip 1.0 \
|
||||
--num_workers 2 \
|
||||
--logging_freq 20\
|
||||
--eval_freq 1000 \
|
||||
--device "gpu"
|
||||
```
|
||||
也可以直接运行脚本：
|
||||
|
||||
```
|
||||
sh scripts/run_pretrain_static.sh
|
||||
```
|
||||
|
||||
其中参数释义如下：
|
||||
- `model_name_or_path` 要训练的模型或者之前训练的checkpoint。
|
||||
- `input_dir` 指定输入文件，可以使用目录，指定目录时将包括目录中的所有文件。
|
||||
- `output_dir` 指定输出文件。
|
||||
- `max_seq_len` 输入文本序列的长度。
|
||||
- `micro_batch_size` 单卡单次的 batch size大小。即单张卡运行一次前向网络的 batch size大小。
|
||||
- `global_batch_size` 全局的batch size大小，即一次参数更新等效的batch size。
|
||||
- `sharding_degree` 切参数切分的分组大小（如 sharding_degree=4 表示参数分为4组，分别到4个设备）。
|
||||
- `dp_degree` 数据并行参数。
|
||||
- `use_sharding` 开启sharding策略，sharding_degree > 1时，请设置为True。
|
||||
- `use_amp` 开启混合精度策略。
|
||||
- `use_recompute` 开启重计算策略。暂时未支持，后续将支持。
|
||||
- `max_lr` 训练学习率。
|
||||
- `min_lr` 学习率衰减的最小值。
|
||||
- `max_steps` 最大训练步数。
|
||||
- `save_steps` 保存模型间隔。
|
||||
- `checkpoint_steps` 模型checkpoint间隔，用于模型断点重启训练。
|
||||
- `weight_decay` 权重衰减参数。
|
||||
- `warmup_rate` 学习率warmup参数。
|
||||
- `grad_clip` 梯度裁剪范围。
|
||||
- `logging_freq` 日志输出间隔。
|
||||
- `eval_freq` 模型评估间隔。
|
||||
- `device` 训练设备。
|
||||
|
||||
注：
|
||||
- 一般而言，需要设置 `mp_degree * sharding_degree` = 训练机器的总卡数。
|
||||
- 一般而言， `global_batch_size = micro_batch_size * sharding_degree * dp_degree`。可以使用梯度累积的方式增大`global_batch_size`。设置`global_batch_size`为理论值的整数倍是，默认启用梯度累积。
|
||||
- 训练断点重启，直接启动即可，程序会找到最新的checkpoint，开始重启训练。
|
||||
|
||||
<a name="模型转换"></a>
|
||||
|
||||
## 6. 模型转换
|
||||
|
||||
### 静态图转动态图
|
||||
|
||||
修改代码中的路径：
|
||||
|
||||
```
|
||||
static_model_path="./output/ERNIE 1.0-dp8-gb1024/model_last/static_vars"
|
||||
```
|
||||
然后运行
|
||||
```
|
||||
python ernie_static_to_dynamic.py
|
||||
```
|
||||
运行结束后，动态图的模型就会保存到ernie_checkpoint文件夹里，也可以根据情况，修改代码，保存到自己的指定路径
|
||||
|
||||
### 参考文献
|
||||
|
||||
- [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/pdf/1904.09223.pdf)
|
||||
@@ -0,0 +1,115 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
|
||||
import paddle
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
|
||||
def str2bool(v):
|
||||
if v.lower() in ('yes', 'true', 't', 'y', '1'):
|
||||
return True
|
||||
elif v.lower() in ('no', 'false', 'f', 'n', '0'):
|
||||
return False
|
||||
else:
|
||||
raise argparse.ArgumentTypeError('Unsupported value encountered.')
|
||||
|
||||
|
||||
def parse_args(MODEL_CLASSES):
|
||||
parser = argparse.ArgumentParser()
|
||||
# yapf: disable
|
||||
parser.add_argument("--model_type", default=None, type=str, required=True, help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()), )
|
||||
parser.add_argument("--model_name_or_path", default=None, type=str, required=True,
|
||||
help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(
|
||||
sum([ list(classes[-1].pretrained_init_configuration.keys()) for classes in MODEL_CLASSES.values() ], [])),)
|
||||
|
||||
# Train I/O config
|
||||
parser.add_argument("--input_dir", default=None, type=str, required=True, help="The input directory where the data will be read from.", )
|
||||
parser.add_argument("--output_dir", default=None, type=str, required=True, help="The output directory where the training logs and checkpoints will be written.")
|
||||
parser.add_argument("--split", type=str, default='949,50,1', help="Train/valid/test data split.")
|
||||
|
||||
parser.add_argument("--max_seq_len", type=int, default=1024, help="Max sequence length.")
|
||||
parser.add_argument("--micro_batch_size", default=8, type=int, help="Batch size per device for one step training.", )
|
||||
parser.add_argument("--global_batch_size", default=None, type=int, help="Global batch size for all training process. None for not check the size is valid. If we only use data parallelism, it should be device_num * micro_batch_size.")
|
||||
|
||||
# Default training config
|
||||
parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
|
||||
parser.add_argument("--grad_clip", default=0.0, type=float, help="Grad clip for the parameter.")
|
||||
parser.add_argument("--max_lr", default=1e-5, type=float, help="The initial max learning rate for Adam.")
|
||||
parser.add_argument("--min_lr", default=5e-5, type=float, help="The initial min learning rate for Adam.")
|
||||
parser.add_argument("--warmup_rate", default=0.01, type=float, help="Linear warmup over warmup_steps for learing rate.")
|
||||
|
||||
# Adam optimizer config
|
||||
parser.add_argument("--adam_beta1", default=0.9, type=float, help="The beta1 for Adam optimizer. The exponential decay rate for the 1st moment estimates.")
|
||||
parser.add_argument("--adam_beta2", default=0.999, type=float, help="The bate2 for Adam optimizer. The exponential decay rate for the 2nd moment estimates.")
|
||||
parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
|
||||
|
||||
# Training steps config
|
||||
parser.add_argument("--num_train_epochs", default=1, type=int, help="Total number of training epochs to perform.", )
|
||||
parser.add_argument("--max_steps", default=500000, type=int, help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
|
||||
parser.add_argument("--checkpoint_steps", type=int, default=500, help="Save checkpoint every X updates steps to the model_last folder.")
|
||||
parser.add_argument("--save_steps", type=int, default=500, help="Save checkpoint every X updates steps.")
|
||||
parser.add_argument("--decay_steps", default=360000, type=int, help="The steps use to control the learing rate. If the step > decay_steps, will use the min_lr.")
|
||||
parser.add_argument("--logging_freq", type=int, default=1, help="Log every X updates steps.")
|
||||
parser.add_argument("--eval_freq", type=int, default=500, help="Evaluate for every X updates steps.")
|
||||
parser.add_argument("--eval_iters", type=int, default=10, help="Evaluate the model use X steps data.")
|
||||
|
||||
# Config for 4D Parallelism
|
||||
parser.add_argument("--use_sharding", type=str2bool, nargs='?', const=False, help="Use sharding Parallelism to training.")
|
||||
parser.add_argument("--sharding_degree", type=int, default=1, help="Sharding degree. Share the parameters to many cards.")
|
||||
parser.add_argument("--dp_degree", type=int, default=1, help="Data Parallelism degree.")
|
||||
parser.add_argument("--mp_degree", type=int, default=1, help="Model Parallelism degree. Spliting the linear layers to many cards.")
|
||||
parser.add_argument("--pp_degree", type=int, default=1, help="Pipeline Parallelism degree. Spliting the the model layers to different parts.")
|
||||
parser.add_argument("--use_recompute", type=str2bool, nargs='?', const=False, help="Using the recompute to save the memory.")
|
||||
|
||||
# AMP config
|
||||
parser.add_argument("--use_amp", type=str2bool, nargs='?', const=False, help="Enable mixed precision training.")
|
||||
parser.add_argument("--enable_addto", type=str2bool, nargs='?', const=True, default=True, help="Whether to enable the addto strategy for gradient accumulation or not. This is only used for AMP training.")
|
||||
parser.add_argument("--scale_loss", type=float, default=128, help="The value of scale_loss for fp16. This is only used for AMP training.")
|
||||
parser.add_argument("--hidden_dropout_prob", type=float, default=0.1, help="The hidden dropout prob.")
|
||||
parser.add_argument("--attention_probs_dropout_prob", type=float, default=0.1, help="The attention probs dropout prob.")
|
||||
|
||||
# Other config
|
||||
parser.add_argument("--seed", type=int, default=1234, help="Random seed for initialization.")
|
||||
parser.add_argument("--num_workers", type=int, default=2, help="Num of workers for DataLoader.")
|
||||
parser.add_argument("--check_accuracy", type=str2bool, nargs='?', const=False, help="Check accuracy for training process.")
|
||||
parser.add_argument("--device", type=str, default="gpu", choices=["cpu", "gpu", "xpu"], help="select cpu, gpu, xpu devices.")
|
||||
parser.add_argument("--lr_decay_style", type=str, default="cosine", choices=["cosine", "none"], help="Learning rate decay style.")
|
||||
parser.add_argument("--share_folder", type=str2bool, nargs='?', const=False, help="Use share folder for data dir and output dir on multi machine.")
|
||||
|
||||
# Argument for bert
|
||||
parser.add_argument("--masked_lm_prob", type=float, default=0.15, help="Mask token prob.")
|
||||
parser.add_argument("--short_seq_prob", type=float, default=0.1, help="Short sequence prob.")
|
||||
# yapf: enable
|
||||
|
||||
args = parser.parse_args()
|
||||
args.test_iters = args.eval_iters * 10
|
||||
|
||||
if args.check_accuracy:
|
||||
if args.hidden_dropout_prob != 0:
|
||||
args.hidden_dropout_prob = .0
|
||||
logger.warning(
|
||||
"The hidden_dropout_prob should set to 0 for accuracy checking.")
|
||||
if args.attention_probs_dropout_prob != 0:
|
||||
args.attention_probs_dropout_prob = .0
|
||||
logger.warning(
|
||||
"The attention_probs_dropout_prob should set to 0 for accuracy checking."
|
||||
)
|
||||
|
||||
logger.info('{:20}:{}'.format("paddle commit id", paddle.version.commit))
|
||||
for arg in vars(args):
|
||||
logger.info('{:20}:{}'.format(arg, getattr(args, arg)))
|
||||
|
||||
return args
|
||||
@@ -0,0 +1,9 @@
|
||||
CXXFLAGS += -O3 -Wall -shared -std=c++11 -fPIC -fdiagnostics-color
|
||||
CPPFLAGS += $(shell python3 -m pybind11 --includes)
|
||||
LIBNAME = helpers
|
||||
LIBEXT = $(shell python3-config --extension-suffix)
|
||||
|
||||
default: $(LIBNAME)$(LIBEXT)
|
||||
|
||||
%$(LIBEXT): %.cpp
|
||||
$(CXX) $(CXXFLAGS) $(CPPFLAGS) $< -o $@
|
||||
@@ -0,0 +1,192 @@
|
||||
# PaddleNLP 预训练数据流程
|
||||
|
||||
本示例致力于打造基于PaddleNLP预训练模型的最佳实践。
|
||||
|
||||
我们将预训练数据过程划分为以下部分
|
||||
|
||||
- 原始数据转换，原始文本转换为jsonl的json字符串格式。
|
||||
- 数据ID化，断句、分词、tokenize转化为token id格式。
|
||||
- 训练index文件生成，生成train、valid、test的每个样本索引。
|
||||
- token动态mask(可选)，python 层实时mask文本。
|
||||
|
||||
本目录下主要包含一下文件：
|
||||
```
|
||||
../data_tools
|
||||
├── create_pretraining_data.py
|
||||
├── dataset_utils.py
|
||||
├── ernie_dataset.py
|
||||
├── helpers.cpp
|
||||
├── Makefile
|
||||
├── README.md
|
||||
└── trans_to_json.py
|
||||
```
|
||||
其中，`trans_to_json.py`是原始数据转化的脚本，将数据转化为json串格式。
|
||||
`create_pretraining_data.py`将jsonl文本，断句、分词后，tokenizer转化为token id。
|
||||
`dataset_utils.py`中包含了index生成、动态mask的实现。
|
||||
`ernie_dataset.py`通过调用`dataset_utils.py`的一些函数，产生ernie的输入dataset。
|
||||
|
||||
### 环境依赖
|
||||
|
||||
- tqdm
|
||||
- numpy
|
||||
- pybind11
|
||||
- lac (可选)
|
||||
- zstandard (可选)
|
||||
|
||||
安装命令`pip install tqdm numpy pybind11 lac zstandard`。另，部分功能需要`g++>=4.8`编译支持
|
||||
|
||||
|
||||
## 训练全流程数据Pipeline
|
||||
|
||||
|步骤|阶段|数据格式| 样例|
|
||||
|-|-|-|-|
|
||||
| - |-|原始数据： <br/> 每个doc之间用空行间隔开 <br/> - 中文，默认每句换行符，作为句子结束。<br/> - 英文，默认使用nltk判断句子结束 | ```百度，是一家中国互联网公司。``` <br/> ```百度为用户提供搜索服务。``` <br/><br/> ```PaddleNLP是自然语言处理领域的优秀工具。``` |
|
||||
|原始数据转换<br/>`trans_to_json.py`|预处理|jsonl格式：每个doc对应一行json字符串| ```{"text": "百度是一家中国互联网公司。百度为..."}```<br/>```{"text": "PaddleNLP是自然语言..."}```
|
||||
|数据ID化<br/>`create_pretrain_data.py`|预处理| npy格式：数据id化后的token id <br/>npz格式：数据句子、文章位置索引 | -
|
||||
|训练index文件生成|训练启动|npy格式：<br/> 根据训练步数max_steps生成<br/>train、valid、test的每个样本索引文件| -
|
||||
|token动态mask（可选）| Dataset取数据 | 无 |-
|
||||
|
||||
|
||||
## ERNIE预训练例子
|
||||
|
||||
下面以ERNIE预训练为例，简要介绍一下预训练的全流程。
|
||||
|
||||
### 原始数据
|
||||
首先下载样例数据：
|
||||
```
|
||||
mkdir data && cd data
|
||||
wget https://paddlenlp.bj.bcebos.com/models/transformers/data_tools/baike.txt
|
||||
cd ..
|
||||
```
|
||||
### 原始数据转换 jsonl 格式
|
||||
使用`trans_to_json.py`转化为json串格式，下面是脚本的使用说明
|
||||
```
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--input_path INPUT_PATH
|
||||
Path to you raw files. Folder or file path.
|
||||
必须设置，可以是文件夹或者单个文件。文件夹中的目录默认最多搜索两层子目录。
|
||||
--output_path OUTPUT_PATH
|
||||
Path to save the output json files.
|
||||
必须设置，输出文件的名字。
|
||||
--json_key JSON_KEY The content key of json file.
|
||||
建议不修改，默认的key是text
|
||||
--doc_spliter DOC_SPLITER
|
||||
Spliter between documents. We will strip the line, if you use blank line to split doc, leave it blank.
|
||||
根据实际情况修改，默认空行作为文章换行符。
|
||||
--min_doc_length MIN_DOC_LENGTH
|
||||
Minimal char of a documment.
|
||||
可选。过滤掉长度多短的文章，默认值10
|
||||
--workers WORKERS Number of worker processes to launch
|
||||
可选。多进程转化文件，适用于 input_path 中包含的文件数据较多的情况。每个文件，分配给不同worker处理
|
||||
--log_interval LOG_INTERVAL
|
||||
Interval between progress updates.
|
||||
可选。此处的interval是值处理完文件个数的间隔。
|
||||
--no-merge Don't merge the file.
|
||||
可选。默认不开启这个选项，默认每个文件转换的jsonl文本，会拼接成到同一个文件。
|
||||
--no-shuffle Don't shuffle the file.
|
||||
可选。默认不开启这个选项，默认对处理完进行shuffle。
|
||||
```
|
||||
根据说明，我们使用下面简单命令，可以得到`baike_sample.jsonl`文件。此处，我们对文章所有doc进行了shuffle。
|
||||
```shell
|
||||
python trans_to_json.py --input_path ./data --output_path baike_sample
|
||||
|
||||
#查看数据
|
||||
head -1 baike_sample.jsonl
|
||||
{"text": "百度手机助手：最具人气的应用商店\n百度手机助手是Android手机的权威资源平台，分发市场份额连续十个季度排名市场第一，拥有最全最好的应用、游戏、壁纸资源，帮助用户在海量资源中精准搜索、高速下载、轻松管理，万千汇聚，一触即得。\n"}
|
||||
```
|
||||
|
||||
### 数据ID化
|
||||
本部分，我们使用 `create_pretraining_data.py` 脚本将前面得到的 `baike_sample.jsonl` 进行tokenize id化处理。
|
||||
```
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--model_name MODEL_NAME
|
||||
What model to use.
|
||||
必须设置，如：ernie-1.0, 可以参考已有的模型名称 https://github.com/PaddlePaddle/PaddleNLP/blob/develop/docs/model_zoo/transformers.rst
|
||||
--tokenizer_name {ErnieTokenizer,BertTokenizer,GPTTokenizer,GPTChineseTokenizer}
|
||||
What type of tokenizer to use.
|
||||
模型对应的tokenizer, 目前暂时只支持 Ernie，Bert，GPT
|
||||
data input/output:
|
||||
--input_path INPUT_PATH
|
||||
Path to input JSON files.
|
||||
必须设置，输入文件jsonl的目录
|
||||
--output_prefix OUTPUT_PREFIX
|
||||
Output prefix to store output file.
|
||||
必须设置，输出文件的名称。
|
||||
假设名称为XXX，则会输出 XXX_ids.npy, XXX_idx.npz 两个文件。
|
||||
npy文件，数据id化后的token ids; npz文件，数据句子、文章位置索引。
|
||||
--data_format {JSON} Only support json format for now. One document per line.
|
||||
不需要设置。目前默认处理jsonl数据格式
|
||||
--json_key JSON_KEY For JSON format. Space separate listed of keys to extract from json
|
||||
文本串json的key值。同前面trans_to_json.py的json_key，默认text为key
|
||||
--split_sentences Split documents into sentences.
|
||||
是否需要将文章划分成句子。一般而言，GPT不需要，Bert/Ernie模型需要
|
||||
|
||||
chinese words:
|
||||
--chinese Is corpus need words segmentation step for chinese words.
|
||||
中文情形必须设置。处理的文本类型是否是中文。
|
||||
--cn_whole_word_segment
|
||||
Is corpus need words segmentation step for chinese words WWM.
|
||||
可选。是否需要WWM策略。一般而言，Bert/Ernie模型需要，GPT不需要。
|
||||
--cn_seg_func {lac,seg,jieba}
|
||||
Words segment function for chinese words.
|
||||
默认lac，jieba速度较快
|
||||
--cn_splited Is chinese corpus is splited in to words.
|
||||
分词后的文本，可选。设置此选项则，cn_seg_func不起作用。
|
||||
例如分词后文本串 "百度 手机助手 是 Android 手机 的 权威 资源平台"
|
||||
--cn_split_dimer CN_SPLIT_DIMER
|
||||
Split dimer between chinese words.
|
||||
配合cn_splited使用，默认空格表示分词间隔。
|
||||
|
||||
common config:
|
||||
--append_eos Append an <eos> token to the end of a document.
|
||||
gpt模型专用，gpt设置此选项，表示doc结束。
|
||||
--log_interval LOG_INTERVAL
|
||||
Interval between progress updates
|
||||
打印日志间隔，interval表示处理 文本行数/doc数的 间隔。
|
||||
--workers WORKERS Number of worker processes to launch
|
||||
处理文本id化的进程个数。
|
||||
```
|
||||
同过下面脚本转化，我们可以得到处理好的预训练数据，token ids:`baike_sample_ids.npy`, 文章索引信息`baike_sample_idx.npz`.
|
||||
```
|
||||
python -u create_pretraining_data.py \
|
||||
--model_name ernie-1.0 \
|
||||
--tokenizer_name ErnieTokenizer \
|
||||
--input_path baike_sample.jsonl \
|
||||
--split_sentences\
|
||||
--chinese \
|
||||
--cn_whole_word_segment \
|
||||
--output_prefix baike_sample \
|
||||
--workers 1 \
|
||||
--log_interval 5
|
||||
```
|
||||
|
||||
### Ernie预训练开始
|
||||
得到了处理好的训练数据，就可以开始Ernie模型的预训练了。ernie预训练的代码在`examples/language_model/ernie-1.0`。
|
||||
简单将预处理好的数据，拷贝到data目录，即可开始Ernie模型预训练。
|
||||
```
|
||||
cd ..
|
||||
mkdir data
|
||||
mv ./data_tools/baike_sample* ./data 或者运行 sh mv_data.sh
|
||||
sh run_static.sh
|
||||
# 建议修改 run_static.sh 中的配置，将max_steps设置小一些。
|
||||
```
|
||||
代码说明：
|
||||
|
||||
- ernie预训练使用的 dataset 代码文件在 `./data_tools/ernie_dataset.py`
|
||||
- 数据集index生成，动态mask相关代码实现在`./data_tools/dataset_utils.py`
|
||||
|
||||
用户可以根据自己的需求，灵活修改mask方式。具体可以参考`dataset_utils.py`中`create_masked_lm_predictions`函数。
|
||||
可以自定义的选项有do_whole_word_mask, favor_longer_ngram, do_permutation, geometric_dist等，
|
||||
可以参考[Megatron](https://github.com/NVIDIA/Megatron-LM)使用这些lm_mask策略。
|
||||
|
||||
### FAQ
|
||||
|
||||
#### C++代码编译失败怎么办？
|
||||
- 请先检查pybind11包是否安装，g++、make工具是否正常。
|
||||
- 编译失败可能是本文件夹下的Makefile命令出现了一些问题。可以将Makefile中的python3、python3-config设置成完全的路径，如/usr/bin/python3.7。
|
||||
|
||||
## 参考内容
|
||||
|
||||
注: 大部分数据流程，参考自[Megatron](https://github.com/NVIDIA/Megatron-LM)，特此表达感谢。
|
||||
@@ -0,0 +1,14 @@
|
||||
# wanfangdata/
|
||||
# python trans_to_json.py --input_path ./data --output_path baike_sample
|
||||
python trans_to_json.py --input_path ./wanfangdata --output_path baike_sample
|
||||
|
||||
python -u create_pretraining_data.py \
|
||||
--model_name ernie-1.0 \
|
||||
--tokenizer_name ErnieTokenizer \
|
||||
--input_path baike_sample.jsonl \
|
||||
--split_sentences\
|
||||
--chinese \
|
||||
--cn_whole_word_segment \
|
||||
--output_prefix baike_sample \
|
||||
--workers 1 \
|
||||
--log_interval 5
|
||||
@@ -0,0 +1,398 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import io
|
||||
import re
|
||||
import argparse
|
||||
import json
|
||||
import multiprocessing
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
import paddlenlp.transformers as tfs
|
||||
|
||||
try:
|
||||
import nltk
|
||||
nltk_available = True
|
||||
except ImportError:
|
||||
nltk_available = False
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'--model_name', type=str, required=True, help='What model to use.')
|
||||
parser.add_argument(
|
||||
'--tokenizer_name',
|
||||
type=str,
|
||||
required=True,
|
||||
choices=[
|
||||
'ErnieTokenizer', 'BertTokenizer', 'GPTTokenizer',
|
||||
'GPTChineseTokenizer'
|
||||
],
|
||||
help='What type of tokenizer to use.')
|
||||
group = parser.add_argument_group(title='data input/output')
|
||||
group.add_argument(
|
||||
'--input_path',
|
||||
type=str,
|
||||
required=True,
|
||||
help='Path to input JSON files.')
|
||||
group.add_argument(
|
||||
'--output_prefix',
|
||||
type=str,
|
||||
required=True,
|
||||
help='Output prefix to store output file.')
|
||||
group.add_argument(
|
||||
'--data_format',
|
||||
type=str,
|
||||
default='text',
|
||||
choices=['JSON'],
|
||||
help='Only support json format for now. One document per line.')
|
||||
group.add_argument(
|
||||
'--json_key',
|
||||
type=str,
|
||||
default='text',
|
||||
help='For JSON format. Space separate listed of keys to extract from json'
|
||||
)
|
||||
group.add_argument(
|
||||
'--split_sentences',
|
||||
action='store_true',
|
||||
help='Split documents into sentences.')
|
||||
|
||||
group = parser.add_argument_group(title='chinese words')
|
||||
group.add_argument(
|
||||
'--chinese',
|
||||
action='store_true',
|
||||
help="Is corpus need words segmentation step for chinese words.")
|
||||
group.add_argument(
|
||||
'--cn_whole_word_segment',
|
||||
action='store_true',
|
||||
help="Is corpus need words segmentation step for chinese words WWM.")
|
||||
group.add_argument(
|
||||
'--cn_seg_func',
|
||||
type=str,
|
||||
default='lac',
|
||||
choices=['lac', 'seg', 'jieba'],
|
||||
help='Words segment function for chinese words.')
|
||||
group.add_argument(
|
||||
'--cn_splited',
|
||||
action='store_true',
|
||||
help="Is chinese corpus is splited in to words.")
|
||||
group.add_argument(
|
||||
'--cn_split_dimer',
|
||||
type=str,
|
||||
default=' ',
|
||||
help="Split dimer between chinese words.")
|
||||
|
||||
group = parser.add_argument_group(title='common config')
|
||||
group.add_argument(
|
||||
'--append_eos',
|
||||
action='store_true',
|
||||
help='Append an <eos> token to the end of a document.')
|
||||
group.add_argument(
|
||||
'--log_interval',
|
||||
type=int,
|
||||
default=100,
|
||||
help='Interval between progress updates')
|
||||
group.add_argument(
|
||||
'--workers',
|
||||
type=int,
|
||||
default=1,
|
||||
help='Number of worker processes to launch')
|
||||
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def lexical_analysis_fn():
|
||||
from LAC import LAC
|
||||
lac = LAC(mode="lac")
|
||||
|
||||
def process(line):
|
||||
words, _ = lac.run(line)
|
||||
return words
|
||||
|
||||
return process
|
||||
|
||||
|
||||
def chinese_segmentation_fn():
|
||||
from LAC import LAC
|
||||
lac_cws = LAC(mode='seg')
|
||||
|
||||
def process(line):
|
||||
words = lac.run(line)
|
||||
return words
|
||||
|
||||
return process
|
||||
|
||||
|
||||
def jieba_segmentation_fn():
|
||||
import jieba
|
||||
|
||||
def process(line):
|
||||
words = jieba.cut(line)
|
||||
return list(words)
|
||||
|
||||
return process
|
||||
|
||||
|
||||
CHINESE_SEG_FUNC = {
|
||||
'lac': lexical_analysis_fn(),
|
||||
'seg': chinese_segmentation_fn(),
|
||||
'jieba': jieba_segmentation_fn(),
|
||||
}
|
||||
|
||||
|
||||
def get_whole_word_mask_tokens(tokens, words, max_word_length=4):
|
||||
"""
|
||||
Do whole word mask on Chinese word.
|
||||
First, we do Chinese word segmentation on the sequence of tokens, which are from the WordPiece tokenization.
|
||||
Then, we add the '##' mark on chinese characters which are in the middle of Chinese words.
|
||||
And if the tokens are not chinese characters, we just exploit the results of WordPiece tokenization as words.
|
||||
Such as,
|
||||
- text line : 通过利用mercer核，将样本从输入空间映射到高维特征空间，使原来没有显现的特征突现出来，取得了很好的图像分割效果。
|
||||
- the input tokens (after WordPiece):
|
||||
['通', '过', '利', '用', 'me', '##rc', '##er', '核', '，', '将', '样', '本', '从', '输', '入', '空', '间', '映',
|
||||
'射', '到', '高', '维', '特', '征', '空', '间', '，', '使', '原', '来', '没', '有', '显', '现', '的', '特', '征',
|
||||
'突', '现', '出', '来', '，', '取', '得', '了', '很', '好', '的', '图', '像', '分', '割', '效', '果', '。']
|
||||
- the Chinese words (after Chinese word segmentation like jieba)
|
||||
['通过', '利用', 'mercer', '核', '，', '将', '样本', '从', '输入', '空间', '映射', '到', '高维', '特征',
|
||||
'空间', '，', '使', '原来', '没有', '显现', '的', '特征', '突现', '出来', '，', '取得', '了', '很', '好',
|
||||
'的', '图像', '分割', '效果', '。']
|
||||
- the output whole word mask tokens:
|
||||
['通', '##过', '利', '##用', 'me', '##rc', '##er', '核', '，', '将', '样', '##本', '从', '输', '##入',
|
||||
'空', '##间', '映', '##射', '到', '高', '##维', '特', '##征', '空', '##间', '，', '使', '原', '##来',
|
||||
'没', '##有', '显', '##现', '的', '特', '##征', '突', '##现', '出', '##来', '，', '取', '##得', '了',
|
||||
'很', '好', '的', '图', '##像', '分', '##割', '效', '##果', '。']
|
||||
|
||||
Args:
|
||||
tokens(list(str)): The sequence of tokens, which are from the WordPiece tokenization.
|
||||
words(list(str)): The sequence of Chinese words.
|
||||
max_word_length(int, optional):
|
||||
The maximum chinese character in Chinese words. It avoids too long Chinese word to be masked.
|
||||
Defaults as 4.
|
||||
|
||||
Returns:
|
||||
new_tokens(list(str)): The new token will be done with whole word masking strategy.
|
||||
|
||||
"""
|
||||
|
||||
new_tokens = []
|
||||
# opt for long document
|
||||
words_set = set(words)
|
||||
i = 0
|
||||
while i < len(tokens):
|
||||
# non-chinese character, then do word piece
|
||||
if len(re.findall('[\u4E00-\u9FA5]', tokens[i])) == 0:
|
||||
new_tokens.append(tokens[i])
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# add "##" mark on the middel tokens of Chinese words
|
||||
# such as ["通过", "利用"] -> ["通", "##过"， "利", "##用"]
|
||||
has_add = False
|
||||
for length in range(max_word_length, 0, -1):
|
||||
if i + length > len(tokens):
|
||||
continue
|
||||
if ''.join(tokens[i:i + length]) in words_set:
|
||||
new_tokens.append(tokens[i])
|
||||
for l in range(1, length):
|
||||
new_tokens.append('##' + tokens[i + l])
|
||||
i += length
|
||||
has_add = True
|
||||
break
|
||||
|
||||
if not has_add:
|
||||
new_tokens.append(tokens[i])
|
||||
i += 1
|
||||
return new_tokens
|
||||
|
||||
|
||||
class IdentitySplitter(object):
|
||||
def tokenize(self, *text):
|
||||
return text
|
||||
|
||||
|
||||
class NewlineSplitter():
|
||||
def tokenize(self, text):
|
||||
return text.split("\n")
|
||||
|
||||
|
||||
class Converter(object):
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
|
||||
def initializer(self):
|
||||
Converter.tokenizer = getattr(
|
||||
tfs, self.args.tokenizer_name).from_pretrained(self.args.model_name)
|
||||
|
||||
# Split document to sentence.
|
||||
if self.args.split_sentences:
|
||||
if self.args.chinese:
|
||||
Converter.splitter = NewlineSplitter()
|
||||
else:
|
||||
if not nltk_available:
|
||||
print("NLTK is not available to split sentences.")
|
||||
exit()
|
||||
splitter = nltk.load("tokenizers/punkt/english.pickle")
|
||||
Converter.splitter = splitter
|
||||
else:
|
||||
Converter.splitter = IdentitySplitter()
|
||||
|
||||
# Split sentence whole words mask for chinese
|
||||
if self.args.cn_whole_word_segment:
|
||||
if self.args.cn_splited:
|
||||
Converter.segment_func = lambda text: text.split(self.args.cn_split_dimer)
|
||||
else:
|
||||
Converter.segment_func = CHINESE_SEG_FUNC[self.args.cn_seg_func]
|
||||
Converter.whole_word_mask = get_whole_word_mask_tokens
|
||||
else:
|
||||
Converter.segment_func = lambda x: x
|
||||
Converter.whole_word_mask = lambda x, y: x
|
||||
|
||||
def process(text):
|
||||
words = Converter.segment_func(text)
|
||||
tokens = Converter.tokenizer.tokenize("".join(words))
|
||||
tokens = Converter.whole_word_mask(tokens, words)
|
||||
tokens = Converter.tokenizer.convert_tokens_to_ids(tokens)
|
||||
return tokens
|
||||
|
||||
Converter.process = process
|
||||
|
||||
def encode(self, json_line):
|
||||
text = json.loads(json_line)[self.args.json_key]
|
||||
doc_ids = []
|
||||
for sentence in Converter.splitter.tokenize(text):
|
||||
sentence_ids = Converter.process(sentence.strip())
|
||||
if len(sentence_ids) > 0:
|
||||
doc_ids.append(sentence_ids)
|
||||
|
||||
if len(doc_ids) > 0 and self.args.append_eos:
|
||||
doc_ids[-1].append(Converter.tokenizer.eos_token_id)
|
||||
|
||||
return doc_ids, len(text.encode("utf-8"))
|
||||
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
|
||||
file_paths = []
|
||||
if os.path.isfile(args.input_path):
|
||||
file_paths.append(args.input_path)
|
||||
else:
|
||||
for root, _, fs in os.walk(args.input_path):
|
||||
for f in fs:
|
||||
file_paths.append(os.path.join(root, f))
|
||||
convert = Converter(args)
|
||||
|
||||
# Try tokenizer is availiable
|
||||
sample_tokenizer = getattr(
|
||||
tfs, args.tokenizer_name).from_pretrained(args.model_name)
|
||||
if sample_tokenizer.vocab_size < 2**16 - 1:
|
||||
save_dtype = np.uint16
|
||||
else:
|
||||
save_dtype = np.int32
|
||||
|
||||
pool = multiprocessing.Pool(args.workers, initializer=convert.initializer)
|
||||
|
||||
# We use BytesIO to store the ids.
|
||||
token_ids_stream = io.BytesIO()
|
||||
sentlens_stream = io.BytesIO()
|
||||
# Cumsum on tokens num
|
||||
sent_cumsum_stream = io.BytesIO()
|
||||
sent_cumsum_stream.write((0).to_bytes(8, byteorder='little', signed=True))
|
||||
# Cunsum on document on every sentence num, type=np.int64
|
||||
doc_cumsum_stream = io.BytesIO()
|
||||
doc_cumsum_stream.write((0).to_bytes(8, byteorder='little', signed=True))
|
||||
|
||||
sent_count = 0
|
||||
token_count = 0
|
||||
|
||||
file_paths.sort()
|
||||
|
||||
step = 0
|
||||
total_bytes_processed = 0
|
||||
startup_start = time.time()
|
||||
for file_path in tqdm(file_paths):
|
||||
if file_path.endswith(".zst"):
|
||||
import zstandard
|
||||
cctx = zstandard.ZstdDecompressor()
|
||||
fh = open(file_path, 'rb')
|
||||
text = io.BufferedReader(cctx.stream_reader(fh))
|
||||
elif file_path.endswith(".jsonl"):
|
||||
text = open(file_path, 'r', encoding='utf-8')
|
||||
else:
|
||||
print("Unexpected data format, skiped %s" % file_path)
|
||||
continue
|
||||
|
||||
encoded_docs = pool.imap(convert.encode, text, 256)
|
||||
print("Processing %s" % file_path)
|
||||
for i, (doc, bytes_processed) in enumerate(encoded_docs, start=1):
|
||||
step += 1
|
||||
total_bytes_processed += bytes_processed
|
||||
if len(doc) == 0:
|
||||
continue
|
||||
|
||||
for sentence in doc:
|
||||
sentence_len = len(sentence)
|
||||
if sentence_len == 0:
|
||||
continue
|
||||
sentlens_stream.write(
|
||||
sentence_len.to_bytes(
|
||||
4, byteorder='little', signed=True))
|
||||
token_count += sentence_len
|
||||
sent_cumsum_stream.write(
|
||||
token_count.to_bytes(
|
||||
8, byteorder='little', signed=True))
|
||||
sent_count += 1
|
||||
token_ids_stream.write(
|
||||
np.array(
|
||||
sentence, dtype=save_dtype).tobytes(order='C'))
|
||||
|
||||
doc_cumsum_stream.write(
|
||||
sent_count.to_bytes(
|
||||
8, byteorder='little', signed=True))
|
||||
|
||||
if step % args.log_interval == 0:
|
||||
current = time.time()
|
||||
elapsed = current - startup_start
|
||||
mbs = total_bytes_processed / elapsed / 1024 / 1024
|
||||
print(
|
||||
f"Processed {step} documents",
|
||||
f"({step/elapsed:.2f} docs/s, {mbs:.4f} MB/s).",
|
||||
file=sys.stderr)
|
||||
|
||||
pool.close()
|
||||
print("Saving tokens to files...")
|
||||
all_doc_ids = np.frombuffer(token_ids_stream.getbuffer(), dtype=save_dtype)
|
||||
lens = np.frombuffer(sentlens_stream.getbuffer(), dtype=np.int32)
|
||||
sents = np.frombuffer(sent_cumsum_stream.getbuffer(), dtype=np.int64)
|
||||
docs = np.frombuffer(doc_cumsum_stream.getbuffer(), dtype=np.int64)
|
||||
np.save(args.output_prefix + "_ids.npy", all_doc_ids)
|
||||
np.savez(args.output_prefix + "_idx.npz", lens=lens, sents=sents, docs=docs)
|
||||
|
||||
print("Total sentences num: %d" % len(lens))
|
||||
print("Total documents num: %d" % (len(docs) - 1))
|
||||
print("Total tokens num: %d" % len(all_doc_ids))
|
||||
print("Average tokens per sentence: %.2f" % (len(all_doc_ids) / len(lens)))
|
||||
print("Average tokens per document: %.2f" % (len(all_doc_ids) /
|
||||
(len(docs) - 1)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,775 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The Google AI Language Team Authors, and NVIDIA, and PaddlePaddle Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Most of the code here has been copied from:
|
||||
# https://github.com/google-research/albert/blob/master/create_pretraining_data.py
|
||||
# with some modifications.
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import collections
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.distributed.fleet as fleet
|
||||
|
||||
print_rank_0 = print
|
||||
|
||||
#from megatron.data.indexed_dataset import make_dataset as make_indexed_dataset
|
||||
COMPILED = False
|
||||
DSET_TYPE_BERT = 'standard_bert'
|
||||
DSET_TYPE_T5 = 't5'
|
||||
DSET_TYPE_ERNIE = 'ernie'
|
||||
|
||||
DSET_TYPES = [DSET_TYPE_BERT, DSET_TYPE_T5, DSET_TYPE_ERNIE]
|
||||
|
||||
|
||||
class MMapIndexedDataset(paddle.io.Dataset):
|
||||
def __init__(self, path, skip_warmup=False):
|
||||
super().__init__()
|
||||
|
||||
self._path = path
|
||||
|
||||
# All documment ids, extend as 1-D array.
|
||||
|
||||
for suffix in ["_ids.npy", "_idx.npz"]:
|
||||
if not os.path.isfile(path + suffix):
|
||||
raise ValueError("File Not found, %s" % (path + suffix))
|
||||
|
||||
self._token_ids = np.load(
|
||||
path + "_ids.npy", mmap_mode="r", allow_pickle=True)
|
||||
process_datas = np.load(path + "_idx.npz")
|
||||
self._sizes = process_datas["lens"]
|
||||
self._pointers = process_datas["sents"]
|
||||
self._doc_idx = process_datas["docs"]
|
||||
|
||||
def __getstate__(self):
|
||||
return self._path
|
||||
|
||||
def __len__(self):
|
||||
return len(self._sizes)
|
||||
|
||||
# @lru_cache(maxsize=8)
|
||||
def __getitem__(self, idx):
|
||||
if isinstance(idx, int):
|
||||
size = self._sizes[idx]
|
||||
ptr = self._pointers[idx]
|
||||
np_array = self._token_ids[ptr:ptr + size]
|
||||
return np_array
|
||||
|
||||
elif isinstance(idx, slice):
|
||||
start, stop, step = idx.indices(len(self))
|
||||
if step != 1:
|
||||
raise ValueError(
|
||||
"Slices into indexed_dataset must be contiguous")
|
||||
ptr = self._pointers[start]
|
||||
sizes = self._sizes[idx]
|
||||
offsets = list(accumulate(sizes))
|
||||
total_size = sum(sizes)
|
||||
np_array = self._token_ids[ptr:ptr + total_size]
|
||||
sents = np.split(np_array, offsets[:-1])
|
||||
return sents
|
||||
|
||||
def get(self, idx, offset=0, length=None):
|
||||
""" Retrieves a single item from the dataset with the option to only
|
||||
return a portion of the item.
|
||||
|
||||
get(idx) is the same as [idx] but get() does not support slicing.
|
||||
"""
|
||||
size = self._sizes[idx]
|
||||
ptr = self._pointers[idx]
|
||||
|
||||
if length is None:
|
||||
length = size - offset
|
||||
ptr += offset
|
||||
np_array = self._token_ids[ptr:prt + length]
|
||||
return np_array
|
||||
|
||||
@property
|
||||
def sizes(self):
|
||||
return self._sizes
|
||||
|
||||
@property
|
||||
def doc_idx(self):
|
||||
return self._doc_idx
|
||||
|
||||
def get_doc_idx(self):
|
||||
return self._doc_idx
|
||||
|
||||
def set_doc_idx(self, doc_idx_):
|
||||
self._doc_idx = doc_idx_
|
||||
|
||||
|
||||
def make_indexed_dataset(data_prefix, data_impl=None, skip_warmup=False):
|
||||
return MMapIndexedDataset(data_prefix)
|
||||
|
||||
|
||||
def compile_helper():
|
||||
"""Compile helper function ar runtime. Make sure this
|
||||
is invoked on a single process."""
|
||||
import os
|
||||
import subprocess
|
||||
path = os.path.abspath(os.path.dirname(__file__))
|
||||
ret = subprocess.run(['make', '-C', path])
|
||||
if ret.returncode != 0:
|
||||
print("Making C++ dataset helpers module failed, exiting.")
|
||||
import sys
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def get_a_and_b_segments(sample, np_rng):
|
||||
"""Divide sample into a and b segments."""
|
||||
|
||||
# Number of sentences in the sample.
|
||||
n_sentences = len(sample)
|
||||
# Make sure we always have two sentences.
|
||||
assert n_sentences > 1, 'make sure each sample has at least two sentences.'
|
||||
|
||||
# First part:
|
||||
# `a_end` is how many sentences go into the `A`.
|
||||
a_end = 1
|
||||
if n_sentences >= 3:
|
||||
# Note that randin in numpy is exclusive.
|
||||
a_end = np_rng.randint(1, n_sentences)
|
||||
tokens_a = []
|
||||
for j in range(a_end):
|
||||
tokens_a.extend(sample[j])
|
||||
|
||||
# Second part:
|
||||
tokens_b = []
|
||||
for j in range(a_end, n_sentences):
|
||||
tokens_b.extend(sample[j])
|
||||
|
||||
# Random next:
|
||||
is_next_random = False
|
||||
if np_rng.random() < 0.5:
|
||||
is_next_random = True
|
||||
tokens_a, tokens_b = tokens_b, tokens_a
|
||||
|
||||
return tokens_a, tokens_b, is_next_random
|
||||
|
||||
|
||||
def truncate_segments(tokens_a, tokens_b, len_a, len_b, max_num_tokens, np_rng):
|
||||
"""Truncates a pair of sequences to a maximum sequence length."""
|
||||
#print(len_a, len_b, max_num_tokens)
|
||||
assert len_a > 0
|
||||
if len_a + len_b <= max_num_tokens:
|
||||
return False
|
||||
while len_a + len_b > max_num_tokens:
|
||||
if len_a > len_b:
|
||||
len_a -= 1
|
||||
tokens = tokens_a
|
||||
else:
|
||||
len_b -= 1
|
||||
tokens = tokens_b
|
||||
if np_rng.random() < 0.5:
|
||||
del tokens[0]
|
||||
else:
|
||||
tokens.pop()
|
||||
return True
|
||||
|
||||
|
||||
def create_tokens_and_tokentypes(tokens_a, tokens_b, cls_id, sep_id):
|
||||
"""Merge segments A and B, add [CLS] and [SEP] and build tokentypes."""
|
||||
|
||||
tokens = []
|
||||
tokentypes = []
|
||||
# [CLS].
|
||||
tokens.append(cls_id)
|
||||
tokentypes.append(0)
|
||||
# Segment A.
|
||||
for token in tokens_a:
|
||||
tokens.append(token)
|
||||
tokentypes.append(0)
|
||||
# [SEP].
|
||||
tokens.append(sep_id)
|
||||
tokentypes.append(0)
|
||||
# Segment B.
|
||||
for token in tokens_b:
|
||||
tokens.append(token)
|
||||
tokentypes.append(1)
|
||||
if tokens_b:
|
||||
# [SEP].
|
||||
tokens.append(sep_id)
|
||||
tokentypes.append(1)
|
||||
|
||||
return tokens, tokentypes
|
||||
|
||||
|
||||
MaskedLmInstance = collections.namedtuple("MaskedLmInstance",
|
||||
["index", "label"])
|
||||
|
||||
|
||||
def is_start_piece(piece):
|
||||
"""Check if the current word piece is the starting piece (BERT)."""
|
||||
# When a word has been split into
|
||||
# WordPieces, the first token does not have any marker and any subsequence
|
||||
# tokens are prefixed with ##. So whenever we see the ## token, we
|
||||
# append it to the previous set of word indexes.
|
||||
return not piece.startswith("##")
|
||||
|
||||
|
||||
def create_masked_lm_predictions(tokens,
|
||||
vocab_id_list,
|
||||
vocab_id_to_token_dict,
|
||||
masked_lm_prob,
|
||||
cls_id,
|
||||
sep_id,
|
||||
mask_id,
|
||||
max_predictions_per_seq,
|
||||
np_rng,
|
||||
max_ngrams=3,
|
||||
vocab_token_to_id_dict=None,
|
||||
do_whole_word_mask=True,
|
||||
favor_longer_ngram=False,
|
||||
do_permutation=False,
|
||||
geometric_dist=False,
|
||||
to_chinese_char=False,
|
||||
inplace_random_mask=False,
|
||||
masking_style="bert"):
|
||||
"""Creates the predictions for the masked LM objective.
|
||||
Note: Tokens here are vocab ids and not text tokens."""
|
||||
|
||||
cand_indexes = []
|
||||
# Note(mingdachen): We create a list for recording if the piece is
|
||||
# the starting piece of current token, where 1 means true, so that
|
||||
# on-the-fly whole word masking is possible.
|
||||
token_boundary = [0] * len(tokens)
|
||||
|
||||
for (i, token) in enumerate(tokens):
|
||||
if token == cls_id or token == sep_id:
|
||||
token_boundary[i] = 1
|
||||
continue
|
||||
# Whole Word Masking means that if we mask all of the wordpieces
|
||||
# corresponding to an original word.
|
||||
#
|
||||
# Note that Whole Word Masking does *not* change the training code
|
||||
# at all -- we still predict each WordPiece independently, softmaxed
|
||||
# over the entire vocabulary.
|
||||
vocab_id = vocab_id_to_token_dict[token]
|
||||
if (do_whole_word_mask and len(cand_indexes) >= 1 and
|
||||
not is_start_piece(vocab_id)):
|
||||
cand_indexes[-1].append(i)
|
||||
else:
|
||||
cand_indexes.append([i])
|
||||
if is_start_piece(vocab_id_to_token_dict[token]):
|
||||
token_boundary[i] = 1
|
||||
|
||||
if to_chinese_char:
|
||||
char_tokens = []
|
||||
assert vocab_token_to_id_dict is not None
|
||||
for i, b in enumerate(token_boundary):
|
||||
if b == 0:
|
||||
vocab_id = vocab_id_to_token_dict[tokens[i]]
|
||||
new_vocab_id = vocab_id[2:] if len(
|
||||
re.findall('##[\u4E00-\u9FA5]', vocab_id)) > 0 else vocab_id
|
||||
char_tokens.append(vocab_token_to_id_dict[new_vocab_id]
|
||||
if new_vocab_id in vocab_token_to_id_dict
|
||||
else token)
|
||||
else:
|
||||
char_tokens.append(tokens[i])
|
||||
output_tokens = list(char_tokens)
|
||||
else:
|
||||
output_tokens = list(tokens)
|
||||
|
||||
masked_lm_positions = []
|
||||
masked_lm_labels = []
|
||||
|
||||
if masked_lm_prob == 0:
|
||||
return (output_tokens, masked_lm_positions, masked_lm_labels,
|
||||
token_boundary)
|
||||
|
||||
num_to_predict = min(max_predictions_per_seq,
|
||||
max(1, int(round(len(tokens) * masked_lm_prob))))
|
||||
|
||||
ngrams = np.arange(1, max_ngrams + 1, dtype=np.int64)
|
||||
if not geometric_dist:
|
||||
# Note(mingdachen):
|
||||
# By default, we set the probilities to favor shorter ngram sequences.
|
||||
pvals = 1. / np.arange(1, max_ngrams + 1)
|
||||
pvals /= pvals.sum(keepdims=True)
|
||||
if favor_longer_ngram:
|
||||
pvals = pvals[::-1]
|
||||
|
||||
ngram_indexes = []
|
||||
for idx in range(len(cand_indexes)):
|
||||
ngram_index = []
|
||||
for n in ngrams:
|
||||
ngram_index.append(cand_indexes[idx:idx + n])
|
||||
ngram_indexes.append(ngram_index)
|
||||
|
||||
np_rng.shuffle(ngram_indexes)
|
||||
|
||||
(masked_lms, masked_spans) = ([], [])
|
||||
covered_indexes = set()
|
||||
backup_output_tokens = list(output_tokens)
|
||||
for cand_index_set in ngram_indexes:
|
||||
if len(masked_lms) >= num_to_predict:
|
||||
break
|
||||
if not cand_index_set:
|
||||
continue
|
||||
# Note(mingdachen):
|
||||
# Skip current piece if they are covered in lm masking or previous ngrams.
|
||||
for index_set in cand_index_set[0]:
|
||||
for index in index_set:
|
||||
if index in covered_indexes:
|
||||
continue
|
||||
|
||||
if not geometric_dist:
|
||||
n = np_rng.choice(
|
||||
ngrams[:len(cand_index_set)],
|
||||
p=pvals[:len(cand_index_set)] /
|
||||
pvals[:len(cand_index_set)].sum(keepdims=True))
|
||||
else:
|
||||
# Sampling "n" from the geometric distribution and clipping it to
|
||||
# the max_ngrams. Using p=0.2 default from the SpanBERT paper
|
||||
# https://arxiv.org/pdf/1907.10529.pdf (Sec 3.1)
|
||||
n = min(np_rng.geometric(0.2), max_ngrams)
|
||||
|
||||
index_set = sum(cand_index_set[n - 1], [])
|
||||
n -= 1
|
||||
# Note(mingdachen):
|
||||
# Repeatedly looking for a candidate that does not exceed the
|
||||
# maximum number of predictions by trying shorter ngrams.
|
||||
while len(masked_lms) + len(index_set) > num_to_predict:
|
||||
if n == 0:
|
||||
break
|
||||
index_set = sum(cand_index_set[n - 1], [])
|
||||
n -= 1
|
||||
# If adding a whole-word mask would exceed the maximum number of
|
||||
# predictions, then just skip this candidate.
|
||||
if len(masked_lms) + len(index_set) > num_to_predict:
|
||||
continue
|
||||
is_any_index_covered = False
|
||||
for index in index_set:
|
||||
if index in covered_indexes:
|
||||
is_any_index_covered = True
|
||||
break
|
||||
if is_any_index_covered:
|
||||
continue
|
||||
for index in index_set:
|
||||
covered_indexes.add(index)
|
||||
masked_token = None
|
||||
if masking_style == "bert":
|
||||
# 80% of the time, replace with [MASK]
|
||||
if np_rng.random() < 0.8:
|
||||
masked_token = mask_id
|
||||
else:
|
||||
# 10% of the time, keep original
|
||||
if np_rng.random() < 0.5:
|
||||
masked_token = output_tokens[index]
|
||||
# 10% of the time, replace with random word
|
||||
else:
|
||||
if inplace_random_mask:
|
||||
masked_token = backup_output_tokens[np_rng.randint(
|
||||
0, len(output_tokens))]
|
||||
else:
|
||||
masked_token = vocab_id_list[np_rng.randint(
|
||||
0, len(vocab_id_list))]
|
||||
elif masking_style == "t5":
|
||||
masked_token = mask_id
|
||||
else:
|
||||
raise ValueError("invalid value of masking style")
|
||||
|
||||
output_tokens[index] = masked_token
|
||||
masked_lms.append(
|
||||
MaskedLmInstance(
|
||||
index=index, label=tokens[index]))
|
||||
|
||||
masked_spans.append(
|
||||
MaskedLmInstance(
|
||||
index=index_set, label=[tokens[index] for index in index_set]))
|
||||
|
||||
assert len(masked_lms) <= num_to_predict
|
||||
np_rng.shuffle(ngram_indexes)
|
||||
|
||||
select_indexes = set()
|
||||
if do_permutation:
|
||||
for cand_index_set in ngram_indexes:
|
||||
if len(select_indexes) >= num_to_predict:
|
||||
break
|
||||
if not cand_index_set:
|
||||
continue
|
||||
# Note(mingdachen):
|
||||
# Skip current piece if they are covered in lm masking or previous ngrams.
|
||||
for index_set in cand_index_set[0]:
|
||||
for index in index_set:
|
||||
if index in covered_indexes or index in select_indexes:
|
||||
continue
|
||||
|
||||
n = np.random.choice(
|
||||
ngrams[:len(cand_index_set)],
|
||||
p=pvals[:len(cand_index_set)] /
|
||||
pvals[:len(cand_index_set)].sum(keepdims=True))
|
||||
index_set = sum(cand_index_set[n - 1], [])
|
||||
n -= 1
|
||||
|
||||
while len(select_indexes) + len(index_set) > num_to_predict:
|
||||
if n == 0:
|
||||
break
|
||||
index_set = sum(cand_index_set[n - 1], [])
|
||||
n -= 1
|
||||
# If adding a whole-word mask would exceed the maximum number of
|
||||
# predictions, then just skip this candidate.
|
||||
if len(select_indexes) + len(index_set) > num_to_predict:
|
||||
continue
|
||||
is_any_index_covered = False
|
||||
for index in index_set:
|
||||
if index in covered_indexes or index in select_indexes:
|
||||
is_any_index_covered = True
|
||||
break
|
||||
if is_any_index_covered:
|
||||
continue
|
||||
for index in index_set:
|
||||
select_indexes.add(index)
|
||||
assert len(select_indexes) <= num_to_predict
|
||||
|
||||
select_indexes = sorted(select_indexes)
|
||||
permute_indexes = list(select_indexes)
|
||||
np_rng.shuffle(permute_indexes)
|
||||
orig_token = list(output_tokens)
|
||||
|
||||
for src_i, tgt_i in zip(select_indexes, permute_indexes):
|
||||
output_tokens[src_i] = orig_token[tgt_i]
|
||||
masked_lms.append(
|
||||
MaskedLmInstance(
|
||||
index=src_i, label=orig_token[src_i]))
|
||||
|
||||
masked_lms = sorted(masked_lms, key=lambda x: x.index)
|
||||
# Sort the spans by the index of the first span
|
||||
masked_spans = sorted(masked_spans, key=lambda x: x.index[0])
|
||||
|
||||
for p in masked_lms:
|
||||
masked_lm_positions.append(p.index)
|
||||
masked_lm_labels.append(p.label)
|
||||
return (output_tokens, masked_lm_positions, masked_lm_labels,
|
||||
token_boundary, masked_spans)
|
||||
|
||||
|
||||
def pad_and_convert_to_numpy(tokens, tokentypes, masked_positions,
|
||||
masked_labels, pad_id, max_seq_length):
|
||||
"""Pad sequences and convert them to numpy."""
|
||||
|
||||
# Some checks.
|
||||
num_tokens = len(tokens)
|
||||
padding_length = max_seq_length - num_tokens
|
||||
assert padding_length >= 0
|
||||
assert len(tokentypes) == num_tokens
|
||||
assert len(masked_positions) == len(masked_labels)
|
||||
|
||||
# Tokens and token types.
|
||||
filler = [pad_id] * padding_length
|
||||
tokens_np = np.array(tokens + filler, dtype=np.int64)
|
||||
tokentypes_np = np.array(tokentypes + filler, dtype=np.int64)
|
||||
|
||||
# Padding mask.
|
||||
padding_mask_np = np.array(
|
||||
[1] * num_tokens + [0] * padding_length, dtype=np.int64)
|
||||
|
||||
# Lables and loss mask.
|
||||
labels = [-1] * max_seq_length
|
||||
loss_mask = [0] * max_seq_length
|
||||
for i in range(len(masked_positions)):
|
||||
assert masked_positions[i] < num_tokens
|
||||
labels[masked_positions[i]] = masked_labels[i]
|
||||
loss_mask[masked_positions[i]] = 1
|
||||
labels_np = np.array(labels, dtype=np.int64)
|
||||
loss_mask_np = np.array(loss_mask, dtype=np.int64)
|
||||
|
||||
return tokens_np, tokentypes_np, labels_np, padding_mask_np, loss_mask_np
|
||||
|
||||
|
||||
def build_train_valid_test_datasets(data_prefix,
|
||||
args,
|
||||
tokenizer,
|
||||
splits_string,
|
||||
train_valid_test_num_samples,
|
||||
max_seq_length,
|
||||
masked_lm_prob,
|
||||
short_seq_prob,
|
||||
seed,
|
||||
skip_warmup,
|
||||
binary_head=False,
|
||||
max_seq_length_dec=None,
|
||||
dataset_type='standard_bert'):
|
||||
|
||||
if len(data_prefix) == 1:
|
||||
return _build_train_valid_test_datasets(
|
||||
data_prefix[0],
|
||||
args,
|
||||
tokenizer,
|
||||
splits_string,
|
||||
train_valid_test_num_samples,
|
||||
max_seq_length,
|
||||
masked_lm_prob,
|
||||
short_seq_prob,
|
||||
seed,
|
||||
skip_warmup,
|
||||
binary_head,
|
||||
max_seq_length_dec,
|
||||
dataset_type=dataset_type)
|
||||
|
||||
|
||||
def _build_train_valid_test_datasets(data_prefix,
|
||||
args,
|
||||
tokenizer,
|
||||
splits_string,
|
||||
train_valid_test_num_samples,
|
||||
max_seq_length,
|
||||
masked_lm_prob,
|
||||
short_seq_prob,
|
||||
seed,
|
||||
skip_warmup,
|
||||
binary_head,
|
||||
max_seq_length_dec,
|
||||
dataset_type='standard_bert'):
|
||||
|
||||
if dataset_type not in DSET_TYPES:
|
||||
raise ValueError("Invalid dataset_type: ", dataset_type)
|
||||
|
||||
# Indexed dataset.
|
||||
indexed_dataset = get_indexed_dataset_(data_prefix, None, skip_warmup)
|
||||
|
||||
# Get start and end indices of train/valid/train into doc-idx
|
||||
# Note that doc-idx is desinged to be num-docs + 1 so we can
|
||||
# easily iterate over it.
|
||||
total_num_of_documents = indexed_dataset.doc_idx.shape[0] - 1
|
||||
splits = get_train_valid_test_split_(splits_string, total_num_of_documents)
|
||||
print(splits)
|
||||
# Print stats about the splits.
|
||||
print_rank_0(' > dataset split:')
|
||||
|
||||
def print_split_stats(name, index):
|
||||
print_rank_0(' {}:'.format(name))
|
||||
print_rank_0(' document indices in [{}, {}) total of {} '
|
||||
'documents'.format(splits[index], splits[index + 1],
|
||||
splits[index + 1] - splits[index]))
|
||||
start_index = indexed_dataset.doc_idx[splits[index]]
|
||||
end_index = indexed_dataset.doc_idx[splits[index + 1]]
|
||||
print_rank_0(' sentence indices in [{}, {}) total of {} '
|
||||
'sentences'.format(start_index, end_index, end_index -
|
||||
start_index))
|
||||
|
||||
print_split_stats('train', 0)
|
||||
print_split_stats('validation', 1)
|
||||
print_split_stats('test', 2)
|
||||
|
||||
def build_dataset(index, name):
|
||||
# from megatron.data.bert_dataset import BertDataset
|
||||
# from megatron.data.t5_dataset import T5Dataset
|
||||
from .ernie_dataset import ErnieDataset
|
||||
dataset = None
|
||||
if splits[index + 1] > splits[index]:
|
||||
# Get the pointer to the original doc-idx so we can set it later.
|
||||
doc_idx_ptr = indexed_dataset.get_doc_idx()
|
||||
# Slice the doc-idx
|
||||
start_index = splits[index]
|
||||
# Add +1 so we can index into the dataset to get the upper bound.
|
||||
end_index = splits[index + 1] + 1
|
||||
# New doc_idx view.
|
||||
indexed_dataset.set_doc_idx(doc_idx_ptr[start_index:end_index])
|
||||
# Build the dataset accordingly.
|
||||
kwargs = dict(
|
||||
name=name,
|
||||
data_prefix=data_prefix,
|
||||
num_epochs=None,
|
||||
max_num_samples=train_valid_test_num_samples[index],
|
||||
max_seq_length=max_seq_length,
|
||||
seed=seed,
|
||||
share_folder=args.share_folder, )
|
||||
if dataset_type == DSET_TYPE_T5:
|
||||
dataset = T5Dataset(
|
||||
indexed_dataset=indexed_dataset,
|
||||
tokenizer=tokenizer,
|
||||
masked_lm_prob=masked_lm_prob,
|
||||
max_seq_length_dec=max_seq_length_dec,
|
||||
short_seq_prob=short_seq_prob,
|
||||
**kwargs)
|
||||
elif dataset_type == DSET_TYPE_BERT:
|
||||
dataset = BertDataset(
|
||||
indexed_dataset=indexed_dataset,
|
||||
tokenizer=tokenizer,
|
||||
masked_lm_prob=masked_lm_prob,
|
||||
short_seq_prob=short_seq_prob,
|
||||
binary_head=binary_head,
|
||||
**kwargs)
|
||||
elif dataset_type == DSET_TYPE_ERNIE:
|
||||
dataset = ErnieDataset(
|
||||
indexed_dataset=indexed_dataset,
|
||||
tokenizer=tokenizer, #ErnieTokenizer.from_pretrained("ernie-1.0"),
|
||||
masked_lm_prob=masked_lm_prob,
|
||||
short_seq_prob=short_seq_prob,
|
||||
binary_head=binary_head,
|
||||
**kwargs)
|
||||
else:
|
||||
raise NotImplementedError("Dataset type not fully implemented.")
|
||||
|
||||
# Set the original pointer so dataset remains the main dataset.
|
||||
indexed_dataset.set_doc_idx(doc_idx_ptr)
|
||||
# Checks.
|
||||
assert indexed_dataset.doc_idx[0] == 0
|
||||
assert indexed_dataset.doc_idx.shape[0] == \
|
||||
(total_num_of_documents + 1)
|
||||
return dataset
|
||||
|
||||
train_dataset = build_dataset(0, 'train')
|
||||
valid_dataset = build_dataset(1, 'valid')
|
||||
test_dataset = build_dataset(2, 'test')
|
||||
|
||||
return (train_dataset, valid_dataset, test_dataset)
|
||||
|
||||
|
||||
def get_indexed_dataset_(data_prefix, data_impl, skip_warmup):
|
||||
|
||||
print_rank_0(' > building dataset index ...')
|
||||
|
||||
start_time = time.time()
|
||||
indexed_dataset = make_indexed_dataset(data_prefix, data_impl, skip_warmup)
|
||||
assert indexed_dataset.sizes.shape[0] == indexed_dataset.doc_idx[-1]
|
||||
print_rank_0(' > finished creating indexed dataset in {:4f} '
|
||||
'seconds'.format(time.time() - start_time))
|
||||
|
||||
print_rank_0(' > indexed dataset stats:')
|
||||
print_rank_0(' number of documents: {}'.format(
|
||||
indexed_dataset.doc_idx.shape[0] - 1))
|
||||
print_rank_0(' number of sentences: {}'.format(
|
||||
indexed_dataset.sizes.shape[0]))
|
||||
|
||||
return indexed_dataset
|
||||
|
||||
|
||||
def get_train_valid_test_split_(splits_string, size):
|
||||
""" Get dataset splits from comma or '/' separated string list."""
|
||||
print(splits_string)
|
||||
splits = []
|
||||
if splits_string.find(',') != -1:
|
||||
splits = [float(s) for s in splits_string.split(',')]
|
||||
elif splits_string.find('/') != -1:
|
||||
splits = [float(s) for s in splits_string.split('/')]
|
||||
else:
|
||||
splits = [float(splits_string)]
|
||||
while len(splits) < 3:
|
||||
splits.append(0.)
|
||||
splits = splits[:3]
|
||||
splits_sum = sum(splits)
|
||||
assert splits_sum > 0.0
|
||||
splits = [split / splits_sum for split in splits]
|
||||
splits_index = [0]
|
||||
for index, split in enumerate(splits):
|
||||
splits_index.append(splits_index[index] + int(
|
||||
round(split * float(size))))
|
||||
diff = splits_index[-1] - size
|
||||
for index in range(1, len(splits_index)):
|
||||
splits_index[index] -= diff
|
||||
assert len(splits_index) == 4
|
||||
assert splits_index[-1] == size
|
||||
return splits_index
|
||||
|
||||
|
||||
def get_samples_mapping(indexed_dataset, data_prefix, num_epochs,
|
||||
max_num_samples, max_seq_length, short_seq_prob, seed,
|
||||
name, binary_head, share_folder):
|
||||
"""Get a list that maps a sample index to a starting sentence index, end sentence index, and length"""
|
||||
|
||||
if not num_epochs:
|
||||
if not max_num_samples:
|
||||
raise ValueError("Need to specify either max_num_samples "
|
||||
"or num_epochs")
|
||||
num_epochs = np.iinfo(np.int32).max - 1
|
||||
if not max_num_samples:
|
||||
max_num_samples = np.iinfo(np.int64).max - 1
|
||||
|
||||
# Filename of the index mapping
|
||||
indexmap_filename = data_prefix
|
||||
indexmap_filename += '_{}_indexmap'.format(name)
|
||||
if num_epochs != (np.iinfo(np.int32).max - 1):
|
||||
indexmap_filename += '_{}ep'.format(num_epochs)
|
||||
if max_num_samples != (np.iinfo(np.int64).max - 1):
|
||||
indexmap_filename += '_{}mns'.format(max_num_samples)
|
||||
indexmap_filename += '_{}msl'.format(max_seq_length)
|
||||
indexmap_filename += '_{:0.2f}ssp'.format(short_seq_prob)
|
||||
indexmap_filename += '_{}s'.format(seed)
|
||||
indexmap_filename += '.npy'
|
||||
|
||||
local_rank = 0 if fleet.local_rank() is None else int(fleet.local_rank())
|
||||
if share_folder:
|
||||
local_rank = fleet.worker_index()
|
||||
# Build the indexed mapping if not exist.
|
||||
|
||||
if local_rank == 0 and \
|
||||
not os.path.isfile(indexmap_filename):
|
||||
print(' > WARNING: could not find index map file {}, building '
|
||||
'the indices on rank 0 ...'.format(indexmap_filename))
|
||||
|
||||
# Make sure the types match the helpers input types.
|
||||
assert indexed_dataset.doc_idx.dtype == np.int64
|
||||
print(indexed_dataset.sizes.dtype)
|
||||
assert indexed_dataset.sizes.dtype == np.int32
|
||||
|
||||
# Build samples mapping
|
||||
verbose = local_rank == 0
|
||||
start_time = time.time()
|
||||
print_rank_0(' > building sapmles index mapping for {} ...'.format(
|
||||
name))
|
||||
# First compile and then import.
|
||||
if local_rank == 0:
|
||||
compile_helper()
|
||||
import data_tools.helpers as helpers
|
||||
samples_mapping = helpers.build_mapping(
|
||||
indexed_dataset.doc_idx, indexed_dataset.sizes, num_epochs,
|
||||
max_num_samples, max_seq_length, short_seq_prob, seed, verbose, 2
|
||||
if binary_head else 1)
|
||||
print_rank_0(' > done building sapmles index maping')
|
||||
np.save(indexmap_filename, samples_mapping, allow_pickle=True)
|
||||
print_rank_0(' > saved the index mapping in {}'.format(
|
||||
indexmap_filename))
|
||||
# Make sure all the ranks have built the mapping
|
||||
print_rank_0(' > elasped time to build and save samples mapping '
|
||||
'(seconds): {:4f}'.format(time.time() - start_time))
|
||||
|
||||
else:
|
||||
while True:
|
||||
if (not os.path.isfile(indexmap_filename)):
|
||||
time.sleep(3)
|
||||
else:
|
||||
try:
|
||||
np.load(indexmap_filename, allow_pickle=True, mmap_mode='r')
|
||||
break
|
||||
except Exception as e:
|
||||
print(
|
||||
"%s file is still writing or damaged, please wait a moment."
|
||||
% indexmap_filename)
|
||||
time.sleep(3)
|
||||
|
||||
# This should be a barrier but nccl barrier assumes
|
||||
# device_index=rank which is not the case for model
|
||||
# parallel case
|
||||
if paddle.distributed.get_world_size() > 1:
|
||||
if paddle.in_dynamic_mode():
|
||||
paddle.distributed.barrier()
|
||||
|
||||
# Load indexed dataset.
|
||||
print_rank_0(' > loading indexed mapping from {}'.format(indexmap_filename))
|
||||
start_time = time.time()
|
||||
samples_mapping = np.load(
|
||||
indexmap_filename, allow_pickle=True, mmap_mode='r')
|
||||
print_rank_0(' loaded indexed file in {:3.3f} seconds'.format(time.time(
|
||||
) - start_time))
|
||||
print_rank_0(' total number of samples: {}'.format(samples_mapping.shape[
|
||||
0]))
|
||||
|
||||
return samples_mapping
|
||||
@@ -0,0 +1,217 @@
|
||||
# Copyright (c) 2021, PadddlePaddle authors. All rights reserved.
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""BERT Style dataset."""
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import re
|
||||
|
||||
from .dataset_utils import (
|
||||
get_samples_mapping,
|
||||
get_a_and_b_segments,
|
||||
truncate_segments,
|
||||
create_tokens_and_tokentypes,
|
||||
create_masked_lm_predictions,
|
||||
make_indexed_dataset,
|
||||
get_indexed_dataset_, )
|
||||
|
||||
from paddlenlp.transformers import ErnieTokenizer
|
||||
|
||||
|
||||
class ErnieDataset(paddle.io.Dataset):
|
||||
def __init__(self,
|
||||
name,
|
||||
tokenizer,
|
||||
indexed_dataset,
|
||||
data_prefix,
|
||||
num_epochs,
|
||||
max_num_samples,
|
||||
masked_lm_prob,
|
||||
max_seq_length,
|
||||
short_seq_prob,
|
||||
seed,
|
||||
binary_head,
|
||||
share_folder=False):
|
||||
|
||||
# Params to store.
|
||||
self.name = name
|
||||
self.seed = seed
|
||||
self.masked_lm_prob = masked_lm_prob
|
||||
self.max_seq_length = max_seq_length
|
||||
self.binary_head = binary_head
|
||||
self.share_folder = share_folder
|
||||
|
||||
# Dataset.
|
||||
self.indexed_dataset = indexed_dataset
|
||||
|
||||
# Build the samples mapping.
|
||||
self.samples_mapping = get_samples_mapping(
|
||||
self.indexed_dataset,
|
||||
data_prefix,
|
||||
num_epochs,
|
||||
max_num_samples,
|
||||
self.max_seq_length - 3, # account for added tokens
|
||||
short_seq_prob,
|
||||
self.seed,
|
||||
self.name,
|
||||
self.binary_head,
|
||||
self.share_folder)
|
||||
|
||||
# Vocab stuff.
|
||||
# tokenizer = get_tokenizer()
|
||||
# self.vocab_id_list = list(tokenizer.inv_vocab.keys())
|
||||
# self.vocab_id_to_token_dict = tokenizer.inv_vocab
|
||||
self.vocab_id_list = list(tokenizer.vocab.idx_to_token.keys())
|
||||
self.vocab_id_to_token_dict = tokenizer.vocab.idx_to_token
|
||||
self.vocab_token_to_id_dict = tokenizer.vocab.token_to_idx
|
||||
|
||||
self.cls_id = tokenizer.cls_token_id
|
||||
self.sep_id = tokenizer.sep_token_id
|
||||
self.mask_id = tokenizer.mask_token_id
|
||||
self.pad_id = tokenizer.pad_token_id
|
||||
|
||||
def __len__(self):
|
||||
return self.samples_mapping.shape[0]
|
||||
|
||||
def __getitem__(self, idx):
|
||||
start_idx, end_idx, seq_length = self.samples_mapping[idx]
|
||||
sample = [self.indexed_dataset[i] for i in range(start_idx, end_idx)]
|
||||
# Note that this rng state should be numpy and not python since
|
||||
# python randint is inclusive whereas the numpy one is exclusive.
|
||||
# We % 2**32 since numpy requres the seed to be between 0 and 2**32 - 1
|
||||
np_rng = np.random.RandomState(seed=((self.seed + idx) % 2**32))
|
||||
return build_training_sample(
|
||||
sample,
|
||||
seq_length,
|
||||
self.max_seq_length, # needed for padding
|
||||
self.vocab_id_list,
|
||||
self.vocab_id_to_token_dict,
|
||||
self.vocab_token_to_id_dict,
|
||||
self.cls_id,
|
||||
self.sep_id,
|
||||
self.mask_id,
|
||||
self.pad_id,
|
||||
self.masked_lm_prob,
|
||||
np_rng,
|
||||
self.binary_head)
|
||||
|
||||
|
||||
def build_training_sample(sample, target_seq_length, max_seq_length,
|
||||
vocab_id_list, vocab_id_to_token_dict,
|
||||
vocab_token_to_id_dict, cls_id, sep_id, mask_id,
|
||||
pad_id, masked_lm_prob, np_rng, binary_head):
|
||||
"""Biuld training sample.
|
||||
|
||||
Arguments:
|
||||
sample: A list of sentences in which each sentence is a list token ids.
|
||||
target_seq_length: Desired sequence length.
|
||||
max_seq_length: Maximum length of the sequence. All values are padded to
|
||||
this length.
|
||||
vocab_id_list: List of vocabulary ids. Used to pick a random id.
|
||||
vocab_id_to_token_dict: A dictionary from vocab ids to text tokens.
|
||||
vocab_token_to_id_dict: A dictionary from text tokens to vocab ids.
|
||||
cls_id: Start of example id.
|
||||
sep_id: Separator id.
|
||||
mask_id: Mask token id.
|
||||
pad_id: Padding token id.
|
||||
masked_lm_prob: Probability to mask tokens.
|
||||
np_rng: Random number genenrator. Note that this rng state should be
|
||||
numpy and not python since python randint is inclusive for
|
||||
the opper bound whereas the numpy one is exclusive.
|
||||
"""
|
||||
|
||||
if binary_head:
|
||||
# We assume that we have at least two sentences in the sample
|
||||
assert len(sample) > 1, "The sentence num should be large than 1."
|
||||
assert target_seq_length <= max_seq_length
|
||||
|
||||
# Divide sample into two segments (A and B).
|
||||
if binary_head:
|
||||
tokens_a, tokens_b, is_next_random = get_a_and_b_segments(sample,
|
||||
np_rng)
|
||||
else:
|
||||
tokens_a = []
|
||||
for j in range(len(sample)):
|
||||
tokens_a.extend(sample[j])
|
||||
tokens_b = []
|
||||
is_next_random = False
|
||||
|
||||
# Truncate to `target_sequence_length`.
|
||||
max_num_tokens = target_seq_length
|
||||
truncated = truncate_segments(tokens_a, tokens_b,
|
||||
len(tokens_a),
|
||||
len(tokens_b), max_num_tokens, np_rng)
|
||||
|
||||
# Build tokens and toketypes.
|
||||
tokens, tokentypes = create_tokens_and_tokentypes(tokens_a, tokens_b,
|
||||
cls_id, sep_id)
|
||||
|
||||
# Masking.
|
||||
max_predictions_per_seq = masked_lm_prob * max_num_tokens
|
||||
(tokens, masked_positions, masked_labels, _,
|
||||
_) = create_masked_lm_predictions(
|
||||
tokens,
|
||||
vocab_id_list,
|
||||
vocab_id_to_token_dict,
|
||||
masked_lm_prob,
|
||||
cls_id,
|
||||
sep_id,
|
||||
mask_id,
|
||||
max_predictions_per_seq,
|
||||
np_rng,
|
||||
vocab_token_to_id_dict=vocab_token_to_id_dict,
|
||||
to_chinese_char=True,
|
||||
inplace_random_mask=False, )
|
||||
|
||||
# Padding.
|
||||
tokens_np, tokentypes_np, labels_np, padding_mask_np, loss_mask_np \
|
||||
= pad_and_convert_to_numpy(tokens, tokentypes, masked_positions,
|
||||
masked_labels, pad_id, max_seq_length)
|
||||
|
||||
return tokens_np, tokentypes_np, padding_mask_np, masked_positions, masked_labels, int(
|
||||
is_next_random)
|
||||
|
||||
|
||||
def pad_and_convert_to_numpy(tokens, tokentypes, masked_positions,
|
||||
masked_labels, pad_id, max_seq_length):
|
||||
"""Pad sequences and convert them to numpy."""
|
||||
|
||||
# Some checks.
|
||||
num_tokens = len(tokens)
|
||||
padding_length = max_seq_length - num_tokens
|
||||
assert padding_length >= 0
|
||||
assert len(tokentypes) == num_tokens
|
||||
assert len(masked_positions) == len(masked_labels)
|
||||
|
||||
# Tokens and token types.
|
||||
filler = [pad_id] * padding_length
|
||||
tokens_np = np.array(tokens + filler, dtype=np.int64)
|
||||
tokentypes_np = np.array(tokentypes + filler, dtype=np.int64)
|
||||
|
||||
# Padding mask.
|
||||
padding_mask_np = np.array(
|
||||
[1] * num_tokens + [0] * padding_length, dtype=np.float32)
|
||||
padding_mask_np = padding_mask_np.reshape([1, 1, -1])
|
||||
# Lables and loss mask.
|
||||
labels = [-1] * max_seq_length
|
||||
loss_mask = [0] * max_seq_length
|
||||
for i in range(len(masked_positions)):
|
||||
assert masked_positions[i] < num_tokens
|
||||
labels[masked_positions[i]] = masked_labels[i]
|
||||
loss_mask[masked_positions[i]] = 1
|
||||
labels_np = np.array(labels, dtype=np.int64)
|
||||
loss_mask_np = np.array(loss_mask, dtype=np.int64)
|
||||
|
||||
return tokens_np, tokentypes_np, labels_np, padding_mask_np, loss_mask_np
|
||||
@@ -0,0 +1,736 @@
|
||||
/*
|
||||
coding=utf-8
|
||||
Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
/* Helper methods for fast index mapping builds */
|
||||
|
||||
#include <math.h>
|
||||
#include <pybind11/numpy.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <random>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace py = pybind11;
|
||||
using namespace std;
|
||||
|
||||
const int32_t LONG_SENTENCE_LEN = 512;
|
||||
|
||||
|
||||
void build_blending_indices(py::array_t<uint8_t>& dataset_index,
|
||||
py::array_t<int64_t>& dataset_sample_index,
|
||||
const py::array_t<double>& weights,
|
||||
const int32_t num_datasets,
|
||||
const int64_t size,
|
||||
const bool verbose) {
|
||||
/* Given multiple datasets and a weighting array, build samples
|
||||
such that it follows those wieghts.*/
|
||||
|
||||
if (verbose) {
|
||||
std::cout << "> building indices for blendable datasets ..." << std::endl;
|
||||
}
|
||||
|
||||
// Get the pointer access without the checks.
|
||||
auto dataset_index_ptr = dataset_index.mutable_unchecked<1>();
|
||||
auto dataset_sample_index_ptr = dataset_sample_index.mutable_unchecked<1>();
|
||||
auto weights_ptr = weights.unchecked<1>();
|
||||
|
||||
// Initialize buffer for number of samples used for each dataset.
|
||||
int64_t current_samples[num_datasets];
|
||||
for (int64_t i = 0; i < num_datasets; ++i) {
|
||||
current_samples[i] = 0;
|
||||
}
|
||||
|
||||
// For each sample:
|
||||
for (int64_t sample_idx = 0; sample_idx < size; ++sample_idx) {
|
||||
// Determine where the max error in sampling is happening.
|
||||
auto sample_idx_double = std::max(static_cast<double>(sample_idx), 1.0);
|
||||
int64_t max_error_index = 0;
|
||||
double max_error = weights_ptr[0] * sample_idx_double -
|
||||
static_cast<double>(current_samples[0]);
|
||||
for (int64_t dataset_idx = 1; dataset_idx < num_datasets; ++dataset_idx) {
|
||||
double error = weights_ptr[dataset_idx] * sample_idx_double -
|
||||
static_cast<double>(current_samples[dataset_idx]);
|
||||
if (error > max_error) {
|
||||
max_error = error;
|
||||
max_error_index = dataset_idx;
|
||||
}
|
||||
}
|
||||
|
||||
// Populate the indices.
|
||||
dataset_index_ptr[sample_idx] = static_cast<uint8_t>(max_error_index);
|
||||
dataset_sample_index_ptr[sample_idx] = current_samples[max_error_index];
|
||||
|
||||
// Update the total samples.
|
||||
current_samples[max_error_index] += 1;
|
||||
}
|
||||
|
||||
// print info
|
||||
if (verbose) {
|
||||
std::cout << " > sample ratios:" << std::endl;
|
||||
for (int64_t dataset_idx = 0; dataset_idx < num_datasets; ++dataset_idx) {
|
||||
auto ratio = static_cast<double>(current_samples[dataset_idx]) /
|
||||
static_cast<double>(size);
|
||||
std::cout << " dataset " << dataset_idx
|
||||
<< ", input: " << weights_ptr[dataset_idx]
|
||||
<< ", achieved: " << ratio << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
py::array build_sample_idx(const py::array_t<int32_t>& sizes_,
|
||||
const py::array_t<int32_t>& doc_idx_,
|
||||
const int32_t seq_length,
|
||||
const int32_t num_epochs,
|
||||
const int64_t tokens_per_epoch) {
|
||||
/* Sample index (sample_idx) is used for gpt2 like dataset for which
|
||||
the documents are flattened and the samples are built based on this
|
||||
1-D flatten array. It is a 2D array with sizes [number-of-samples + 1, 2]
|
||||
where [..., 0] contains the index into `doc_idx` and [..., 1] is the
|
||||
starting offset in that document.*/
|
||||
|
||||
// Consistency checks.
|
||||
assert(seq_length > 1);
|
||||
assert(num_epochs > 0);
|
||||
assert(tokens_per_epoch > 1);
|
||||
|
||||
// Remove bound checks.
|
||||
auto sizes = sizes_.unchecked<1>();
|
||||
auto doc_idx = doc_idx_.unchecked<1>();
|
||||
|
||||
// Mapping and it's length (1D).
|
||||
int64_t num_samples = (num_epochs * tokens_per_epoch - 1) / seq_length;
|
||||
int32_t* sample_idx = new int32_t[2 * (num_samples + 1)];
|
||||
|
||||
cout << " using:" << endl << std::flush;
|
||||
cout << " number of documents: " << doc_idx_.shape(0) / num_epochs
|
||||
<< endl
|
||||
<< std::flush;
|
||||
cout << " number of epochs: " << num_epochs << endl
|
||||
<< std::flush;
|
||||
cout << " sequence length: " << seq_length << endl
|
||||
<< std::flush;
|
||||
cout << " total number of samples: " << num_samples << endl
|
||||
<< std::flush;
|
||||
|
||||
// Index into sample_idx.
|
||||
int64_t sample_index = 0;
|
||||
// Index into doc_idx.
|
||||
int64_t doc_idx_index = 0;
|
||||
// Begining offset for each document.
|
||||
int32_t doc_offset = 0;
|
||||
// Start with first document and no offset.
|
||||
sample_idx[2 * sample_index] = doc_idx_index;
|
||||
sample_idx[2 * sample_index + 1] = doc_offset;
|
||||
++sample_index;
|
||||
|
||||
while (sample_index <= num_samples) {
|
||||
// Start with a fresh sequence.
|
||||
int32_t remaining_seq_length = seq_length + 1;
|
||||
while (remaining_seq_length != 0) {
|
||||
// Get the document length.
|
||||
auto doc_id = doc_idx[doc_idx_index];
|
||||
auto doc_length = sizes[doc_id] - doc_offset;
|
||||
// And add it to the current sequence.
|
||||
remaining_seq_length -= doc_length;
|
||||
// If we have more than a full sequence, adjust offset and set
|
||||
// remaining length to zero so we return from the while loop.
|
||||
// Note that -1 here is for the same reason we have -1 in
|
||||
// `_num_epochs` calculations.
|
||||
if (remaining_seq_length <= 0) {
|
||||
doc_offset += (remaining_seq_length + doc_length - 1);
|
||||
remaining_seq_length = 0;
|
||||
} else {
|
||||
// Otherwise, start from the begining of the next document.
|
||||
++doc_idx_index;
|
||||
doc_offset = 0;
|
||||
}
|
||||
}
|
||||
// Record the sequence.
|
||||
sample_idx[2 * sample_index] = doc_idx_index;
|
||||
sample_idx[2 * sample_index + 1] = doc_offset;
|
||||
++sample_index;
|
||||
}
|
||||
|
||||
// Method to deallocate memory.
|
||||
py::capsule free_when_done(sample_idx, [](void* mem_) {
|
||||
int32_t* mem = reinterpret_cast<int32_t*>(mem_);
|
||||
delete[] mem;
|
||||
});
|
||||
|
||||
// Return the numpy array.
|
||||
const auto byte_size = sizeof(int32_t);
|
||||
return py::array(std::vector<int64_t>{num_samples + 1, 2}, // shape
|
||||
{2 * byte_size, byte_size}, // C-style contiguous strides
|
||||
sample_idx, // the data pointer
|
||||
free_when_done); // numpy array references
|
||||
}
|
||||
|
||||
|
||||
inline int32_t get_target_sample_len(const int32_t short_seq_ratio,
|
||||
const int32_t max_length,
|
||||
std::mt19937& rand32_gen) {
|
||||
/* Training sample length. */
|
||||
if (short_seq_ratio == 0) {
|
||||
return max_length;
|
||||
}
|
||||
const auto random_number = rand32_gen();
|
||||
if ((random_number % short_seq_ratio) == 0) {
|
||||
return 2 + random_number % (max_length - 1);
|
||||
}
|
||||
return max_length;
|
||||
}
|
||||
|
||||
|
||||
template <typename DocIdx>
|
||||
py::array build_mapping_impl(const py::array_t<int64_t>& docs_,
|
||||
const py::array_t<int32_t>& sizes_,
|
||||
const int32_t num_epochs,
|
||||
const uint64_t max_num_samples,
|
||||
const int32_t max_seq_length,
|
||||
const double short_seq_prob,
|
||||
const int32_t seed,
|
||||
const bool verbose,
|
||||
const int32_t min_num_sent) {
|
||||
/* Build a mapping of (start-index, end-index, sequence-length) where
|
||||
start and end index are the indices of the sentences in the sample
|
||||
and sequence-length is the target sequence length.
|
||||
*/
|
||||
|
||||
// Consistency checks.
|
||||
assert(num_epochs > 0);
|
||||
assert(max_seq_length > 1);
|
||||
assert(short_seq_prob >= 0.0);
|
||||
assert(short_seq_prob <= 1.0);
|
||||
assert(seed > 0);
|
||||
|
||||
// Remove bound checks.
|
||||
auto docs = docs_.unchecked<1>();
|
||||
auto sizes = sizes_.unchecked<1>();
|
||||
|
||||
// For efficiency, convert probability to ratio. Note: rand() generates int.
|
||||
int32_t short_seq_ratio = 0;
|
||||
if (short_seq_prob > 0) {
|
||||
short_seq_ratio = static_cast<int32_t>(round(1.0 / short_seq_prob));
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
const auto sent_start_index = docs[0];
|
||||
const auto sent_end_index = docs[docs_.shape(0) - 1];
|
||||
const auto num_sentences = sent_end_index - sent_start_index;
|
||||
cout << " using:" << endl << std::flush;
|
||||
cout << " number of documents: " << docs_.shape(0) - 1
|
||||
<< endl
|
||||
<< std::flush;
|
||||
cout << " sentences range: [" << sent_start_index << ", "
|
||||
<< sent_end_index << ")" << endl
|
||||
<< std::flush;
|
||||
cout << " total number of sentences: " << num_sentences << endl
|
||||
<< std::flush;
|
||||
cout << " number of epochs: " << num_epochs << endl
|
||||
<< std::flush;
|
||||
cout << " maximum number of samples: " << max_num_samples << endl
|
||||
<< std::flush;
|
||||
cout << " maximum sequence length: " << max_seq_length << endl
|
||||
<< std::flush;
|
||||
cout << " short sequence probability: " << short_seq_prob << endl
|
||||
<< std::flush;
|
||||
cout << " short sequence ration (1/prob): " << short_seq_ratio << endl
|
||||
<< std::flush;
|
||||
cout << " seed: " << seed << endl
|
||||
<< std::flush;
|
||||
}
|
||||
|
||||
// Mapping and it's length (1D).
|
||||
int64_t num_samples = -1;
|
||||
DocIdx* maps = NULL;
|
||||
|
||||
// Perform two iterations, in the first iteration get the size
|
||||
// and allocate memory and in the second iteration populate the map.
|
||||
bool second = false;
|
||||
for (int32_t iteration = 0; iteration < 2; ++iteration) {
|
||||
// Set the seed so both iterations produce the same results.
|
||||
std::mt19937 rand32_gen(seed);
|
||||
|
||||
// Set the flag on second iteration.
|
||||
second = (iteration == 1);
|
||||
|
||||
// Counters:
|
||||
uint64_t empty_docs = 0;
|
||||
uint64_t one_sent_docs = 0;
|
||||
uint64_t long_sent_docs = 0;
|
||||
|
||||
// Current map index.
|
||||
uint64_t map_index = 0;
|
||||
|
||||
// For each epoch:
|
||||
for (int32_t epoch = 0; epoch < num_epochs; ++epoch) {
|
||||
if (map_index >= max_num_samples) {
|
||||
if (verbose && (!second)) {
|
||||
cout << " reached " << max_num_samples << " samples after "
|
||||
<< epoch << " epochs ..." << endl
|
||||
<< std::flush;
|
||||
}
|
||||
break;
|
||||
}
|
||||
// For each document:
|
||||
for (int32_t doc = 0; doc < (docs.shape(0) - 1); ++doc) {
|
||||
// Document sentences are in [sent_index_first, sent_index_last)
|
||||
const auto sent_index_first = docs[doc];
|
||||
const auto sent_index_last = docs[doc + 1];
|
||||
|
||||
// At the begining of the document previous index is the
|
||||
// start index.
|
||||
auto prev_start_index = sent_index_first;
|
||||
|
||||
// Remaining documents.
|
||||
auto num_remain_sent = sent_index_last - sent_index_first;
|
||||
|
||||
// Some bookkeeping
|
||||
if ((epoch == 0) && (!second)) {
|
||||
if (num_remain_sent == 0) {
|
||||
++empty_docs;
|
||||
}
|
||||
if (num_remain_sent == 1) {
|
||||
++one_sent_docs;
|
||||
}
|
||||
}
|
||||
|
||||
// Detect documents with long sentences.
|
||||
bool contains_long_sentence = false;
|
||||
if (num_remain_sent > 1) {
|
||||
for (auto sent_index = sent_index_first; sent_index < sent_index_last;
|
||||
++sent_index) {
|
||||
if (sizes[sent_index] > LONG_SENTENCE_LEN) {
|
||||
if ((epoch == 0) && (!second)) {
|
||||
++long_sent_docs;
|
||||
}
|
||||
contains_long_sentence = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we have more than two sentences.
|
||||
if ((num_remain_sent >= min_num_sent) && (!contains_long_sentence)) {
|
||||
// Set values.
|
||||
auto seq_len = int32_t{0};
|
||||
auto num_sent = int32_t{0};
|
||||
auto target_seq_len = get_target_sample_len(
|
||||
short_seq_ratio, max_seq_length, rand32_gen);
|
||||
|
||||
// Loop through sentences.
|
||||
for (auto sent_index = sent_index_first; sent_index < sent_index_last;
|
||||
++sent_index) {
|
||||
// Add the size and number of sentences.
|
||||
seq_len += sizes[sent_index];
|
||||
++num_sent;
|
||||
--num_remain_sent;
|
||||
|
||||
// If we have reached the target length.
|
||||
// and if not only one sentence is left in the document.
|
||||
// and if we have at least two sentneces.
|
||||
// and if we have reached end of the document.
|
||||
if (((seq_len >= target_seq_len) && (num_remain_sent > 1) &&
|
||||
(num_sent >= min_num_sent)) ||
|
||||
(num_remain_sent == 0)) {
|
||||
// Check for overflow.
|
||||
if ((3 * map_index + 2) > std::numeric_limits<int64_t>::max()) {
|
||||
cout << "number of samples exceeded maximum "
|
||||
<< "allowed by type int64: "
|
||||
<< std::numeric_limits<int64_t>::max() << endl;
|
||||
throw std::overflow_error("Number of samples");
|
||||
}
|
||||
|
||||
// Populate the map.
|
||||
if (second) {
|
||||
const auto map_index_0 = 3 * map_index;
|
||||
maps[map_index_0] = static_cast<DocIdx>(prev_start_index);
|
||||
maps[map_index_0 + 1] = static_cast<DocIdx>(sent_index + 1);
|
||||
maps[map_index_0 + 2] = static_cast<DocIdx>(target_seq_len);
|
||||
}
|
||||
|
||||
// Update indices / counters.
|
||||
++map_index;
|
||||
prev_start_index = sent_index + 1;
|
||||
target_seq_len = get_target_sample_len(
|
||||
short_seq_ratio, max_seq_length, rand32_gen);
|
||||
seq_len = 0;
|
||||
num_sent = 0;
|
||||
}
|
||||
|
||||
} // for (auto sent_index=sent_index_first; ...
|
||||
} // if (num_remain_sent > 1) {
|
||||
} // for (int doc=0; doc < num_docs; ++doc) {
|
||||
} // for (int epoch=0; epoch < num_epochs; ++epoch) {
|
||||
|
||||
if (!second) {
|
||||
if (verbose) {
|
||||
cout << " number of empty documents: " << empty_docs << endl
|
||||
<< std::flush;
|
||||
cout << " number of documents with one sentence: " << one_sent_docs
|
||||
<< endl
|
||||
<< std::flush;
|
||||
cout << " number of documents with long sentences: " << long_sent_docs
|
||||
<< endl
|
||||
<< std::flush;
|
||||
cout << " will create mapping for " << map_index << " samples" << endl
|
||||
<< std::flush;
|
||||
}
|
||||
assert(maps == NULL);
|
||||
assert(num_samples < 0);
|
||||
maps = new DocIdx[3 * map_index];
|
||||
num_samples = static_cast<int64_t>(map_index);
|
||||
}
|
||||
|
||||
} // for (int iteration=0; iteration < 2; ++iteration) {
|
||||
|
||||
// Shuffle.
|
||||
// We need a 64 bit random number generator as we might have more
|
||||
// than 2 billion samples.
|
||||
std::mt19937_64 rand64_gen(seed + 1);
|
||||
for (auto i = (num_samples - 1); i > 0; --i) {
|
||||
const auto j = static_cast<int64_t>(rand64_gen() % (i + 1));
|
||||
const auto i0 = 3 * i;
|
||||
const auto j0 = 3 * j;
|
||||
// Swap values.
|
||||
swap(maps[i0], maps[j0]);
|
||||
swap(maps[i0 + 1], maps[j0 + 1]);
|
||||
swap(maps[i0 + 2], maps[j0 + 2]);
|
||||
}
|
||||
|
||||
// Method to deallocate memory.
|
||||
py::capsule free_when_done(maps, [](void* mem_) {
|
||||
DocIdx* mem = reinterpret_cast<DocIdx*>(mem_);
|
||||
delete[] mem;
|
||||
});
|
||||
|
||||
// Return the numpy array.
|
||||
const auto byte_size = sizeof(DocIdx);
|
||||
return py::array(std::vector<int64_t>{num_samples, 3}, // shape
|
||||
{3 * byte_size, byte_size}, // C-style contiguous strides
|
||||
maps, // the data pointer
|
||||
free_when_done); // numpy array references
|
||||
}
|
||||
|
||||
|
||||
py::array build_mapping(const py::array_t<int64_t>& docs_,
|
||||
const py::array_t<int>& sizes_,
|
||||
const int num_epochs,
|
||||
const uint64_t max_num_samples,
|
||||
const int max_seq_length,
|
||||
const double short_seq_prob,
|
||||
const int seed,
|
||||
const bool verbose,
|
||||
const int32_t min_num_sent) {
|
||||
if (sizes_.size() > std::numeric_limits<uint32_t>::max()) {
|
||||
if (verbose) {
|
||||
cout << " using uint64 for data mapping..." << endl << std::flush;
|
||||
}
|
||||
return build_mapping_impl<uint64_t>(docs_,
|
||||
sizes_,
|
||||
num_epochs,
|
||||
max_num_samples,
|
||||
max_seq_length,
|
||||
short_seq_prob,
|
||||
seed,
|
||||
verbose,
|
||||
min_num_sent);
|
||||
} else {
|
||||
if (verbose) {
|
||||
cout << " using uint32 for data mapping..." << endl << std::flush;
|
||||
}
|
||||
return build_mapping_impl<uint32_t>(docs_,
|
||||
sizes_,
|
||||
num_epochs,
|
||||
max_num_samples,
|
||||
max_seq_length,
|
||||
short_seq_prob,
|
||||
seed,
|
||||
verbose,
|
||||
min_num_sent);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename DocIdx>
|
||||
py::array build_blocks_mapping_impl(const py::array_t<int64_t>& docs_,
|
||||
const py::array_t<int32_t>& sizes_,
|
||||
const py::array_t<int32_t>& titles_sizes_,
|
||||
const int32_t num_epochs,
|
||||
const uint64_t max_num_samples,
|
||||
const int32_t max_seq_length,
|
||||
const int32_t seed,
|
||||
const bool verbose,
|
||||
const bool use_one_sent_blocks) {
|
||||
/* Build a mapping of (start-index, end-index, sequence-length) where
|
||||
start and end index are the indices of the sentences in the sample
|
||||
and sequence-length is the target sequence length.
|
||||
*/
|
||||
|
||||
// Consistency checks.
|
||||
assert(num_epochs > 0);
|
||||
assert(max_seq_length > 1);
|
||||
assert(seed > 0);
|
||||
|
||||
// Remove bound checks.
|
||||
auto docs = docs_.unchecked<1>();
|
||||
auto sizes = sizes_.unchecked<1>();
|
||||
auto titles_sizes = titles_sizes_.unchecked<1>();
|
||||
|
||||
if (verbose) {
|
||||
const auto sent_start_index = docs[0];
|
||||
const auto sent_end_index = docs[docs_.shape(0) - 1];
|
||||
const auto num_sentences = sent_end_index - sent_start_index;
|
||||
cout << " using:" << endl << std::flush;
|
||||
cout << " number of documents: " << docs_.shape(0) - 1
|
||||
<< endl
|
||||
<< std::flush;
|
||||
cout << " sentences range: [" << sent_start_index << ", "
|
||||
<< sent_end_index << ")" << endl
|
||||
<< std::flush;
|
||||
cout << " total number of sentences: " << num_sentences << endl
|
||||
<< std::flush;
|
||||
cout << " number of epochs: " << num_epochs << endl
|
||||
<< std::flush;
|
||||
cout << " maximum number of samples: " << max_num_samples << endl
|
||||
<< std::flush;
|
||||
cout << " maximum sequence length: " << max_seq_length << endl
|
||||
<< std::flush;
|
||||
cout << " seed: " << seed << endl
|
||||
<< std::flush;
|
||||
}
|
||||
|
||||
// Mapping and its length (1D).
|
||||
int64_t num_samples = -1;
|
||||
DocIdx* maps = NULL;
|
||||
|
||||
// Acceptable number of sentences per block.
|
||||
int min_num_sent = 2;
|
||||
if (use_one_sent_blocks) {
|
||||
min_num_sent = 1;
|
||||
}
|
||||
|
||||
// Perform two iterations, in the first iteration get the size
|
||||
// and allocate memory and in the second iteration populate the map.
|
||||
bool second = false;
|
||||
for (int32_t iteration = 0; iteration < 2; ++iteration) {
|
||||
// Set the flag on second iteration.
|
||||
second = (iteration == 1);
|
||||
|
||||
// Current map index.
|
||||
uint64_t map_index = 0;
|
||||
|
||||
uint64_t empty_docs = 0;
|
||||
uint64_t one_sent_docs = 0;
|
||||
uint64_t long_sent_docs = 0;
|
||||
// For each epoch:
|
||||
for (int32_t epoch = 0; epoch < num_epochs; ++epoch) {
|
||||
// assign every block a unique id
|
||||
int32_t block_id = 0;
|
||||
|
||||
if (map_index >= max_num_samples) {
|
||||
if (verbose && (!second)) {
|
||||
cout << " reached " << max_num_samples << " samples after "
|
||||
<< epoch << " epochs ..." << endl
|
||||
<< std::flush;
|
||||
}
|
||||
break;
|
||||
}
|
||||
// For each document:
|
||||
for (int32_t doc = 0; doc < (docs.shape(0) - 1); ++doc) {
|
||||
// Document sentences are in [sent_index_first, sent_index_last)
|
||||
const auto sent_index_first = docs[doc];
|
||||
const auto sent_index_last = docs[doc + 1];
|
||||
const auto target_seq_len = max_seq_length - titles_sizes[doc];
|
||||
|
||||
// At the begining of the document previous index is the
|
||||
// start index.
|
||||
auto prev_start_index = sent_index_first;
|
||||
|
||||
// Remaining documents.
|
||||
auto num_remain_sent = sent_index_last - sent_index_first;
|
||||
|
||||
// Some bookkeeping
|
||||
if ((epoch == 0) && (!second)) {
|
||||
if (num_remain_sent == 0) {
|
||||
++empty_docs;
|
||||
}
|
||||
if (num_remain_sent == 1) {
|
||||
++one_sent_docs;
|
||||
}
|
||||
}
|
||||
// Detect documents with long sentences.
|
||||
bool contains_long_sentence = false;
|
||||
if (num_remain_sent >= min_num_sent) {
|
||||
for (auto sent_index = sent_index_first; sent_index < sent_index_last;
|
||||
++sent_index) {
|
||||
if (sizes[sent_index] > LONG_SENTENCE_LEN) {
|
||||
if ((epoch == 0) && (!second)) {
|
||||
++long_sent_docs;
|
||||
}
|
||||
contains_long_sentence = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// If we have enough sentences and no long sentences.
|
||||
if ((num_remain_sent >= min_num_sent) && (!contains_long_sentence)) {
|
||||
// Set values.
|
||||
auto seq_len = int32_t{0};
|
||||
auto num_sent = int32_t{0};
|
||||
|
||||
// Loop through sentences.
|
||||
for (auto sent_index = sent_index_first; sent_index < sent_index_last;
|
||||
++sent_index) {
|
||||
// Add the size and number of sentences.
|
||||
seq_len += sizes[sent_index];
|
||||
++num_sent;
|
||||
--num_remain_sent;
|
||||
|
||||
// If we have reached the target length.
|
||||
// and there are an acceptable number of sentences left
|
||||
// and if we have at least the minimum number of sentences.
|
||||
// or if we have reached end of the document.
|
||||
if (((seq_len >= target_seq_len) &&
|
||||
(num_remain_sent >= min_num_sent) &&
|
||||
(num_sent >= min_num_sent)) ||
|
||||
(num_remain_sent == 0)) {
|
||||
// Populate the map.
|
||||
if (second) {
|
||||
const auto map_index_0 = 4 * map_index;
|
||||
// Each sample has 4 items: the starting sentence index, ending
|
||||
// sentence index,
|
||||
// the index of the document from which the block comes (used
|
||||
// for fetching titles)
|
||||
// and the unique id of the block (used for creating block
|
||||
// indexes)
|
||||
|
||||
maps[map_index_0] = static_cast<DocIdx>(prev_start_index);
|
||||
maps[map_index_0 + 1] = static_cast<DocIdx>(sent_index + 1);
|
||||
maps[map_index_0 + 2] = static_cast<DocIdx>(doc);
|
||||
maps[map_index_0 + 3] = static_cast<DocIdx>(block_id);
|
||||
}
|
||||
|
||||
// Update indices / counters.
|
||||
++map_index;
|
||||
++block_id;
|
||||
prev_start_index = sent_index + 1;
|
||||
seq_len = 0;
|
||||
num_sent = 0;
|
||||
}
|
||||
} // for (auto sent_index=sent_index_first; ...
|
||||
} // if (num_remain_sent > 1) {
|
||||
} // for (int doc=0; doc < num_docs; ++doc) {
|
||||
} // for (int epoch=0; epoch < num_epochs; ++epoch) {
|
||||
|
||||
if (!second) {
|
||||
if (verbose) {
|
||||
cout << " number of empty documents: " << empty_docs << endl
|
||||
<< std::flush;
|
||||
cout << " number of documents with one sentence: " << one_sent_docs
|
||||
<< endl
|
||||
<< std::flush;
|
||||
cout << " number of documents with long sentences: " << long_sent_docs
|
||||
<< endl
|
||||
<< std::flush;
|
||||
cout << " will create mapping for " << map_index << " samples" << endl
|
||||
<< std::flush;
|
||||
}
|
||||
assert(maps == NULL);
|
||||
assert(num_samples < 0);
|
||||
maps = new DocIdx[4 * map_index];
|
||||
num_samples = static_cast<int64_t>(map_index);
|
||||
}
|
||||
|
||||
} // for (int iteration=0; iteration < 2; ++iteration) {
|
||||
|
||||
// Shuffle.
|
||||
// We need a 64 bit random number generator as we might have more
|
||||
// than 2 billion samples.
|
||||
std::mt19937_64 rand64_gen(seed + 1);
|
||||
for (auto i = (num_samples - 1); i > 0; --i) {
|
||||
const auto j = static_cast<int64_t>(rand64_gen() % (i + 1));
|
||||
const auto i0 = 4 * i;
|
||||
const auto j0 = 4 * j;
|
||||
// Swap values.
|
||||
swap(maps[i0], maps[j0]);
|
||||
swap(maps[i0 + 1], maps[j0 + 1]);
|
||||
swap(maps[i0 + 2], maps[j0 + 2]);
|
||||
swap(maps[i0 + 3], maps[j0 + 3]);
|
||||
}
|
||||
|
||||
// Method to deallocate memory.
|
||||
py::capsule free_when_done(maps, [](void* mem_) {
|
||||
DocIdx* mem = reinterpret_cast<DocIdx*>(mem_);
|
||||
delete[] mem;
|
||||
});
|
||||
|
||||
// Return the numpy array.
|
||||
const auto byte_size = sizeof(DocIdx);
|
||||
return py::array(std::vector<int64_t>{num_samples, 4}, // shape
|
||||
{4 * byte_size, byte_size}, // C-style contiguous strides
|
||||
maps, // the data pointer
|
||||
free_when_done); // numpy array references
|
||||
}
|
||||
|
||||
py::array build_blocks_mapping(const py::array_t<int64_t>& docs_,
|
||||
const py::array_t<int>& sizes_,
|
||||
const py::array_t<int>& titles_sizes_,
|
||||
const int num_epochs,
|
||||
const uint64_t max_num_samples,
|
||||
const int max_seq_length,
|
||||
const int seed,
|
||||
const bool verbose,
|
||||
const bool use_one_sent_blocks) {
|
||||
if (sizes_.size() > std::numeric_limits<uint32_t>::max()) {
|
||||
if (verbose) {
|
||||
cout << " using uint64 for data mapping..." << endl << std::flush;
|
||||
}
|
||||
return build_blocks_mapping_impl<uint64_t>(docs_,
|
||||
sizes_,
|
||||
titles_sizes_,
|
||||
num_epochs,
|
||||
max_num_samples,
|
||||
max_seq_length,
|
||||
seed,
|
||||
verbose,
|
||||
use_one_sent_blocks);
|
||||
} else {
|
||||
if (verbose) {
|
||||
cout << " using uint32 for data mapping..." << endl << std::flush;
|
||||
}
|
||||
return build_blocks_mapping_impl<uint32_t>(docs_,
|
||||
sizes_,
|
||||
titles_sizes_,
|
||||
num_epochs,
|
||||
max_num_samples,
|
||||
max_seq_length,
|
||||
seed,
|
||||
verbose,
|
||||
use_one_sent_blocks);
|
||||
}
|
||||
}
|
||||
|
||||
PYBIND11_MODULE(helpers, m) {
|
||||
m.def("build_mapping", &build_mapping);
|
||||
m.def("build_blocks_mapping", &build_blocks_mapping);
|
||||
m.def("build_sample_idx", &build_sample_idx);
|
||||
m.def("build_blending_indices", &build_blending_indices);
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
import argparse
|
||||
import json
|
||||
import multiprocessing
|
||||
import sys
|
||||
import time
|
||||
import shutil
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'--input_path',
|
||||
type=str,
|
||||
required=True,
|
||||
help='Path to you raw files. Folder or file path.')
|
||||
parser.add_argument(
|
||||
'--output_path',
|
||||
type=str,
|
||||
required=True,
|
||||
help='Path to save the output json files.')
|
||||
parser.add_argument(
|
||||
'--json_key',
|
||||
type=str,
|
||||
default='text',
|
||||
help='The content key of json file.')
|
||||
parser.add_argument(
|
||||
'--doc_spliter',
|
||||
type=str,
|
||||
default='',
|
||||
help="Spliter between documents. We will strip the line, if you use blank line to split doc, leave it blank."
|
||||
)
|
||||
parser.add_argument(
|
||||
'--min_doc_length',
|
||||
type=int,
|
||||
default=10,
|
||||
help="Minimal char of a documment.")
|
||||
parser.add_argument(
|
||||
'--workers',
|
||||
type=int,
|
||||
default=1,
|
||||
help='Number of worker processes to launch')
|
||||
parser.add_argument(
|
||||
'--log_interval',
|
||||
type=int,
|
||||
default=1,
|
||||
help='Interval between progress updates.')
|
||||
parser.add_argument(
|
||||
'--no-merge', action='store_true', help='Don\'t merge the file.')
|
||||
parser.add_argument(
|
||||
'--no-shuffle', action='store_true', help='Don\'t shuffle the file.')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def raw_text_to_json(path, doc_spliter="", json_key="text", min_doc_length=10):
|
||||
path = os.path.abspath(path)
|
||||
if not os.path.exists(path):
|
||||
print("No found file %s" % path)
|
||||
return 0, None
|
||||
|
||||
out_filepath = path + ".jsonl"
|
||||
fout = open(out_filepath, "w", encoding="utf-8")
|
||||
len_files = 0
|
||||
with open(path, "r") as f:
|
||||
doc = ""
|
||||
line = f.readline()
|
||||
while line:
|
||||
len_files += len(line)
|
||||
if line.strip() == doc_spliter:
|
||||
if len(doc) > min_doc_length:
|
||||
fout.write(
|
||||
json.dumps(
|
||||
{
|
||||
json_key: doc
|
||||
}, ensure_ascii=False) + "\n")
|
||||
doc = ""
|
||||
else:
|
||||
doc += line
|
||||
line = f.readline()
|
||||
|
||||
if len(doc) > min_doc_length:
|
||||
fout.write(json.dumps({json_key: doc}, ensure_ascii=False) + "\n")
|
||||
doc = ""
|
||||
|
||||
return len_files, out_filepath
|
||||
|
||||
|
||||
def merge_file(file_paths, output_path):
|
||||
if not output_path.endswith(".jsonl"):
|
||||
output_path = output_path + ".jsonl"
|
||||
print("Merging files into %s" % output_path)
|
||||
with open(output_path, 'wb') as wfd:
|
||||
for f in file_paths:
|
||||
if f is not None and os.path.exists(f):
|
||||
with open(f, 'rb') as fd:
|
||||
shutil.copyfileobj(fd, wfd)
|
||||
os.remove(f)
|
||||
print("File save in %s" % output_path)
|
||||
return output_path
|
||||
|
||||
|
||||
def shuffle_file(output_path):
|
||||
print("Shuffling the jsonl file...")
|
||||
if os.path.exists(output_path):
|
||||
os.system("shuf %s -o %s" % (output_path, output_path))
|
||||
print("File shuffled!!!")
|
||||
else:
|
||||
raise ValueError("File not found: %s" % output_path)
|
||||
|
||||
|
||||
def main():
|
||||
args = get_args()
|
||||
startup_start = time.time()
|
||||
|
||||
file_paths = []
|
||||
if os.path.isfile(args.input_path):
|
||||
file_paths.append(args.input_path)
|
||||
else:
|
||||
for root, _, fs in os.walk(args.input_path):
|
||||
for f in fs:
|
||||
file_paths.append(os.path.join(root, f))
|
||||
|
||||
pool = multiprocessing.Pool(args.workers)
|
||||
|
||||
startup_end = time.time()
|
||||
proc_start = time.time()
|
||||
total_bytes_processed = 0
|
||||
print("Time to startup:", startup_end - startup_start)
|
||||
|
||||
trans_json = partial(
|
||||
raw_text_to_json,
|
||||
doc_spliter=args.doc_spliter,
|
||||
json_key=args.json_key,
|
||||
min_doc_length=args.min_doc_length)
|
||||
encoded_files = pool.imap(trans_json, file_paths, 1)
|
||||
|
||||
out_paths = []
|
||||
for i, (bytes_processed, out_path) in enumerate(encoded_files, start=1):
|
||||
total_bytes_processed += bytes_processed
|
||||
out_paths.append(out_path)
|
||||
master_start = time.time()
|
||||
|
||||
if i % args.log_interval == 0:
|
||||
current = time.time()
|
||||
elapsed = current - proc_start
|
||||
mbs = total_bytes_processed / elapsed / 1024 / 1024
|
||||
print(
|
||||
f"Processed {i} files",
|
||||
f"({i/elapsed} files/s, {mbs} MB/s).",
|
||||
file=sys.stderr)
|
||||
|
||||
if not args.no_merge:
|
||||
output_path = merge_file(out_paths, args.output_path)
|
||||
if not args.no_shuffle:
|
||||
shuffle_file(output_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
#profile.run("main()", "testprof")
|
||||
@@ -0,0 +1,40 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from paddlenlp.utils.tools import static_params_to_dygraph
|
||||
import paddle
|
||||
from paddlenlp.transformers import ErnieModel, ErnieForPretraining, ErniePretrainingCriterion, ErnieTokenizer
|
||||
import os
|
||||
import paddle
|
||||
import paddle.static as static
|
||||
import paddle.nn as nn
|
||||
|
||||
|
||||
def load_ernie_model(static_model_path):
|
||||
|
||||
model = ErnieModel.from_pretrained('ernie-1.0')
|
||||
program_state = static.load_program_state(static_model_path)
|
||||
ret_dict = static_params_to_dygraph(model, program_state)
|
||||
|
||||
print('转换前的参数：')
|
||||
print(model.embeddings.word_embeddings.weight)
|
||||
model.load_dict(ret_dict)
|
||||
print('转换后的参数：')
|
||||
print(model.embeddings.word_embeddings.weight)
|
||||
model.save_pretrained("./ernie_checkpoint")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
static_model_path = "./output/ernie-1.0-dp8-gb1024/model_last/static_vars"
|
||||
load_ernie_model(static_model_path)
|
||||
@@ -0,0 +1,676 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Pretrain ERNIE in static graph mode.
|
||||
"""
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
import yaml
|
||||
import shutil
|
||||
import collections
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.distributed.fleet as fleet
|
||||
from paddle.distributed.fleet.meta_optimizers.sharding.utils import save_persistables
|
||||
from paddle.io import DataLoader, Dataset
|
||||
from paddlenlp.utils.batch_sampler import DistributedBatchSampler
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
from paddlenlp.transformers import ErnieModel, ErnieForPretraining, ErniePretrainingCriterion, ErnieTokenizer
|
||||
|
||||
from paddlenlp.transformers import CosineAnnealingWithWarmupDecay, LinearDecayWithWarmup
|
||||
|
||||
from paddlenlp.ops import guard, Topology, get_rng_state_tracker
|
||||
from paddlenlp.utils.log import logger
|
||||
import paddlenlp.ops as ops
|
||||
from visualdl import LogWriter
|
||||
|
||||
from args import parse_args
|
||||
import sys
|
||||
sys.path.insert(0, "../")
|
||||
from data_tools.dataset_utils import build_train_valid_test_datasets
|
||||
|
||||
MODEL_CLASSES = {
|
||||
"ernie": (ErnieModel, ErnieForPretraining, ErniePretrainingCriterion,
|
||||
ErnieTokenizer),
|
||||
}
|
||||
|
||||
|
||||
def create_pretrained_dataset(
|
||||
args,
|
||||
data_file,
|
||||
tokenizer,
|
||||
data_world_size,
|
||||
data_world_rank,
|
||||
max_seq_len,
|
||||
places,
|
||||
data_holders,
|
||||
current_step=0, ):
|
||||
|
||||
train_valid_test_num_samples = [
|
||||
args.global_batch_size * args.max_steps, args.micro_batch_size *
|
||||
(args.max_steps // args.eval_freq + 1) * args.eval_iters *
|
||||
data_world_size, args.micro_batch_size * args.test_iters
|
||||
]
|
||||
train_ds, valid_ds, test_ds = build_train_valid_test_datasets(
|
||||
data_prefix=data_file,
|
||||
args=args,
|
||||
tokenizer=tokenizer,
|
||||
splits_string=args.split,
|
||||
train_valid_test_num_samples=train_valid_test_num_samples,
|
||||
max_seq_length=args.max_seq_len,
|
||||
masked_lm_prob=args.masked_lm_prob,
|
||||
short_seq_prob=args.short_seq_prob,
|
||||
seed=args.seed,
|
||||
skip_warmup=True,
|
||||
binary_head=True,
|
||||
max_seq_length_dec=None,
|
||||
dataset_type='ernie')
|
||||
|
||||
# 测试集没有
|
||||
# test_ds=valid_ds
|
||||
def _collate_data(data, stack_fn=Stack()):
|
||||
num_fields = len(data[0])
|
||||
out = [None] * num_fields
|
||||
# 0. input_ids,
|
||||
# 1. segment_ids,
|
||||
# 2. input_mask,
|
||||
# 3. masked_lm_positions,
|
||||
# 4. masked_lm_labels,
|
||||
# 5. next_sentence_labels
|
||||
for i in (0, 1, 2, 5):
|
||||
out[i] = stack_fn([x[i] for x in data])
|
||||
out[5] = out[5].reshape([-1, 1])
|
||||
batch_size, seq_length = out[0].shape
|
||||
size = num_mask = sum(len(x[3]) for x in data)
|
||||
# masked_lm_positions
|
||||
# Organize as a 1D tensor for gather or use gather_nd
|
||||
if size % 8 != 0:
|
||||
size += 8 - (size % 8)
|
||||
out[3] = np.full(size, 0, dtype=np.int32)
|
||||
# masked_lm_labels
|
||||
out[4] = np.full([size, 1], -1, dtype=np.int64)
|
||||
mask_token_num = 0
|
||||
for i, x in enumerate(data):
|
||||
for j, pos in enumerate(x[3]):
|
||||
out[3][mask_token_num] = i * seq_length + pos
|
||||
out[4][mask_token_num] = x[4][j]
|
||||
mask_token_num += 1
|
||||
|
||||
return out
|
||||
|
||||
def loader(dataset, consumed_samples=0):
|
||||
# print(dataset[0])
|
||||
batch_sampler = DistributedBatchSampler(
|
||||
dataset,
|
||||
batch_size=args.micro_batch_size,
|
||||
num_replicas=data_world_size,
|
||||
rank=data_world_rank,
|
||||
shuffle=False,
|
||||
drop_last=True,
|
||||
consumed_samples=consumed_samples)
|
||||
data_loader = paddle.io.DataLoader(
|
||||
dataset=dataset,
|
||||
places=places,
|
||||
feed_list=data_holders,
|
||||
batch_sampler=batch_sampler,
|
||||
num_workers=args.num_workers,
|
||||
worker_init_fn=None,
|
||||
collate_fn=_collate_data,
|
||||
return_list=False)
|
||||
return data_loader
|
||||
|
||||
train_dl = loader(train_ds, args.global_batch_size * current_step)
|
||||
valid_dl = loader(valid_ds, args.micro_batch_size * (
|
||||
(current_step + 1) // args.eval_freq) * args.eval_iters *
|
||||
data_world_size)
|
||||
test_dl = loader(test_ds, 0)
|
||||
|
||||
return train_dl, valid_dl, test_dl
|
||||
|
||||
|
||||
def create_data_holder(args=None):
|
||||
input_ids = paddle.static.data(
|
||||
name="input_ids", shape=[-1, -1], dtype="int64")
|
||||
segment_ids = paddle.static.data(
|
||||
name="segment_ids", shape=[-1, -1], dtype="int64")
|
||||
input_mask = paddle.static.data(
|
||||
name="input_mask", shape=[-1, 1, 1, -1], dtype="float32")
|
||||
masked_lm_positions = paddle.static.data(
|
||||
name="masked_lm_positions", shape=[-1], dtype="int32")
|
||||
masked_lm_labels = paddle.static.data(
|
||||
name="masked_lm_labels", shape=[-1, 1], dtype="int64")
|
||||
next_sentence_labels = paddle.static.data(
|
||||
name="next_sentence_labels", shape=[-1, 1], dtype="int64")
|
||||
|
||||
return [
|
||||
input_ids, segment_ids, input_mask, masked_lm_positions,
|
||||
masked_lm_labels, next_sentence_labels
|
||||
]
|
||||
|
||||
|
||||
def dist_optimizer(args, topo):
|
||||
default_global_batch_size = topo.data_info.size * args.micro_batch_size
|
||||
if args.global_batch_size is None:
|
||||
args.global_batch_size = default_global_batch_size
|
||||
|
||||
bsz_per_dp = args.global_batch_size // topo.data_info.size
|
||||
micro_batch_size = args.micro_batch_size
|
||||
assert args.global_batch_size % micro_batch_size == 0, \
|
||||
"cannot do gradient accumulate, global_batch_size: {} micro_batch_size: {}".format(
|
||||
args.global_batch_size, micro_batch_size)
|
||||
accumulate_steps = bsz_per_dp // micro_batch_size
|
||||
|
||||
exec_strategy = paddle.fluid.ExecutionStrategy()
|
||||
exec_strategy.num_threads = 1
|
||||
exec_strategy.num_iteration_per_drop_scope = 10000
|
||||
|
||||
build_strategy = paddle.static.BuildStrategy()
|
||||
#build_strategy.enable_sequential_execution = True # for profile
|
||||
build_strategy.fuse_broadcast_ops = True
|
||||
build_strategy.enable_inplace = True
|
||||
build_strategy.enable_addto = args.enable_addto
|
||||
|
||||
dist_strategy = fleet.DistributedStrategy()
|
||||
dist_strategy.execution_strategy = exec_strategy
|
||||
dist_strategy.build_strategy = build_strategy
|
||||
dist_strategy.nccl_comm_num = 3
|
||||
dist_strategy.fuse_grad_size_in_MB = 16
|
||||
|
||||
dist_strategy.recompute = args.use_recompute
|
||||
dist_strategy.pipeline = args.pp_degree > 1
|
||||
|
||||
if args.pp_degree <= 1 and args.sharding_degree <= 1 and accumulate_steps > 1:
|
||||
dist_strategy.gradient_merge = True
|
||||
dist_strategy.gradient_merge_configs = {'k_steps': accumulate_steps}
|
||||
args.eval_iters *= accumulate_steps
|
||||
args.test_iters *= accumulate_steps
|
||||
|
||||
if args.use_amp:
|
||||
dist_strategy.amp = True
|
||||
dist_strategy.amp_configs = {
|
||||
"custom_white_list": [
|
||||
'softmax',
|
||||
'layer_norm',
|
||||
'gelu',
|
||||
],
|
||||
"custom_black_list": ['c_softmax_with_cross_entropy'],
|
||||
"init_loss_scaling": 32768,
|
||||
"use_dynamic_loss_scaling": True,
|
||||
}
|
||||
if args.use_sharding:
|
||||
dist_strategy.sharding = True
|
||||
dist_strategy.sharding_configs = {
|
||||
"segment_broadcast_MB": 32,
|
||||
"sharding_degree": args.sharding_degree,
|
||||
"mp_degree": args.mp_degree,
|
||||
"pp_degree": args.pp_degree,
|
||||
"dp_degree": args.dp_degree,
|
||||
"gradient_merge_acc_step": accumulate_steps
|
||||
if args.sharding_degree > 1 else 1,
|
||||
"optimize_offload": False,
|
||||
}
|
||||
if args.pp_degree > 1:
|
||||
dist_strategy.pipeline_configs = {
|
||||
"schedule_mode": "1F1B",
|
||||
"micro_micro_batch_size": micro_batch_size,
|
||||
"accumulate_steps": accumulate_steps,
|
||||
}
|
||||
|
||||
args.accumulate_steps = accumulate_steps
|
||||
return dist_strategy
|
||||
|
||||
|
||||
def get_train_data_file(args):
|
||||
files = [
|
||||
os.path.join(args.input_dir, f) for f in os.listdir(args.input_dir)
|
||||
if (os.path.isfile(os.path.join(args.input_dir, f)) and "_idx.npz" in
|
||||
str(f))
|
||||
]
|
||||
files = [x.replace("_idx.npz", "") for x in files]
|
||||
return files
|
||||
|
||||
|
||||
def run_evaluate(data_loader,
|
||||
exe,
|
||||
program,
|
||||
iter_steps,
|
||||
log_writer,
|
||||
global_step,
|
||||
args,
|
||||
is_last,
|
||||
eval_fetch,
|
||||
task_name="valid"):
|
||||
all_loss, all_lm_loss, all_sop_loss = [], [], []
|
||||
local_time = time.time()
|
||||
|
||||
for eval_step, batch in enumerate(data_loader):
|
||||
ret = exe.run(program, feed=batch, fetch_list=eval_fetch)
|
||||
loss_return, lm_loss_return, sop_loss_return = ret
|
||||
if is_last:
|
||||
all_loss.append(float(loss_return[0]))
|
||||
all_lm_loss.append(float(lm_loss_return[0]))
|
||||
all_sop_loss.append(float(sop_loss_return[0]))
|
||||
|
||||
if eval_step >= iter_steps - 1:
|
||||
if not is_last:
|
||||
break
|
||||
average_loss = sum(all_loss) / len(all_loss)
|
||||
average_lm_loss = sum(all_lm_loss) / len(all_lm_loss)
|
||||
average_sop_loss = sum(all_sop_loss) / len(all_sop_loss)
|
||||
logger.info(
|
||||
"%s step %d, batch: %d, loss: %f, lm_loss: %.6f, sop_loss: %.6f, speed: %.0f tokens/s"
|
||||
% (task_name, global_step, eval_step, average_loss,
|
||||
average_lm_loss, average_sop_loss,
|
||||
iter_steps * args.micro_batch_size * args.max_seq_len /
|
||||
(time.time() - local_time)))
|
||||
|
||||
log_writer.add_scalar(task_name + "_loss", average_loss,
|
||||
global_step)
|
||||
log_writer.add_scalar(task_name + "_lm_loss", average_lm_loss,
|
||||
global_step)
|
||||
log_writer.add_scalar(task_name + "_sop_loss", average_sop_loss,
|
||||
global_step)
|
||||
|
||||
break
|
||||
|
||||
|
||||
def do_train(args):
|
||||
# Initialize the paddle and paddle fleet execute environment
|
||||
paddle.enable_static()
|
||||
fleet.init(is_collective=True)
|
||||
|
||||
# Create the random seed for the worker
|
||||
random.seed(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
paddle.seed(args.seed)
|
||||
get_rng_state_tracker().add('global_seed', args.seed)
|
||||
get_rng_state_tracker().add('local_seed',
|
||||
args.seed + fleet.worker_index() + 2021)
|
||||
|
||||
assert args.device in [
|
||||
"cpu", "gpu", "xpu"
|
||||
], "Invalid device! Available device should be cpu, gpu, or xpu."
|
||||
place = paddle.set_device(args.device)
|
||||
|
||||
worker_num = fleet.worker_num()
|
||||
worker_index = fleet.worker_index()
|
||||
assert args.dp_degree * args.sharding_degree * args.mp_degree * args.pp_degree == worker_num, \
|
||||
"The product of degree num should be equal to worker_num."
|
||||
|
||||
topo = Topology(
|
||||
device_rank=worker_index,
|
||||
world_size=worker_num,
|
||||
dp_degree=args.dp_degree,
|
||||
pp_degree=args.pp_degree,
|
||||
sharding_degree=args.sharding_degree,
|
||||
mp_degree=args.mp_degree)
|
||||
|
||||
logger.info("The topo of hybrid parallelism:\n{}".format(topo))
|
||||
|
||||
dist_strategy = dist_optimizer(args, topo)
|
||||
|
||||
# Create log write, train results show on last card of pipeline.
|
||||
if topo.is_last:
|
||||
log_writer_path = os.path.join(
|
||||
args.output_dir, "train_log",
|
||||
"{}_globalbsz_{}_amp_{}_recompute_{}_card_{}".format(
|
||||
args.model_name_or_path, args.global_batch_size, args.use_amp,
|
||||
args.use_recompute, worker_index).lower())
|
||||
# if os.path.exists(log_writer_path):
|
||||
# shutil.rmtree(log_writer_path)
|
||||
log_writer = LogWriter(log_writer_path)
|
||||
|
||||
# Define the input data in the static mode
|
||||
base_class, model_class, criterion_class, tokenizer_class = MODEL_CLASSES[
|
||||
args.model_type]
|
||||
pretrained_models_list = list(
|
||||
model_class.pretrained_init_configuration.keys())
|
||||
|
||||
# load config in checkpoint
|
||||
global_step = 0
|
||||
consumed_samples = 0
|
||||
checkpoint_dir = os.path.join(args.output_dir, "model_last")
|
||||
if os.path.exists(checkpoint_dir):
|
||||
if os.path.isfile(os.path.join(checkpoint_dir, "./config.yml")):
|
||||
with open(os.path.join(checkpoint_dir, "./config.yml"), "r") as f:
|
||||
step_config = yaml.load(f, Loader=yaml.FullLoader)
|
||||
assert step_config[
|
||||
"global_batch_size"] == args.global_batch_size, "Please ensure checkpoint global batch size is the same. Folder: {}".format(
|
||||
checkpoint_dir)
|
||||
consumed_samples = step_config["consumed_samples"]
|
||||
global_step = step_config["global_step"]
|
||||
|
||||
data_file = get_train_data_file(args)
|
||||
main_program = paddle.static.default_main_program()
|
||||
startup_program = paddle.static.default_startup_program()
|
||||
with paddle.static.program_guard(main_program, startup_program):
|
||||
data_holders = create_data_holder(args)
|
||||
# 0. input_ids,
|
||||
# 1. segment_ids,
|
||||
# 2. input_mask,
|
||||
# 3. masked_lm_positions,
|
||||
# 4. masked_lm_labels,
|
||||
# 5. next_sentence_labels
|
||||
|
||||
[
|
||||
input_ids, segment_ids, input_mask, masked_lm_positions,
|
||||
masked_lm_labels, next_sentence_labels
|
||||
] = data_holders
|
||||
|
||||
tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)
|
||||
|
||||
train_data_loader, valid_data_loader, test_data_loader = create_pretrained_dataset(
|
||||
args,
|
||||
data_file,
|
||||
tokenizer,
|
||||
data_world_size=topo.data_info.size,
|
||||
data_world_rank=topo.data_info.rank,
|
||||
max_seq_len=args.max_seq_len,
|
||||
places=paddle.static.cuda_places(),
|
||||
data_holders=data_holders,
|
||||
current_step=global_step)
|
||||
fleet.init(is_collective=True)
|
||||
|
||||
if args.model_name_or_path in pretrained_models_list:
|
||||
model_config = model_class.pretrained_init_configuration[
|
||||
args.model_name_or_path]
|
||||
if model_config["vocab_size"] % 8 != 0:
|
||||
model_config["vocab_size"] += 8 - (model_config["vocab_size"] %
|
||||
8)
|
||||
model_config["hidden_dropout_prob"] = args.hidden_dropout_prob
|
||||
model_config[
|
||||
"attention_probs_dropout_prob"] = args.attention_probs_dropout_prob
|
||||
model = model_class(base_class(**model_config))
|
||||
else:
|
||||
model, _ = model_class.from_pretrained(
|
||||
args.model_name_or_path,
|
||||
hidden_dropout_prob=args.hidden_dropout_prob,
|
||||
attention_probs_dropout_prob=args.attention_probs_dropout_prob,
|
||||
)
|
||||
|
||||
# Create the model for the gpt pretrain
|
||||
prediction_scores, seq_relationship_score = model(
|
||||
input_ids=input_ids,
|
||||
token_type_ids=segment_ids,
|
||||
position_ids=None,
|
||||
attention_mask=input_mask,
|
||||
masked_positions=masked_lm_positions)
|
||||
|
||||
criterion = criterion_class()
|
||||
lm_loss, sop_loss = criterion(prediction_scores, seq_relationship_score,
|
||||
masked_lm_labels, next_sentence_labels)
|
||||
loss = lm_loss + sop_loss
|
||||
|
||||
# Create the learning_rate sheduler and optimizer
|
||||
if args.decay_steps is None:
|
||||
args.decay_steps = args.max_steps
|
||||
|
||||
# lr_scheduler = CosineAnnealingWithWarmupDecay(
|
||||
# max_lr=args.max_lr,
|
||||
# min_lr=args.min_lr,
|
||||
# warmup_step=args.warmup_rate * args.max_steps,
|
||||
# decay_step=args.decay_steps, last_epoch=global_step)
|
||||
|
||||
lr_scheduler = LinearDecayWithWarmup(
|
||||
args.max_lr,
|
||||
args.max_steps,
|
||||
args.warmup_rate,
|
||||
last_epoch=global_step)
|
||||
|
||||
clip = None
|
||||
if args.grad_clip > 0:
|
||||
clip = paddle.fluid.clip.GradientClipByGlobalNorm(
|
||||
clip_norm=args.grad_clip)
|
||||
|
||||
decay_param = [
|
||||
p.name for n, p in model.named_parameters()
|
||||
if not any(nd in n for nd in ["bias", "norm"])
|
||||
]
|
||||
logger.info("Using paddle.optimizer.AdamW.")
|
||||
optimizer = paddle.optimizer.AdamW(
|
||||
learning_rate=lr_scheduler,
|
||||
beta1=args.adam_beta1,
|
||||
beta2=args.adam_beta2,
|
||||
epsilon=args.adam_epsilon,
|
||||
grad_clip=clip,
|
||||
weight_decay=args.weight_decay,
|
||||
apply_decay_param_fun=lambda x: x in decay_param)
|
||||
# alias
|
||||
optimizer.apply_optimize = optimizer._apply_optimize
|
||||
|
||||
# if args.use_recompute:
|
||||
# dist_strategy.recompute = True
|
||||
# dist_strategy.recompute_configs = {
|
||||
# "checkpoints": model.bert.checkpoints
|
||||
# }
|
||||
|
||||
# Use the fleet api to compile the distributed optimizer
|
||||
optimizer = fleet.distributed_optimizer(
|
||||
optimizer, strategy=dist_strategy)
|
||||
|
||||
optimizer.minimize(loss)
|
||||
logger.info(f'final strategy: {fleet._final_strategy()}')
|
||||
logger.info("The training meta optimizer is/are %s" %
|
||||
fleet._get_applied_meta_list())
|
||||
|
||||
program_desc_dir = os.path.join(args.output_dir, "program_desc")
|
||||
if not os.path.isdir(program_desc_dir):
|
||||
os.mkdir(program_desc_dir)
|
||||
|
||||
with open(program_desc_dir + "/main_program.txt.%d" % worker_index,
|
||||
'w') as f:
|
||||
f.write(str(main_program))
|
||||
|
||||
with open(program_desc_dir + "/startup_program.txt.%d" % worker_index,
|
||||
'w') as f:
|
||||
f.write(str(startup_program))
|
||||
|
||||
# Define the Executor for running the static model
|
||||
exe = paddle.static.Executor(place)
|
||||
exe.run(startup_program)
|
||||
|
||||
test_program = main_program.clone(for_test=True)
|
||||
|
||||
if args.model_name_or_path not in pretrained_models_list:
|
||||
logger.info("Try to load checkpoint from %s " % args.model_name_or_path)
|
||||
dygrah_path = os.path.join(args.model_name_or_path,
|
||||
"model_state.pdparams")
|
||||
static_path = os.path.join(args.model_name_or_path, "static_vars")
|
||||
|
||||
flag_loaded = False
|
||||
if os.path.exists(static_path):
|
||||
if args.mp_degree > 1:
|
||||
logger.warning("MP should init with dygraph params")
|
||||
else:
|
||||
logger.info("Loading parameters from %s" % static_path)
|
||||
paddle.static.load(main_program, static_path, exe)
|
||||
flag_loaded = True
|
||||
|
||||
if not flag_loaded and os.path.exists(dygrah_path):
|
||||
if args.sharding_degree > 1:
|
||||
logger.warning("Sharding should init with static vars")
|
||||
else:
|
||||
logger.info("Loading parameters from %s" % dygrah_path)
|
||||
init_static_with_params(
|
||||
model,
|
||||
paddle.load(
|
||||
dygrah_path, return_numpy=True),
|
||||
topo,
|
||||
main_program)
|
||||
flag_loaded = True
|
||||
|
||||
if not flag_loaded:
|
||||
logger.error("No checkpoint load.")
|
||||
|
||||
# load checkpoint vars
|
||||
if os.path.exists(checkpoint_dir):
|
||||
if os.path.isfile(os.path.join(checkpoint_dir, "./config.yml")):
|
||||
paddle.static.load(main_program,
|
||||
os.path.join(checkpoint_dir, "static_vars"), exe)
|
||||
|
||||
fetch_vars = collections.OrderedDict()
|
||||
fetch_vars["loss"] = loss
|
||||
fetch_vars["lm_loss"] = lm_loss
|
||||
fetch_vars["sop_loss"] = sop_loss
|
||||
fetch_vars["learning_rate"] = main_program.global_block().vars[
|
||||
"learning_rate_0"]
|
||||
|
||||
additional_vars = collections.OrderedDict()
|
||||
if args.use_amp:
|
||||
for key in ["loss_scaling", "num_good_steps", "num_bad_steps"]:
|
||||
additional_vars[key] = main_program.global_block().vars[key + "_0"]
|
||||
|
||||
tic_train = time.time()
|
||||
while True:
|
||||
fetchs = []
|
||||
fetchs_keys = []
|
||||
if topo.is_last:
|
||||
fetchs = list(fetch_vars.values()) + list(additional_vars.values())
|
||||
fetchs_keys = list(fetch_vars.keys()) + list(additional_vars.keys())
|
||||
|
||||
# Bug fix, if not call valid_data_loader, the enumerate will call valid_data_loader
|
||||
# many times. and start a new random dataloader.
|
||||
valid_data_loader = valid_data_loader()
|
||||
test_data_loader = test_data_loader()
|
||||
|
||||
for step, batch in enumerate(train_data_loader()):
|
||||
ret = exe.run(main_program,
|
||||
feed=batch,
|
||||
fetch_list=fetchs,
|
||||
use_program_cache=True)
|
||||
# Skip for accumulate_steps in global step
|
||||
if (step + 1) % args.accumulate_steps != 0:
|
||||
continue
|
||||
global_step += 1
|
||||
# In the new 2.0 api, must call this function to change the learning_rate
|
||||
lr_scheduler.step()
|
||||
|
||||
if global_step % args.logging_freq == 0:
|
||||
if topo.is_last:
|
||||
res = {}
|
||||
for k, v in zip(fetchs_keys, ret):
|
||||
res[k] = v[0]
|
||||
|
||||
speed = args.logging_freq / (time.time() - tic_train)
|
||||
common_loginfo = "global step %d, loss: %.9f, lm_loss: %.6f, sop_loss: %.6f, speed: %.2f steps/s, ips: %.2f seqs/s, learning rate: %.5e" % (
|
||||
global_step, res["loss"], res["lm_loss"],
|
||||
res["sop_loss"], speed, speed * args.global_batch_size,
|
||||
res["learning_rate"])
|
||||
additional_loginfo = ", ".join([
|
||||
"{}: {}".format(k, res[k])
|
||||
for k in additional_vars.keys()
|
||||
])
|
||||
if additional_loginfo:
|
||||
common_loginfo += ", " + additional_loginfo
|
||||
logger.info(common_loginfo)
|
||||
for k, v in res.items():
|
||||
log_writer.add_scalar(k, v, global_step)
|
||||
|
||||
tic_train = time.time()
|
||||
|
||||
#if args.check_accuracy:
|
||||
# if global_step >= args.max_steps:
|
||||
# return
|
||||
# else:
|
||||
# continue
|
||||
|
||||
if global_step % args.eval_freq == 0:
|
||||
# TODO, check the input data of validation
|
||||
eval_fetch = []
|
||||
if topo.is_last:
|
||||
eval_fetch = [loss, lm_loss, sop_loss]
|
||||
|
||||
run_evaluate(valid_data_loader, exe, test_program,
|
||||
args.eval_iters, log_writer, global_step, args,
|
||||
topo.is_last, eval_fetch, "valid")
|
||||
tic_train = time.time()
|
||||
|
||||
if global_step % args.save_steps == 0 or global_step >= args.max_steps:
|
||||
output_dir = os.path.join(args.output_dir,
|
||||
"model_%d" % global_step)
|
||||
logger.debug("saving models to {}".format(output_dir))
|
||||
save_persistables(exe,
|
||||
os.path.join(output_dir, "static_vars"),
|
||||
main_program)
|
||||
if global_step == args.save_steps:
|
||||
model.init_config["init_args"][0].init_config.pop("topo",
|
||||
None)
|
||||
model.save_pretrained(output_dir)
|
||||
tokenizer.save_pretrained(output_dir)
|
||||
tic_train = time.time()
|
||||
|
||||
if global_step % args.checkpoint_steps == 0:
|
||||
output_dir = os.path.join(args.output_dir, "model_last")
|
||||
if worker_index == 0:
|
||||
if not os.path.exists(output_dir):
|
||||
os.mkdir(output_dir)
|
||||
output_dir_bak = os.path.join(args.output_dir,
|
||||
"model_last_bak")
|
||||
if os.path.exists(output_dir):
|
||||
if os.path.exists(output_dir_bak):
|
||||
shutil.rmtree(output_dir_bak)
|
||||
shutil.move(output_dir, output_dir_bak)
|
||||
os.mkdir(output_dir)
|
||||
|
||||
step_config = {
|
||||
"model_name": args.model_name_or_path,
|
||||
"global_step": global_step,
|
||||
"global_batch_size": args.global_batch_size,
|
||||
"consumed_samples":
|
||||
global_step * args.global_batch_size,
|
||||
}
|
||||
|
||||
with open(os.path.join(output_dir, "config.yml"), "w") as f:
|
||||
yaml.dump(
|
||||
step_config,
|
||||
f,
|
||||
encoding='utf-8',
|
||||
allow_unicode=True)
|
||||
|
||||
fleet.barrier_worker()
|
||||
|
||||
logger.debug("saving models to {}".format(output_dir))
|
||||
if args.sharding_degree <= 1:
|
||||
# Save on the first worker by default.
|
||||
if worker_index == 0:
|
||||
paddle.static.save(main_program,
|
||||
os.path.join(output_dir,
|
||||
"static_vars"))
|
||||
else:
|
||||
# Use save_persistables in sharding, but more slower
|
||||
save_persistables(exe,
|
||||
os.path.join(output_dir, "static_vars"),
|
||||
main_program)
|
||||
|
||||
if global_step >= args.max_steps:
|
||||
eval_fetch = []
|
||||
if topo.is_last:
|
||||
eval_fetch = [loss, lm_loss, sop_loss]
|
||||
|
||||
run_evaluate(test_data_loader, exe, test_program,
|
||||
args.test_iters, log_writer, global_step, args,
|
||||
topo.is_last, eval_fetch, "test")
|
||||
del train_data_loader
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
config = parse_args(MODEL_CLASSES)
|
||||
do_train(config)
|
||||
@@ -0,0 +1,34 @@
|
||||
unset CUDA_VISIBLE_DEVICES
|
||||
|
||||
task_name="ernie-1.0-dp8-gb1024"
|
||||
rm -rf output/$task_name/log
|
||||
|
||||
PYTHONPATH=../../../ python -u -m paddle.distributed.launch \
|
||||
--gpus "0,1,2,3" \
|
||||
--log_dir "output/$task_name/log" \
|
||||
run_pretrain_static.py \
|
||||
--model_type "ernie" \
|
||||
--model_name_or_path "ernie-1.0" \
|
||||
--input_dir "./data" \
|
||||
--output_dir "output/$task_name" \
|
||||
--max_seq_len 512 \
|
||||
--micro_batch_size 32 \
|
||||
--global_batch_size 128 \
|
||||
--sharding_degree 1\
|
||||
--dp_degree 4 \
|
||||
--use_sharding false \
|
||||
--use_amp true \
|
||||
--use_recompute false \
|
||||
--max_lr 0.0001 \
|
||||
--min_lr 0.00001 \
|
||||
--max_steps 200000 \
|
||||
--save_steps 100000 \
|
||||
--checkpoint_steps 5000 \
|
||||
--decay_steps 1980000 \
|
||||
--weight_decay 0.01\
|
||||
--warmup_rate 0.01 \
|
||||
--grad_clip 1.0 \
|
||||
--num_workers 2 \
|
||||
--logging_freq 20\
|
||||
--eval_freq 1000 \
|
||||
--device "gpu"
|
||||
577
applications/neural_search/recall/in_batch_negative/README.md
Normal file
577
applications/neural_search/recall/in_batch_negative/README.md
Normal file
@@ -0,0 +1,577 @@
|
||||
# In-batch Negatives
|
||||
|
||||
**目录**
|
||||
|
||||
* [背景介绍](#背景介绍)
|
||||
* [In-batch Negatives](#In-batchNegatives)
|
||||
* [1. 技术方案和评估指标](#技术方案)
|
||||
* [2. 环境依赖](#环境依赖)
|
||||
* [3. 代码结构](#代码结构)
|
||||
* [4. 数据准备](#数据准备)
|
||||
* [5. 模型训练](#模型训练)
|
||||
* [6. 评估](#开始评估)
|
||||
* [7. 预测](#预测)
|
||||
* [8. 部署](#部署)
|
||||
|
||||
<a name="背景介绍"></a>
|
||||
|
||||
# 背景介绍
|
||||
|
||||
语义索引（可通俗理解为向量索引）技术是搜索引擎、推荐系统、广告系统在召回阶段的核心技术之一。语义索引模型的目标是：给定输入文本，模型可以从海量候选召回库中**快速、准确**地召回一批语义相关文本。语义索引模型的效果直接决定了语义相关的物料能否被成功召回进入系统参与上层排序，从基础层面影响整个系统的效果。
|
||||
|
||||
在召回阶段，最常见的方式是通过双塔模型，学习Document(简写为Doc)的向量表示，对Doc端建立索引，用ANN召回。我们在这种方式的基础上，引入语义索引策略 [In-batch Negatives](https://arxiv.org/abs/2004.04906)，以如下Batch size=4的训练数据为例：
|
||||
|
||||
|
||||
```
|
||||
我手机丢了，我想换个手机 我想买个新手机，求推荐
|
||||
求秋色之空漫画全集 求秋色之空全集漫画
|
||||
学日语软件手机上的 手机学日语的软件
|
||||
侠盗飞车罪恶都市怎样改车 侠盗飞车罪恶都市怎么改车
|
||||
```
|
||||
|
||||
In-batch Negatives 策略的训练数据为语义相似的 Pair 对，策略核心是在 1 个 Batch 内同时基于 N 个负例进行梯度更新，将Batch 内除自身之外其它所有 Source Text 的相似文本 Target Text 作为负例，例如: 上例中“我手机丢了，我想换个手机” 有 1 个正例(”我想买个新手机，求推荐“)，3 个负例(1.求秋色之空全集漫画，2.手机学日语的软件，3.侠盗飞车罪恶都市怎么改车)。
|
||||
|
||||
|
||||
<a name="In-batch Negatives"></a>
|
||||
|
||||
# In-batch Negatives
|
||||
|
||||
<a name="技术方案"></a>
|
||||
|
||||
## 1. 技术方案和评估指标
|
||||
|
||||
### 技术方案
|
||||
|
||||
双塔模型，采用ERNIE1.0热启，在召回训练阶段引入In-batch Negatives 策略，使用hnswlib建立索引库，进行召回测试。
|
||||
|
||||
|
||||
### 评估指标
|
||||
|
||||
采用 Recall@1，Recall@5 ，Recall@10 ，Recall@20 和 Recall@50 指标来评估语义索引模型的召回效果。
|
||||
|
||||
Recall@K召回率是指预测的前topK（top-k是指从最后的按得分排序的召回列表中返回前k个结果）结果中检索出的相关结果数和库中所有的相关结果数的比率，衡量的是检索系统的查全率。
|
||||
|
||||
**效果评估**
|
||||
|
||||
| 模型 | Recall@1 | Recall@5 |Recall@10 |Recall@20 |Recall@50 |策略简要说明|
|
||||
| ------------ | ------------ | ------------ |--------- |--------- |--------- |--------- |
|
||||
| In-batch Negatives | 51.301 | 65.309| 69.878| 73.996|78.881| Inbatch-negative有监督训练|
|
||||
|
||||
|
||||
|
||||
<a name="环境依赖"></a>
|
||||
|
||||
## 2. 环境依赖
|
||||
|
||||
推荐使用GPU进行训练，在预测阶段使用CPU或者GPU均可。
|
||||
|
||||
**环境依赖**
|
||||
* python >= 3.6
|
||||
* paddlepaddle >= 2.1.3
|
||||
* paddlenlp >= 2.2
|
||||
* [hnswlib](https://github.com/nmslib/hnswlib) >= 0.5.2
|
||||
* visualdl >= 2.2.2
|
||||
|
||||
<a name="代码结构"></a>
|
||||
|
||||
## 3. 代码结构
|
||||
|
||||
```
|
||||
|—— data.py # 数据读取、数据转换等预处理逻辑
|
||||
|—— base_model.py # 语义索引模型基类
|
||||
|—— train_batch_neg.py # In-batch Negatives 策略的训练主脚本
|
||||
|—— batch_negative
|
||||
|—— model.py # In-batch Negatives 策略核心网络结构
|
||||
|—— ann_util.py # Ann 建索引库相关函数
|
||||
|
||||
|
||||
|—— recall.py # 基于训练好的语义索引模型，从召回库中召回给定文本的相似文本
|
||||
|—— evaluate.py # 根据召回结果和评估集计算评估指标
|
||||
|—— predict.py # 给定输入文件，计算文本 pair 的相似度
|
||||
|—— export_model.py # 动态图转换成静态图
|
||||
|—— scripts
|
||||
|—— export_model.sh # 动态图转换成静态图脚本
|
||||
|—— predict.sh # 预测 bash 版本
|
||||
|—— evaluate.sh # 评估 bash 版本
|
||||
|—— run_build_index.sh # 构建索引 bash 版本
|
||||
|—— train_batch_neg.sh # 训练 bash 版本
|
||||
|—— export_to_serving.sh # Paddle Inference 转 Serving 的 bash 脚本
|
||||
|—— deploy
|
||||
|—— python
|
||||
|—— predict.py # PaddleInference
|
||||
|—— deploy.sh # Paddle Inference 部署脚本
|
||||
|—— rpc_client.py # Paddle Serving 的 Client 端
|
||||
|—— web_service.py # Paddle Serving 的 Serving 端
|
||||
|—— config_nlp.yml # Paddle Serving 的配置文件
|
||||
|—— inference.py # 动态图抽取向量
|
||||
|—— export_to_serving.py # 静态图转 Serving
|
||||
|
||||
```
|
||||
|
||||
<a name="数据准备"></a>
|
||||
|
||||
## 4. 数据准备
|
||||
|
||||
### 数据集说明
|
||||
|
||||
我们基于某文献检索平台数据，构造面向语义索引的训练集、测试集、召回库。
|
||||
|
||||
**训练集** 和 **验证集** 格式一致，训练集4k条，测试集2w条，每行由一对语义相似的文本Pair构成，以tab符分割，第一列是检索query，第二列由相关文献标题（+关键词）构成。样例数据如下:
|
||||
|
||||
```
|
||||
宁夏社区图书馆服务体系布局现状分析 宁夏社区图书馆服务体系布局现状分析社区图书馆,社区图书馆服务,社区图书馆服务体系
|
||||
人口老龄化对京津冀经济 京津冀人口老龄化对区域经济增长的影响京津冀,人口老龄化,区域经济增长,固定效应模型
|
||||
英语广告中的模糊语 模糊语在英语广告中的应用及其功能模糊语,英语广告,表现形式,语用功能
|
||||
甘氨酸二肽的合成 甘氨酸二肽合成中缩合剂的选择甘氨酸,缩合剂,二肽
|
||||
```
|
||||
|
||||
**召回库** 用于模拟业务线上的全量语料库，评估模型的召回效果，计算相应的Recall指标。召回库总共30万条样本，每行由一列构成，文献标题（+关键词），样例数据如下：
|
||||
```
|
||||
陕西省贫困地区城乡青春期少女生长发育调查青春期,生长发育,贫困地区
|
||||
五丈岩水库溢洪道加固工程中的新材料应用碳纤维布,粘钢加固技术,超细水泥,灌浆技术
|
||||
木塑复合材料在儿童卫浴家具中的应用探索木塑复合材料,儿童,卫浴家具
|
||||
泡沫铝准静态轴向压缩有限元仿真泡沫铝,准静态,轴向压缩,力学特性
|
||||
```
|
||||
|
||||
|
||||
### 数据集下载
|
||||
|
||||
|
||||
- [literature_search_data](https://bj.bcebos.com/v1/paddlenlp/data/literature_search_data.zip)
|
||||
|
||||
```
|
||||
├── milvus # milvus建库数据集
|
||||
├── milvus_data.csv. # 构建召回库的数据
|
||||
├── recall # 召回（语义索引）数据集
|
||||
├── corpus.csv # 用于测试的召回库
|
||||
├── dev.csv # 召回验证集
|
||||
├── test.csv # 召回测试集
|
||||
├── train.csv # 召回训练集
|
||||
├── train_unsupervised.csv # 无监督训练集
|
||||
├── sort # 排序数据集
|
||||
├── test_pairwise.csv # 排序测试集
|
||||
├── dev_pairwise.csv # 排序验证集
|
||||
└── train_pairwise.csv # 排序训练集
|
||||
|
||||
```
|
||||
|
||||
<a name="模型训练"></a>
|
||||
|
||||
## 5. 模型训练
|
||||
|
||||
**语义索引训练模型下载链接：**
|
||||
|
||||
以下模型结构参数为: `TrasformerLayer:12, Hidden:768, Heads:12, OutputEmbSize: 256`
|
||||
|
||||
|Model|训练参数配置|硬件|MD5|
|
||||
| ------------ | ------------ | ------------ |-----------|
|
||||
|[batch_neg](https://bj.bcebos.com/v1/paddlenlp/models/inbatch_model.zip)|<div style="width: 150pt">margin:0.2 scale:30 epoch:3 lr:5E-5 bs:64 max_len:64 </div>|<div style="width: 100pt">4卡 v100-16g</div>|f3e5c7d7b0b718c2530c5e1b136b2d74|
|
||||
|
||||
### 训练环境说明
|
||||
|
||||
|
||||
- NVIDIA Driver Version: 440.64.00
|
||||
- Ubuntu 16.04.6 LTS (Docker)
|
||||
- Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz
|
||||
|
||||
|
||||
### 单机单卡训练/单机多卡训练
|
||||
|
||||
这里采用单机多卡方式进行训练，通过如下命令，指定 GPU 0,1,2,3 卡, 基于 In-batch Negatives 策略训练模型，数据量比较小，几分钟就可以完成。如果采用单机单卡训练，只需要把`--gpus`参数设置成单卡的卡号即可。
|
||||
|
||||
如果使用CPU进行训练，则需要吧`--gpus`参数去除，然后吧`device`设置成cpu即可，详细请参考train_batch_neg.sh文件的训练设置
|
||||
|
||||
然后运行下面的命令使用GPU训练，得到语义索引模型：
|
||||
|
||||
```
|
||||
root_path=recall
|
||||
python -u -m paddle.distributed.launch --gpus "0,1,2,3" \
|
||||
train_batch_neg.py \
|
||||
--device gpu \
|
||||
--save_dir ./checkpoints/${root_path} \
|
||||
--batch_size 64 \
|
||||
--learning_rate 5E-5 \
|
||||
--epochs 3 \
|
||||
--output_emb_size 256 \
|
||||
--save_steps 10 \
|
||||
--max_seq_length 64 \
|
||||
--margin 0.2 \
|
||||
--train_set_file recall/train.csv
|
||||
|
||||
```
|
||||
|
||||
参数含义说明
|
||||
|
||||
* `device`: 使用 cpu/gpu 进行训练
|
||||
* `batch_size`: 训练的batch size的大小
|
||||
* `learning_rate`: 训练的学习率的大小
|
||||
* `epochs`: 训练的epoch数
|
||||
* `save_dir`: 模型存储路径
|
||||
* `output_emb_size`: Transformer 顶层输出的文本向量维度
|
||||
* `save_steps`： 模型存储 checkpoint 的间隔 steps 个数
|
||||
* `max_seq_length`: 输入序列的最大长度
|
||||
* `margin`: 正样本相似度与负样本之间的目标 Gap
|
||||
* `train_set_file`: 训练集文件
|
||||
|
||||
|
||||
也可以使用bash脚本：
|
||||
|
||||
```
|
||||
sh scripts/train_batch_neg.sh
|
||||
```
|
||||
|
||||
|
||||
|
||||
<a name="评估"></a>
|
||||
|
||||
## 6. 评估
|
||||
|
||||
效果评估分为 4 个步骤:
|
||||
|
||||
a. 获取Doc端Embedding
|
||||
|
||||
基于语义索引模型抽取出Doc样本库的文本向量。
|
||||
|
||||
b. 采用hnswlib对Doc端Embedding建库
|
||||
|
||||
使用 ANN 引擎构建索引库(这里基于 [hnswlib](https://github.com/nmslib/hnswlib) 进行 ANN 索引)
|
||||
|
||||
c. 获取Query的Embedding并查询相似结果
|
||||
|
||||
基于语义索引模型抽取出评估集 *Source Text* 的文本向量，在第 2 步中建立的索引库中进行 ANN 查询，召回 Top50 最相似的 *Target Text*, 产出评估集中 *Source Text* 的召回结果 `recall_result` 文件。
|
||||
|
||||
d. 评估
|
||||
|
||||
基于评估集 `dev.csv` 和召回结果 `recall_result` 计算评估指标 Recall@k，其中k取值1，5，10，20，50。
|
||||
|
||||
运行如下命令进行 ANN 建库、召回，产出召回结果数据 `recall_result`
|
||||
|
||||
```
|
||||
root_dir="checkpoints/inbatch"
|
||||
python -u -m paddle.distributed.launch --gpus "3" --log_dir "recall_log/" \
|
||||
recall.py \
|
||||
--device gpu \
|
||||
--recall_result_dir "recall_result_dir" \
|
||||
--recall_result_file "recall_result.txt" \
|
||||
--params_path "${root_dir}/model_40/model_state.pdparams" \
|
||||
--hnsw_m 100 \
|
||||
--hnsw_ef 100 \
|
||||
--batch_size 64 \
|
||||
--output_emb_size 256\
|
||||
--max_seq_length 60 \
|
||||
--recall_num 50 \
|
||||
--similar_text_pair "recall/dev.csv" \
|
||||
--corpus_file "recall/corpus.csv"
|
||||
```
|
||||
参数含义说明
|
||||
* `device`: 使用 cpu/gpu 进行训练
|
||||
* `recall_result_dir`: 召回结果存储目录
|
||||
* `recall_result_file`: 召回结果的文件名
|
||||
* `params_path`： 待评估模型的参数文件名
|
||||
* `hnsw_m`: hnsw 算法相关参数，保持默认即可
|
||||
* `hnsw_ef`: hnsw 算法相关参数，保持默认即可
|
||||
* `output_emb_size`: Transformer 顶层输出的文本向量维度
|
||||
* `recall_num`: 对 1 个文本召回的相似文本数量
|
||||
* `similar_text_pair`: 由相似文本对构成的评估集
|
||||
* `corpus_file`: 召回库数据 corpus_file
|
||||
|
||||
也可以使用下面的bash脚本：
|
||||
|
||||
```
|
||||
sh scripts/run_build_index.sh
|
||||
```
|
||||
|
||||
run_build_index.sh还包含cpu和gpu运行的脚本，默认是gpu的脚本
|
||||
|
||||
成功运行结束后，会在 `./recall_result_dir/` 目录下产出 `recall_result.txt` 文件
|
||||
|
||||
```
|
||||
热处理对尼龙6 及其与聚酰胺嵌段共聚物共混体系晶体熔融行为和结晶结构的影响 热处理对尼龙6及其与聚酰胺嵌段共聚物共混体系晶体熔融行为和结晶结构的影响尼龙6,聚酰胺嵌段共聚物,芳香聚酰胺,热处理 0.9831992387771606
|
||||
热处理对尼龙6 及其与聚酰胺嵌段共聚物共混体系晶体熔融行为和结晶结构的影响 热处理方法对高强高模聚乙烯醇纤维性能的影响聚乙烯醇纤维,热处理,性能,热拉伸,热定型 0.8438636660575867
|
||||
热处理对尼龙6 及其与聚酰胺嵌段共聚物共混体系晶体熔融行为和结晶结构的影响 制备工艺对PVC/ABS合金力学性能和维卡软化温度的影响PVC,ABS,正交试验,力学性能,维卡软化温度 0.8130228519439697
|
||||
.....
|
||||
```
|
||||
|
||||
|
||||
接下来，运行如下命令进行效果评估，产出Recall@1, Recall@5, Recall@10, Recall@20 和 Recall@50 指标:
|
||||
```
|
||||
python -u evaluate.py \
|
||||
--similar_text_pair "recall/dev.csv" \
|
||||
--recall_result_file "./recall_result_dir/recall_result.txt" \
|
||||
--recall_num 50
|
||||
```
|
||||
也可以使用下面的bash脚本：
|
||||
|
||||
```
|
||||
sh scripts/evaluate.sh
|
||||
```
|
||||
|
||||
参数含义说明
|
||||
* `similar_text_pair`: 由相似文本对构成的评估集 semantic_similar_pair.tsv
|
||||
* `recall_result_file`: 针对评估集中第一列文本 *Source Text* 的召回结果
|
||||
* `recall_num`: 对 1 个文本召回的相似文本数量
|
||||
|
||||
成功运行结束后，会输出如下评估指标:
|
||||
|
||||
```
|
||||
recall@1=51.261
|
||||
recall@5=65.279
|
||||
recall@10=69.848
|
||||
recall@20=73.971
|
||||
recall@50=78.84
|
||||
```
|
||||
|
||||
<a name="预测"></a>
|
||||
|
||||
## 7. 预测
|
||||
|
||||
我们可以基于语义索引模型预测文本的语义向量或者计算文本 Pair 的语义相似度。
|
||||
|
||||
### 7.1 功能一：抽取文本的语义向量
|
||||
|
||||
修改 inference.py 文件里面输入文本 id2corpus 和模型路径 params_path ：
|
||||
|
||||
```
|
||||
params_path='checkpoints/inbatch/model_40/model_state.pdparams'
|
||||
id2corpus={0:'国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
|
||||
```
|
||||
然后运行：
|
||||
```
|
||||
python inference.py
|
||||
```
|
||||
预测结果为256维的向量：
|
||||
|
||||
```
|
||||
[1, 256]
|
||||
[[ 0.07766181 -0.13780491 0.03388524 -0.14910668 -0.0334941 0.06780092
|
||||
0.0104043 0.03168401 0.02605671 0.02088691 0.05520441 -0.0852212
|
||||
.....
|
||||
```
|
||||
|
||||
### 7.2 功能二：计算文本 Pair 的语义相似度
|
||||
|
||||
|
||||
### 准备预测数据
|
||||
|
||||
待预测数据为 tab 分隔的 csv 文件，每一行为 1 个文本 Pair，部分示例如下:
|
||||
```
|
||||
试论我国海岸带经济开发的问题与前景 试论我国海岸带经济开发的问题与前景海岸带,经济开发,问题,前景
|
||||
外语阅读焦虑与英语成绩及性别的关系 外语阅读焦虑与英语成绩及性别的关系外语阅读焦虑,外语课堂焦虑,英语成绩,性别
|
||||
数字图书馆 智能化图书馆
|
||||
网络健康可信性研究 网络成瘾少年
|
||||
```
|
||||
|
||||
### 开始预测
|
||||
|
||||
以上述 demo 数据为例，运行如下命令基于我们开源的 [In-batch Negatives](https://arxiv.org/abs/2004.04906) 策略语义索引模型开始计算文本 Pair 的语义相似度:
|
||||
```
|
||||
root_dir="checkpoints/inbatch"
|
||||
|
||||
python -u -m paddle.distributed.launch --gpus "3" \
|
||||
predict.py \
|
||||
--device gpu \
|
||||
--params_path "${root_dir}/model_40/model_state.pdparams" \
|
||||
--output_emb_size 256 \
|
||||
--batch_size 128 \
|
||||
--max_seq_length 64 \
|
||||
--text_pair_file "recall/test.csv"
|
||||
```
|
||||
|
||||
参数含义说明
|
||||
* `device`: 使用 cpu/gpu 进行训练
|
||||
* `params_path`： 预训练模型的参数文件名
|
||||
* `output_emb_size`: Transformer 顶层输出的文本向量维度
|
||||
* `text_pair_file`: 由文本 Pair 构成的待预测数据集
|
||||
|
||||
也可以运行下面的bash脚本：
|
||||
|
||||
```
|
||||
sh scripts/predict.sh
|
||||
```
|
||||
predict.sh文件包含了cpu和gpu运行的脚本，默认是gpu运行的脚本
|
||||
|
||||
产出如下结果
|
||||
```
|
||||
0.9717282652854919
|
||||
0.9371012449264526
|
||||
0.7968897223472595
|
||||
0.30377304553985596
|
||||
```
|
||||
|
||||
<a name="部署"></a>
|
||||
|
||||
## 8. 部署
|
||||
|
||||
### 动转静导出
|
||||
|
||||
首先把动态图模型转换为静态图：
|
||||
|
||||
```
|
||||
python export_model.py --params_path checkpoints/inbatch/model_40/model_state.pdparams --output_path=./output
|
||||
```
|
||||
也可以运行下面的bash脚本：
|
||||
|
||||
```
|
||||
sh scripts/export_model.sh
|
||||
```
|
||||
|
||||
### Paddle Inference预测
|
||||
|
||||
预测既可以抽取向量也可以计算两个文本的相似度。
|
||||
|
||||
修改id2corpus的样本：
|
||||
|
||||
```
|
||||
# 抽取向量
|
||||
id2corpus={0:'国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
|
||||
# 计算相似度
|
||||
corpus_list=[['中西方语言与文化的差异','中西方文化差异以及语言体现中西方文化,差异,语言体现'],
|
||||
['中西方语言与文化的差异','飞桨致力于让深度学习技术的创新与应用更简单']]
|
||||
|
||||
```
|
||||
|
||||
然后使用PaddleInference
|
||||
|
||||
```
|
||||
python deploy/python/predict.py --model_dir=./output
|
||||
```
|
||||
也可以运行下面的bash脚本：
|
||||
|
||||
```
|
||||
sh deploy.sh
|
||||
```
|
||||
最终输出的是256维度的特征向量和句子对的预测概率：
|
||||
|
||||
```
|
||||
(1, 256)
|
||||
[[-0.0394925 -0.04474756 -0.065534 0.00939134 0.04359895 0.14659195
|
||||
-0.0091779 -0.07303623 0.09413272 -0.01255222 -0.08685658 0.02762237
|
||||
0.10138468 0.00962821 0.10888419 0.04553023 0.05898942 0.00694253
|
||||
....
|
||||
|
||||
[0.959269642829895, 0.04725276678800583]
|
||||
```
|
||||
|
||||
### Paddle Serving部署
|
||||
|
||||
Paddle Serving 的详细文档请参考 [Pipeline_Design](https://github.com/PaddlePaddle/Serving/blob/v0.7.0/doc/Python_Pipeline/Pipeline_Design_CN.md)和[Serving_Design](https://github.com/PaddlePaddle/Serving/blob/v0.7.0/doc/Serving_Design_CN.md),首先把静态图模型转换成Serving的格式：
|
||||
|
||||
```
|
||||
python export_to_serving.py \
|
||||
--dirname "output" \
|
||||
--model_filename "inference.get_pooled_embedding.pdmodel" \
|
||||
--params_filename "inference.get_pooled_embedding.pdiparams" \
|
||||
--server_path "./serving_server" \
|
||||
--client_path "./serving_client" \
|
||||
--fetch_alias_names "output_embedding"
|
||||
|
||||
```
|
||||
|
||||
参数含义说明
|
||||
* `dirname`: 需要转换的模型文件存储路径，Program 结构文件和参数文件均保存在此目录。
|
||||
* `model_filename`： 存储需要转换的模型 Inference Program 结构的文件名称。如果设置为 None ，则使用 `__model__` 作为默认的文件名
|
||||
* `params_filename`: 存储需要转换的模型所有参数的文件名称。当且仅当所有模型参数被保>存在一个单独的二进制文件中，它才需要被指定。如果模型参数是存储在各自分离的文件中，设置它的值为 None
|
||||
* `server_path`: 转换后的模型文件和配置文件的存储路径。默认值为 serving_server
|
||||
* `client_path`: 转换后的客户端配置文件存储路径。默认值为 serving_client
|
||||
* `fetch_alias_names`: 模型输出的别名设置，比如输入的 input_ids 等，都可以指定成其他名字，默认不指定
|
||||
* `feed_alias_names`: 模型输入的别名设置，比如输出 pooled_out 等，都可以重新指定成其他模型，默认不指定
|
||||
|
||||
也可以运行下面的 bash 脚本：
|
||||
```
|
||||
sh scripts/export_to_serving.sh
|
||||
```
|
||||
|
||||
Paddle Serving的部署有两种方式，第一种方式是Pipeline的方式，第二种是C++的方式，下面分别介绍这两种方式的用法：
|
||||
|
||||
#### Pipeline方式
|
||||
|
||||
启动 Pipeline Server:
|
||||
|
||||
```
|
||||
python web_service.py
|
||||
```
|
||||
|
||||
启动客户端调用 Server。
|
||||
|
||||
首先修改rpc_client.py中需要预测的样本：
|
||||
|
||||
```
|
||||
list_data = [
|
||||
"国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据",
|
||||
"试论翻译过程中的文化差异与语言空缺翻译过程,文化差异,语言空缺,文化对比"
|
||||
]
|
||||
```
|
||||
然后运行：
|
||||
|
||||
```
|
||||
python rpc_client.py
|
||||
```
|
||||
模型的输出为：
|
||||
|
||||
```
|
||||
{'0': '国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据', '1': '试论翻译过程中的文化差异与语言空缺翻译过程,文化差异,语言空缺,文化对比'}
|
||||
PipelineClient::predict pack_data time:1641450851.3752182
|
||||
PipelineClient::predict before time:1641450851.375738
|
||||
['output_embedding']
|
||||
(2, 256)
|
||||
[[ 0.07830612 -0.14036864 0.03433796 -0.14967982 -0.03386067 0.06630666
|
||||
0.01357943 0.03531194 0.02411093 0.02000859 0.05724002 -0.08119463
|
||||
......
|
||||
```
|
||||
|
||||
可以看到客户端发送了2条文本，返回了2个 embedding 向量
|
||||
|
||||
#### C++的方式
|
||||
|
||||
启动C++的Serving：
|
||||
|
||||
```
|
||||
python -m paddle_serving_server.serve --model serving_server --port 9393 --gpu_id 2 --thread 5 --ir_optim True --use_trt --precision FP16
|
||||
```
|
||||
也可以使用脚本：
|
||||
|
||||
```
|
||||
sh deploy/C++/start_server.sh
|
||||
```
|
||||
Client 可以使用 http 或者 rpc 两种方式，rpc 的方式为：
|
||||
|
||||
```
|
||||
python deploy/C++/rpc_client.py
|
||||
```
|
||||
运行的输出为：
|
||||
```
|
||||
I0209 20:40:07.978225 20896 general_model.cpp:490] [client]logid=0,client_cost=395.695ms,server_cost=392.559ms.
|
||||
time to cost :0.3960278034210205 seconds
|
||||
{'output_embedding': array([[ 9.01343748e-02, -1.21870913e-01, 1.32834800e-02,
|
||||
-1.57673359e-01, -2.60387752e-02, 6.98455423e-02,
|
||||
1.58108603e-02, 3.89952064e-02, 3.22783105e-02,
|
||||
3.49135026e-02, 7.66086206e-02, -9.12970975e-02,
|
||||
6.25643134e-02, 7.21886680e-02, 7.03565404e-02,
|
||||
5.44054210e-02, 3.25332815e-03, 5.01751155e-02,
|
||||
......
|
||||
```
|
||||
可以看到服务端返回了向量
|
||||
|
||||
或者使用 http 的客户端访问模式：
|
||||
|
||||
```
|
||||
python deploy/C++/http_client.py
|
||||
```
|
||||
运行的输出为：
|
||||
|
||||
```
|
||||
(2, 64)
|
||||
(2, 64)
|
||||
outputs {
|
||||
tensor {
|
||||
float_data: 0.09013437479734421
|
||||
float_data: -0.12187091261148453
|
||||
float_data: 0.01328347995877266
|
||||
float_data: -0.15767335891723633
|
||||
......
|
||||
```
|
||||
可以看到服务端返回了向量
|
||||
|
||||
## Reference
|
||||
|
||||
[1] Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, Wen-tau Yih, Dense Passage Retrieval for Open-Domain Question Answering, Preprint 2020.
|
||||
@@ -0,0 +1,60 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# coding=UTF-8
|
||||
|
||||
import numpy as np
|
||||
import hnswlib
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
|
||||
def build_index(args, data_loader, model):
|
||||
|
||||
index = hnswlib.Index(space='ip', dim=args.output_emb_size)
|
||||
|
||||
# Initializing index
|
||||
# max_elements - the maximum number of elements (capacity). Will throw an exception if exceeded
|
||||
# during insertion of an element.
|
||||
# The capacity can be increased by saving/loading the index, see below.
|
||||
#
|
||||
# ef_construction - controls index search speed/build speed tradeoff
|
||||
#
|
||||
# M - is tightly connected with internal dimensionality of the data. Strongly affects memory consumption (~M)
|
||||
# Higher M leads to higher accuracy/run_time at fixed ef/efConstruction
|
||||
index.init_index(
|
||||
max_elements=args.hnsw_max_elements,
|
||||
ef_construction=args.hnsw_ef,
|
||||
M=args.hnsw_m)
|
||||
|
||||
# Controlling the recall by setting ef:
|
||||
# higher ef leads to better accuracy, but slower search
|
||||
index.set_ef(args.hnsw_ef)
|
||||
|
||||
# Set number of threads used during batch search/construction
|
||||
# By default using all available cores
|
||||
index.set_num_threads(16)
|
||||
|
||||
logger.info("start build index..........")
|
||||
|
||||
all_embeddings = []
|
||||
|
||||
for text_embeddings in model.get_semantic_embedding(data_loader):
|
||||
all_embeddings.append(text_embeddings.numpy())
|
||||
|
||||
all_embeddings = np.concatenate(all_embeddings, axis=0)
|
||||
index.add_items(all_embeddings)
|
||||
|
||||
logger.info("Total index number:{}".format(index.get_current_count()))
|
||||
|
||||
return index
|
||||
@@ -0,0 +1,187 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import abc
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
import paddle
|
||||
import paddle.nn as nn
|
||||
import paddle.nn.functional as F
|
||||
|
||||
|
||||
class SemanticIndexBase(nn.Layer):
|
||||
def __init__(self, pretrained_model, dropout=None, output_emb_size=None):
|
||||
super().__init__()
|
||||
self.ptm = pretrained_model
|
||||
self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)
|
||||
|
||||
# if output_emb_size is not None, then add Linear layer to reduce embedding_size,
|
||||
# we recommend set output_emb_size = 256 considering the trade-off beteween
|
||||
# recall performance and efficiency
|
||||
|
||||
self.output_emb_size = output_emb_size
|
||||
if output_emb_size > 0:
|
||||
weight_attr = paddle.ParamAttr(
|
||||
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
|
||||
self.emb_reduce_linear = paddle.nn.Linear(
|
||||
768, output_emb_size, weight_attr=weight_attr)
|
||||
|
||||
@paddle.jit.to_static(input_spec=[
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64'), paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64')
|
||||
])
|
||||
def get_pooled_embedding(self,
|
||||
input_ids,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
attention_mask=None):
|
||||
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
|
||||
attention_mask)
|
||||
|
||||
if self.output_emb_size > 0:
|
||||
cls_embedding = self.emb_reduce_linear(cls_embedding)
|
||||
cls_embedding = self.dropout(cls_embedding)
|
||||
cls_embedding = F.normalize(cls_embedding, p=2, axis=-1)
|
||||
|
||||
return cls_embedding
|
||||
|
||||
def get_semantic_embedding(self, data_loader):
|
||||
self.eval()
|
||||
with paddle.no_grad():
|
||||
for batch_data in data_loader:
|
||||
input_ids, token_type_ids = batch_data
|
||||
input_ids = paddle.to_tensor(input_ids)
|
||||
token_type_ids = paddle.to_tensor(token_type_ids)
|
||||
|
||||
text_embeddings = self.get_pooled_embedding(
|
||||
input_ids, token_type_ids=token_type_ids)
|
||||
|
||||
yield text_embeddings
|
||||
|
||||
def cosine_sim(self,
|
||||
query_input_ids,
|
||||
title_input_ids,
|
||||
query_token_type_ids=None,
|
||||
query_position_ids=None,
|
||||
query_attention_mask=None,
|
||||
title_token_type_ids=None,
|
||||
title_position_ids=None,
|
||||
title_attention_mask=None):
|
||||
|
||||
query_cls_embedding = self.get_pooled_embedding(
|
||||
query_input_ids, query_token_type_ids, query_position_ids,
|
||||
query_attention_mask)
|
||||
|
||||
title_cls_embedding = self.get_pooled_embedding(
|
||||
title_input_ids, title_token_type_ids, title_position_ids,
|
||||
title_attention_mask)
|
||||
|
||||
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
|
||||
axis=-1)
|
||||
return cosine_sim
|
||||
|
||||
@abc.abstractmethod
|
||||
def forward(self):
|
||||
pass
|
||||
|
||||
|
||||
class SemanticIndexBaseStatic(nn.Layer):
|
||||
def __init__(self, pretrained_model, dropout=None, output_emb_size=None):
|
||||
super().__init__()
|
||||
self.ptm = pretrained_model
|
||||
self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)
|
||||
|
||||
# if output_emb_size is not None, then add Linear layer to reduce embedding_size,
|
||||
# we recommend set output_emb_size = 256 considering the trade-off beteween
|
||||
# recall performance and efficiency
|
||||
|
||||
self.output_emb_size = output_emb_size
|
||||
if output_emb_size > 0:
|
||||
weight_attr = paddle.ParamAttr(
|
||||
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
|
||||
self.emb_reduce_linear = paddle.nn.Linear(
|
||||
768, output_emb_size, weight_attr=weight_attr)
|
||||
|
||||
@paddle.jit.to_static(input_spec=[
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64'), paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64')
|
||||
])
|
||||
def get_pooled_embedding(self,
|
||||
input_ids,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
attention_mask=None):
|
||||
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
|
||||
attention_mask)
|
||||
|
||||
if self.output_emb_size > 0:
|
||||
cls_embedding = self.emb_reduce_linear(cls_embedding)
|
||||
cls_embedding = self.dropout(cls_embedding)
|
||||
cls_embedding = F.normalize(cls_embedding, p=2, axis=-1)
|
||||
|
||||
return cls_embedding
|
||||
|
||||
def get_semantic_embedding(self, data_loader):
|
||||
self.eval()
|
||||
with paddle.no_grad():
|
||||
for batch_data in data_loader:
|
||||
input_ids, token_type_ids = batch_data
|
||||
input_ids = paddle.to_tensor(input_ids)
|
||||
token_type_ids = paddle.to_tensor(token_type_ids)
|
||||
|
||||
text_embeddings = self.get_pooled_embedding(
|
||||
input_ids, token_type_ids=token_type_ids)
|
||||
|
||||
yield text_embeddings
|
||||
|
||||
def cosine_sim(self,
|
||||
query_input_ids,
|
||||
title_input_ids,
|
||||
query_token_type_ids=None,
|
||||
query_position_ids=None,
|
||||
query_attention_mask=None,
|
||||
title_token_type_ids=None,
|
||||
title_position_ids=None,
|
||||
title_attention_mask=None):
|
||||
|
||||
query_cls_embedding = self.get_pooled_embedding(
|
||||
query_input_ids, query_token_type_ids, query_position_ids,
|
||||
query_attention_mask)
|
||||
|
||||
title_cls_embedding = self.get_pooled_embedding(
|
||||
title_input_ids, title_token_type_ids, title_position_ids,
|
||||
title_attention_mask)
|
||||
|
||||
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
|
||||
axis=-1)
|
||||
return cosine_sim
|
||||
|
||||
def forward(self,
|
||||
input_ids,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
attention_mask=None):
|
||||
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
|
||||
attention_mask)
|
||||
|
||||
if self.output_emb_size > 0:
|
||||
cls_embedding = self.emb_reduce_linear(cls_embedding)
|
||||
cls_embedding = self.dropout(cls_embedding)
|
||||
cls_embedding = F.normalize(cls_embedding, p=2, axis=-1)
|
||||
|
||||
return cls_embedding
|
||||
@@ -0,0 +1,75 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn as nn
|
||||
import paddle.nn.functional as F
|
||||
|
||||
from base_model import SemanticIndexBase
|
||||
|
||||
|
||||
class SemanticIndexBatchNeg(SemanticIndexBase):
|
||||
def __init__(self,
|
||||
pretrained_model,
|
||||
dropout=None,
|
||||
margin=0.3,
|
||||
scale=30,
|
||||
output_emb_size=None):
|
||||
super().__init__(pretrained_model, dropout, output_emb_size)
|
||||
|
||||
self.margin = margin
|
||||
# Used scaling cosine similarity to ease converge
|
||||
self.sacle = scale
|
||||
|
||||
def forward(self,
|
||||
query_input_ids,
|
||||
title_input_ids,
|
||||
query_token_type_ids=None,
|
||||
query_position_ids=None,
|
||||
query_attention_mask=None,
|
||||
title_token_type_ids=None,
|
||||
title_position_ids=None,
|
||||
title_attention_mask=None):
|
||||
|
||||
query_cls_embedding = self.get_pooled_embedding(
|
||||
query_input_ids, query_token_type_ids, query_position_ids,
|
||||
query_attention_mask)
|
||||
|
||||
title_cls_embedding = self.get_pooled_embedding(
|
||||
title_input_ids, title_token_type_ids, title_position_ids,
|
||||
title_attention_mask)
|
||||
|
||||
cosine_sim = paddle.matmul(
|
||||
query_cls_embedding, title_cls_embedding, transpose_y=True)
|
||||
|
||||
# substract margin from all positive samples cosine_sim()
|
||||
margin_diag = paddle.full(
|
||||
shape=[query_cls_embedding.shape[0]],
|
||||
fill_value=self.margin,
|
||||
dtype=paddle.get_default_dtype())
|
||||
|
||||
cosine_sim = cosine_sim - paddle.diag(margin_diag)
|
||||
|
||||
# scale cosine to ease training converge
|
||||
cosine_sim *= self.sacle
|
||||
|
||||
labels = paddle.arange(0, query_cls_embedding.shape[0], dtype='int64')
|
||||
labels = paddle.reshape(labels, shape=[-1, 1])
|
||||
|
||||
loss = F.cross_entropy(input=cosine_sim, label=labels)
|
||||
|
||||
return loss
|
||||
182
applications/neural_search/recall/in_batch_negative/data.py
Normal file
182
applications/neural_search/recall/in_batch_negative/data.py
Normal file
@@ -0,0 +1,182 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import paddle
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
|
||||
def create_dataloader(dataset,
|
||||
mode='train',
|
||||
batch_size=1,
|
||||
batchify_fn=None,
|
||||
trans_fn=None):
|
||||
if trans_fn:
|
||||
dataset = dataset.map(trans_fn)
|
||||
shuffle = True if mode == 'train' else False
|
||||
if mode == 'train':
|
||||
batch_sampler = paddle.io.DistributedBatchSampler(
|
||||
dataset, batch_size=batch_size, shuffle=shuffle)
|
||||
else:
|
||||
batch_sampler = paddle.io.BatchSampler(
|
||||
dataset, batch_size=batch_size, shuffle=shuffle)
|
||||
|
||||
return paddle.io.DataLoader(
|
||||
dataset=dataset,
|
||||
batch_sampler=batch_sampler,
|
||||
collate_fn=batchify_fn,
|
||||
return_list=True)
|
||||
|
||||
|
||||
def convert_example(example,
|
||||
tokenizer,
|
||||
max_seq_length=512,
|
||||
pad_to_max_seq_len=False):
|
||||
"""
|
||||
Builds model inputs from a sequence.
|
||||
|
||||
A BERT sequence has the following format:
|
||||
|
||||
- single sequence: ``[CLS] X [SEP]``
|
||||
|
||||
Args:
|
||||
example(obj:`list(str)`): The list of text to be converted to ids.
|
||||
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
|
||||
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
|
||||
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
|
||||
Sequences longer than this will be truncated, sequences shorter will be padded.
|
||||
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
|
||||
|
||||
Returns:
|
||||
input_ids(obj:`list[int]`): The list of query token ids.
|
||||
token_type_ids(obj: `list[int]`): List of query sequence pair mask.
|
||||
"""
|
||||
|
||||
result = []
|
||||
for key, text in example.items():
|
||||
encoded_inputs = tokenizer(
|
||||
text=text,
|
||||
max_seq_len=max_seq_length,
|
||||
pad_to_max_seq_len=pad_to_max_seq_len)
|
||||
input_ids = encoded_inputs["input_ids"]
|
||||
token_type_ids = encoded_inputs["token_type_ids"]
|
||||
result += [input_ids, token_type_ids]
|
||||
return result
|
||||
|
||||
|
||||
def read_text_pair(data_path):
|
||||
"""Reads data."""
|
||||
with open(data_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
data = line.rstrip().split("\t")
|
||||
if len(data) != 2:
|
||||
continue
|
||||
yield {'text_a': data[0], 'text_b': data[1]}
|
||||
|
||||
|
||||
def read_text_triplet(data_path):
|
||||
"""Reads data."""
|
||||
with open(data_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
data = line.rstrip().split("\t")
|
||||
if len(data) != 3:
|
||||
continue
|
||||
yield {
|
||||
'text': data[0],
|
||||
'pos_sample': data[1],
|
||||
'neg_sample': data[2]
|
||||
}
|
||||
|
||||
|
||||
# ANN - active learning ------------------------------------------------------
|
||||
def get_latest_checkpoint(args):
|
||||
"""
|
||||
Return: (latest_checkpint_path, global_step)
|
||||
"""
|
||||
if not os.path.exists(args.save_dir):
|
||||
return args.init_from_ckpt, 0
|
||||
|
||||
subdirectories = list(next(os.walk(args.save_dir))[1])
|
||||
|
||||
def valid_checkpoint(checkpoint):
|
||||
chk_path = os.path.join(args.save_dir, checkpoint)
|
||||
scheduler_path = os.path.join(chk_path, "model_state.pdparams")
|
||||
succeed_flag_file = os.path.join(chk_path, "succeed_flag_file")
|
||||
return os.path.exists(scheduler_path) and os.path.exists(
|
||||
succeed_flag_file)
|
||||
|
||||
trained_steps = [int(s) for s in subdirectories if valid_checkpoint(s)]
|
||||
|
||||
if len(trained_steps) > 0:
|
||||
return os.path.join(args.save_dir,
|
||||
str(max(trained_steps)),
|
||||
"model_state.pdparams"), max(trained_steps)
|
||||
|
||||
return args.init_from_ckpt, 0
|
||||
|
||||
|
||||
# ANN - active learning ------------------------------------------------------
|
||||
def get_latest_ann_data(ann_data_dir):
|
||||
if not os.path.exists(ann_data_dir):
|
||||
return None, -1
|
||||
|
||||
subdirectories = list(next(os.walk(ann_data_dir))[1])
|
||||
|
||||
def valid_checkpoint(step):
|
||||
ann_data_file = os.path.join(ann_data_dir, step, "new_ann_data")
|
||||
# succed_flag_file is an empty file that indicates ann data has been generated
|
||||
succeed_flag_file = os.path.join(ann_data_dir, step,
|
||||
"succeed_flag_file")
|
||||
return os.path.exists(succeed_flag_file) and os.path.exists(
|
||||
ann_data_file)
|
||||
|
||||
ann_data_steps = [int(s) for s in subdirectories if valid_checkpoint(s)]
|
||||
|
||||
if len(ann_data_steps) > 0:
|
||||
latest_ann_data_file = os.path.join(ann_data_dir,
|
||||
str(max(ann_data_steps)),
|
||||
"new_ann_data")
|
||||
logger.info("Using lateset ann_data_file:{}".format(
|
||||
latest_ann_data_file))
|
||||
return latest_ann_data_file, max(ann_data_steps)
|
||||
|
||||
logger.info("no new ann_data, return (None, -1)")
|
||||
return None, -1
|
||||
|
||||
|
||||
def gen_id2corpus(corpus_file):
|
||||
id2corpus = {}
|
||||
with open(corpus_file, 'r', encoding='utf-8') as f:
|
||||
for idx, line in enumerate(f):
|
||||
id2corpus[idx] = line.rstrip()
|
||||
return id2corpus
|
||||
|
||||
|
||||
def gen_text_file(similar_text_pair_file):
|
||||
text2similar_text = {}
|
||||
texts = []
|
||||
with open(similar_text_pair_file, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
splited_line = line.rstrip().split("\t")
|
||||
if len(splited_line) != 2:
|
||||
continue
|
||||
|
||||
text, similar_text = line.rstrip().split("\t")
|
||||
|
||||
if not text or not similar_text:
|
||||
continue
|
||||
|
||||
text2similar_text[text] = similar_text
|
||||
texts.append({"text": text})
|
||||
return texts, text2similar_text
|
||||
@@ -0,0 +1,81 @@
|
||||
# coding:utf-8
|
||||
# pylint: disable=doc-string-missing
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
import requests
|
||||
import json
|
||||
|
||||
from paddle_serving_client import HttpClient
|
||||
import paddlenlp as ppnlp
|
||||
|
||||
|
||||
def convert_example(example,
|
||||
tokenizer,
|
||||
max_seq_length=512,
|
||||
pad_to_max_seq_len=True):
|
||||
list_input_ids = []
|
||||
list_token_type_ids = []
|
||||
for text in example:
|
||||
encoded_inputs = tokenizer(
|
||||
text=text,
|
||||
max_seq_len=max_seq_length,
|
||||
pad_to_max_seq_len=pad_to_max_seq_len)
|
||||
input_ids = encoded_inputs["input_ids"]
|
||||
token_type_ids = encoded_inputs["token_type_ids"]
|
||||
|
||||
list_input_ids.append(input_ids)
|
||||
list_token_type_ids.append(token_type_ids)
|
||||
return list_input_ids, list_token_type_ids
|
||||
|
||||
|
||||
# 启动python客户端
|
||||
endpoint_list = ['127.0.0.1:9393']
|
||||
client = HttpClient()
|
||||
client.load_client_config('serving_client')
|
||||
client.connect(endpoint_list)
|
||||
feed_names = client.feed_names_
|
||||
fetch_names = client.fetch_names_
|
||||
print(feed_names)
|
||||
print(fetch_names)
|
||||
|
||||
# 创建tokenizer
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
max_seq_len = 64
|
||||
|
||||
# 数据预处理
|
||||
|
||||
list_data = ['国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据.', '面向生态系统服务的生态系统分类方案研发与应用']
|
||||
# for i in range(5):
|
||||
# list_data.extend(list_data)
|
||||
# print(len(list_data))
|
||||
examples = convert_example(list_data, tokenizer, max_seq_length=max_seq_len)
|
||||
print(examples)
|
||||
|
||||
feed_dict = {}
|
||||
feed_dict['input_ids'] = np.array(examples[0])
|
||||
feed_dict['token_type_ids'] = np.array(examples[1])
|
||||
|
||||
print(feed_dict['input_ids'].shape)
|
||||
print(feed_dict['token_type_ids'].shape)
|
||||
|
||||
# batch设置为True表示的是批量预测
|
||||
b_start = time.time()
|
||||
result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)
|
||||
b_end = time.time()
|
||||
print(result)
|
||||
print("time to cost :{} seconds".format(b_end - b_start))
|
||||
@@ -0,0 +1,77 @@
|
||||
# coding:utf-8
|
||||
# pylint: disable=doc-string-missing
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
|
||||
from paddle_serving_client import Client
|
||||
import paddlenlp as ppnlp
|
||||
|
||||
|
||||
def convert_example(example,
|
||||
tokenizer,
|
||||
max_seq_length=512,
|
||||
pad_to_max_seq_len=True):
|
||||
list_input_ids = []
|
||||
list_token_type_ids = []
|
||||
for text in example:
|
||||
encoded_inputs = tokenizer(
|
||||
text=text,
|
||||
max_seq_len=max_seq_length,
|
||||
pad_to_max_seq_len=pad_to_max_seq_len)
|
||||
input_ids = encoded_inputs["input_ids"]
|
||||
token_type_ids = encoded_inputs["token_type_ids"]
|
||||
list_input_ids.append(input_ids)
|
||||
list_token_type_ids.append(token_type_ids)
|
||||
return list_input_ids, list_token_type_ids
|
||||
|
||||
|
||||
# 启动python客户端
|
||||
endpoint_list = ['127.0.0.1:9393']
|
||||
client = Client()
|
||||
client.load_client_config('serving_client')
|
||||
client.connect(endpoint_list)
|
||||
feed_names = client.feed_names_
|
||||
fetch_names = client.fetch_names_
|
||||
print(feed_names)
|
||||
print(fetch_names)
|
||||
|
||||
# 创建tokenizer
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
max_seq_len = 64
|
||||
|
||||
# 数据预处理
|
||||
|
||||
list_data = ['国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据.', '面向生态系统服务的生态系统分类方案研发与应用']
|
||||
# for i in range(5):
|
||||
# list_data.extend(list_data)
|
||||
# print(len(list_data))
|
||||
examples = convert_example(list_data, tokenizer, max_seq_length=max_seq_len)
|
||||
print(examples)
|
||||
|
||||
feed_dict = {}
|
||||
feed_dict['input_ids'] = np.array(examples[0])
|
||||
feed_dict['token_type_ids'] = np.array(examples[1])
|
||||
|
||||
print(feed_dict['input_ids'].shape)
|
||||
print(feed_dict['token_type_ids'].shape)
|
||||
# batch设置为True表示的是批量预测
|
||||
b_start = time.time()
|
||||
result = client.predict(feed=feed_dict, fetch=fetch_names, batch=True)
|
||||
b_end = time.time()
|
||||
print("time to cost :{} seconds".format(b_end - b_start))
|
||||
print(result)
|
||||
@@ -0,0 +1 @@
|
||||
python -m paddle_serving_server.serve --model serving_server --port 9393 --gpu_id 2 --thread 5 --ir_optim True --use_trt --precision FP16
|
||||
@@ -0,0 +1 @@
|
||||
python predict.py --model_dir=../../output
|
||||
@@ -0,0 +1,292 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddlenlp as ppnlp
|
||||
from scipy.special import softmax
|
||||
from scipy import spatial
|
||||
from paddle import inference
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.datasets import load_dataset
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
sys.path.append('.')
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model_dir", type=str, required=True,
|
||||
help="The directory to static model.")
|
||||
|
||||
parser.add_argument("--max_seq_length", default=128, type=int,
|
||||
help="The maximum total input sequence length after tokenization. Sequences "
|
||||
"longer than this will be truncated, sequences shorter will be padded.")
|
||||
parser.add_argument("--batch_size", default=15, type=int,
|
||||
help="Batch size per GPU/CPU for training.")
|
||||
parser.add_argument('--device', choices=['cpu', 'gpu', 'xpu'], default="gpu",
|
||||
help="Select which device to train model, defaults to gpu.")
|
||||
|
||||
parser.add_argument('--use_tensorrt', default=False, type=eval, choices=[True, False],
|
||||
help='Enable to use tensorrt to speed up.')
|
||||
parser.add_argument("--precision", default="fp32", type=str, choices=["fp32", "fp16", "int8"],
|
||||
help='The tensorrt precision.')
|
||||
|
||||
parser.add_argument('--cpu_threads', default=10, type=int,
|
||||
help='Number of threads to predict when using cpu.')
|
||||
parser.add_argument('--enable_mkldnn', default=False, type=eval, choices=[True, False],
|
||||
help='Enable to use mkldnn to speed up when using cpu.')
|
||||
|
||||
parser.add_argument("--benchmark", type=eval, default=False,
|
||||
help="To log some information about environment and running.")
|
||||
parser.add_argument("--save_log_path", type=str, default="./log_output/",
|
||||
help="The file path to save log.")
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def convert_example(example,
|
||||
tokenizer,
|
||||
max_seq_length=512,
|
||||
pad_to_max_seq_len=False):
|
||||
"""
|
||||
Builds model inputs from a sequence.
|
||||
|
||||
A BERT sequence has the following format:
|
||||
|
||||
- single sequence: ``[CLS] X [SEP]``
|
||||
|
||||
Args:
|
||||
example(obj:`list(str)`): The list of text to be converted to ids.
|
||||
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
|
||||
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
|
||||
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
|
||||
Sequences longer than this will be truncated, sequences shorter will be padded.
|
||||
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
|
||||
|
||||
Returns:
|
||||
input_ids(obj:`list[int]`): The list of query token ids.
|
||||
token_type_ids(obj: `list[int]`): List of query sequence pair mask.
|
||||
"""
|
||||
|
||||
result = []
|
||||
for key, text in example.items():
|
||||
encoded_inputs = tokenizer(
|
||||
text=text,
|
||||
max_seq_len=max_seq_length,
|
||||
pad_to_max_seq_len=pad_to_max_seq_len)
|
||||
input_ids = encoded_inputs["input_ids"]
|
||||
token_type_ids = encoded_inputs["token_type_ids"]
|
||||
result += [input_ids, token_type_ids]
|
||||
return result
|
||||
|
||||
|
||||
class Predictor(object):
|
||||
def __init__(self,
|
||||
model_dir,
|
||||
device="gpu",
|
||||
max_seq_length=128,
|
||||
batch_size=32,
|
||||
use_tensorrt=False,
|
||||
precision="fp32",
|
||||
cpu_threads=10,
|
||||
enable_mkldnn=False):
|
||||
self.max_seq_length = max_seq_length
|
||||
self.batch_size = batch_size
|
||||
|
||||
model_file = model_dir + "/inference.pdmodel"
|
||||
params_file = model_dir + "/inference.pdiparams"
|
||||
if not os.path.exists(model_file):
|
||||
raise ValueError("not find model file path {}".format(model_file))
|
||||
if not os.path.exists(params_file):
|
||||
raise ValueError("not find params file path {}".format(params_file))
|
||||
config = paddle.inference.Config(model_file, params_file)
|
||||
|
||||
if device == "gpu":
|
||||
# set GPU configs accordingly
|
||||
# such as intialize the gpu memory, enable tensorrt
|
||||
config.enable_use_gpu(100, 0)
|
||||
precision_map = {
|
||||
"fp16": inference.PrecisionType.Half,
|
||||
"fp32": inference.PrecisionType.Float32,
|
||||
"int8": inference.PrecisionType.Int8
|
||||
}
|
||||
precision_mode = precision_map[precision]
|
||||
|
||||
if args.use_tensorrt:
|
||||
config.enable_tensorrt_engine(
|
||||
max_batch_size=batch_size,
|
||||
min_subgraph_size=30,
|
||||
precision_mode=precision_mode)
|
||||
elif device == "cpu":
|
||||
# set CPU configs accordingly,
|
||||
# such as enable_mkldnn, set_cpu_math_library_num_threads
|
||||
config.disable_gpu()
|
||||
if args.enable_mkldnn:
|
||||
# cache 10 different shapes for mkldnn to avoid memory leak
|
||||
config.set_mkldnn_cache_capacity(10)
|
||||
config.enable_mkldnn()
|
||||
config.set_cpu_math_library_num_threads(args.cpu_threads)
|
||||
elif device == "xpu":
|
||||
# set XPU configs accordingly
|
||||
config.enable_xpu(100)
|
||||
|
||||
config.switch_use_feed_fetch_ops(False)
|
||||
self.predictor = paddle.inference.create_predictor(config)
|
||||
self.input_handles = [
|
||||
self.predictor.get_input_handle(name)
|
||||
for name in self.predictor.get_input_names()
|
||||
]
|
||||
self.output_handle = self.predictor.get_output_handle(
|
||||
self.predictor.get_output_names()[0])
|
||||
|
||||
if args.benchmark:
|
||||
import auto_log
|
||||
pid = os.getpid()
|
||||
self.autolog = auto_log.AutoLogger(
|
||||
model_name="ernie-1.0",
|
||||
model_precision=precision,
|
||||
batch_size=self.batch_size,
|
||||
data_shape="dynamic",
|
||||
save_path=args.save_log_path,
|
||||
inference_config=config,
|
||||
pids=pid,
|
||||
process_name=None,
|
||||
gpu_ids=0,
|
||||
time_keys=[
|
||||
'preprocess_time', 'inference_time', 'postprocess_time'
|
||||
],
|
||||
warmup=0,
|
||||
logger=logger)
|
||||
|
||||
def extract_embedding(self, data, tokenizer):
|
||||
"""
|
||||
Predicts the data labels.
|
||||
|
||||
Args:
|
||||
data (obj:`List(str)`): The batch data whose each element is a raw text.
|
||||
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
|
||||
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
|
||||
|
||||
Returns:
|
||||
results(obj:`dict`): All the feature vectors.
|
||||
"""
|
||||
if args.benchmark:
|
||||
self.autolog.times.start()
|
||||
|
||||
examples = []
|
||||
for text in data:
|
||||
input_ids, segment_ids = convert_example(text, tokenizer)
|
||||
examples.append((input_ids, segment_ids))
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # segment
|
||||
): fn(samples)
|
||||
|
||||
if args.benchmark:
|
||||
self.autolog.times.stamp()
|
||||
|
||||
input_ids, segment_ids = batchify_fn(examples)
|
||||
self.input_handles[0].copy_from_cpu(input_ids)
|
||||
self.input_handles[1].copy_from_cpu(segment_ids)
|
||||
self.predictor.run()
|
||||
logits = self.output_handle.copy_to_cpu()
|
||||
if args.benchmark:
|
||||
self.autolog.times.stamp()
|
||||
|
||||
if args.benchmark:
|
||||
self.autolog.times.end(stamp=True)
|
||||
|
||||
return logits
|
||||
|
||||
def predict(self, data, tokenizer):
|
||||
"""
|
||||
Predicts the data labels.
|
||||
|
||||
Args:
|
||||
data (obj:`List(str)`): The batch data whose each element is a raw text.
|
||||
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
|
||||
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
|
||||
|
||||
Returns:
|
||||
results(obj:`dict`): All the predictions probs.
|
||||
"""
|
||||
if args.benchmark:
|
||||
self.autolog.times.start()
|
||||
|
||||
examples = []
|
||||
for idx, text in enumerate(data):
|
||||
input_ids, segment_ids = convert_example({idx: text[0]}, tokenizer)
|
||||
title_ids, title_segment_ids = convert_example({
|
||||
idx: text[1]
|
||||
}, tokenizer)
|
||||
examples.append(
|
||||
(input_ids, segment_ids, title_ids, title_segment_ids))
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # segment
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # segment
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # segment
|
||||
): fn(samples)
|
||||
|
||||
if args.benchmark:
|
||||
self.autolog.times.stamp()
|
||||
|
||||
query_ids, query_segment_ids, title_ids, title_segment_ids = batchify_fn(
|
||||
examples)
|
||||
self.input_handles[0].copy_from_cpu(query_ids)
|
||||
self.input_handles[1].copy_from_cpu(query_segment_ids)
|
||||
self.predictor.run()
|
||||
query_logits = self.output_handle.copy_to_cpu()
|
||||
|
||||
self.input_handles[0].copy_from_cpu(title_ids)
|
||||
self.input_handles[1].copy_from_cpu(title_segment_ids)
|
||||
self.predictor.run()
|
||||
title_logits = self.output_handle.copy_to_cpu()
|
||||
|
||||
if args.benchmark:
|
||||
self.autolog.times.stamp()
|
||||
|
||||
if args.benchmark:
|
||||
self.autolog.times.end(stamp=True)
|
||||
result = [
|
||||
float(1 - spatial.distance.cosine(arr1, arr2))
|
||||
for arr1, arr2 in zip(query_logits, title_logits)
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Define predictor to do prediction.
|
||||
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
|
||||
args.batch_size, args.use_tensorrt, args.precision,
|
||||
args.cpu_threads, args.enable_mkldnn)
|
||||
|
||||
# ErnieTinyTokenizer is special for ernie-tiny pretained model.
|
||||
output_emb_size = 256
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
id2corpus = {0: '国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
|
||||
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
|
||||
res = predictor.extract_embedding(corpus_list, tokenizer)
|
||||
print(res.shape)
|
||||
print(res)
|
||||
corpus_list = [['中西方语言与文化的差异', '中西方文化差异以及语言体现中西方文化,差异,语言体现'],
|
||||
['中西方语言与文化的差异', '飞桨致力于让深度学习技术的创新与应用更简单']]
|
||||
res = predictor.predict(corpus_list, tokenizer)
|
||||
print(res)
|
||||
@@ -0,0 +1,39 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import time
|
||||
import numpy as np
|
||||
|
||||
from paddle_serving_server.pipeline import PipelineClient
|
||||
|
||||
client = PipelineClient()
|
||||
client.connect(['127.0.0.1:8080'])
|
||||
|
||||
list_data = [
|
||||
"国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据",
|
||||
"试论翻译过程中的文化差异与语言空缺翻译过程,文化差异,语言空缺,文化对比"
|
||||
]
|
||||
feed = {}
|
||||
for i, item in enumerate(list_data):
|
||||
feed[str(i)] = item
|
||||
|
||||
print(feed)
|
||||
start_time = time.time()
|
||||
ret = client.predict(feed_dict=feed)
|
||||
end_time = time.time()
|
||||
print("time to cost :{} seconds".format(end_time - start_time))
|
||||
|
||||
result = np.array(eval(ret.value[0]))
|
||||
print(ret.key)
|
||||
print(result.shape)
|
||||
print(result)
|
||||
@@ -0,0 +1,82 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import numpy as np
|
||||
import sys
|
||||
|
||||
from paddle_serving_server.web_service import WebService, Op
|
||||
|
||||
_LOGGER = logging.getLogger()
|
||||
|
||||
|
||||
def convert_example(example,
|
||||
tokenizer,
|
||||
max_seq_length=512,
|
||||
pad_to_max_seq_len=False):
|
||||
result = []
|
||||
for text in example:
|
||||
encoded_inputs = tokenizer(
|
||||
text=text,
|
||||
max_seq_len=max_seq_length,
|
||||
pad_to_max_seq_len=pad_to_max_seq_len)
|
||||
input_ids = encoded_inputs["input_ids"]
|
||||
token_type_ids = encoded_inputs["token_type_ids"]
|
||||
result += [input_ids, token_type_ids]
|
||||
return result
|
||||
|
||||
|
||||
class ErnieOp(Op):
|
||||
def init_op(self):
|
||||
import paddlenlp as ppnlp
|
||||
self.tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained(
|
||||
'ernie-1.0')
|
||||
|
||||
def preprocess(self, input_dicts, data_id, log_id):
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
|
||||
(_, input_dict), = input_dicts.items()
|
||||
print("input dict", input_dict)
|
||||
batch_size = len(input_dict.keys())
|
||||
examples = []
|
||||
for i in range(batch_size):
|
||||
input_ids, segment_ids = convert_example([input_dict[str(i)]],
|
||||
self.tokenizer)
|
||||
examples.append((input_ids, segment_ids))
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=self.tokenizer.pad_token_id), # input
|
||||
Pad(axis=0, pad_val=self.tokenizer.pad_token_id), # segment
|
||||
): fn(samples)
|
||||
input_ids, segment_ids = batchify_fn(examples)
|
||||
feed_dict = {}
|
||||
feed_dict['input_ids'] = input_ids
|
||||
feed_dict['token_type_ids'] = segment_ids
|
||||
return feed_dict, False, None, ""
|
||||
|
||||
def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
|
||||
new_dict = {}
|
||||
new_dict["output_embedding"] = str(fetch_dict["output_embedding"]
|
||||
.tolist())
|
||||
return new_dict, None, ""
|
||||
|
||||
|
||||
class ErnieService(WebService):
|
||||
def get_pipeline_response(self, read_op):
|
||||
ernie_op = ErnieOp(name="ernie", input_ops=[read_op])
|
||||
return ernie_op
|
||||
|
||||
|
||||
ernie_service = ErnieService(name="ernie")
|
||||
ernie_service.prepare_pipeline_config("config_nlp.yml")
|
||||
ernie_service.run_service()
|
||||
@@ -0,0 +1,97 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
|
||||
from paddlenlp.utils.log import logger
|
||||
import time
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--similar_text_pair", type=str,
|
||||
default='', help="The full path of similat pair file")
|
||||
parser.add_argument("--recall_result_file", type=str,
|
||||
default='', help="The full path of recall result file")
|
||||
parser.add_argument("--recall_num", type=int, default=10,
|
||||
help="Most similair number of doc recalled from corpus per query")
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def recall(rs, N=10):
|
||||
"""
|
||||
Ratio of recalled Ground Truth at topN Recalled Docs
|
||||
>>> rs = [[0, 0, 1], [0, 1, 0], [1, 0, 0]]
|
||||
>>> recall(rs, N=1)
|
||||
0.333333
|
||||
>>> recall(rs, N=2)
|
||||
>>> 0.6666667
|
||||
>>> recall(rs, N=3)
|
||||
>>> 1.0
|
||||
Args:
|
||||
rs: Iterator of recalled flag()
|
||||
Returns:
|
||||
Recall@N
|
||||
"""
|
||||
|
||||
recall_flags = [np.sum(r[0:N]) for r in rs]
|
||||
return np.mean(recall_flags)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
text2similar = {}
|
||||
with open(args.similar_text_pair, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
text, similar_text = line.rstrip().split("\t")
|
||||
text2similar[text] = similar_text
|
||||
|
||||
rs = []
|
||||
|
||||
with open(args.recall_result_file, 'r', encoding='utf-8') as f:
|
||||
relevance_labels = []
|
||||
for index, line in enumerate(f):
|
||||
|
||||
if index % args.recall_num == 0 and index != 0:
|
||||
rs.append(relevance_labels)
|
||||
relevance_labels = []
|
||||
|
||||
text, recalled_text, cosine_sim = line.rstrip().split("\t")
|
||||
if text == recalled_text:
|
||||
continue
|
||||
if text2similar[text] == recalled_text:
|
||||
relevance_labels.append(1)
|
||||
else:
|
||||
relevance_labels.append(0)
|
||||
# print(len(rs))
|
||||
# print(rs[:50])
|
||||
|
||||
recall_N = []
|
||||
recall_num = [1, 5, 10, 20, 50]
|
||||
for topN in recall_num:
|
||||
R = round(100 * recall(rs, N=topN), 3)
|
||||
recall_N.append(str(R))
|
||||
result = open('result.tsv', 'a')
|
||||
res = []
|
||||
timestamp = time.strftime('%Y%m%d-%H%M%S', time.localtime())
|
||||
res.append(timestamp)
|
||||
for key, val in zip(recall_num, recall_N):
|
||||
print('recall@{}={}'.format(key, val))
|
||||
res.append(str(val))
|
||||
result.write('\t'.join(res) + '\n')
|
||||
# print("\t".join(recall_N))
|
||||
@@ -0,0 +1,65 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
|
||||
from base_model import SemanticIndexBase, SemanticIndexBaseStatic
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--params_path", type=str, required=True,
|
||||
default='./checkpoint/model_900/model_state.pdparams', help="The path to model parameters to be loaded.")
|
||||
parser.add_argument("--output_path", type=str, default='./output',
|
||||
help="The path of model parameter in static graph to be saved.")
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
if __name__ == "__main__":
|
||||
# If you want to use ernie1.0 model, plesace uncomment the following code
|
||||
output_emb_size = 256
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained(
|
||||
"ernie-1.0")
|
||||
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
model = SemanticIndexBaseStatic(
|
||||
pretrained_model, output_emb_size=output_emb_size)
|
||||
|
||||
if args.params_path and os.path.isfile(args.params_path):
|
||||
state_dict = paddle.load(args.params_path)
|
||||
model.set_dict(state_dict)
|
||||
print("Loaded parameters from %s" % args.params_path)
|
||||
|
||||
model.eval()
|
||||
|
||||
# Convert to static graph with specific input description
|
||||
model = paddle.jit.to_static(
|
||||
model,
|
||||
input_spec=[
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None], dtype="int64"), # input_ids
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None], dtype="int64") # segment_ids
|
||||
])
|
||||
# Save in static graph model.
|
||||
save_path = os.path.join(args.output_path, "inference")
|
||||
paddle.jit.save(model, save_path)
|
||||
@@ -0,0 +1,47 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import paddle_serving_client.io as serving_io
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dirname", type=str, required=True,
|
||||
default='./output', help="Path of saved model files. Program file and parameter files are saved in this directory.")
|
||||
parser.add_argument("--model_filename", type=str, required=True,
|
||||
default='inference.get_pooled_embedding.pdmodel', help="The name of file to load the inference program. If it is None, the default filename __model__ will be used.")
|
||||
parser.add_argument("--params_filename", type=str, required=True,
|
||||
default='inference.get_pooled_embedding.pdiparams', help="The name of file to load all parameters. It is only used for the case that all parameters were saved in a single binary file. If parameters were saved in separate files, set it as None. Default: None.")
|
||||
parser.add_argument("--server_path", type=str, default='./serving_server',
|
||||
help="The path of server parameter in static graph to be saved.")
|
||||
parser.add_argument("--client_path", type=str, default='./serving_client',
|
||||
help="The path of client parameter in static graph to be saved.")
|
||||
parser.add_argument("--feed_alias_names", type=str, default=None,
|
||||
help='set alias names for feed vars, split by comma \',\', you should run --show_proto to check the number of feed vars')
|
||||
parser.add_argument("--fetch_alias_names", type=str, default=None,
|
||||
help='set alias names for feed vars, split by comma \',\', you should run --show_proto to check the number of fetch vars')
|
||||
parser.add_argument("--show_proto", type=bool, default=False,
|
||||
help='If yes, you can preview the proto and then determine your feed var alias name and fetch var alias name.')
|
||||
# yapf: enable
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parser.parse_args()
|
||||
serving_io.inference_model_to_serving(
|
||||
dirname=args.dirname,
|
||||
serving_server=args.server_path,
|
||||
serving_client=args.client_path,
|
||||
model_filename=args.model_filename,
|
||||
params_filename=args.params_filename,
|
||||
show_proto=args.show_proto,
|
||||
feed_alias_names=args.feed_alias_names,
|
||||
fetch_alias_names=args.fetch_alias_names)
|
||||
@@ -0,0 +1,79 @@
|
||||
from functools import partial
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.datasets import load_dataset, MapDataset, load_dataset
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
from base_model import SemanticIndexBase, SemanticIndexBaseStatic
|
||||
from data import convert_example, create_dataloader
|
||||
from data import gen_id2corpus, gen_text_file
|
||||
from ann_util import build_index
|
||||
from tqdm import tqdm
|
||||
|
||||
if __name__ == "__main__":
|
||||
device = 'gpu'
|
||||
max_seq_length = 64
|
||||
output_emb_size = 256
|
||||
batch_size = 1
|
||||
params_path = 'checkpoints/inbatch/model_40/model_state.pdparams'
|
||||
id2corpus = {0: '国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
|
||||
paddle.set_device(device)
|
||||
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
trans_func = partial(
|
||||
convert_example, tokenizer=tokenizer, max_seq_length=max_seq_length)
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # text_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # text_segment
|
||||
): [data for data in fn(samples)]
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained(
|
||||
"ernie-1.0")
|
||||
|
||||
model = SemanticIndexBaseStatic(
|
||||
pretrained_model, output_emb_size=output_emb_size)
|
||||
|
||||
# Load pretrained semantic model
|
||||
if params_path and os.path.isfile(params_path):
|
||||
state_dict = paddle.load(params_path)
|
||||
model.set_dict(state_dict)
|
||||
print("Loaded parameters from %s" % params_path)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Please set --params_path with correct pretrained model file")
|
||||
|
||||
# conver_example function's input must be dict
|
||||
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
|
||||
corpus_ds = MapDataset(corpus_list)
|
||||
|
||||
corpus_data_loader = create_dataloader(
|
||||
corpus_ds,
|
||||
mode='predict',
|
||||
batch_size=batch_size,
|
||||
batchify_fn=batchify_fn,
|
||||
trans_fn=trans_func)
|
||||
|
||||
all_embeddings = []
|
||||
model.eval()
|
||||
with paddle.no_grad():
|
||||
for batch_data in corpus_data_loader:
|
||||
input_ids, token_type_ids = batch_data
|
||||
input_ids = paddle.to_tensor(input_ids)
|
||||
token_type_ids = paddle.to_tensor(token_type_ids)
|
||||
|
||||
text_embeddings = model(input_ids, token_type_ids)
|
||||
all_embeddings.append(text_embeddings)
|
||||
|
||||
text_embedding = all_embeddings[0]
|
||||
print(text_embedding.shape)
|
||||
print(text_embedding.numpy())
|
||||
133
applications/neural_search/recall/in_batch_negative/predict.py
Normal file
133
applications/neural_search/recall/in_batch_negative/predict.py
Normal file
@@ -0,0 +1,133 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from functools import partial
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.datasets import load_dataset
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
|
||||
from data import read_text_pair, convert_example, create_dataloader
|
||||
from base_model import SemanticIndexBase
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--text_pair_file", type=str,
|
||||
required=True, help="The full path of input file")
|
||||
parser.add_argument("--params_path", type=str, required=True,
|
||||
help="The path to model parameters to be loaded.")
|
||||
parser.add_argument("--max_seq_length", default=64, type=int, help="The maximum total input sequence length after tokenization. "
|
||||
"Sequences longer than this will be truncated, sequences shorter will be padded.")
|
||||
parser.add_argument("--batch_size", default=32, type=int,
|
||||
help="Batch size per GPU/CPU for training.")
|
||||
parser.add_argument("--output_emb_size", default=None,
|
||||
type=int, help="output_embedding_size")
|
||||
parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu",
|
||||
help="Select which device to train model, defaults to gpu.")
|
||||
parser.add_argument("--pad_to_max_seq_len", action="store_true",
|
||||
help="Whether to pad to max seq length.")
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def predict(model, data_loader):
|
||||
"""
|
||||
Predicts the data labels.
|
||||
|
||||
Args:
|
||||
model (obj:`SemanticIndexBase`): A model to extract text embedding or calculate similarity of text pair.
|
||||
data_loaer (obj:`List(Example)`): The processed data ids of text pair: [query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids]
|
||||
Returns:
|
||||
results(obj:`List`): cosine similarity of text pairs.
|
||||
"""
|
||||
cosine_sims = []
|
||||
|
||||
model.eval()
|
||||
|
||||
with paddle.no_grad():
|
||||
for batch_data in data_loader:
|
||||
query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids = batch_data
|
||||
|
||||
query_input_ids = paddle.to_tensor(query_input_ids)
|
||||
query_token_type_ids = paddle.to_tensor(query_token_type_ids)
|
||||
title_input_ids = paddle.to_tensor(title_input_ids)
|
||||
title_token_type_ids = paddle.to_tensor(title_token_type_ids)
|
||||
|
||||
batch_cosine_sim = model.cosine_sim(
|
||||
query_input_ids=query_input_ids,
|
||||
title_input_ids=title_input_ids,
|
||||
query_token_type_ids=query_token_type_ids,
|
||||
title_token_type_ids=title_token_type_ids).numpy()
|
||||
|
||||
cosine_sims.append(batch_cosine_sim)
|
||||
|
||||
cosine_sims = np.concatenate(cosine_sims, axis=0)
|
||||
|
||||
return cosine_sims
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
paddle.set_device(args.device)
|
||||
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
|
||||
trans_func = partial(
|
||||
convert_example,
|
||||
tokenizer=tokenizer,
|
||||
max_seq_length=args.max_seq_length,
|
||||
pad_to_max_seq_len=args.pad_to_max_seq_len)
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # query_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # query_segment
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # title_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # tilte_segment
|
||||
): [data for data in fn(samples)]
|
||||
|
||||
valid_ds = load_dataset(
|
||||
read_text_pair, data_path=args.text_pair_file, lazy=False)
|
||||
|
||||
valid_data_loader = create_dataloader(
|
||||
valid_ds,
|
||||
mode='predict',
|
||||
batch_size=args.batch_size,
|
||||
batchify_fn=batchify_fn,
|
||||
trans_fn=trans_func)
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained(
|
||||
"ernie-1.0")
|
||||
|
||||
model = SemanticIndexBase(
|
||||
pretrained_model, output_emb_size=args.output_emb_size)
|
||||
|
||||
if args.params_path and os.path.isfile(args.params_path):
|
||||
state_dict = paddle.load(args.params_path)
|
||||
model.set_dict(state_dict)
|
||||
print("Loaded parameters from %s" % args.params_path)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Please set --params_path with correct pretrained model file")
|
||||
|
||||
cosin_sim = predict(model, valid_data_loader)
|
||||
|
||||
for idx, cosine in enumerate(cosin_sim):
|
||||
print('{}'.format(cosine))
|
||||
153
applications/neural_search/recall/in_batch_negative/recall.py
Normal file
153
applications/neural_search/recall/in_batch_negative/recall.py
Normal file
@@ -0,0 +1,153 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# coding=UTF-8
|
||||
|
||||
from functools import partial
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import hnswlib
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.datasets import load_dataset, MapDataset, load_dataset
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
from base_model import SemanticIndexBase
|
||||
from data import convert_example, create_dataloader
|
||||
from data import gen_id2corpus, gen_text_file
|
||||
from ann_util import build_index
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--corpus_file", type=str, required=True,
|
||||
help="The full path of input file")
|
||||
parser.add_argument("--similar_text_pair_file", type=str,
|
||||
required=True, help="The full path of similar text pair file")
|
||||
parser.add_argument("--recall_result_dir", type=str, default='recall_result',
|
||||
help="The full path of recall result file to save")
|
||||
parser.add_argument("--recall_result_file", type=str,
|
||||
default='recall_result_file', help="The file name of recall result")
|
||||
parser.add_argument("--params_path", type=str, required=True,
|
||||
help="The path to model parameters to be loaded.")
|
||||
parser.add_argument("--max_seq_length", default=64, type=int, help="The maximum total input sequence length after tokenization. "
|
||||
"Sequences longer than this will be truncated, sequences shorter will be padded.")
|
||||
parser.add_argument("--batch_size", default=32, type=int,
|
||||
help="Batch size per GPU/CPU for training.")
|
||||
parser.add_argument("--output_emb_size", default=None,
|
||||
type=int, help="output_embedding_size")
|
||||
parser.add_argument("--recall_num", default=10, type=int,
|
||||
help="Recall number for each query from Ann index.")
|
||||
|
||||
parser.add_argument("--hnsw_m", default=100, type=int,
|
||||
help="Recall number for each query from Ann index.")
|
||||
parser.add_argument("--hnsw_ef", default=100, type=int,
|
||||
help="Recall number for each query from Ann index.")
|
||||
parser.add_argument("--hnsw_max_elements", default=1000000,
|
||||
type=int, help="Recall number for each query from Ann index.")
|
||||
|
||||
parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu",
|
||||
help="Select which device to train model, defaults to gpu.")
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
if __name__ == "__main__":
|
||||
paddle.set_device(args.device)
|
||||
rank = paddle.distributed.get_rank()
|
||||
if paddle.distributed.get_world_size() > 1:
|
||||
paddle.distributed.init_parallel_env()
|
||||
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
|
||||
trans_func = partial(
|
||||
convert_example,
|
||||
tokenizer=tokenizer,
|
||||
max_seq_length=args.max_seq_length)
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # text_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # text_segment
|
||||
): [data for data in fn(samples)]
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained(
|
||||
"ernie-1.0")
|
||||
|
||||
model = SemanticIndexBase(
|
||||
pretrained_model, output_emb_size=args.output_emb_size)
|
||||
model = paddle.DataParallel(model)
|
||||
|
||||
# Load pretrained semantic model
|
||||
if args.params_path and os.path.isfile(args.params_path):
|
||||
state_dict = paddle.load(args.params_path)
|
||||
model.set_dict(state_dict)
|
||||
logger.info("Loaded parameters from %s" % args.params_path)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Please set --params_path with correct pretrained model file")
|
||||
|
||||
id2corpus = gen_id2corpus(args.corpus_file)
|
||||
|
||||
# conver_example function's input must be dict
|
||||
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
|
||||
corpus_ds = MapDataset(corpus_list)
|
||||
|
||||
corpus_data_loader = create_dataloader(
|
||||
corpus_ds,
|
||||
mode='predict',
|
||||
batch_size=args.batch_size,
|
||||
batchify_fn=batchify_fn,
|
||||
trans_fn=trans_func)
|
||||
|
||||
# Need better way to get inner model of DataParallel
|
||||
inner_model = model._layers
|
||||
|
||||
final_index = build_index(args, corpus_data_loader, inner_model)
|
||||
|
||||
text_list, text2similar_text = gen_text_file(args.similar_text_pair_file)
|
||||
|
||||
query_ds = MapDataset(text_list)
|
||||
|
||||
query_data_loader = create_dataloader(
|
||||
query_ds,
|
||||
mode='predict',
|
||||
batch_size=args.batch_size,
|
||||
batchify_fn=batchify_fn,
|
||||
trans_fn=trans_func)
|
||||
|
||||
query_embedding = inner_model.get_semantic_embedding(query_data_loader)
|
||||
|
||||
if not os.path.exists(args.recall_result_dir):
|
||||
os.mkdir(args.recall_result_dir)
|
||||
|
||||
recall_result_file = os.path.join(args.recall_result_dir,
|
||||
args.recall_result_file)
|
||||
with open(recall_result_file, 'w', encoding='utf-8') as f:
|
||||
for batch_index, batch_query_embedding in enumerate(query_embedding):
|
||||
recalled_idx, cosine_sims = final_index.knn_query(
|
||||
batch_query_embedding.numpy(), args.recall_num)
|
||||
|
||||
batch_size = len(cosine_sims)
|
||||
|
||||
for row_index in range(batch_size):
|
||||
text_index = args.batch_size * batch_index + row_index
|
||||
for idx, doc_idx in enumerate(recalled_idx[row_index]):
|
||||
f.write("{}\t{}\t{}\n".format(text_list[text_index][
|
||||
"text"], id2corpus[doc_idx], 1.0 - cosine_sims[
|
||||
row_index][idx]))
|
||||
@@ -0,0 +1,4 @@
|
||||
python -u evaluate.py \
|
||||
--similar_text_pair "recall/dev.csv" \
|
||||
--recall_result_file "./recall_result_dir/recall_result.txt" \
|
||||
--recall_num 50
|
||||
@@ -0,0 +1 @@
|
||||
python export_model.py --params_path checkpoints/inbatch/model_40/model_state.pdparams --output_path=./output
|
||||
@@ -0,0 +1,7 @@
|
||||
python export_to_serving.py \
|
||||
--dirname "output" \
|
||||
--model_filename "inference.get_pooled_embedding.pdmodel" \
|
||||
--params_filename "inference.get_pooled_embedding.pdiparams" \
|
||||
--server_path "serving_server" \
|
||||
--client_path "serving_client" \
|
||||
--fetch_alias_names "output_embedding"
|
||||
@@ -0,0 +1,22 @@
|
||||
# gpu version
|
||||
|
||||
root_dir="checkpoints/inbatch"
|
||||
python -u -m paddle.distributed.launch --gpus "3" \
|
||||
predict.py \
|
||||
--device gpu \
|
||||
--params_path "${root_dir}/model_40/model_state.pdparams" \
|
||||
--output_emb_size 256 \
|
||||
--batch_size 128 \
|
||||
--max_seq_length 64 \
|
||||
--text_pair_file "recall/test.csv"
|
||||
|
||||
|
||||
# cpu
|
||||
# root_dir="checkpoints/inbatch"
|
||||
# python predict.py \
|
||||
# --device cpu \
|
||||
# --params_path "${root_dir}/model_40/model_state.pdparams" \
|
||||
# --output_emb_size 256 \
|
||||
# --batch_size 128 \
|
||||
# --max_seq_length 64 \
|
||||
# --text_pair_file "recall/test.csv"
|
||||
@@ -0,0 +1,31 @@
|
||||
# GPU version
|
||||
root_dir="checkpoints/inbatch"
|
||||
python -u -m paddle.distributed.launch --gpus "3" --log_dir "recall_log/" \
|
||||
recall.py \
|
||||
--device gpu \
|
||||
--recall_result_dir "recall_result_dir" \
|
||||
--recall_result_file "recall_result.txt" \
|
||||
--params_path "${root_dir}/model_40/model_state.pdparams" \
|
||||
--hnsw_m 100 \
|
||||
--hnsw_ef 100 \
|
||||
--batch_size 64 \
|
||||
--output_emb_size 256\
|
||||
--max_seq_length 60 \
|
||||
--recall_num 50 \
|
||||
--similar_text_pair "recall/dev.csv" \
|
||||
--corpus_file "recall/corpus.csv"
|
||||
|
||||
# CPU version
|
||||
# python recall.py \
|
||||
# --device cpu \
|
||||
# --recall_result_dir "recall_result_dir" \
|
||||
# --recall_result_file "recall_result.txt" \
|
||||
# --params_path "${root_dir}/model_40/model_state.pdparams" \
|
||||
# --hnsw_m 100 \
|
||||
# --hnsw_ef 100 \
|
||||
# --batch_size 64 \
|
||||
# --output_emb_size 256\
|
||||
# --max_seq_length 60 \
|
||||
# --recall_num 50 \
|
||||
# --similar_text_pair "recall/dev.csv" \
|
||||
# --corpus_file "recall/corpus.csv"
|
||||
@@ -0,0 +1,61 @@
|
||||
# GPU training
|
||||
root_path=inbatch
|
||||
python -u -m paddle.distributed.launch --gpus "0,1,2,3" \
|
||||
train_batch_neg.py \
|
||||
--device gpu \
|
||||
--save_dir ./checkpoints/${root_path} \
|
||||
--batch_size 64 \
|
||||
--learning_rate 5E-5 \
|
||||
--epochs 3 \
|
||||
--output_emb_size 256 \
|
||||
--save_steps 10 \
|
||||
--max_seq_length 64 \
|
||||
--margin 0.2 \
|
||||
--train_set_file recall/train.csv
|
||||
|
||||
|
||||
# cpu training
|
||||
# root_path=inbatch
|
||||
# python train_batch_neg.py \
|
||||
# --device cpu \
|
||||
# --save_dir ./checkpoints/${root_path} \
|
||||
# --batch_size 64 \
|
||||
# --learning_rate 5E-5 \
|
||||
# --epochs 3 \
|
||||
# --output_emb_size 256 \
|
||||
# --save_steps 10 \
|
||||
# --max_seq_length 64 \
|
||||
# --margin 0.2 \
|
||||
# --train_set_file recall/train.csv
|
||||
|
||||
|
||||
|
||||
# 加载simcse训练的模型，模型放在simcse/model_20000
|
||||
# python -u -m paddle.distributed.launch --gpus "0,1,2,3" \
|
||||
# train_batch_neg.py \
|
||||
# --device gpu \
|
||||
# --save_dir ./checkpoints/simcse_inbatch_negative \
|
||||
# --batch_size 64 \
|
||||
# --learning_rate 5E-5 \
|
||||
# --epochs 3 \
|
||||
# --output_emb_size 256 \
|
||||
# --save_steps 10 \
|
||||
# --max_seq_length 64 \
|
||||
# --margin 0.2 \
|
||||
# --train_set_file data/${root_path}/train.csv \
|
||||
# --init_from_ckpt simcse/model_20000/model_state.pdparams
|
||||
|
||||
# 加载post training的模型，模型放在simcse/post_model_10000
|
||||
# python -u -m paddle.distributed.launch --gpus "0,1,2,3" \
|
||||
# train_batch_neg.py \
|
||||
# --device gpu \
|
||||
# --save_dir ./checkpoints/post_simcse_inbatch_negative \
|
||||
# --batch_size 64 \
|
||||
# --learning_rate 5E-5 \
|
||||
# --epochs 3 \
|
||||
# --output_emb_size 256 \
|
||||
# --save_steps 10 \
|
||||
# --max_seq_length 64 \
|
||||
# --margin 0.2 \
|
||||
# --train_set_file data/${root_path}/train.csv \
|
||||
# --init_from_ckpt simcse/post_model_10000/model_state.pdparams
|
||||
@@ -0,0 +1,177 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from functools import partial
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.datasets import load_dataset
|
||||
from paddlenlp.transformers import LinearDecayWithWarmup
|
||||
|
||||
from batch_negative.model import SemanticIndexBatchNeg
|
||||
from data import read_text_pair, convert_example, create_dataloader
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--save_dir", default='./checkpoint', type=str,
|
||||
help="The output directory where the model checkpoints will be written.")
|
||||
parser.add_argument("--max_seq_length", default=128, type=int, help="The maximum total input sequence length after tokenization. "
|
||||
"Sequences longer than this will be truncated, sequences shorter will be padded.")
|
||||
parser.add_argument("--batch_size", default=32, type=int,
|
||||
help="Batch size per GPU/CPU for training.")
|
||||
parser.add_argument("--output_emb_size", default=None,
|
||||
type=int, help="output_embedding_size")
|
||||
parser.add_argument("--learning_rate", default=1e-5, type=float,
|
||||
help="The initial learning rate for Adam.")
|
||||
parser.add_argument("--weight_decay", default=0.0, type=float,
|
||||
help="Weight decay if we apply some.")
|
||||
parser.add_argument("--epochs", default=10, type=int,
|
||||
help="Total number of training epochs to perform.")
|
||||
parser.add_argument("--warmup_proportion", default=0.0, type=float,
|
||||
help="Linear warmup proption over the training process.")
|
||||
parser.add_argument("--init_from_ckpt", type=str, default=None,
|
||||
help="The path of checkpoint to be loaded.")
|
||||
parser.add_argument("--seed", type=int, default=1000,
|
||||
help="random seed for initialization")
|
||||
parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu",
|
||||
help="Select which device to train model, defaults to gpu.")
|
||||
parser.add_argument('--save_steps', type=int, default=10000,
|
||||
help="Inteval steps to save checkpoint")
|
||||
parser.add_argument("--train_set_file", type=str,
|
||||
required=True, help="The full path of train_set_file")
|
||||
parser.add_argument("--margin", default=0.3, type=float,
|
||||
help="Margin beteween pos_sample and neg_samples")
|
||||
parser.add_argument("--scale", default=30, type=int,
|
||||
help="Scale for pair-wise margin_rank_loss")
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
|
||||
def set_seed(seed):
|
||||
"""sets random seed"""
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
paddle.seed(seed)
|
||||
|
||||
|
||||
def do_train():
|
||||
paddle.set_device(args.device)
|
||||
rank = paddle.distributed.get_rank()
|
||||
if paddle.distributed.get_world_size() > 1:
|
||||
paddle.distributed.init_parallel_env()
|
||||
|
||||
set_seed(args.seed)
|
||||
|
||||
train_ds = load_dataset(
|
||||
read_text_pair, data_path=args.train_set_file, lazy=False)
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained(
|
||||
'ernie-1.0')
|
||||
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
|
||||
trans_func = partial(
|
||||
convert_example,
|
||||
tokenizer=tokenizer,
|
||||
max_seq_length=args.max_seq_length)
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # query_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # query_segment
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # title_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # tilte_segment
|
||||
): [data for data in fn(samples)]
|
||||
|
||||
train_data_loader = create_dataloader(
|
||||
train_ds,
|
||||
mode='train',
|
||||
batch_size=args.batch_size,
|
||||
batchify_fn=batchify_fn,
|
||||
trans_fn=trans_func)
|
||||
|
||||
model = SemanticIndexBatchNeg(
|
||||
pretrained_model,
|
||||
margin=args.margin,
|
||||
scale=args.scale,
|
||||
output_emb_size=args.output_emb_size)
|
||||
|
||||
if args.init_from_ckpt and os.path.isfile(args.init_from_ckpt):
|
||||
state_dict = paddle.load(args.init_from_ckpt)
|
||||
model.set_dict(state_dict)
|
||||
print("warmup from:{}".format(args.init_from_ckpt))
|
||||
|
||||
model = paddle.DataParallel(model)
|
||||
|
||||
num_training_steps = len(train_data_loader) * args.epochs
|
||||
|
||||
lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps,
|
||||
args.warmup_proportion)
|
||||
|
||||
# Generate parameter names needed to perform weight decay.
|
||||
# All bias and LayerNorm parameters are excluded.
|
||||
decay_params = [
|
||||
p.name for n, p in model.named_parameters()
|
||||
if not any(nd in n for nd in ["bias", "norm"])
|
||||
]
|
||||
optimizer = paddle.optimizer.AdamW(
|
||||
learning_rate=lr_scheduler,
|
||||
parameters=model.parameters(),
|
||||
weight_decay=args.weight_decay,
|
||||
apply_decay_param_fun=lambda x: x in decay_params)
|
||||
|
||||
global_step = 0
|
||||
tic_train = time.time()
|
||||
for epoch in range(1, args.epochs + 1):
|
||||
for step, batch in enumerate(train_data_loader, start=1):
|
||||
query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids = batch
|
||||
|
||||
loss = model(
|
||||
query_input_ids=query_input_ids,
|
||||
title_input_ids=title_input_ids,
|
||||
query_token_type_ids=query_token_type_ids,
|
||||
title_token_type_ids=title_token_type_ids)
|
||||
|
||||
global_step += 1
|
||||
if global_step % 50 == 0 and rank == 0:
|
||||
print(
|
||||
"global step %d, epoch: %d, batch: %d, loss: %.5f, speed: %.2f step/s"
|
||||
% (global_step, epoch, step, loss,
|
||||
10 / (time.time() - tic_train)))
|
||||
tic_train = time.time()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
lr_scheduler.step()
|
||||
optimizer.clear_grad()
|
||||
if global_step % args.save_steps == 0 and rank == 0:
|
||||
save_dir = os.path.join(args.save_dir, "model_%d" % global_step)
|
||||
if not os.path.exists(save_dir):
|
||||
os.makedirs(save_dir)
|
||||
save_param_path = os.path.join(save_dir, 'model_state.pdparams')
|
||||
paddle.save(model.state_dict(), save_param_path)
|
||||
tokenizer.save_pretrained(save_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
do_train()
|
||||
206
applications/neural_search/recall/milvus/README.md
Normal file
206
applications/neural_search/recall/milvus/README.md
Normal file
@@ -0,0 +1,206 @@
|
||||
**目录**
|
||||
|
||||
* [背景介绍](#背景介绍)
|
||||
* [Milvus召回](#Milvus召回)
|
||||
* [1. 技术方案和评估指标](#技术方案)
|
||||
* [2. 环境依赖](#环境依赖)
|
||||
* [3. 代码结构](#代码结构)
|
||||
* [4. 数据准备](#数据准备)
|
||||
* [5. 向量检索](#向量检索)
|
||||
|
||||
|
||||
<a name="背景介绍"></a>
|
||||
|
||||
# 背景介绍
|
||||
|
||||
基于某检索平台开源的数据集构造生成了面向语义索引的召回库。
|
||||
|
||||
<a name="Milvus召回"></a>
|
||||
|
||||
# Milvus召回
|
||||
|
||||
<a name="技术方案"></a>
|
||||
|
||||
## 1. 技术方案和评估指标
|
||||
|
||||
### 技术方案
|
||||
|
||||
使用 Milvus 搭建召回系统，然后使用训练好的语义索引模型，抽取向量，插入到 Milvus 中，然后进行检索。
|
||||
|
||||
<a name="环境依赖"></a>
|
||||
|
||||
## 2. 环境依赖和安装说明
|
||||
|
||||
**环境依赖**
|
||||
* python >= 3.6
|
||||
* paddlepaddle >= 2.2
|
||||
* paddlenlp >= 2.2
|
||||
* milvus >= 1.1.1
|
||||
* pymilvus >= 1.1.2
|
||||
|
||||
<a name="代码结构"></a>
|
||||
|
||||
## 3. 代码结构
|
||||
|
||||
## 代码结构：
|
||||
|
||||
```
|
||||
|—— scripts
|
||||
|—— feature_extract.sh 提取特征向量的bash脚本
|
||||
├── base_model.py # 语义索引模型基类
|
||||
├── config.py # milvus配置文件
|
||||
├── data.py # 数据处理函数
|
||||
├── embedding_insert.py # 插入向量
|
||||
├── embedding_recall.py # 检索topK相似结果 / ANN
|
||||
├── inference.py # 动态图模型向量抽取脚本
|
||||
├── feature_extract.py # 批量抽取向量脚本
|
||||
├── milvus_insert.py # 插入向量工具类
|
||||
├── milvus_recall.py # 向量召回工具类
|
||||
├── README.md
|
||||
└── server_config.yml # milvus的config文件，本项目所用的配置
|
||||
```
|
||||
<a name="数据准备"></a>
|
||||
|
||||
## 4. 数据准备
|
||||
|
||||
数据集的样例如下，有两种，第一种是 title+keywords 进行拼接；第二种是一句话。
|
||||
|
||||
```
|
||||
煤矸石-污泥基活性炭介导强化污水厌氧消化煤矸石,污泥,复合基活性炭,厌氧消化,直接种间电子传递
|
||||
睡眠障碍与常见神经系统疾病的关系睡眠觉醒障碍,神经系统疾病,睡眠,快速眼运动,细胞增殖,阿尔茨海默病
|
||||
城市道路交通流中观仿真研究智能运输系统;城市交通管理;计算机仿真;城市道路;交通流;路径选择
|
||||
....
|
||||
```
|
||||
|
||||
### 数据集下载
|
||||
|
||||
|
||||
- [literature_search_data](https://bj.bcebos.com/v1/paddlenlp/data/literature_search_data.zip)
|
||||
|
||||
```
|
||||
├── milvus # milvus建库数据集
|
||||
├── milvus_data.csv. # 构建召回库的数据
|
||||
├── recall # 召回（语义索引）数据集
|
||||
├── corpus.csv # 用于测试的召回库
|
||||
├── dev.csv # 召回验证集
|
||||
├── test.csv # 召回测试集
|
||||
├── train.csv # 召回训练集
|
||||
├── train_unsupervised.csv # 无监督训练集
|
||||
├── sort # 排序数据集
|
||||
├── test_pairwise.csv # 排序测试集
|
||||
├── dev_pairwise.csv # 排序验证集
|
||||
└── train_pairwise.csv # 排序训练集
|
||||
|
||||
```
|
||||
|
||||
<a name="向量检索"></a>
|
||||
|
||||
## 5. 向量检索
|
||||
|
||||
|
||||
数据准备结束以后，我们开始搭建 Milvus 的语义检索引擎，用于语义向量的快速检索，我们使用[Milvus](https://milvus.io/)开源工具进行召回，Milvus 的搭建教程请参考官方教程 [Milvus官方安装教程](https://milvus.io/cn/docs/v1.1.1/milvus_docker-cpu.md)本案例使用的是 Milvus 的1.1.1 CPU版本，建议使用官方的 Docker 安装方式，简单快捷。
|
||||
|
||||
Milvus 搭建完系统以后就可以插入和检索向量了，首先生成 embedding 向量，每个样本生成256维度的向量，使用的是32GB的V100的卡进行的提取：
|
||||
|
||||
```
|
||||
CUDA_VISIBLE_DEVICES=2 python feature_extract.py \
|
||||
--model_dir=./output \
|
||||
--corpus_file "data/milvus_data.csv"
|
||||
```
|
||||
其中 output 目录下存放的是召回的 Paddle Inference 静态图模型。
|
||||
|
||||
| 数据量 | 时间 |
|
||||
| ------------ | ------------ |
|
||||
|1000万条|3hour40min39s|
|
||||
|
||||
运行结束后会生成 corpus_embedding.npy
|
||||
|
||||
生成了向量后，需要把数据插入到 Milvus 库中，首先修改配置：
|
||||
|
||||
修改 config.py 的配置 ip 和端口，本项目使用的是8530端口，而 Milvus 默认的是19530，需要根据情况进行修改：
|
||||
|
||||
```
|
||||
MILVUS_HOST='your milvus ip'
|
||||
MILVUS_PORT = 8530
|
||||
```
|
||||
|
||||
然后运行下面的命令把向量插入到Milvus库中：
|
||||
|
||||
```
|
||||
python3 embedding_insert.py
|
||||
```
|
||||
|
||||
|
||||
| 数据量 | 时间 |
|
||||
| ------------ | ------------ |
|
||||
|1000万条|12min24s|
|
||||
|
||||
另外，Milvus提供了可视化的管理界面，可以很方便的查看数据，安装地址为[Milvus Enterprise Manager](https://zilliz.com/products/em).
|
||||
|
||||
|
||||
|
||||
|
||||
运行召回脚本：
|
||||
|
||||
```
|
||||
python3 embedding_recall.py
|
||||
|
||||
```
|
||||
运行的结果为，表示的是召回的 id 和与当前的 query 计算的距离：
|
||||
|
||||
```
|
||||
10000000
|
||||
time cost 0.5410025119781494 s
|
||||
Status(code=0, message='Search vectors successfully!')
|
||||
[
|
||||
[
|
||||
(id:1, distance:0.0),
|
||||
(id:7109733, distance:0.832247257232666),
|
||||
(id:6770053, distance:0.8488889932632446),
|
||||
(id:2653227, distance:0.9032443761825562),
|
||||
...
|
||||
```
|
||||
|
||||
第一次检索的时间大概是18s左右，需要把数据从磁盘加载到内存，后面检索就很快，下面是测试的速度：
|
||||
|
||||
| 数据量 | 时间 |
|
||||
| ------------ | ------------ |
|
||||
|100条|0.15351247787475586|
|
||||
|
||||
如果测试的速度过慢，可以修改 Milvus 配置里面的 cache 参数：
|
||||
|
||||
```
|
||||
cache:
|
||||
cache_size: 32GB
|
||||
insert_buffer_size: 8GB
|
||||
preload_collection:
|
||||
|
||||
```
|
||||
把 cache_size，insert_buffer_size 调的越大，速度越快，调完后重启 Milvus
|
||||
|
||||
|
||||
修改代码的模型路径和样本：
|
||||
|
||||
```
|
||||
params_path='checkpoints/model_40/model_state.pdparams'
|
||||
id2corpus={0:'国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
|
||||
```
|
||||
|
||||
运行命令
|
||||
|
||||
```
|
||||
python3 inference.py
|
||||
|
||||
```
|
||||
运行的输出为，分别是抽取的向量和召回的结果：
|
||||
|
||||
```
|
||||
[1, 256]
|
||||
[[ 0.06374735 -0.08051944 0.05118101 -0.05855767 -0.06969483 0.05318566
|
||||
0.079629 0.02667932 -0.04501902 -0.01187392 0.09590752 -0.05831281
|
||||
....
|
||||
5677638 国有股权参股对家族企业创新投入的影响混合所有制改革,国有股权,家族企业,创新投入 0.5417419672012329
|
||||
1321645 高管政治联系对民营企业创新绩效的影响——董事会治理行为的非线性中介效应高管政治联系,创新绩效,民营上市公司,董事会治理行为,中介效应 0.5445536375045776
|
||||
1340319 国有控股上市公司资产并购重组风险探讨国有控股上市公司,并购重组,防范对策 0.5515031218528748
|
||||
....
|
||||
```
|
||||
186
applications/neural_search/recall/milvus/base_model.py
Normal file
186
applications/neural_search/recall/milvus/base_model.py
Normal file
@@ -0,0 +1,186 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import abc
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
import paddle
|
||||
import paddle.nn as nn
|
||||
import paddle.nn.functional as F
|
||||
|
||||
|
||||
class SemanticIndexBase(nn.Layer):
|
||||
def __init__(self, pretrained_model, dropout=None, output_emb_size=None):
|
||||
super().__init__()
|
||||
self.ptm = pretrained_model
|
||||
self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)
|
||||
|
||||
# if output_emb_size is not None, then add Linear layer to reduce embedding_size,
|
||||
# we recommend set output_emb_size = 256 considering the trade-off beteween
|
||||
# recall performance and efficiency
|
||||
|
||||
self.output_emb_size = output_emb_size
|
||||
if output_emb_size > 0:
|
||||
weight_attr = paddle.ParamAttr(
|
||||
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
|
||||
self.emb_reduce_linear = paddle.nn.Linear(
|
||||
768, output_emb_size, weight_attr=weight_attr)
|
||||
|
||||
@paddle.jit.to_static(input_spec=[
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64'), paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64')
|
||||
])
|
||||
def get_pooled_embedding(self,
|
||||
input_ids,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
attention_mask=None):
|
||||
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
|
||||
attention_mask)
|
||||
|
||||
if self.output_emb_size > 0:
|
||||
cls_embedding = self.emb_reduce_linear(cls_embedding)
|
||||
cls_embedding = self.dropout(cls_embedding)
|
||||
cls_embedding = F.normalize(cls_embedding, p=2, axis=-1)
|
||||
|
||||
return cls_embedding
|
||||
|
||||
def get_semantic_embedding(self, data_loader):
|
||||
self.eval()
|
||||
with paddle.no_grad():
|
||||
for batch_data in data_loader:
|
||||
input_ids, token_type_ids = batch_data
|
||||
input_ids = paddle.to_tensor(input_ids)
|
||||
token_type_ids = paddle.to_tensor(token_type_ids)
|
||||
|
||||
text_embeddings = self.get_pooled_embedding(
|
||||
input_ids, token_type_ids=token_type_ids)
|
||||
|
||||
yield text_embeddings
|
||||
|
||||
def cosine_sim(self,
|
||||
query_input_ids,
|
||||
title_input_ids,
|
||||
query_token_type_ids=None,
|
||||
query_position_ids=None,
|
||||
query_attention_mask=None,
|
||||
title_token_type_ids=None,
|
||||
title_position_ids=None,
|
||||
title_attention_mask=None):
|
||||
|
||||
query_cls_embedding = self.get_pooled_embedding(
|
||||
query_input_ids, query_token_type_ids, query_position_ids,
|
||||
query_attention_mask)
|
||||
|
||||
title_cls_embedding = self.get_pooled_embedding(
|
||||
title_input_ids, title_token_type_ids, title_position_ids,
|
||||
title_attention_mask)
|
||||
|
||||
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
|
||||
axis=-1)
|
||||
return cosine_sim
|
||||
|
||||
@abc.abstractmethod
|
||||
def forward(self):
|
||||
pass
|
||||
|
||||
|
||||
class SemanticIndexBaseStatic(nn.Layer):
|
||||
def __init__(self, pretrained_model, dropout=None, output_emb_size=None):
|
||||
super().__init__()
|
||||
self.ptm = pretrained_model
|
||||
self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)
|
||||
|
||||
# if output_emb_size is not None, then add Linear layer to reduce embedding_size,
|
||||
# we recommend set output_emb_size = 256 considering the trade-off beteween
|
||||
# recall performance and efficiency
|
||||
|
||||
self.output_emb_size = output_emb_size
|
||||
if output_emb_size > 0:
|
||||
weight_attr = paddle.ParamAttr(
|
||||
initializer=paddle.nn.initializer.TruncatedNormal(std=0.02))
|
||||
self.emb_reduce_linear = paddle.nn.Linear(
|
||||
768, output_emb_size, weight_attr=weight_attr)
|
||||
|
||||
@paddle.jit.to_static(input_spec=[
|
||||
paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64'), paddle.static.InputSpec(
|
||||
shape=[None, None], dtype='int64')
|
||||
])
|
||||
def get_pooled_embedding(self,
|
||||
input_ids,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
attention_mask=None):
|
||||
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
|
||||
attention_mask)
|
||||
|
||||
if self.output_emb_size > 0:
|
||||
cls_embedding = self.emb_reduce_linear(cls_embedding)
|
||||
cls_embedding = self.dropout(cls_embedding)
|
||||
cls_embedding = F.normalize(cls_embedding, p=2, axis=-1)
|
||||
|
||||
return cls_embedding
|
||||
|
||||
def get_semantic_embedding(self, data_loader):
|
||||
self.eval()
|
||||
with paddle.no_grad():
|
||||
for batch_data in data_loader:
|
||||
input_ids, token_type_ids = batch_data
|
||||
input_ids = paddle.to_tensor(input_ids)
|
||||
token_type_ids = paddle.to_tensor(token_type_ids)
|
||||
|
||||
text_embeddings = self.get_pooled_embedding(
|
||||
input_ids, token_type_ids=token_type_ids)
|
||||
|
||||
yield text_embeddings
|
||||
|
||||
def cosine_sim(self,
|
||||
query_input_ids,
|
||||
title_input_ids,
|
||||
query_token_type_ids=None,
|
||||
query_position_ids=None,
|
||||
query_attention_mask=None,
|
||||
title_token_type_ids=None,
|
||||
title_position_ids=None,
|
||||
title_attention_mask=None):
|
||||
|
||||
query_cls_embedding = self.get_pooled_embedding(
|
||||
query_input_ids, query_token_type_ids, query_position_ids,
|
||||
query_attention_mask)
|
||||
|
||||
title_cls_embedding = self.get_pooled_embedding(
|
||||
title_input_ids, title_token_type_ids, title_position_ids,
|
||||
title_attention_mask)
|
||||
|
||||
cosine_sim = paddle.sum(query_cls_embedding * title_cls_embedding,
|
||||
axis=-1)
|
||||
return cosine_sim
|
||||
|
||||
def forward(self,
|
||||
input_ids,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
attention_mask=None):
|
||||
_, cls_embedding = self.ptm(input_ids, token_type_ids, position_ids,
|
||||
attention_mask)
|
||||
if self.output_emb_size > 0:
|
||||
cls_embedding = self.emb_reduce_linear(cls_embedding)
|
||||
cls_embedding = self.dropout(cls_embedding)
|
||||
cls_embedding = F.normalize(cls_embedding, p=2, axis=-1)
|
||||
|
||||
return cls_embedding
|
||||
31
applications/neural_search/recall/milvus/config.py
Normal file
31
applications/neural_search/recall/milvus/config.py
Normal file
@@ -0,0 +1,31 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from milvus import MetricType, IndexType
|
||||
|
||||
MILVUS_HOST = '10.21.226.173'
|
||||
MILVUS_PORT = 8530
|
||||
|
||||
collection_param = {
|
||||
'dimension': 256,
|
||||
'index_file_size': 256,
|
||||
'metric_type': MetricType.L2
|
||||
}
|
||||
|
||||
index_type = IndexType.IVF_FLAT
|
||||
index_param = {'nlist': 1000}
|
||||
|
||||
top_k = 100
|
||||
search_param = {'nprobe': 20}
|
||||
184
applications/neural_search/recall/milvus/data.py
Normal file
184
applications/neural_search/recall/milvus/data.py
Normal file
@@ -0,0 +1,184 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
import paddle
|
||||
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
|
||||
def create_dataloader(dataset,
|
||||
mode='train',
|
||||
batch_size=1,
|
||||
batchify_fn=None,
|
||||
trans_fn=None):
|
||||
if trans_fn:
|
||||
dataset = dataset.map(trans_fn)
|
||||
shuffle = True if mode == 'train' else False
|
||||
if mode == 'train':
|
||||
batch_sampler = paddle.io.DistributedBatchSampler(
|
||||
dataset, batch_size=batch_size, shuffle=shuffle)
|
||||
else:
|
||||
batch_sampler = paddle.io.BatchSampler(
|
||||
dataset, batch_size=batch_size, shuffle=shuffle)
|
||||
|
||||
return paddle.io.DataLoader(
|
||||
dataset=dataset,
|
||||
batch_sampler=batch_sampler,
|
||||
collate_fn=batchify_fn,
|
||||
return_list=True)
|
||||
|
||||
|
||||
def convert_example(example,
|
||||
tokenizer,
|
||||
max_seq_length=512,
|
||||
pad_to_max_seq_len=False):
|
||||
"""
|
||||
Builds model inputs from a sequence.
|
||||
|
||||
A BERT sequence has the following format:
|
||||
|
||||
- single sequence: ``[CLS] X [SEP]``
|
||||
|
||||
Args:
|
||||
example(obj:`list(str)`): The list of text to be converted to ids.
|
||||
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
|
||||
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
|
||||
max_seq_len(obj:`int`): The maximum total input sequence length after tokenization.
|
||||
Sequences longer than this will be truncated, sequences shorter will be padded.
|
||||
is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.
|
||||
|
||||
Returns:
|
||||
input_ids(obj:`list[int]`): The list of query token ids.
|
||||
token_type_ids(obj: `list[int]`): List of query sequence pair mask.
|
||||
"""
|
||||
|
||||
result = []
|
||||
for key, text in example.items():
|
||||
encoded_inputs = tokenizer(
|
||||
text=text,
|
||||
max_seq_len=max_seq_length,
|
||||
pad_to_max_seq_len=pad_to_max_seq_len)
|
||||
input_ids = encoded_inputs["input_ids"]
|
||||
token_type_ids = encoded_inputs["token_type_ids"]
|
||||
result += [input_ids, token_type_ids]
|
||||
return result
|
||||
|
||||
|
||||
def read_text_pair(data_path):
|
||||
"""Reads data."""
|
||||
with open(data_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
data = line.rstrip().split("\t")
|
||||
if len(data) != 2:
|
||||
continue
|
||||
yield {'text_a': data[0], 'text_b': data[1]}
|
||||
|
||||
|
||||
def read_text_triplet(data_path):
|
||||
"""Reads data."""
|
||||
with open(data_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
data = line.rstrip().split("\t")
|
||||
if len(data) != 3:
|
||||
continue
|
||||
yield {
|
||||
'text': data[0],
|
||||
'pos_sample': data[1],
|
||||
'neg_sample': data[2]
|
||||
}
|
||||
|
||||
|
||||
# ANN - active learning ------------------------------------------------------
|
||||
def get_latest_checkpoint(args):
|
||||
"""
|
||||
Return: (latest_checkpint_path, global_step)
|
||||
"""
|
||||
if not os.path.exists(args.save_dir):
|
||||
return args.init_from_ckpt, 0
|
||||
|
||||
subdirectories = list(next(os.walk(args.save_dir))[1])
|
||||
|
||||
def valid_checkpoint(checkpoint):
|
||||
chk_path = os.path.join(args.save_dir, checkpoint)
|
||||
scheduler_path = os.path.join(chk_path, "model_state.pdparams")
|
||||
succeed_flag_file = os.path.join(chk_path, "succeed_flag_file")
|
||||
return os.path.exists(scheduler_path) and os.path.exists(
|
||||
succeed_flag_file)
|
||||
|
||||
trained_steps = [int(s) for s in subdirectories if valid_checkpoint(s)]
|
||||
|
||||
if len(trained_steps) > 0:
|
||||
return os.path.join(args.save_dir,
|
||||
str(max(trained_steps)),
|
||||
"model_state.pdparams"), max(trained_steps)
|
||||
|
||||
return args.init_from_ckpt, 0
|
||||
|
||||
|
||||
# ANN - active learning ------------------------------------------------------
|
||||
def get_latest_ann_data(ann_data_dir):
|
||||
if not os.path.exists(ann_data_dir):
|
||||
return None, -1
|
||||
|
||||
subdirectories = list(next(os.walk(ann_data_dir))[1])
|
||||
|
||||
def valid_checkpoint(step):
|
||||
ann_data_file = os.path.join(ann_data_dir, step, "new_ann_data")
|
||||
# succed_flag_file is an empty file that indicates ann data has been generated
|
||||
succeed_flag_file = os.path.join(ann_data_dir, step,
|
||||
"succeed_flag_file")
|
||||
return os.path.exists(succeed_flag_file) and os.path.exists(
|
||||
ann_data_file)
|
||||
|
||||
ann_data_steps = [int(s) for s in subdirectories if valid_checkpoint(s)]
|
||||
|
||||
if len(ann_data_steps) > 0:
|
||||
latest_ann_data_file = os.path.join(ann_data_dir,
|
||||
str(max(ann_data_steps)),
|
||||
"new_ann_data")
|
||||
logger.info("Using lateset ann_data_file:{}".format(
|
||||
latest_ann_data_file))
|
||||
return latest_ann_data_file, max(ann_data_steps)
|
||||
|
||||
logger.info("no new ann_data, return (None, -1)")
|
||||
return None, -1
|
||||
|
||||
|
||||
def gen_id2corpus(corpus_file):
|
||||
id2corpus = {}
|
||||
with open(corpus_file, 'r', encoding='utf-8') as f:
|
||||
for idx, line in enumerate(f):
|
||||
id2corpus[idx] = line.rstrip()
|
||||
return id2corpus
|
||||
|
||||
|
||||
def gen_text_file(similar_text_pair_file):
|
||||
text2similar_text = {}
|
||||
texts = []
|
||||
with open(similar_text_pair_file, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
splited_line = line.rstrip().split("\t")
|
||||
if len(splited_line) != 2:
|
||||
continue
|
||||
|
||||
text, similar_text = line.rstrip().split("\t")
|
||||
|
||||
if not text or not similar_text:
|
||||
continue
|
||||
|
||||
text2similar_text[text] = similar_text
|
||||
texts.append({"text": text})
|
||||
return texts, text2similar_text
|
||||
41
applications/neural_search/recall/milvus/embedding_insert.py
Normal file
41
applications/neural_search/recall/milvus/embedding_insert.py
Normal file
@@ -0,0 +1,41 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import numpy as np
|
||||
from milvus_insert import VecToMilvus
|
||||
import random
|
||||
from tqdm import tqdm
|
||||
|
||||
embeddings = np.load('corpus_embedding.npy')
|
||||
print(embeddings.shape)
|
||||
|
||||
embedding_ids = [i for i in range(embeddings.shape[0])]
|
||||
print(len(embedding_ids))
|
||||
client = VecToMilvus()
|
||||
collection_name = 'literature_search'
|
||||
partition_tag = 'partition_2'
|
||||
data_size = len(embedding_ids)
|
||||
batch_size = 100000
|
||||
for i in tqdm(range(0, data_size, batch_size)):
|
||||
cur_end = i + batch_size
|
||||
if (cur_end > data_size):
|
||||
cur_end = data_size
|
||||
batch_emb = embeddings[np.arange(i, cur_end)]
|
||||
status, ids = client.insert(
|
||||
collection_name=collection_name,
|
||||
vectors=batch_emb.tolist(),
|
||||
ids=embedding_ids[i:i + batch_size],
|
||||
partition_tag=partition_tag)
|
||||
# print(status)
|
||||
# print(ids)
|
||||
43
applications/neural_search/recall/milvus/embedding_recall.py
Normal file
43
applications/neural_search/recall/milvus/embedding_recall.py
Normal file
@@ -0,0 +1,43 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import numpy as np
|
||||
from milvus_insert import VecToMilvus
|
||||
import random
|
||||
from tqdm import tqdm
|
||||
from milvus_recall import RecallByMilvus
|
||||
import time
|
||||
|
||||
embeddings = np.load('corpus_embedding.npy')
|
||||
print(embeddings.shape)
|
||||
|
||||
embedding_ids = [i for i in range(embeddings.shape[0])]
|
||||
print(len(embedding_ids))
|
||||
client = VecToMilvus()
|
||||
collection_name = 'literature_search'
|
||||
partition_tag = 'partition_2'
|
||||
data_size = len(embedding_ids)
|
||||
client = RecallByMilvus()
|
||||
embeddings = embeddings[np.arange(1, 2)]
|
||||
time_start = time.time() #开始计时
|
||||
status, resultes = client.search(
|
||||
collection_name=collection_name,
|
||||
vectors=embeddings,
|
||||
partition_tag=partition_tag)
|
||||
time_end = time.time() #结束计时
|
||||
|
||||
sum_t = time_end - time_start #运行所花时间
|
||||
print('time cost', sum_t, 's')
|
||||
print(status)
|
||||
print(resultes)
|
||||
187
applications/neural_search/recall/milvus/feature_extract.py
Normal file
187
applications/neural_search/recall/milvus/feature_extract.py
Normal file
@@ -0,0 +1,187 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from tqdm import tqdm
|
||||
import numpy as np
|
||||
from scipy.special import softmax
|
||||
|
||||
import paddle
|
||||
from paddle import inference
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.datasets import load_dataset
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
sys.path.append('.')
|
||||
|
||||
from base_model import SemanticIndexBase, SemanticIndexBaseStatic
|
||||
|
||||
from data import convert_example
|
||||
|
||||
# yapf: disable
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model_dir", type=str, required=True,
|
||||
help="The directory to static model.")
|
||||
|
||||
parser.add_argument("--corpus_file", type=str, required=True,
|
||||
help="The corpus_file path.")
|
||||
|
||||
parser.add_argument("--max_seq_length", default=64, type=int,
|
||||
help="The maximum total input sequence length after tokenization. Sequences "
|
||||
"longer than this will be truncated, sequences shorter will be padded.")
|
||||
parser.add_argument("--batch_size", default=32, type=int,
|
||||
help="Batch size per GPU/CPU for training.")
|
||||
parser.add_argument('--device', choices=['cpu', 'gpu', 'xpu'], default="gpu",
|
||||
help="Select which device to train model, defaults to gpu.")
|
||||
|
||||
parser.add_argument('--use_tensorrt', default=False, type=eval, choices=[True, False],
|
||||
help='Enable to use tensorrt to speed up.')
|
||||
parser.add_argument("--precision", default="fp32", type=str, choices=["fp32", "fp16", "int8"],
|
||||
help='The tensorrt precision.')
|
||||
|
||||
parser.add_argument('--cpu_threads', default=10, type=int,
|
||||
help='Number of threads to predict when using cpu.')
|
||||
parser.add_argument('--enable_mkldnn', default=False, type=eval, choices=[True, False],
|
||||
help='Enable to use mkldnn to speed up when using cpu.')
|
||||
|
||||
args = parser.parse_args()
|
||||
# yapf: enable
|
||||
|
||||
|
||||
class Predictor(object):
|
||||
def __init__(self,
|
||||
model_dir,
|
||||
device="gpu",
|
||||
max_seq_length=128,
|
||||
batch_size=32,
|
||||
use_tensorrt=False,
|
||||
precision="fp32",
|
||||
cpu_threads=10,
|
||||
enable_mkldnn=False):
|
||||
self.max_seq_length = max_seq_length
|
||||
self.batch_size = batch_size
|
||||
|
||||
model_file = model_dir + "/inference.pdmodel"
|
||||
params_file = model_dir + "/inference.pdiparams"
|
||||
if not os.path.exists(model_file):
|
||||
raise ValueError("not find model file path {}".format(model_file))
|
||||
if not os.path.exists(params_file):
|
||||
raise ValueError("not find params file path {}".format(params_file))
|
||||
config = paddle.inference.Config(model_file, params_file)
|
||||
|
||||
if device == "gpu":
|
||||
# set GPU configs accordingly
|
||||
# such as intialize the gpu memory, enable tensorrt
|
||||
config.enable_use_gpu(100, 0)
|
||||
precision_map = {
|
||||
"fp16": inference.PrecisionType.Half,
|
||||
"fp32": inference.PrecisionType.Float32,
|
||||
"int8": inference.PrecisionType.Int8
|
||||
}
|
||||
precision_mode = precision_map[precision]
|
||||
|
||||
if args.use_tensorrt:
|
||||
config.enable_tensorrt_engine(
|
||||
max_batch_size=batch_size,
|
||||
min_subgraph_size=30,
|
||||
precision_mode=precision_mode)
|
||||
elif device == "cpu":
|
||||
# set CPU configs accordingly,
|
||||
# such as enable_mkldnn, set_cpu_math_library_num_threads
|
||||
config.disable_gpu()
|
||||
if args.enable_mkldnn:
|
||||
# cache 10 different shapes for mkldnn to avoid memory leak
|
||||
config.set_mkldnn_cache_capacity(10)
|
||||
config.enable_mkldnn()
|
||||
config.set_cpu_math_library_num_threads(args.cpu_threads)
|
||||
elif device == "xpu":
|
||||
# set XPU configs accordingly
|
||||
config.enable_xpu(100)
|
||||
|
||||
config.switch_use_feed_fetch_ops(False)
|
||||
self.predictor = paddle.inference.create_predictor(config)
|
||||
self.input_handles = [
|
||||
self.predictor.get_input_handle(name)
|
||||
for name in self.predictor.get_input_names()
|
||||
]
|
||||
self.output_handle = self.predictor.get_output_handle(
|
||||
self.predictor.get_output_names()[0])
|
||||
|
||||
def predict(self, data, tokenizer):
|
||||
"""
|
||||
Predicts the data labels.
|
||||
|
||||
Args:
|
||||
data (obj:`List(str)`): The batch data whose each element is a raw text.
|
||||
tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
|
||||
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
|
||||
|
||||
Returns:
|
||||
results(obj:`dict`): All the predictions labels.
|
||||
"""
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # segment
|
||||
): fn(samples)
|
||||
|
||||
all_embeddings = []
|
||||
examples = []
|
||||
for idx, text in enumerate(tqdm(data)):
|
||||
input_ids, segment_ids = convert_example(
|
||||
text,
|
||||
tokenizer,
|
||||
max_seq_length=self.max_seq_length,
|
||||
pad_to_max_seq_len=True)
|
||||
examples.append((input_ids, segment_ids))
|
||||
if (len(examples) > 100):
|
||||
input_ids, segment_ids = batchify_fn(examples)
|
||||
self.input_handles[0].copy_from_cpu(input_ids)
|
||||
self.input_handles[1].copy_from_cpu(segment_ids)
|
||||
self.predictor.run()
|
||||
logits = self.output_handle.copy_to_cpu()
|
||||
all_embeddings.append(logits)
|
||||
examples = []
|
||||
if (len(examples) > 0):
|
||||
input_ids, segment_ids = batchify_fn(examples)
|
||||
self.input_handles[0].copy_from_cpu(input_ids)
|
||||
self.input_handles[1].copy_from_cpu(segment_ids)
|
||||
self.predictor.run()
|
||||
logits = self.output_handle.copy_to_cpu()
|
||||
all_embeddings.append(logits)
|
||||
all_embeddings = np.concatenate(all_embeddings, axis=0)
|
||||
np.save('corpus_embedding', all_embeddings)
|
||||
|
||||
|
||||
def read_text(file_path):
|
||||
file = open(file_path)
|
||||
id2corpus = {}
|
||||
for idx, data in enumerate(file.readlines()):
|
||||
id2corpus[idx] = data.strip()
|
||||
return id2corpus
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
predictor = Predictor(args.model_dir, args.device, args.max_seq_length,
|
||||
args.batch_size, args.use_tensorrt, args.precision,
|
||||
args.cpu_threads, args.enable_mkldnn)
|
||||
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
id2corpus = read_text(args.corpus_file)
|
||||
|
||||
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
|
||||
predictor.predict(corpus_list, tokenizer)
|
||||
105
applications/neural_search/recall/milvus/inference.py
Normal file
105
applications/neural_search/recall/milvus/inference.py
Normal file
@@ -0,0 +1,105 @@
|
||||
from functools import partial
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import paddle
|
||||
import paddle.nn.functional as F
|
||||
import paddlenlp as ppnlp
|
||||
from paddlenlp.data import Stack, Tuple, Pad
|
||||
from paddlenlp.datasets import load_dataset, MapDataset, load_dataset
|
||||
from paddlenlp.utils.log import logger
|
||||
|
||||
from base_model import SemanticIndexBaseStatic
|
||||
from data import convert_example, create_dataloader
|
||||
from data import gen_id2corpus, gen_text_file
|
||||
from tqdm import tqdm
|
||||
from milvus_recall import RecallByMilvus
|
||||
|
||||
|
||||
def search_in_milvus(text_embedding):
|
||||
collection_name = 'literature_search'
|
||||
partition_tag = 'partition_2'
|
||||
client = RecallByMilvus()
|
||||
status, results = client.search(
|
||||
collection_name=collection_name,
|
||||
vectors=text_embedding.tolist(),
|
||||
partition_tag=partition_tag)
|
||||
# print(status)
|
||||
# print(resultes)
|
||||
corpus_file = "milvus/milvus_data.csv"
|
||||
id2corpus = gen_id2corpus(corpus_file)
|
||||
# print(status)
|
||||
# print(results)
|
||||
for line in results:
|
||||
for item in line:
|
||||
idx = item.id
|
||||
distance = item.distance
|
||||
text = id2corpus[idx]
|
||||
print(idx, text, distance)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
device = 'gpu'
|
||||
max_seq_length = 64
|
||||
output_emb_size = 256
|
||||
batch_size = 1
|
||||
params_path = 'checkpoints/model_40/model_state.pdparams'
|
||||
id2corpus = {0: '国有企业引入非国有资本对创新绩效的影响——基于制造业国有上市公司的经验证据'}
|
||||
paddle.set_device(device)
|
||||
|
||||
tokenizer = ppnlp.transformers.ErnieTokenizer.from_pretrained('ernie-1.0')
|
||||
trans_func = partial(
|
||||
convert_example, tokenizer=tokenizer, max_seq_length=max_seq_length)
|
||||
|
||||
batchify_fn = lambda samples, fn=Tuple(
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_id), # text_input
|
||||
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # text_segment
|
||||
): [data for data in fn(samples)]
|
||||
|
||||
pretrained_model = ppnlp.transformers.ErnieModel.from_pretrained(
|
||||
"ernie-1.0")
|
||||
|
||||
model = SemanticIndexBaseStatic(
|
||||
pretrained_model, output_emb_size=output_emb_size)
|
||||
|
||||
# Load pretrained semantic model
|
||||
if params_path and os.path.isfile(params_path):
|
||||
state_dict = paddle.load(params_path)
|
||||
model.set_dict(state_dict)
|
||||
print("Loaded parameters from %s" % params_path)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Please set --params_path with correct pretrained model file")
|
||||
|
||||
# conver_example function's input must be dict
|
||||
corpus_list = [{idx: text} for idx, text in id2corpus.items()]
|
||||
corpus_ds = MapDataset(corpus_list)
|
||||
|
||||
corpus_data_loader = create_dataloader(
|
||||
corpus_ds,
|
||||
mode='predict',
|
||||
batch_size=batch_size,
|
||||
batchify_fn=batchify_fn,
|
||||
trans_fn=trans_func)
|
||||
|
||||
# Need better way to get inner model of DataParallel
|
||||
|
||||
all_embeddings = []
|
||||
|
||||
with paddle.no_grad():
|
||||
for batch_data in corpus_data_loader:
|
||||
input_ids, token_type_ids = batch_data
|
||||
input_ids = paddle.to_tensor(input_ids)
|
||||
token_type_ids = paddle.to_tensor(token_type_ids)
|
||||
|
||||
text_embeddings = model(input_ids, token_type_ids)
|
||||
all_embeddings.append(text_embeddings)
|
||||
|
||||
text_embedding = all_embeddings[0]
|
||||
print(text_embedding.shape)
|
||||
print(text_embedding)
|
||||
search_in_milvus(text_embedding)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user