From 88ca820e2b03fd811a0f34a998054252688d45f4 Mon Sep 17 00:00:00 2001 From: Zhong Hui Date: Mon, 31 Mar 2025 12:06:23 +0800 Subject: [PATCH] [Deprecated] Delete few used model and tools. (#10279) * delete few used model and tools. * fix --- docs/zh/index.rst | 2 - docs/zh/locale/en/LC_MESSAGES/index.po | 4 - .../en/LC_MESSAGES/model_zoo/embeddings.po | 219 -- .../source/paddlenlp.embeddings.po | 23 - .../paddlenlp.embeddings.token_embedding.po | 200 -- docs/zh/model_zoo/embeddings.md | 307 --- docs/zh/source/paddlenlp.embeddings.rst | 14 - .../paddlenlp.embeddings.token_embedding.rst | 7 - docs/zh/source/paddlenlp.rst | 1 - paddlenlp/__init__.py | 1 - paddlenlp/embeddings/__init__.py | 15 - paddlenlp/embeddings/constant.py | 98 - paddlenlp/embeddings/token_embedding.py | 378 ---- paddlenlp/transformers/__init__.py | 9 +- paddlenlp/transformers/auto/configuration.py | 2 - paddlenlp/transformers/auto/modeling.py | 1 - paddlenlp/transformers/auto/tokenizer.py | 1 - paddlenlp/transformers/gau_alpha/__init__.py | 13 - .../transformers/gau_alpha/configuration.py | 161 -- paddlenlp/transformers/gau_alpha/modeling.py | 810 -------- paddlenlp/transformers/gau_alpha/tokenizer.py | 292 --- pyproject.toml | 1 - .../unsupervised_qa/README.md | 2 +- slm/examples/README.md | 12 +- slm/examples/RLHF/README.md | 241 --- slm/examples/RLHF/comm_utils.py | 403 ---- slm/examples/RLHF/data/__init__.py | 33 - slm/examples/RLHF/data/alpaca.py | 42 - slm/examples/RLHF/data/base.py | 520 ----- slm/examples/RLHF/data/preference.py | 113 -- slm/examples/RLHF/data/prompt_only.py | 78 - slm/examples/RLHF/data/safe_rlhf.py | 88 - slm/examples/RLHF/data/supervised.py | 128 -- slm/examples/RLHF/infer_utils.py | 379 ---- slm/examples/RLHF/models/__init__.py | 19 - slm/examples/RLHF/models/infer_model_utils.py | 185 -- slm/examples/RLHF/models/model_pp.py | 261 --- slm/examples/RLHF/models/pp_model_utils.py | 92 - slm/examples/RLHF/models/ppo_model.py | 117 -- slm/examples/RLHF/models/ppo_model_utils.py | 274 --- slm/examples/RLHF/models/score_model.py | 140 -- slm/examples/RLHF/models/score_model_utils.py | 417 ---- slm/examples/RLHF/ppo_config.json | 57 - slm/examples/RLHF/ppo_main.py | 511 ----- slm/examples/RLHF/ppo_trainer.py | 1803 ----------------- slm/examples/RLHF/reward_config.json | 40 - slm/examples/RLHF/reward_main.py | 202 -- slm/examples/RLHF/reward_trainer.py | 219 -- slm/examples/RLHF/run.sh | 15 - slm/examples/RLHF/tests/run_model.py | 125 -- slm/examples/RLHF/tests/test_export.py | 68 - slm/examples/RLHF/trainer_utils.py | 662 ------ slm/examples/dialogue/README_DGU.md | 145 -- slm/examples/dialogue/README_LIC2021.md | 148 -- slm/examples/dialogue/README_PLATO-2.md | 69 - slm/examples/dialogue/README_PLATO-XL.md | 150 -- .../dialogue/README_UnifiedTransformer.md | 219 -- slm/examples/dialogue/dgu/README.md | 143 -- slm/examples/dialogue/dgu/args.py | 118 -- slm/examples/dialogue/dgu/data.py | 509 ----- slm/examples/dialogue/dgu/main.py | 293 --- slm/examples/dialogue/dgu/metric.py | 244 --- .../dialogue/lic2021_baseline/README.md | 146 -- .../dialogue/lic2021_baseline/args.py | 58 - .../dialogue/lic2021_baseline/data.py | 258 --- .../dialogue/lic2021_baseline/finetune.py | 149 -- .../dialogue/lic2021_baseline/infer.py | 88 - slm/examples/dialogue/plato-2/README.md | 67 - slm/examples/dialogue/plato-2/imgs/case.jpg | Bin 101382 -> 0 bytes .../dialogue/plato-2/imgs/network.png | Bin 113024 -> 0 bytes slm/examples/dialogue/plato-2/interaction.py | 104 - slm/examples/dialogue/plato-2/model.py | 458 ----- .../dialogue/plato-2/readers/dialog_reader.py | 436 ---- .../dialogue/plato-2/readers/nsp_reader.py | 152 -- .../dialogue/plato-2/readers/plato_reader.py | 84 - .../dialogue/plato-2/utils/__init__.py | 51 - slm/examples/dialogue/plato-2/utils/args.py | 88 - .../dialogue/plato-2/utils/masking.py | 119 -- .../dialogue/plato-2/utils/tokenization.py | 189 -- slm/examples/dialogue/plato-xl | 1 - .../dialogue/unified_transformer/README.md | 217 -- .../dialogue/unified_transformer/finetune.py | 165 -- .../dialogue/unified_transformer/infer.py | 146 -- .../unified_transformer/interaction.py | 107 - .../dialogue/unified_transformer/utils.py | 265 --- .../waybill_ie/model.py | 3 +- .../model_compression/distill_lstm/README.md | 2 +- .../model_compression/distill_lstm/small.py | 6 +- slm/examples/question_generation/README.md | 20 - slm/examples/question_generation/t5/README.md | 208 -- .../question_generation/t5/predict.py | 160 -- .../question_generation/t5/requirements.txt | 2 - slm/examples/question_generation/t5/train.py | 236 --- slm/examples/question_generation/t5/utils.py | 166 -- .../question_generation/unimo-text/README.md | 343 ---- .../deploy/paddle_inference/README.md | 54 - .../deploy/paddle_inference/infer_utils.py | 260 --- .../deploy/paddle_inference/inference.py | 223 -- .../deploy/paddle_serving/README.md | 142 -- .../deploy/paddle_serving/config.yml | 59 - .../deploy/paddle_serving/infer_utils.py | 260 --- .../deploy/paddle_serving/pipeline_client.py | 50 - .../deploy/paddle_serving/pipeline_service.py | 74 - .../unimo-text/export_model.py | 99 - .../unimo-text/gen_utils.py | 316 --- .../question_generation/unimo-text/predict.py | 141 -- .../unimo-text/requirements.txt | 3 - .../question_generation/unimo-text/train.py | 281 --- slm/examples/text_classification/README.md | 7 - .../pretrained_models/README.md | 1 - .../text_correction/ernie-csc/README.md | 163 -- .../ernie-csc/change_sgml_to_txt.py | 47 - .../text_correction/ernie-csc/download.py | 33 - .../text_correction/ernie-csc/export_model.py | 57 - .../text_correction/ernie-csc/model.py | 123 -- .../ernie-csc/pinyin_vocab.txt | 417 ---- .../text_correction/ernie-csc/predict.py | 148 -- .../ernie-csc/predict_sighan.py | 122 -- .../ernie-csc/requirements.txt | 1 - .../ernie-csc/run_sighan_predict.sh | 40 - .../ernie-csc/sighan_evaluate.py | 116 -- .../text_correction/ernie-csc/train.py | 207 -- .../text_correction/ernie-csc/utils.py | 116 -- .../text_generation/couplet/README.md | 22 - slm/examples/text_generation/ctrl/README.md | 52 - slm/examples/text_generation/ernie-gen | 1 - slm/examples/text_generation/opt/README.md | 62 - slm/examples/text_generation/opt/demo.py | 67 - .../text_generation/reformer/README.md | 13 - .../text_generation/unimo-text/README.md | 144 -- .../text_generation/unimo-text/gen_utils.py | 201 -- .../text_generation/unimo-text/run_gen.py | 242 --- .../unimo-text/scripts/lcsts_train.sh | 39 - .../unimo-text/scripts/qg_train.sh | 39 - .../unimo-text/scripts/table_train.sh | 39 - .../text_generation/vae-seq2seq/README.md | 30 - tests/embeddings/__init__.py | 0 tests/embeddings/test_token_embedding.py | 119 -- tests/transformers/gau_alpha/__init__.py | 13 - tests/transformers/gau_alpha/test_modeling.py | 296 --- .../transformers/gau_alpha/test_tokenizer.py | 241 --- 141 files changed, 13 insertions(+), 21504 deletions(-) delete mode 100644 docs/zh/locale/en/LC_MESSAGES/model_zoo/embeddings.po delete mode 100644 docs/zh/locale/en/LC_MESSAGES/source/paddlenlp.embeddings.po delete mode 100644 docs/zh/locale/en/LC_MESSAGES/source/paddlenlp.embeddings.token_embedding.po delete mode 100644 docs/zh/model_zoo/embeddings.md delete mode 100644 docs/zh/source/paddlenlp.embeddings.rst delete mode 100644 docs/zh/source/paddlenlp.embeddings.token_embedding.rst delete mode 100644 paddlenlp/embeddings/__init__.py delete mode 100644 paddlenlp/embeddings/constant.py delete mode 100644 paddlenlp/embeddings/token_embedding.py delete mode 100644 paddlenlp/transformers/gau_alpha/__init__.py delete mode 100644 paddlenlp/transformers/gau_alpha/configuration.py delete mode 100644 paddlenlp/transformers/gau_alpha/modeling.py delete mode 100644 paddlenlp/transformers/gau_alpha/tokenizer.py delete mode 100644 slm/examples/RLHF/README.md delete mode 100644 slm/examples/RLHF/comm_utils.py delete mode 100644 slm/examples/RLHF/data/__init__.py delete mode 100644 slm/examples/RLHF/data/alpaca.py delete mode 100644 slm/examples/RLHF/data/base.py delete mode 100644 slm/examples/RLHF/data/preference.py delete mode 100644 slm/examples/RLHF/data/prompt_only.py delete mode 100644 slm/examples/RLHF/data/safe_rlhf.py delete mode 100644 slm/examples/RLHF/data/supervised.py delete mode 100644 slm/examples/RLHF/infer_utils.py delete mode 100644 slm/examples/RLHF/models/__init__.py delete mode 100644 slm/examples/RLHF/models/infer_model_utils.py delete mode 100644 slm/examples/RLHF/models/model_pp.py delete mode 100644 slm/examples/RLHF/models/pp_model_utils.py delete mode 100644 slm/examples/RLHF/models/ppo_model.py delete mode 100644 slm/examples/RLHF/models/ppo_model_utils.py delete mode 100644 slm/examples/RLHF/models/score_model.py delete mode 100644 slm/examples/RLHF/models/score_model_utils.py delete mode 100644 slm/examples/RLHF/ppo_config.json delete mode 100644 slm/examples/RLHF/ppo_main.py delete mode 100644 slm/examples/RLHF/ppo_trainer.py delete mode 100644 slm/examples/RLHF/reward_config.json delete mode 100644 slm/examples/RLHF/reward_main.py delete mode 100644 slm/examples/RLHF/reward_trainer.py delete mode 100644 slm/examples/RLHF/run.sh delete mode 100644 slm/examples/RLHF/tests/run_model.py delete mode 100644 slm/examples/RLHF/tests/test_export.py delete mode 100644 slm/examples/RLHF/trainer_utils.py delete mode 100644 slm/examples/dialogue/README_DGU.md delete mode 100644 slm/examples/dialogue/README_LIC2021.md delete mode 100644 slm/examples/dialogue/README_PLATO-2.md delete mode 100644 slm/examples/dialogue/README_PLATO-XL.md delete mode 100644 slm/examples/dialogue/README_UnifiedTransformer.md delete mode 100644 slm/examples/dialogue/dgu/README.md delete mode 100644 slm/examples/dialogue/dgu/args.py delete mode 100644 slm/examples/dialogue/dgu/data.py delete mode 100644 slm/examples/dialogue/dgu/main.py delete mode 100644 slm/examples/dialogue/dgu/metric.py delete mode 100644 slm/examples/dialogue/lic2021_baseline/README.md delete mode 100644 slm/examples/dialogue/lic2021_baseline/args.py delete mode 100644 slm/examples/dialogue/lic2021_baseline/data.py delete mode 100644 slm/examples/dialogue/lic2021_baseline/finetune.py delete mode 100644 slm/examples/dialogue/lic2021_baseline/infer.py delete mode 100644 slm/examples/dialogue/plato-2/README.md delete mode 100644 slm/examples/dialogue/plato-2/imgs/case.jpg delete mode 100644 slm/examples/dialogue/plato-2/imgs/network.png delete mode 100644 slm/examples/dialogue/plato-2/interaction.py delete mode 100644 slm/examples/dialogue/plato-2/model.py delete mode 100644 slm/examples/dialogue/plato-2/readers/dialog_reader.py delete mode 100644 slm/examples/dialogue/plato-2/readers/nsp_reader.py delete mode 100644 slm/examples/dialogue/plato-2/readers/plato_reader.py delete mode 100644 slm/examples/dialogue/plato-2/utils/__init__.py delete mode 100644 slm/examples/dialogue/plato-2/utils/args.py delete mode 100644 slm/examples/dialogue/plato-2/utils/masking.py delete mode 100644 slm/examples/dialogue/plato-2/utils/tokenization.py delete mode 120000 slm/examples/dialogue/plato-xl delete mode 100644 slm/examples/dialogue/unified_transformer/README.md delete mode 100644 slm/examples/dialogue/unified_transformer/finetune.py delete mode 100644 slm/examples/dialogue/unified_transformer/infer.py delete mode 100644 slm/examples/dialogue/unified_transformer/interaction.py delete mode 100644 slm/examples/dialogue/unified_transformer/utils.py delete mode 100644 slm/examples/question_generation/README.md delete mode 100644 slm/examples/question_generation/t5/README.md delete mode 100644 slm/examples/question_generation/t5/predict.py delete mode 100644 slm/examples/question_generation/t5/requirements.txt delete mode 100644 slm/examples/question_generation/t5/train.py delete mode 100644 slm/examples/question_generation/t5/utils.py delete mode 100644 slm/examples/question_generation/unimo-text/README.md delete mode 100644 slm/examples/question_generation/unimo-text/deploy/paddle_inference/README.md delete mode 100644 slm/examples/question_generation/unimo-text/deploy/paddle_inference/infer_utils.py delete mode 100644 slm/examples/question_generation/unimo-text/deploy/paddle_inference/inference.py delete mode 100644 slm/examples/question_generation/unimo-text/deploy/paddle_serving/README.md delete mode 100644 slm/examples/question_generation/unimo-text/deploy/paddle_serving/config.yml delete mode 100644 slm/examples/question_generation/unimo-text/deploy/paddle_serving/infer_utils.py delete mode 100644 slm/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_client.py delete mode 100644 slm/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_service.py delete mode 100644 slm/examples/question_generation/unimo-text/export_model.py delete mode 100644 slm/examples/question_generation/unimo-text/gen_utils.py delete mode 100644 slm/examples/question_generation/unimo-text/predict.py delete mode 100644 slm/examples/question_generation/unimo-text/requirements.txt delete mode 100644 slm/examples/question_generation/unimo-text/train.py delete mode 100644 slm/examples/text_classification/README.md delete mode 100644 slm/examples/text_classification/pretrained_models/README.md delete mode 100644 slm/examples/text_correction/ernie-csc/README.md delete mode 100644 slm/examples/text_correction/ernie-csc/change_sgml_to_txt.py delete mode 100644 slm/examples/text_correction/ernie-csc/download.py delete mode 100644 slm/examples/text_correction/ernie-csc/export_model.py delete mode 100644 slm/examples/text_correction/ernie-csc/model.py delete mode 100644 slm/examples/text_correction/ernie-csc/pinyin_vocab.txt delete mode 100644 slm/examples/text_correction/ernie-csc/predict.py delete mode 100644 slm/examples/text_correction/ernie-csc/predict_sighan.py delete mode 100644 slm/examples/text_correction/ernie-csc/requirements.txt delete mode 100644 slm/examples/text_correction/ernie-csc/run_sighan_predict.sh delete mode 100644 slm/examples/text_correction/ernie-csc/sighan_evaluate.py delete mode 100644 slm/examples/text_correction/ernie-csc/train.py delete mode 100644 slm/examples/text_correction/ernie-csc/utils.py delete mode 100644 slm/examples/text_generation/couplet/README.md delete mode 100644 slm/examples/text_generation/ctrl/README.md delete mode 120000 slm/examples/text_generation/ernie-gen delete mode 100644 slm/examples/text_generation/opt/README.md delete mode 100644 slm/examples/text_generation/opt/demo.py delete mode 100644 slm/examples/text_generation/reformer/README.md delete mode 100644 slm/examples/text_generation/unimo-text/README.md delete mode 100644 slm/examples/text_generation/unimo-text/gen_utils.py delete mode 100644 slm/examples/text_generation/unimo-text/run_gen.py delete mode 100644 slm/examples/text_generation/unimo-text/scripts/lcsts_train.sh delete mode 100644 slm/examples/text_generation/unimo-text/scripts/qg_train.sh delete mode 100644 slm/examples/text_generation/unimo-text/scripts/table_train.sh delete mode 100644 slm/examples/text_generation/vae-seq2seq/README.md delete mode 100644 tests/embeddings/__init__.py delete mode 100644 tests/embeddings/test_token_embedding.py delete mode 100644 tests/transformers/gau_alpha/__init__.py delete mode 100644 tests/transformers/gau_alpha/test_modeling.py delete mode 100644 tests/transformers/gau_alpha/test_tokenizer.py diff --git a/docs/zh/index.rst b/docs/zh/index.rst index 417636a05b..66a8c4283f 100644 --- a/docs/zh/index.rst +++ b/docs/zh/index.rst @@ -89,7 +89,6 @@ :caption: PaddleNLP工具库 一键预测功能 - 预训练词向量 .. toctree:: :maxdepth: 1 @@ -159,7 +158,6 @@ paddlenlp.data paddlenlp.datasets - paddlenlp.embeddings paddlenlp.layers paddlenlp.losses paddlenlp.metrics diff --git a/docs/zh/locale/en/LC_MESSAGES/index.po b/docs/zh/locale/en/LC_MESSAGES/index.po index 1cf3bcec97..ae627e0816 100644 --- a/docs/zh/locale/en/LC_MESSAGES/index.po +++ b/docs/zh/locale/en/LC_MESSAGES/index.po @@ -133,10 +133,6 @@ msgstr "" msgid "paddlenlp.datasets" msgstr "" -#: ../index.rst:88 -msgid "paddlenlp.embeddings" -msgstr "" - #: ../index.rst:88 msgid "paddlenlp.layers" msgstr "" diff --git a/docs/zh/locale/en/LC_MESSAGES/model_zoo/embeddings.po b/docs/zh/locale/en/LC_MESSAGES/model_zoo/embeddings.po deleted file mode 100644 index ad25919694..0000000000 --- a/docs/zh/locale/en/LC_MESSAGES/model_zoo/embeddings.po +++ /dev/null @@ -1,219 +0,0 @@ -# SOME DESCRIPTIVE TITLE. -# Copyright (C) 2021, PaddleNLP -# This file is distributed under the same license as the PaddleNLP package. -# FIRST AUTHOR , 2022. -# -#, fuzzy -msgid "" -msgstr "" -"Project-Id-Version: PaddleNLP \n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-03-18 21:31+0800\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=utf-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.9.0\n" - -#: ../model_zoo/embeddings.md:1 -msgid "PaddleNLP Embedding API" -msgstr "" - -#: ../model_zoo/embeddings.md:3 ../model_zoo/embeddings.md:24 -msgid "介绍" -msgstr "" - -#: ../model_zoo/embeddings.md:4 ../model_zoo/embeddings.md:28 -msgid "用法" -msgstr "" - -#: ../model_zoo/embeddings.md:5 ../model_zoo/embeddings.md:30 -msgid "TokenEmbedding参数" -msgstr "" - -#: ../model_zoo/embeddings.md:6 ../model_zoo/embeddings.md:69 -msgid "初始化" -msgstr "" - -#: ../model_zoo/embeddings.md:7 ../model_zoo/embeddings.md:101 -msgid "查询embedding结果" -msgstr "" - -#: ../model_zoo/embeddings.md:8 ../model_zoo/embeddings.md:115 -msgid "可视化embedding结果" -msgstr "" - -#: ../model_zoo/embeddings.md:9 ../model_zoo/embeddings.md:140 -msgid "计算词向量cosine相似度" -msgstr "" - -#: ../model_zoo/embeddings.md:10 ../model_zoo/embeddings.md:147 -msgid "计算词向量内积" -msgstr "" - -#: ../model_zoo/embeddings.md:11 ../model_zoo/embeddings.md:155 -msgid "训练" -msgstr "" - -#: ../model_zoo/embeddings.md:12 ../model_zoo/embeddings.md:171 -msgid "切词" -msgstr "" - -#: ../model_zoo/embeddings.md:13 ../model_zoo/embeddings.md:183 -msgid "预训练模型" -msgstr "" - -#: ../model_zoo/embeddings.md:14 ../model_zoo/embeddings.md:189 -msgid "中文词向量" -msgstr "" - -#: ../model_zoo/embeddings.md:15 ../model_zoo/embeddings.md:343 -msgid "英文词向量" -msgstr "" - -#: ../model_zoo/embeddings.md:16 ../model_zoo/embeddings.md:345 -msgid "Word2Vec" -msgstr "" - -#: ../model_zoo/embeddings.md:17 ../model_zoo/embeddings.md:362 -msgid "GloVe" -msgstr "" - -#: ../model_zoo/embeddings.md:18 ../model_zoo/embeddings.md:395 -msgid "FastText" -msgstr "" - -#: ../model_zoo/embeddings.md:19 ../model_zoo/embeddings.md:416 -msgid "使用方式" -msgstr "" - -#: ../model_zoo/embeddings.md:20 ../model_zoo/embeddings.md:427 -msgid "模型信息" -msgstr "" - -#: ../model_zoo/embeddings.md:21 ../model_zoo/embeddings.md:751 -msgid "致谢" -msgstr "" - -#: ../model_zoo/embeddings.md:22 ../model_zoo/embeddings.md:756 -msgid "参考论文" -msgstr "" - -#: ../model_zoo/embeddings.md:26 -msgid "PaddleNLP提供多个开源的预训练词向量模型，用户仅需在使用paddlenlp.embeddings.TokenEmbedding时，指定预训练模型的名称，即可加载相对应的预训练模型。以下将介绍TokenEmbeddign详细用法，并列出PaddleNLP所支持的预训练Embedding模型。" -msgstr "" - -#: ../model_zoo/embeddings.md:116 -msgid "使用深度学习可视化工具VisualDL的High Dimensional组件可以对embedding结果进行可视化展示，便于对其直观分析，步骤如下：" -msgstr "" - -#: ../model_zoo/embeddings.md:128 -msgid "执行完毕后会在当前路径下生成一个visualize目录，并将日志存放在其中，我们在命令行启动VisualDL即可进行查看，启动命令为：" -msgstr "" - -#: ../model_zoo/embeddings.md:132 -msgid "启动后打开浏览器即可看到可视化结果" -msgstr "" - -#: ../model_zoo/embeddings.md:138 -msgid "使用VisualDL除可视化embedding结果外，还可以对标量、图片、音频等进行可视化，有效提升训练调参效率。关于VisualDL更多功能和详细介绍，可参考VisualDL使用文档。" -msgstr "" - -#: ../model_zoo/embeddings.md:157 -msgid "" -"以下为TokenEmbedding简单的组网使用方法。有关更多TokenEmbedding训练流程相关的使用方法，请参考Word " -"Embedding with PaddleNLP。" -msgstr "" - -#: ../model_zoo/embeddings.md:185 -msgid "以下将列举PaddleNLP支持的Embedding预训练模型。" -msgstr "" - -#: ../model_zoo/embeddings.md:186 -msgid "模型命名方式为：${训练模型}.${语料}.${词向量类型}.${co-occurrence type}.dim${维度}。" -msgstr "" - -#: ../model_zoo/embeddings.md:187 -msgid "模型有三种，分别是Word2Vec(w2v, skip-gram), GloVe(glove)和FastText(fasttext)。" -msgstr "" - -#: ../model_zoo/embeddings.md:191 -msgid "以下预训练词向量由Chinese-Word-Vectors提供。" -msgstr "" - -#: ../model_zoo/embeddings.md:193 -msgid "根据不同类型的上下文为每个语料训练多个目标词向量，第二列开始表示不同类型的上下文。以下为上下文类别：" -msgstr "" - -#: ../model_zoo/embeddings.md:195 -msgid "Word表示训练时目标词预测的上下文是一个Word。" -msgstr "" - -#: ../model_zoo/embeddings.md:196 -msgid "" -"Word + " -"N-gram表示训练时目标词预测的上下文是一个Word或者Ngram，其中bigram表示2-grams，ngram.1-2表示1-gram或者2-grams。" -msgstr "" - -#: ../model_zoo/embeddings.md:197 -msgid "" -"Word + Character表示训练时目标词预测的上下文是一个Word或者Character，其中word-" -"character.char1-2表示上下文是1个或2个Character。" -msgstr "" - -#: ../model_zoo/embeddings.md:198 -msgid "" -"Word + Character + Ngram表示训练时目标词预测的上下文是一个Word、Character或者Ngram。bigram-" -"char表示上下文是2-grams或者1个Character。" -msgstr "" - -#: ../model_zoo/embeddings.md:284 -msgid "特别地，对于百度百科语料，在不同的 Co-occurrence类型下分别提供了目标词与上下文向量：" -msgstr "" - -#: ../model_zoo/embeddings.md:418 -msgid "" -"以上所述的模型名称可直接以参数形式传入padddlenlp.embeddings.TokenEmbedding，加载相对应的模型。比如要加载语料为Wiki2017，通过FastText训练的预训练模型（fasttext" -".wiki-news.target.word-word.dim300.en），只需执行以下代码：" -msgstr "" - -#: ../model_zoo/embeddings.md:752 -msgid "感谢 Chinese-Word-Vectors提供Word2Vec中文预训练词向量。" -msgstr "" - -#: ../model_zoo/embeddings.md:753 -msgid "感谢 GloVe Project提供的GloVe英文预训练词向量。" -msgstr "" - -#: ../model_zoo/embeddings.md:754 -msgid "感谢 FastText Project提供的英文预训练词向量。" -msgstr "" - -#: ../model_zoo/embeddings.md:757 -msgid "" -"Li, Shen, et al. \"Analogical reasoning on chinese morphological and " -"semantic relations.\" arXiv preprint arXiv:1805.06504 (2018)." -msgstr "" - -#: ../model_zoo/embeddings.md:758 -msgid "" -"Qiu, Yuanyuan, et al. \"Revisiting correlations between intrinsic and " -"extrinsic evaluations of word embeddings.\" Chinese Computational " -"Linguistics and Natural Language Processing Based on Naturally Annotated " -"Big Data. Springer, Cham, 2018. 209-221." -msgstr "" - -#: ../model_zoo/embeddings.md:759 -msgid "" -"Jeffrey Pennington, Richard Socher, and Christopher D. Manning. 2014. " -"GloVe: Global Vectors for Word Representation." -msgstr "" - -#: ../model_zoo/embeddings.md:760 -msgid "" -"T. Mikolov, E. Grave, P. Bojanowski, C. Puhrsch, A. Joulin. Advances in " -"Pre-Training Distributed Word Representations." -msgstr "" - diff --git a/docs/zh/locale/en/LC_MESSAGES/source/paddlenlp.embeddings.po b/docs/zh/locale/en/LC_MESSAGES/source/paddlenlp.embeddings.po deleted file mode 100644 index d3c2572e94..0000000000 --- a/docs/zh/locale/en/LC_MESSAGES/source/paddlenlp.embeddings.po +++ /dev/null @@ -1,23 +0,0 @@ -# SOME DESCRIPTIVE TITLE. -# Copyright (C) 2021, PaddleNLP -# This file is distributed under the same license as the PaddleNLP package. -# FIRST AUTHOR , 2022. -# -#, fuzzy -msgid "" -msgstr "" -"Project-Id-Version: PaddleNLP \n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-03-18 21:31+0800\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=utf-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.9.0\n" - -#: ../source/paddlenlp.embeddings.rst:2 -msgid "paddlenlp.embeddings" -msgstr "" - diff --git a/docs/zh/locale/en/LC_MESSAGES/source/paddlenlp.embeddings.token_embedding.po b/docs/zh/locale/en/LC_MESSAGES/source/paddlenlp.embeddings.token_embedding.po deleted file mode 100644 index 638fc8a0a6..0000000000 --- a/docs/zh/locale/en/LC_MESSAGES/source/paddlenlp.embeddings.token_embedding.po +++ /dev/null @@ -1,200 +0,0 @@ -# SOME DESCRIPTIVE TITLE. -# Copyright (C) 2021, PaddleNLP -# This file is distributed under the same license as the PaddleNLP package. -# FIRST AUTHOR , 2022. -# -#, fuzzy -msgid "" -msgstr "" -"Project-Id-Version: PaddleNLP \n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-03-18 21:31+0800\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=utf-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.9.0\n" - -#: ../source/paddlenlp.embeddings.token_embedding.rst:2 -msgid "token\\_embedding" -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.list_embedding_name:1 -msgid "Lists all names of pretrained embedding models paddlenlp provides." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding:1 -msgid "基类：:class:`paddle.nn.layer.common.Embedding`" -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding:1 -msgid "" -"A `TokenEmbedding` can load pre-trained embedding model which paddlenlp " -"provides by specifying embedding name. Furthermore, a `TokenEmbedding` " -"can load extended vocabulary by specifying extended_vocab_path." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.cosine_sim -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.dot -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_from_word -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_list_from_words -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.search -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.set_trainable -msgid "参数" -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding:5 -msgid "" -"The pre-trained embedding model name. Use " -"`paddlenlp.embeddings.list_embedding_name()` to list the names of all " -"embedding models that we provide. Defaults to " -"`w2v.baidu_encyclopedia.target.word-word.dim300`." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding:9 -msgid "Specifies unknown token. Defaults to `[UNK]`." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding:12 -msgid "" -"To initialize the vector of unknown token. If it's none, use normal " -"distribution to initialize the vector of unknown token. Defaults to " -"`None`." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding:16 -msgid "The file path of extended vocabulary. Defaults to `None`." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding:19 -msgid "Whether the weight of embedding can be trained. Defaults to True." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding:22 -msgid "" -"Whether to keep the extended vocabulary only, will be effective only if " -"provides extended_vocab_path. Defaults to False." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.set_trainable:1 -msgid "Whether or not to set the weights of token embedding to be trainable." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.set_trainable:3 -msgid "" -"The weights can be trained if trainable is set to True, or the weights " -"are fixed if trainable is False." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.search:1 -msgid "Gets the vectors of specifying words." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.search:3 -msgid "The words which need to be searched." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.cosine_sim -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.dot -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_from_word -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_list_from_words -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.search -msgid "返回" -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.search:6 -msgid "The vectors of specifying words." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.cosine_sim -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.dot -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_from_word -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_list_from_words -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.search -msgid "返回类型" -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.search:7 -msgid "`numpy.array`" -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.cosine_sim:13 -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.dot:14 -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_list_from_words:10 -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.search:10 -msgid "实际案例" -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_from_word:1 -msgid "Gets the index of specifying word by searching word_to_idx dict." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_from_word:3 -msgid "The input token word which we want to get the token index converted from." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_from_word:6 -msgid "The index of specifying word." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_from_word:7 -msgid "`int`" -msgstr "" - -#: of -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_list_from_words:1 -msgid "Gets the index list of specifying words by searching word_to_idx dict." -msgstr "" - -#: of -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_list_from_words:3 -msgid "" -"The input token words which we want to get the token indices converted " -"from." -msgstr "" - -#: of -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_list_from_words:6 -msgid "The indexes list of specifying words." -msgstr "" - -#: of -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.get_idx_list_from_words:7 -msgid "`list`" -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.dot:1 -msgid "" -"Calculates the dot product of 2 words. Dot product or scalar product is " -"an algebraic operation that takes two equal-length sequences of numbers " -"(usually coordinate vectors), and returns a single number." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.cosine_sim:4 -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.dot:5 -msgid "The first word string." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.cosine_sim:6 -#: paddlenlp.embeddings.token_embedding.TokenEmbedding.dot:7 -msgid "The second word string." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.dot:10 -msgid "The dot product of 2 words." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.cosine_sim:1 -msgid "" -"Calculates the cosine similarity of 2 word vectors. Cosine similarity is " -"the cosine of the angle between two n-dimensional vectors in an " -"n-dimensional space." -msgstr "" - -#: of paddlenlp.embeddings.token_embedding.TokenEmbedding.cosine_sim:9 -msgid "The cosine similarity of 2 words." -msgstr "" - diff --git a/docs/zh/model_zoo/embeddings.md b/docs/zh/model_zoo/embeddings.md deleted file mode 100644 index 7535eb0693..0000000000 --- a/docs/zh/model_zoo/embeddings.md +++ /dev/null @@ -1,307 +0,0 @@ -# PaddleNLP Embedding API - -- [介绍](#介绍) -- [用法](#用法) - * [TokenEmbedding 参数](#TokenEmbedding 参数) - * [初始化](#初始化) - * [查询 embedding 结果](#查询 embedding 结果) - * [可视化 embedding 结果](#可视化 embedding 结果) - * [计算词向量 cosine 相似度](#计算词向量 cosine 相似度) - * [计算词向量内积](#计算词向量内积) - * [训练](#训练) - * [切词](#切词) -- [预训练模型](#预训练模型) - * [中文词向量](#中文词向量) - * [英文词向量](#英文词向量) - * [Word2Vec](#word2vec) - * [GloVe](#glove) - * [FastText](#fasttext) - * [使用方式](#使用方式) - * [模型信息](#模型信息) -- [致谢](#致谢) -- [参考论文](#参考论文) - -## 介绍 - -PaddleNLP 提供多个开源的预训练词向量模型，用户仅需在使用`paddlenlp.embeddings.TokenEmbedding`时，指定预训练模型的名称，即可加载相对应的预训练模型。以下将介绍`TokenEmbedding`详细用法，并列出 PaddleNLP 所支持的预训练 Embedding 模型。 - -## 用法 - -### TokenEmbedding 参数 - -| 参数 | 类型 | 属性 | -| ------------ | ------------ | ------------ | -| embedding_name | **string** | 预训练 embedding 名称，可通过 paddlenlp.embeddings.list_embedding_name()或[Embedding 模型汇总](#中文词向量)查询。 | -| unknown_token | **string** | unknown token。 | -| unknown_token_vector | **list** 或者 **np.array** | 用来初始化 unknown token 对应的 vector。默认为 None（以正态分布方式初始化 vector）| -| extended_vocab_path | **string** | 扩展词表的文件名路径。词表格式为一行一个词。 | -| trainable | **bool** | 是否可训练。True 表示 Embedding 可以更新参数，False 为不可更新。 | - -### 初始化 -```python -import paddle -from paddlenlp.embeddings import TokenEmbedding, list_embedding_name -paddle.set_device("cpu") - -# 查看预训练embedding名称： -print(list_embedding_name()) # ['w2v.baidu_encyclopedia.target.word-word.dim300'] - -# 初始化TokenEmbedding， 预训练embedding没下载时会自动下载并加载数据 -token_embedding = TokenEmbedding(embedding_name="w2v.baidu_encyclopedia.target.word-word.dim300") - -# 查看token_embedding详情 -print(token_embedding) - -Object type: -Unknown index: 635963 -Unknown token: [UNK] -Padding index: 635964 -Padding token: [PAD] -Parameter containing: -Tensor(shape=[635965, 300], dtype=float32, place=CPUPlace, stop_gradient=False, - [[-0.24200200, 0.13931701, 0.07378800, ..., 0.14103900, 0.05592300, -0.08004800], - [-0.08671700, 0.07770800, 0.09515300, ..., 0.11196400, 0.03082200, -0.12893000], - [-0.11436500, 0.12201900, 0.02833000, ..., 0.11068700, 0.03607300, -0.13763499], - ..., - [ 0.02628800, -0.00008300, -0.00393500, ..., 0.00654000, 0.00024600, -0.00662600], - [-0.00924490, 0.00652097, 0.01049327, ..., -0.01796000, 0.03498908, -0.02209341], - [ 0. , 0. , 0. , ..., 0. , 0. , 0. ]]) - -``` - -### 查询 embedding 结果 - -```python -test_token_embedding = token_embedding.search("中国") -print(test_token_embedding) -[[ 0.260801 0.1047 0.129453 -0.257317 -0.16152 0.19567 -0.074868 - 0.361168 0.245882 -0.219141 -0.388083 0.235189 0.029316 0.154215 - -0.354343 0.017746 0.009028 0.01197 -0.121429 0.096542 0.009255 - ..., - -0.260592 -0.019668 -0.063312 -0.094939 0.657352 0.247547 -0.161621 - 0.289043 -0.284084 0.205076 0.059885 0.055871 0.159309 0.062181 - 0.123634 0.282932 0.140399 -0.076253 -0.087103 0.07262 ]] -``` - -### 可视化 embedding 结果 -使用深度学习可视化工具[VisualDL](https://github.com/PaddlePaddle/VisualDL)的 High Dimensional 组件可以对 embedding 结果进行可视化展示，便于对其直观分析，步骤如下： -```python -# 获取词表中前1000个单词 -labels = token_embedding.vocab.to_tokens(list(range(0,1000))) -test_token_embedding = token_embedding.search(labels) - -# 引入VisualDL的LogWriter记录日志 -from visualdl import LogWriter - -with LogWriter(logdir='./visualize') as writer: - writer.add_embeddings(tag='test', mat=test_token_embedding, metadata=labels) -``` -执行完毕后会在当前路径下生成一个 visualize 目录，并将日志存放在其中，我们在命令行启动 VisualDL 即可进行查看，启动命令为： -```shell -visualdl --logdir ./visualize -``` -启动后打开浏览器即可看到可视化结果 - -

- -

- -使用 VisualDL 除可视化 embedding 结果外，还可以对标量、图片、音频等进行可视化，有效提升训练调参效率。关于 VisualDL 更多功能和详细介绍，可参考[VisualDL 使用文档](https://github.com/PaddlePaddle/VisualDL/tree/develop/docs)。 - -### 计算词向量 cosine 相似度 - -```python -score = token_embedding.cosine_sim("中国", "美国") -print(score) # 0.49586025 -``` - -### 计算词向量内积 - -```python -score = token_embedding.dot("中国", "美国") -print(score) # 8.611071 -``` - - -### 训练 - -以下为`TokenEmbedding`简单的组网使用方法。有关更多`TokenEmbedding`训练流程相关的使用方法，请参考[Word Embedding with PaddleNLP](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/word_embedding)。 - -```python -in_words = paddle.to_tensor([0, 2, 3]) -input_embeddings = token_embedding(in_words) -linear = paddle.nn.Linear(token_embedding.embedding_dim, 20) -input_fc = linear(input_embeddings) -print(input_fc) -Tensor(shape=[3, 20], dtype=float32, place=CPUPlace, stop_gradient=False, - [[ 0. , 0. , 0. , ..., 0. , 0. , 0. ], - [-0.23473957, 0.17878169, 0.07215232, ..., 0.03698236, 0.14291850, 0.05136518], - [-0.42466098, 0.15017235, -0.04780108, ..., -0.04995505, 0.15847842, 0.00025209]]) -``` - -### 切词 - -```python -from paddlenlp.data import JiebaTokenizer -tokenizer = JiebaTokenizer(vocab=token_embedding.vocab) -words = tokenizer.cut("中国人民") -print(words) # ['中国人', '民'] - -tokens = tokenizer.encode("中国人民") -print(tokens) # [12530, 1334] -``` - -## 预训练模型 - -以下将列举 PaddleNLP 支持的 Embedding 预训练模型。 -- 模型命名方式为：\${训练模型}.\${语料}.\${词向量类型}.\${co-occurrence type}.dim\${维度}。 -- 模型有三种，分别是 Word2Vec(w2v, skip-gram), GloVe(glove)和 FastText(fasttext)。 - -### 中文词向量 - -以下预训练词向量由[Chinese-Word-Vectors](https://github.com/Embedding/Chinese-Word-Vectors)提供。 - -根据不同类型的上下文为每个语料训练多个目标词向量，第二列开始表示不同类型的上下文。以下为上下文类别： - -* Word 表示训练时目标词预测的上下文是一个 Word。 -* Word + N-gram 表示训练时目标词预测的上下文是一个 Word 或者 Ngram，其中 bigram 表示2-grams，ngram.1-2表示1-gram 或者2-grams。 -* Word + Character 表示训练时目标词预测的上下文是一个 Word 或者 Character，其中 word-character.char1-2表示上下文是1个或2个 Character。 -* Word + Character + Ngram 表示训练时目标词预测的上下文是一个 Word、Character 或者 Ngram。bigram-char 表示上下文是2-grams 或者1个 Character。 - -| 语料 | Word | Word + N-gram | Word + Character | Word + Character + N-gram | -| ------------------------------------------- | ---- | ---- | ---- | ---- | -| Baidu Encyclopedia 百度百科 | w2v.baidu_encyclopedia.target.word-word.dim300 | w2v.baidu_encyclopedia.target.word-ngram.1-2.dim300 | w2v.baidu_encyclopedia.target.word-character.char1-2.dim300 | w2v.baidu_encyclopedia.target.bigram-char.dim300 | -| Wikipedia_zh 中文维基百科 | w2v.wiki.target.word-word.dim300 | w2v.wiki.target.word-bigram.dim300 | w2v.wiki.target.word-char.dim300 | w2v.wiki.target.bigram-char.dim300 | -| People's Daily News 人民日报 | w2v.people_daily.target.word-word.dim300 | w2v.people_daily.target.word-bigram.dim300 | w2v.people_daily.target.word-char.dim300 | w2v.people_daily.target.bigram-char.dim300 | -| Sogou News 搜狗新闻 | w2v.sogou.target.word-word.dim300 | w2v.sogou.target.word-bigram.dim300 | w2v.sogou.target.word-char.dim300 | w2v.sogou.target.bigram-char.dim300 | -| Financial News 金融新闻 | w2v.financial.target.word-word.dim300 | w2v.financial.target.word-bigram.dim300 | w2v.financial.target.word-char.dim300 | w2v.financial.target.bigram-char.dim300 | -| Zhihu_QA 知乎问答 | w2v.zhihu.target.word-word.dim300 | w2v.zhihu.target.word-bigram.dim300 | w2v.zhihu.target.word-char.dim300 | w2v.zhihu.target.bigram-char.dim300 | -| Weibo 微博 | w2v.weibo.target.word-word.dim300 | w2v.weibo.target.word-bigram.dim300 | w2v.weibo.target.word-char.dim300 | w2v.weibo.target.bigram-char.dim300 | -| Literature 文学作品 | w2v.literature.target.word-word.dim300 | w2v.literature.target.word-bigram.dim300 | w2v.literature.target.word-char.dim300 | w2v.literature.target.bigram-char.dim300 | -| Complete Library in Four Sections 四库全书 | w2v.sikuquanshu.target.word-word.dim300 | w2v.sikuquanshu.target.word-bigram.dim300 | 无 | 无 | -| Mixed-large 综合 | w2v.mixed-large.target.word-word.dim300 | 暂无 | w2v.mixed-large.target.word-word.dim300 | 暂无 | - -特别地，对于百度百科语料，在不同的 Co-occurrence 类型下分别提供了目标词与上下文向量： - -| Co-occurrence 类型 | 目标词向量 | 上下文词向量 | -| --------------------------- | ------ | ---- | -| Word → Word | w2v.baidu_encyclopedia.target.word-word.dim300 | w2v.baidu_encyclopedia.context.word-word.dim300 | -| Word → Ngram (1-2) | w2v.baidu_encyclopedia.target.word-ngram.1-2.dim300 | w2v.baidu_encyclopedia.context.word-ngram.1-2.dim300 | -| Word → Ngram (1-3) | w2v.baidu_encyclopedia.target.word-ngram.1-3.dim300 | w2v.baidu_encyclopedia.context.word-ngram.1-3.dim300 | -| Ngram (1-2) → Ngram (1-2)| w2v.baidu_encyclopedia.target.word-ngram.2-2.dim300 | w2v.baidu_encyclopedia.target.word-ngram.2-2.dim300 | -| Word → Character (1) | w2v.baidu_encyclopedia.target.word-character.char1-1.dim300 | w2v.baidu_encyclopedia.context.word-character.char1-1.dim300 | -| Word → Character (1-2) | w2v.baidu_encyclopedia.target.word-character.char1-2.dim300 | w2v.baidu_encyclopedia.context.word-character.char1-2.dim300 | -| Word → Character (1-4) | w2v.baidu_encyclopedia.target.word-character.char1-4.dim300 | w2v.baidu_encyclopedia.context.word-character.char1-4.dim300 | -| Word → Word (left/right) | w2v.baidu_encyclopedia.target.word-wordLR.dim300 | w2v.baidu_encyclopedia.context.word-wordLR.dim300 | -| Word → Word (distance) | w2v.baidu_encyclopedia.target.word-wordPosition.dim300 | w2v.baidu_encyclopedia.context.word-wordPosition.dim300 | - -### 英文词向量 - -### Word2Vec - -| 语料 | 名称 | -|------|------| -| Google News | w2v.google_news.target.word-word.dim300.en | - -### GloVe - -| 语料 | 25维 | 50维 | 100维 | 200维 | 300 维 | -| ----------------- | ------ | ------ | ------ | ------ | ------ | -| Wiki2014 + GigaWord | 无 | glove.wiki2014-gigaword.target.word-word.dim50.en | glove.wiki2014-gigaword.target.word-word.dim100.en | glove.wiki2014-gigaword.target.word-word.dim200.en | glove.wiki2014-gigaword.target.word-word.dim300.en | -| Twitter | glove.twitter.target.word-word.dim25.en | glove.twitter.target.word-word.dim50.en | glove.twitter.target.word-word.dim100.en | glove.twitter.target.word-word.dim200.en | 无 | - -### FastText - -| 语料 | 名称 | -|------|------| -| Wiki2017 | fasttext.wiki-news.target.word-word.dim300.en | -| Crawl | fasttext.crawl.target.word-word.dim300.en | - -### 使用方式 - -以上所述的模型名称可直接以参数形式传入`padddlenlp.embeddings.TokenEmbedding`，加载相对应的模型。比如要加载语料为 Wiki2017，通过 FastText 训练的预训练模型（`fasttext.wiki-news.target.word-word.dim300.en`），只需执行以下代码： - -```python -import paddle -from paddlenlp.embeddings import TokenEmbedding - -token_embedding = TokenEmbedding(embedding_name="fasttext.wiki-news.target.word-word.dim300.en") -``` - -### 模型信息 - -| 模型 | 文件大小 | 词表大小 | -|-----|---------|---------| -| w2v.baidu_encyclopedia.target.word-word.dim300 | 678.21 MB | 635965 | -| w2v.baidu_encyclopedia.target.word-character.char1-1.dim300 | 679.15 MB | 636038 | -| w2v.baidu_encyclopedia.target.word-character.char1-2.dim300 | 679.30 MB | 636038 | -| w2v.baidu_encyclopedia.target.word-character.char1-4.dim300 | 679.51 MB | 636038 | -| w2v.baidu_encyclopedia.target.word-ngram.1-2.dim300 | 679.48 MB | 635977 | -| w2v.baidu_encyclopedia.target.word-ngram.1-3.dim300 | 671.27 MB | 628669 | -| w2v.baidu_encyclopedia.target.word-ngram.2-2.dim300 | 7.28 GB | 6969069 | -| w2v.baidu_encyclopedia.target.word-wordLR.dim300 | 678.22 MB | 635958 | -| w2v.baidu_encyclopedia.target.word-wordPosition.dim300 | 679.32 MB | 636038 | -| w2v.baidu_encyclopedia.target.bigram-char.dim300 | 679.29 MB | 635976 | -| w2v.baidu_encyclopedia.context.word-word.dim300 | 677.74 MB | 635952 | -| w2v.baidu_encyclopedia.context.word-character.char1-1.dim300 | 678.65 MB | 636200 | -| w2v.baidu_encyclopedia.context.word-character.char1-2.dim300 | 844.23 MB | 792631 | -| w2v.baidu_encyclopedia.context.word-character.char1-4.dim300 | 1.16 GB | 1117461 | -| w2v.baidu_encyclopedia.context.word-ngram.1-2.dim300 | 7.25 GB | 6967598 | -| w2v.baidu_encyclopedia.context.word-ngram.1-3.dim300 | 5.21 GB | 5000001 | -| w2v.baidu_encyclopedia.context.word-ngram.2-2.dim300 | 7.26 GB | 6968998 | -| w2v.baidu_encyclopedia.context.word-wordLR.dim300 | 1.32 GB | 1271031 | -| w2v.baidu_encyclopedia.context.word-wordPosition.dim300 | 6.47 GB | 6293920 | -| w2v.wiki.target.bigram-char.dim300 | 375.98 MB | 352274 | -| w2v.wiki.target.word-char.dim300 | 375.52 MB | 352223 | -| w2v.wiki.target.word-word.dim300 | 374.95 MB | 352219 | -| w2v.wiki.target.word-bigram.dim300 | 375.72 MB | 352219 | -| w2v.people_daily.target.bigram-char.dim300 | 379.96 MB | 356055 | -| w2v.people_daily.target.word-char.dim300 | 379.45 MB | 355998 | -| w2v.people_daily.target.word-word.dim300 | 378.93 MB | 355989 | -| w2v.people_daily.target.word-bigram.dim300 | 379.68 MB | 355991 | -| w2v.weibo.target.bigram-char.dim300 | 208.24 MB | 195199 | -| w2v.weibo.target.word-char.dim300 | 208.03 MB | 195204 | -| w2v.weibo.target.word-word.dim300 | 207.94 MB | 195204 | -| w2v.weibo.target.word-bigram.dim300 | 208.19 MB | 195204 | -| w2v.sogou.target.bigram-char.dim300 | 389.81 MB | 365112 | -| w2v.sogou.target.word-char.dim300 | 389.89 MB | 365078 | -| w2v.sogou.target.word-word.dim300 | 388.66 MB | 364992 | -| w2v.sogou.target.word-bigram.dim300 | 388.66 MB | 364994 | -| w2v.zhihu.target.bigram-char.dim300 | 277.35 MB | 259755 | -| w2v.zhihu.target.word-char.dim300 | 277.40 MB | 259940 | -| w2v.zhihu.target.word-word.dim300 | 276.98 MB | 259871 | -| w2v.zhihu.target.word-bigram.dim300 | 277.53 MB | 259885 | -| w2v.financial.target.bigram-char.dim300 | 499.52 MB | 467163 | -| w2v.financial.target.word-char.dim300 | 499.17 MB | 467343 | -| w2v.financial.target.word-word.dim300 | 498.94 MB | 467324 | -| w2v.financial.target.word-bigram.dim300 | 499.54 MB | 467331 | -| w2v.literature.target.bigram-char.dim300 | 200.69 MB | 187975 | -| w2v.literature.target.word-char.dim300 | 200.44 MB | 187980 | -| w2v.literature.target.word-word.dim300 | 200.28 MB | 187961 | -| w2v.literature.target.word-bigram.dim300 | 200.59 MB | 187962 | -| w2v.sikuquanshu.target.word-word.dim300 | 20.70 MB | 19529 | -| w2v.sikuquanshu.target.word-bigram.dim300 | 20.77 MB | 19529 | -| w2v.mixed-large.target.word-char.dim300 | 1.35 GB | 1292552 | -| w2v.mixed-large.target.word-word.dim300 | 1.35 GB | 1292483 | -| w2v.google_news.target.word-word.dim300.en | 1.61 GB | 3000000 | -| glove.wiki2014-gigaword.target.word-word.dim50.en | 73.45 MB | 400002 | -| glove.wiki2014-gigaword.target.word-word.dim100.en | 143.30 MB | 400002 | -| glove.wiki2014-gigaword.target.word-word.dim200.en | 282.97 MB | 400002 | -| glove.wiki2014-gigaword.target.word-word.dim300.en | 422.83 MB | 400002 | -| glove.twitter.target.word-word.dim25.en | 116.92 MB | 1193516 | -| glove.twitter.target.word-word.dim50.en | 221.64 MB | 1193516 | -| glove.twitter.target.word-word.dim100.en | 431.08 MB | 1193516 | -| glove.twitter.target.word-word.dim200.en | 848.56 MB | 1193516 | -| fasttext.wiki-news.target.word-word.dim300.en | 541.63 MB | 999996 | -| fasttext.crawl.target.word-word.dim300.en | 1.19 GB | 2000002 | - -## 致谢 -- 感谢 [Chinese-Word-Vectors](https://github.com/Embedding/Chinese-Word-Vectors)提供 Word2Vec 中文预训练词向量。 -- 感谢 [GloVe Project](https://nlp.stanford.edu/projects/glove)提供的 GloVe 英文预训练词向量。 -- 感谢 [FastText Project](https://fasttext.cc/docs/en/english-vectors.html)提供的英文预训练词向量。 - -## 参考论文 -- Li, Shen, et al. "Analogical reasoning on chinese morphological and semantic relations." arXiv preprint arXiv:1805.06504 (2018). -- Qiu, Yuanyuan, et al. "Revisiting correlations between intrinsic and extrinsic evaluations of word embeddings." Chinese Computational Linguistics and Natural Language Processing Based on Naturally Annotated Big Data. Springer, Cham, 2018. 209-221. -- Jeffrey Pennington, Richard Socher, and Christopher D. Manning. 2014. GloVe: Global Vectors for Word Representation. -- T. Mikolov, E. Grave, P. Bojanowski, C. Puhrsch, A. Joulin. Advances in Pre-Training Distributed Word Representations. diff --git a/docs/zh/source/paddlenlp.embeddings.rst b/docs/zh/source/paddlenlp.embeddings.rst deleted file mode 100644 index 1c85548c46..0000000000 --- a/docs/zh/source/paddlenlp.embeddings.rst +++ /dev/null @@ -1,14 +0,0 @@ -paddlenlp.embeddings -============================ - -.. automodule:: paddlenlp.embeddings - :members: - :no-undoc-members: - :show-inheritance: - - -.. toctree:: - :maxdepth: 4 - - paddlenlp.embeddings.constant - paddlenlp.embeddings.token_embedding diff --git a/docs/zh/source/paddlenlp.embeddings.token_embedding.rst b/docs/zh/source/paddlenlp.embeddings.token_embedding.rst deleted file mode 100644 index eabb899600..0000000000 --- a/docs/zh/source/paddlenlp.embeddings.token_embedding.rst +++ /dev/null @@ -1,7 +0,0 @@ -token\_embedding -============================================ - -.. automodule:: paddlenlp.embeddings.token_embedding - :members: - :no-undoc-members: - :show-inheritance: diff --git a/docs/zh/source/paddlenlp.rst b/docs/zh/source/paddlenlp.rst index fbc7672055..94a0c7a17f 100644 --- a/docs/zh/source/paddlenlp.rst +++ b/docs/zh/source/paddlenlp.rst @@ -13,7 +13,6 @@ paddlenlp paddlenlp.data paddlenlp.dataaug paddlenlp.datasets - paddlenlp.embeddings paddlenlp.experimental paddlenlp.layers paddlenlp.losses diff --git a/paddlenlp/__init__.py b/paddlenlp/__init__.py index 7718428b44..9f16600f82 100644 --- a/paddlenlp/__init__.py +++ b/paddlenlp/__init__.py @@ -44,7 +44,6 @@ from . import ( data, dataaug, datasets, - embeddings, experimental, layers, losses, diff --git a/paddlenlp/embeddings/__init__.py b/paddlenlp/embeddings/__init__.py deleted file mode 100644 index d0daf27cf8..0000000000 --- a/paddlenlp/embeddings/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .token_embedding import * diff --git a/paddlenlp/embeddings/constant.py b/paddlenlp/embeddings/constant.py deleted file mode 100644 index 600078eb28..0000000000 --- a/paddlenlp/embeddings/constant.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors and Chinese-Word-Vectors Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -URL_ROOT = "https://bj.bcebos.com/paddlenlp" -EMBEDDING_URL_ROOT = URL_ROOT + "/models/embeddings" - -PAD_TOKEN = "[PAD]" -UNK_TOKEN = "[UNK]" - -EMBEDDING_NAME_LIST = [ - # Word2Vec - # baidu_encyclopedia - "w2v.baidu_encyclopedia.target.word-word.dim300", - "w2v.baidu_encyclopedia.target.word-character.char1-1.dim300", - "w2v.baidu_encyclopedia.target.word-character.char1-2.dim300", - "w2v.baidu_encyclopedia.target.word-character.char1-4.dim300", - "w2v.baidu_encyclopedia.target.word-ngram.1-2.dim300", - "w2v.baidu_encyclopedia.target.word-ngram.1-3.dim300", - "w2v.baidu_encyclopedia.target.word-ngram.2-2.dim300", - "w2v.baidu_encyclopedia.target.word-wordLR.dim300", - "w2v.baidu_encyclopedia.target.word-wordPosition.dim300", - "w2v.baidu_encyclopedia.target.bigram-char.dim300", - "w2v.baidu_encyclopedia.context.word-word.dim300", - "w2v.baidu_encyclopedia.context.word-character.char1-1.dim300", - "w2v.baidu_encyclopedia.context.word-character.char1-2.dim300", - "w2v.baidu_encyclopedia.context.word-character.char1-4.dim300", - "w2v.baidu_encyclopedia.context.word-ngram.1-2.dim300", - "w2v.baidu_encyclopedia.context.word-ngram.1-3.dim300", - "w2v.baidu_encyclopedia.context.word-ngram.2-2.dim300", - "w2v.baidu_encyclopedia.context.word-wordLR.dim300", - "w2v.baidu_encyclopedia.context.word-wordPosition.dim300", - # wikipedia - "w2v.wiki.target.bigram-char.dim300", - "w2v.wiki.target.word-char.dim300", - "w2v.wiki.target.word-word.dim300", - "w2v.wiki.target.word-bigram.dim300", - # people_daily - "w2v.people_daily.target.bigram-char.dim300", - "w2v.people_daily.target.word-char.dim300", - "w2v.people_daily.target.word-word.dim300", - "w2v.people_daily.target.word-bigram.dim300", - # weibo - "w2v.weibo.target.bigram-char.dim300", - "w2v.weibo.target.word-char.dim300", - "w2v.weibo.target.word-word.dim300", - "w2v.weibo.target.word-bigram.dim300", - # sogou - "w2v.sogou.target.bigram-char.dim300", - "w2v.sogou.target.word-char.dim300", - "w2v.sogou.target.word-word.dim300", - "w2v.sogou.target.word-bigram.dim300", - # zhihu - "w2v.zhihu.target.bigram-char.dim300", - "w2v.zhihu.target.word-char.dim300", - "w2v.zhihu.target.word-word.dim300", - "w2v.zhihu.target.word-bigram.dim300", - # finacial - "w2v.financial.target.bigram-char.dim300", - "w2v.financial.target.word-char.dim300", - "w2v.financial.target.word-word.dim300", - "w2v.financial.target.word-bigram.dim300", - # literature - "w2v.literature.target.bigram-char.dim300", - "w2v.literature.target.word-char.dim300", - "w2v.literature.target.word-word.dim300", - "w2v.literature.target.word-bigram.dim300", - # siku - "w2v.sikuquanshu.target.word-word.dim300", - "w2v.sikuquanshu.target.word-bigram.dim300", - # Mix-large - "w2v.mixed-large.target.word-char.dim300", - "w2v.mixed-large.target.word-word.dim300", - # GOOGLE NEWS - "w2v.google_news.target.word-word.dim300.en", - # GloVe - "glove.wiki2014-gigaword.target.word-word.dim50.en", - "glove.wiki2014-gigaword.target.word-word.dim100.en", - "glove.wiki2014-gigaword.target.word-word.dim200.en", - "glove.wiki2014-gigaword.target.word-word.dim300.en", - "glove.twitter.target.word-word.dim25.en", - "glove.twitter.target.word-word.dim50.en", - "glove.twitter.target.word-word.dim100.en", - "glove.twitter.target.word-word.dim200.en", - # FastText - "fasttext.wiki-news.target.word-word.dim300.en", - "fasttext.crawl.target.word-word.dim300.en", -] diff --git a/paddlenlp/embeddings/token_embedding.py b/paddlenlp/embeddings/token_embedding.py deleted file mode 100644 index f7d54174c5..0000000000 --- a/paddlenlp/embeddings/token_embedding.py +++ /dev/null @@ -1,378 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os.path as osp - -import numpy as np -import paddle -import paddle.nn as nn -from paddle.utils.download import get_path_from_url - -from paddlenlp.data import Vocab, get_idx_from_word -from paddlenlp.utils.env import MODEL_HOME, _get_sub_home -from paddlenlp.utils.log import logger - -from .constant import EMBEDDING_NAME_LIST, EMBEDDING_URL_ROOT, PAD_TOKEN, UNK_TOKEN - -EMBEDDING_HOME = _get_sub_home("embeddings", parent_home=MODEL_HOME) - -__all__ = ["list_embedding_name", "TokenEmbedding"] - - -def list_embedding_name(): - """ - Lists all names of pretrained embedding models paddlenlp provides. - """ - return list(EMBEDDING_NAME_LIST) - - -class TokenEmbedding(nn.Embedding): - """ - A `TokenEmbedding` can load pre-trained embedding model which paddlenlp provides by - specifying embedding name. Furthermore, a `TokenEmbedding` can load extended vocabulary - by specifying extended_vocab_path. - - Args: - embedding_name (`str`, optional): - The pre-trained embedding model name. Use `paddlenlp.embeddings.list_embedding_name()` to - list the names of all embedding models that we provide. - Defaults to `w2v.baidu_encyclopedia.target.word-word.dim300`. - unknown_token (`str`, optional): - Specifies unknown token. - Defaults to `[UNK]`. - unknown_token_vector (`list`, optional): - To initialize the vector of unknown token. If it's none, use normal distribution to - initialize the vector of unknown token. - Defaults to `None`. - extended_vocab_path (`str`, optional): - The file path of extended vocabulary. - Defaults to `None`. - trainable (`bool`, optional): - Whether the weight of embedding can be trained. - Defaults to True. - keep_extended_vocab_only (`bool`, optional): - Whether to keep the extended vocabulary only, will be effective only if provides extended_vocab_path. - Defaults to False. - """ - - def __init__( - self, - embedding_name=EMBEDDING_NAME_LIST[0], - unknown_token=UNK_TOKEN, - unknown_token_vector=None, - extended_vocab_path=None, - trainable=True, - keep_extended_vocab_only=False, - ): - vector_path = osp.join(EMBEDDING_HOME, embedding_name + ".npz") - if not osp.exists(vector_path): - # download - url = EMBEDDING_URL_ROOT + "/" + embedding_name + ".tar.gz" - get_path_from_url(url, EMBEDDING_HOME) - - logger.info("Loading token embedding...") - vector_np = np.load(vector_path) - self.embedding_dim = vector_np["embedding"].shape[1] - self.unknown_token = unknown_token - if unknown_token_vector is not None: - unk_vector = np.array(unknown_token_vector).astype(paddle.get_default_dtype()) - else: - unk_vector = np.random.normal(scale=0.02, size=self.embedding_dim).astype(paddle.get_default_dtype()) - pad_vector = np.array([0] * self.embedding_dim).astype(paddle.get_default_dtype()) - if extended_vocab_path is not None: - embedding_table = self._extend_vocab( - extended_vocab_path, vector_np, pad_vector, unk_vector, keep_extended_vocab_only - ) - trainable = True - else: - embedding_table = self._init_without_extend_vocab(vector_np, pad_vector, unk_vector) - - self.vocab = Vocab.from_dict(self._word_to_idx, unk_token=unknown_token, pad_token=PAD_TOKEN) - self.num_embeddings = embedding_table.shape[0] - # import embedding - super(TokenEmbedding, self).__init__( - self.num_embeddings, self.embedding_dim, padding_idx=self._word_to_idx[PAD_TOKEN] - ) - self.weight.set_value(embedding_table) - self.set_trainable(trainable) - logger.info("Finish loading embedding vector.") - s = "Token Embedding info:\ - \nUnknown index: {}\ - \nUnknown token: {}\ - \nPadding index: {}\ - \nPadding token: {}\ - \nShape :{}".format( - self._word_to_idx[self.unknown_token], - self.unknown_token, - self._word_to_idx[PAD_TOKEN], - PAD_TOKEN, - self.weight.shape, - ) - logger.info(s) - - def _init_without_extend_vocab(self, vector_np, pad_vector, unk_vector): - """ - Constructs index to word list, word to index dict and embedding weight. - """ - self._idx_to_word = list(vector_np["vocab"]) - self._idx_to_word.append(self.unknown_token) - self._idx_to_word.append(PAD_TOKEN) - self._word_to_idx = self._construct_word_to_idx(self._idx_to_word) - # insert unk, pad embedding - embedding_table = np.append(vector_np["embedding"], [unk_vector, pad_vector], axis=0) - - return embedding_table - - def _read_vocab_list_from_file(self, extended_vocab_path): - # load new vocab table from file - vocab_list = [] - with open(extended_vocab_path, "r", encoding="utf-8") as f: - for line in f.readlines(): - vocab = line.rstrip("\n").split("\t")[0] - vocab_list.append(vocab) - return vocab_list - - def _extend_vocab(self, extended_vocab_path, vector_np, pad_vector, unk_vector, keep_extended_vocab_only): - """ - Constructs index to word list, word to index dict and embedding weight using - extended vocab. - """ - logger.info("Start extending vocab.") - extend_vocab_list = self._read_vocab_list_from_file(extended_vocab_path) - extend_vocab_set = set(extend_vocab_list) - # update idx_to_word - self._idx_to_word = extend_vocab_list - self._word_to_idx = self._construct_word_to_idx(self._idx_to_word) - - # use the Xavier init the embedding - xavier_scale = np.sqrt(6.0 / float(len(self._idx_to_word) + self.embedding_dim)) - embedding_table = np.random.uniform( - low=-1.0 * xavier_scale, high=xavier_scale, size=(len(self._idx_to_word), self.embedding_dim) - ).astype(paddle.get_default_dtype()) - - pretrained_idx_to_word = list(vector_np["vocab"]) - pretrained_word_to_idx = self._construct_word_to_idx(pretrained_idx_to_word) - pretrained_embedding_table = np.array(vector_np["embedding"]) - - pretrained_vocab_set = set(pretrained_idx_to_word) - extend_vocab_set = set(self._idx_to_word) - vocab_intersection = pretrained_vocab_set & extend_vocab_set - vocab_subtraction = pretrained_vocab_set - extend_vocab_set - - # assignment from pretrained_vocab_embedding to extend_vocab_embedding - pretrained_vocab_intersect_index = [pretrained_word_to_idx[word] for word in vocab_intersection] - pretrained_vocab_subtract_index = [pretrained_word_to_idx[word] for word in vocab_subtraction] - extend_vocab_intersect_index = [self._word_to_idx[word] for word in vocab_intersection] - embedding_table[extend_vocab_intersect_index] = pretrained_embedding_table[pretrained_vocab_intersect_index] - if not keep_extended_vocab_only: - for idx in pretrained_vocab_subtract_index: - word = pretrained_idx_to_word[idx] - self._idx_to_word.append(word) - self._word_to_idx[word] = len(self._idx_to_word) - 1 - - embedding_table = np.append( - embedding_table, pretrained_embedding_table[pretrained_vocab_subtract_index], axis=0 - ) - - if self.unknown_token not in extend_vocab_set: - self._idx_to_word.append(self.unknown_token) - self._word_to_idx[self.unknown_token] = len(self._idx_to_word) - 1 - embedding_table = np.append(embedding_table, [unk_vector], axis=0) - else: - unk_idx = self._word_to_idx[self.unknown_token] - embedding_table[unk_idx] = unk_vector - - if PAD_TOKEN not in extend_vocab_set: - self._idx_to_word.append(PAD_TOKEN) - self._word_to_idx[PAD_TOKEN] = len(self._idx_to_word) - 1 - embedding_table = np.append(embedding_table, [pad_vector], axis=0) - else: - embedding_table[self._word_to_idx[PAD_TOKEN]] = pad_vector - - logger.info("Finish extending vocab.") - return embedding_table - - def set_trainable(self, trainable): - """ - Whether or not to set the weights of token embedding to be trainable. - - Args: - trainable (`bool`): - The weights can be trained if trainable is set to True, or the weights are fixed if trainable is False. - - """ - self.weight.stop_gradient = not trainable - - def search(self, words): - """ - Gets the vectors of specifying words. - - Args: - words (`list` or `str` or `int`): The words which need to be searched. - - Returns: - `numpy.array`: The vectors of specifying words. - - Examples: - .. code-block:: - - from paddlenlp.embeddings import TokenEmbedding - - embed = TokenEmbedding() - vector = embed.search('Welcome to use PaddlePaddle and PaddleNLP!') - - """ - idx_list = self.get_idx_list_from_words(words) - idx_tensor = paddle.to_tensor(idx_list) - return self(idx_tensor).cpu().numpy() - - def get_idx_from_word(self, word): - """ - Gets the index of specifying word by searching word_to_idx dict. - - Args: - word (`list` or `str` or `int`): The input token word which we want to get the token index converted from. - - Returns: - `int`: The index of specifying word. - - """ - return get_idx_from_word(word, self.vocab.token_to_idx, self.unknown_token) - - def get_idx_list_from_words(self, words): - """ - Gets the index list of specifying words by searching word_to_idx dict. - - Args: - words (`list` or `str` or `int`): The input token words which we want to get the token indices converted from. - - Returns: - `list`: The indexes list of specifying words. - - Examples: - .. code-block:: - - from paddlenlp.embeddings import TokenEmbedding - - embed = TokenEmbedding() - index = embed.get_idx_from_word('Welcome to use PaddlePaddle and PaddleNLP!') - #635963 - - """ - if isinstance(words, str): - idx_list = [self.get_idx_from_word(words)] - elif isinstance(words, int): - idx_list = [words] - elif isinstance(words, list) or isinstance(words, tuple): - idx_list = [self.get_idx_from_word(word) if isinstance(word, str) else word for word in words] - else: - raise TypeError - return idx_list - - def _dot_np(self, array_a, array_b): - return np.sum(array_a * array_b) - - def _calc_word(self, word_a, word_b, calc_kernel): - embeddings = self.search([word_a, word_b]) - embedding_a = embeddings[0] - embedding_b = embeddings[1] - return calc_kernel(embedding_a, embedding_b) - - def dot(self, word_a, word_b): - """ - Calculates the dot product of 2 words. Dot product or scalar product is an - algebraic operation that takes two equal-length sequences of numbers (usually - coordinate vectors), and returns a single number. - - Args: - word_a (`str`): The first word string. - word_b (`str`): The second word string. - - Returns: - float: The dot product of 2 words. - - Examples: - .. code-block:: - - from paddlenlp.embeddings import TokenEmbedding - - embed = TokenEmbedding() - dot_product = embed.dot('PaddlePaddle', 'PaddleNLP!') - #0.11827179 - - """ - dot = self._dot_np - return self._calc_word(word_a, word_b, lambda x, y: dot(x, y)) - - def cosine_sim(self, word_a, word_b): - """ - Calculates the cosine similarity of 2 word vectors. Cosine similarity is the - cosine of the angle between two n-dimensional vectors in an n-dimensional space. - - Args: - word_a (`str`): The first word string. - word_b (`str`): The second word string. - - Returns: - float: The cosine similarity of 2 words. - - Examples: - .. code-block:: - - from paddlenlp.embeddings import TokenEmbedding - - embed = TokenEmbedding() - cosine_simi = embed.cosine_sim('PaddlePaddle', 'PaddleNLP!') - #0.99999994 - - """ - dot = self._dot_np - return self._calc_word(word_a, word_b, lambda x, y: dot(x, y) / (np.sqrt(dot(x, x)) * np.sqrt(dot(y, y)))) - - def _construct_word_to_idx(self, idx_to_word): - """ - Constructs word to index dict. - - Args: - idx_to_word ('list'): - - Returns: - `Dict`: The word to index dict constructed by idx_to_word. - - """ - word_to_idx = {} - for i, word in enumerate(idx_to_word): - word_to_idx[word] = i - return word_to_idx - - def __repr__(self): - """ - Returns: - `Str`: The token embedding information. - - """ - info = "Object type: {}\ - \nUnknown index: {}\ - \nUnknown token: {}\ - \nPadding index: {}\ - \nPadding token: {}\ - \n{}".format( - super(TokenEmbedding, self).__repr__(), - self._word_to_idx[self.unknown_token], - self.unknown_token, - self._word_to_idx[PAD_TOKEN], - PAD_TOKEN, - self.weight, - ) - return info diff --git a/paddlenlp/transformers/__init__.py b/paddlenlp/transformers/__init__.py index 316bd92293..1a2618dcc7 100644 --- a/paddlenlp/transformers/__init__.py +++ b/paddlenlp/transformers/__init__.py @@ -140,6 +140,9 @@ from .dallebart.tokenizer import * from .deberta.configuration import * from .deberta.modeling import * from .deberta.tokenizer import * +from .nezha.configuration import * +from .nezha.modeling import * +from .nezha.tokenizer import * from .deberta_v2.configuration import * from .deberta_v2.modeling import * from .deberta_v2.tokenizer import * @@ -188,9 +191,6 @@ from .fnet.tokenizer import * from .funnel.configuration import * from .funnel.modeling import * from .funnel.tokenizer import * -from .gau_alpha.configuration import * -from .gau_alpha.modeling import * -from .gau_alpha.tokenizer import * from .gemma import * from .glm.configuration import * from .glm.modeling import * @@ -243,9 +243,6 @@ from .mpnet.modeling import * from .mpnet.tokenizer import * from .mt5.configuration import * from .mt5.modeling import * -from .nezha.configuration import * -from .nezha.modeling import * -from .nezha.tokenizer import * from .nv_embed.modeling import * from .nystromformer.configuration import * from .nystromformer.modeling import * diff --git a/paddlenlp/transformers/auto/configuration.py b/paddlenlp/transformers/auto/configuration.py index ebc33c3cbd..422a0dbb52 100644 --- a/paddlenlp/transformers/auto/configuration.py +++ b/paddlenlp/transformers/auto/configuration.py @@ -71,7 +71,6 @@ CONFIG_MAPPING_NAMES = OrderedDict( ("ernie_vil", "ErnieViLConfig"), ("fnet", "FNetConfig"), ("funnel", "FunnelConfig"), - ("gau_alpha", "GAUAlphaConfig"), ("gemma", "GemmaConfig"), ("glm", "GLMConfig"), ("gpt", "GPTConfig"), @@ -163,7 +162,6 @@ MODEL_NAMES_MAPPING = OrderedDict( ("ernie_vil", "ErnieViL"), ("fnet", "FNet"), ("funnel", "Funnel"), - ("gau_alpha", "GAUAlpha"), ("gemma", "Gemma"), ("glm", "GLM"), ("gpt", "GPT"), diff --git a/paddlenlp/transformers/auto/modeling.py b/paddlenlp/transformers/auto/modeling.py index 88ee24e376..669753408a 100644 --- a/paddlenlp/transformers/auto/modeling.py +++ b/paddlenlp/transformers/auto/modeling.py @@ -104,7 +104,6 @@ MAPPING_NAMES = OrderedDict( ("T5", "t5"), ("Bert", "bert"), ("Bart", "bart"), - ("GAUAlpha", "gau_alpha"), ("CodeGen", "codegen"), ("CLIPVision", "clip"), ("CLIPText", "clip"), diff --git a/paddlenlp/transformers/auto/tokenizer.py b/paddlenlp/transformers/auto/tokenizer.py index 93d2ea633e..40bd800458 100644 --- a/paddlenlp/transformers/auto/tokenizer.py +++ b/paddlenlp/transformers/auto/tokenizer.py @@ -145,7 +145,6 @@ else: "GPTTokenizerFast" if is_tokenizers_available() else None, ), ), - ("gau_alpha", "GAUAlphaTokenizer"), ("artist", "ArtistTokenizer"), ("chineseclip", "ChineseCLIPTokenizer"), ("ernie_vil", "ErnieViLTokenizer"), diff --git a/paddlenlp/transformers/gau_alpha/__init__.py b/paddlenlp/transformers/gau_alpha/__init__.py deleted file mode 100644 index 97043fd7ba..0000000000 --- a/paddlenlp/transformers/gau_alpha/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/paddlenlp/transformers/gau_alpha/configuration.py b/paddlenlp/transformers/gau_alpha/configuration.py deleted file mode 100644 index bf47e202d5..0000000000 --- a/paddlenlp/transformers/gau_alpha/configuration.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from ..configuration_utils import PretrainedConfig - -__all__ = ["GAUAlPHA_PRETRAINED_INIT_CONFIGURATION", "GAUAlphaConfig", "GAUAlPHA_PRETRAINED_RESOURCE_FILES_MAP"] - -GAUAlPHA_PRETRAINED_INIT_CONFIGURATION = { - "chinese_GAU-alpha-char_L-24_H-768": { - "vocab_size": 12000, - "hidden_size": 768, - "intermediate_size": 1536, - "num_hidden_layers": 24, - "max_position_embeddings": 512, - "type_vocab_size": 2, - "attention_key_size": 128, - "norm_eps": 1e-12, - "pad_token_id": 0, - "hidden_dropout_prob": 0.1, - "attention_probs_dropout_prob": 0.1, - "hidden_act": "swish", - "use_bias": False, - "normalization": "softmax_plus", - "attention_scale": True, - }, -} - -GAUAlPHA_PRETRAINED_RESOURCE_FILES_MAP = { - "model_state": { - "chinese_GAU-alpha-char_L-24_H-768": "https://bj.bcebos.com/paddlenlp/models/transformers/gau_alpha/chinese_GAU-alpha-char_L-24_H-768/model_state.pdparams", - } -} - - -class GAUAlphaConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a [`GAUAlphaModel`]. It is used to - instantiate a GAUAlpha model according to the specified arguments, defining the model architecture. Instantiating a - configuration with the defaults will yield a similar configuration to that of the GAUAlpha - chinese_GAU-alpha-char_L-24_H-768 architecture. - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the - documentation from [`PretrainedConfig`] for more information. - Args: - vocab_size (`int`, *optional*, defaults to 30522): - Vocabulary size of the GAUAlpha model. Defines the number of different tokens that can be represented by the - `inputs_ids` passed when calling [`GAUAlphaModel`]. - hidden_size (`int`, *optional*, defaults to 768): - Dimensionality of the encoder layers and the pooler layer. - num_hidden_layers (`int`, *optional*, defaults to 12): - Number of hidden layers in the Transformer encoder. - num_attention_heads (`int`, *optional*, defaults to 12): - Number of attention heads for each attention layer in the Transformer encoder. - intermediate_size (`int`, *optional*, defaults to 3072): - Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. - hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`): - The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"silu"` and `"gelu_new"` are supported. - hidden_dropout_prob (`float`, *optional*, defaults to 0.1): - The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. - attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): - The dropout ratio for the attention probabilities. - max_position_embeddings (`int`, *optional*, defaults to 512): - The maximum sequence length that this model might ever be used with. Typically set this to something large - just in case (e.g., 512 or 1024 or 2048). - type_vocab_size (`int`, *optional*, defaults to 2): - The vocabulary size of the `token_type_ids` passed when calling [`GAUAlphaModel`]. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - layer_norm_eps (`float`, *optional*, defaults to 1e-12): - The epsilon used by the layer normalization layers. - position_embedding_type (`str`, *optional*, defaults to `"absolute"`): - Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For - positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to - [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). - For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models - with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658). - use_cache (`bool`, *optional*, defaults to `True`): - Whether or not the model should return the last key/values attentions (not used by all models). Only - relevant if `config.is_decoder=True`. - classifier_dropout (`float`, *optional*): - The dropout ratio for the classification head. - Examples: - ```python - >>> from paddlenlp.transformers import GAUAlphaModel, GAUAlphaConfig - >>> # Initializing a GAUAlpha chinese_GAU-alpha-char_L-24_H-768style configuration - >>> configuration = GAUAlphaConfig() - >>> # Initializing a model from the style configuration - >>> model = GAUAlphaModel(configuration) - >>> # Accessing the model configuration - >>> configuration = model.config - ```""" - model_type = "gau_alpha" - pretrained_init_configuration = GAUAlPHA_PRETRAINED_INIT_CONFIGURATION - - def __init__( - self, - vocab_size: int = 30522, - hidden_size: int = 768, - num_hidden_layers: int = 12, - num_attention_heads: int = 12, - task_id=0, - intermediate_size: int = 3072, - hidden_act: str = "gelu", - hidden_dropout_prob: float = 0.1, - attention_probs_dropout_prob: float = 0.1, - max_position_embeddings: int = 512, - task_type_vocab_size: int = 3, - type_vocab_size: int = 16, - attention_key_size=128, - initializer_range: float = 0.02, - pad_token_id: int = 0, - pool_act: str = "tanh", - activation: str = "swish", - normalization: str = "softmax_plus", - fuse: bool = False, - layer_norm_eps=1e-12, - norm_eps=1e-12, - use_cache=False, - use_task_id=True, - use_bias=False, - attention_scale=True, - **kwargs - ): - super().__init__(pad_token_id=pad_token_id, **kwargs) - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.task_id = task_id - self.intermediate_size = intermediate_size - self.hidden_act = hidden_act - self.hidden_dropout_prob = hidden_dropout_prob - self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.max_position_embeddings = max_position_embeddings - self.task_type_vocab_size = task_type_vocab_size - self.type_vocab_size = type_vocab_size - self.initializer_range = initializer_range - self.pool_act = pool_act - self.fuse = fuse - self.layer_norm_eps = layer_norm_eps - self.norm_eps = norm_eps - self.use_cache = use_cache - self.use_task_id = use_task_id - self.use_bias = use_bias - self.activation = activation - self.attention_key_size = attention_key_size - self.normalization = normalization - self.attention_scale = attention_scale diff --git a/paddlenlp/transformers/gau_alpha/modeling.py b/paddlenlp/transformers/gau_alpha/modeling.py deleted file mode 100644 index 4a4ab981f0..0000000000 --- a/paddlenlp/transformers/gau_alpha/modeling.py +++ /dev/null @@ -1,810 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np -import paddle -import paddle.nn as nn -import paddle.nn.functional as F -from paddle.nn import Layer - -from paddlenlp.utils.env import CONFIG_NAME - -from .. import PretrainedModel, register_base_model -from ..albert.modeling import ACT2FN -from .configuration import ( - GAUAlPHA_PRETRAINED_INIT_CONFIGURATION, - GAUAlPHA_PRETRAINED_RESOURCE_FILES_MAP, - GAUAlphaConfig, -) - -__all__ = [ - "GAUAlphaModel", - "GAUAlphaForMaskedLM", - "GAUAlphaPretrainedModel", - "GAUAlphaForSequenceClassification", - "GAUAlphaForTokenClassification", - "GAUAlphaForQuestionAnswering", - "GAUAlphaForMultipleChoice", -] - -INF = 1e4 - - -class Norm(Layer): - def __init__(self, epsilon=1e-12): - super().__init__() - self._epsilon = epsilon - - def forward(self, x): - variance = paddle.mean(paddle.square(x), axis=-1, keepdim=True) - return x / paddle.sqrt(variance + self._epsilon) - - -def attention_normalize(a, mask=None, axis=-1, method="softmax"): - if method == "softmax": - return F.softmax(a, axis=axis) - else: - if mask is not None: - l = mask.sum(-1, keepdim=True) - else: - l = paddle.ones_like(a) * a.shape[-2] - if method == "squared_relu": - return F.relu(a) ** 2 / l - elif method == "softmax_plus": - scale = paddle.log(l) / np.log(512) - # mask: 1 for not padding, 0 for padding - # padding position's scale is 1 - if mask is not None: - scale = scale * mask + 1 - mask - return F.softmax(a * scale, axis=axis) - return a - - -class ScaleOffset(Layer): - def __init__( - self, - hidden_size=768, - scale=True, - offset=True, - ): - super().__init__() - self.scale = scale - self.offset = offset - - if self.scale: - self.weight = self.create_parameter((hidden_size,), default_initializer=nn.initializer.Constant(1.0)) - if self.offset: - self.bias = self.create_parameter((hidden_size,), is_bias=True) - - def forward(self, inputs): - if self.scale: - inputs = inputs * self.weight - if self.offset: - inputs = inputs + self.bias - - return inputs - - -class GatedAttentionUnit(Layer): - """ - https://github.com/ZhuiyiTechnology/GAU-alpha/blob/ea15e08a85d35652775c360218090cbaed98da18/models.py#L6-L85 - """ - - def __init__(self, config: GAUAlphaConfig): - super().__init__() - self.activation = ACT2FN[config.activation] - self.intermediate_size = config.intermediate_size - self.attention_key_size = config.attention_key_size - self.use_bias = config.use_bias - self.normalization = config.normalization - self.attention_scale = config.attention_scale - self.attention_dropout = config.attention_probs_dropout_prob - - self.i_dense = nn.Linear( - config.hidden_size, - 2 * config.intermediate_size + config.attention_key_size, - bias_attr=self.use_bias, - ) - self.o_dense = nn.Linear(config.intermediate_size, config.hidden_size, bias_attr=self.use_bias) - - self.q_scaleoffset = ScaleOffset(config.attention_key_size, offset=self.use_bias) - self.k_scaleoffset = ScaleOffset(config.attention_key_size, offset=self.use_bias) - self.rotary = RotaryPositionEmbedding(config) - - def forward(self, hidden_states, attention_mask=None): - x = self.i_dense(hidden_states) - u, v, qk = paddle.split( - self.activation(x), - [self.intermediate_size, self.intermediate_size, self.attention_key_size], - axis=-1, - ) - q, k = self.q_scaleoffset(qk), self.k_scaleoffset(qk) - - # apply_rotary - q, k = self.rotary(q), self.rotary(k) - - # Attention - a = paddle.matmul(q, k, transpose_y=True) - - if self.attention_scale: - a = a / self.attention_key_size**0.5 - - if attention_mask is not None: - a = a * attention_mask + (attention_mask - 1) * INF - - A = attention_normalize(a, attention_mask, axis=-1, method=self.normalization) - - A = F.dropout(A, p=self.attention_dropout, training=self.training) - - o = self.o_dense(u * paddle.matmul(A, v)) - - return o - - -class GAULayer(Layer): - def __init__(self, config: GAUAlphaConfig): - super().__init__() - self.gau = GatedAttentionUnit(config) - self.norm = Norm(config.norm_eps) - self.hidden_dropout = config.hidden_dropout_prob - - def forward(self, hidden_states, attention_mask=None): - gau_output = self.gau(hidden_states, attention_mask=attention_mask) - - # dropout and residual - o = F.dropout(gau_output[0], p=self.hidden_dropout, training=self.training) - o = self.norm(hidden_states + o) - - return o - - -def initializer(tensor, num_hidden_layers=12, order=2, gain=1.0): - """ - https://github.com/bojone/bert4keras/blob/5572ed481a14f5a62be7107e3846c88a5d6b617d/bert4keras/models.py#L1226-L1235 - """ - shape = tensor.shape - if shape[0] > 10000 or shape[0] < 10: - hidden_size = shape[1] - else: - hidden_size = shape[0] - gain *= num_hidden_layers ** (-1.0 / order) - std = 1.13684723 / hidden_size**0.5 * gain - - return nn.initializer.TruncatedNormal(std=std) - - -class RotaryPositionEmbedding(Layer): - def __init__(self, config: GAUAlphaConfig): - super().__init__() - inv_freq = 1.0 / ( - 10000 - ** ( - paddle.arange(0, config.attention_key_size, 2, dtype=paddle.get_default_dtype()) - / config.attention_key_size - ) - ) - t = paddle.arange(config.max_position_embeddings, dtype=paddle.get_default_dtype()) - freqs = paddle.matmul(t.unsqueeze(1), inv_freq.unsqueeze(0)) - self.register_buffer("sin", freqs.sin(), persistable=False) - self.register_buffer("cos", freqs.cos(), persistable=False) - - def forward(self, x, offset=0): - # x shape [batch_size, seqlen, dim] - seqlen = x.shape[-2] - sin, cos = ( - self.sin[offset : offset + seqlen, :], - self.cos[offset : offset + seqlen, :], - ) - x1, x2 = x[..., 0::2], x[..., 1::2] - # [cos_nθ, -sin_nθ] [x1] - # [sin_nθ, cos_nθ] [x2] - # => [x1 * cos_nθ - x2 * sin_nθ, x1 * sin_nθ + x2 * cos_nθ] - return paddle.stack([x1 * cos - x2 * sin, x1 * sin + x2 * cos], axis=-1).flatten(-2, -1) - - -class GAUAlphaPretrainedModel(PretrainedModel): - """ - An abstract class for pretrained GAU-alpha models. It provides GAU-alpha related - `model_config_file`, `pretrained_init_configuration`, `resource_files_names`, - `pretrained_resource_files_map`, `base_model_prefix` for downloading and - loading pretrained models. - See :class:`~paddlenlp.transformers.model_utils.PretrainedModel` for more details. - """ - - model_config_file = CONFIG_NAME - config_class = GAUAlphaConfig - resource_files_names = {"model_state": "model_state.pdparams"} - base_model_prefix = "gau_alpha" - - pretrained_init_configuration = GAUAlPHA_PRETRAINED_INIT_CONFIGURATION - pretrained_resource_files_map = GAUAlPHA_PRETRAINED_RESOURCE_FILES_MAP - - def _init_weights(self, layer): - """Initialization hook""" - if isinstance(layer, (nn.Linear, nn.Embedding)): - # In the dygraph mode, use the `set_value` to reset the parameter directly, - # and reset the `state_dict` to update parameter in static mode. - if isinstance(layer.weight, paddle.Tensor): - initializer(layer.weight, self.config.num_hidden_layers, order=2, gain=1.0) - if isinstance(layer, nn.Linear): - use_bias = self.use_bias if hasattr(self, "use_bias") else self.gau_alpha.config["use_bias"] - if layer.bias is not None and not use_bias: - layer.bias = None - - -@register_base_model -class GAUAlphaModel(GAUAlphaPretrainedModel): - """ - The bare GAUAlpha Model transformer outputting raw hidden-states. - - This model inherits from :class:`~paddlenlp.transformers.model_utils.PretrainedModel`. - Refer to the superclass documentation for the generic methods. - - This model is also a Paddle `paddle.Layer `__ subclass. Use it as a regular Paddle Layer - and refer to the Paddle documentation for all matter related to general usage and behavior. - - Args: - vocab_size (int): - Vocabulary size of `inputs_ids` in `GAUAlphaModel`. Also is the vocab size of token embedding matrix. - Defines the number of different tokens that can be represented by the `inputs_ids` passed when calling `GAUAlphaModel`. - hidden_size (int, optional): - Dimensionality of the, encoder layers and pooler layer. Defaults to `768`. - intermediate_size (int, optional): - Dimensionality of the feed-forward (ff) layer in the encoder. Input tensors - to ff layers are firstly projected from `hidden_size` to `intermediate_size`, - and then projected back to `hidden_size`. Typically `intermediate_size` is larger than `hidden_size`. - Defaults to `3072`. - num_hidden_layers (int, optional): - Number of hidden layers in the gau_alpha encoder. Defaults to `12`. - max_position_embeddings (int, optional): - The maximum value of the dimensionality of position encoding, which dictates the maximum supported length of an input - sequence. Defaults to `512`. - type_vocab_size (int, optional): - The vocabulary size of `token_type_ids`. - Defaults to `2`. - attention_key_size (int, optional): - The dimensionality of the key used in the gau layer. Defaults to `128`. - norm_eps (float, optional): - The epsilon value used in the normalization layer. - Defaults to `1e-12`. - pad_token_id (int, optional): - The index of padding token in the token vocabulary. - Defaults to `0`. - hidden_dropout_prob (float, optional): - The dropout probability for all fully connected layers in the embeddings and encoder. - Defaults to `0.1`. - attention_probs_dropout_prob (float, optional): - The dropout probability used in gau in all encoder layers to drop some attention target. - Defaults to `0.1`. - hidden_act (str, optional): - The activation function used in gau layer. Defaults to `swish`. - use_bias (bool, optional): - Whether or not use bias. - Defaults to `False`. - normalization (str, optional): - The normalization method used in gau layer. - Defaults to `softmax_plus`. - attention_scale (bool, optional): - Whether or not to scale the attention scores. - Defaults to `True`. - """ - - def __init__(self, config: GAUAlphaConfig): - super(GAUAlphaModel, self).__init__(config) - self.pad_token_id = config.pad_token_id - self.norm_eps = config.norm_eps - self.num_hidden_layers = config.num_hidden_layers - self.use_bias = config.use_bias - self.embeddings = GAUAlphaEmbeddings(config) - - self.encoder = GAUAlphaEncoder(config) - - def get_input_embeddings(self): - return self.embeddings.word_embeddings - - def set_input_embeddings(self, value): - self.embeddings.word_embeddings = value - - def forward( - self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - ): - r""" - The GAUAlphaModel forward method, overrides the `__call__()` special method. - - Args: - input_ids (Tensor): - Indices of input sequence tokens in the vocabulary. They are - numerical representations of tokens that build the input sequence. - Its data type should be `int64` and it has a shape of [batch_size, sequence_length]. - attention_mask (Tensor, optional): - Mask used in gau to avoid performing attention on to some unwanted positions, - usually the paddings or the subsequent positions. - Its data type can be int, float and bool. - When the data type is bool, the `masked` tokens have `False` values and the others have `True` values. - When the data type is int, the `masked` tokens have `0` values and the others have `1` values. - It is a tensor with shape broadcasted to `[batch_size, sequence_length, sequence_length]`. - Defaults to `None`, which means nothing needed to be prevented attention to. - token_type_ids (Tensor, optional): - Segment token indices to indicate different portions of the inputs. - Selected in the range ``[0, type_vocab_size - 1]``. - If `type_vocab_size` is 2, which means the inputs have two portions. - Indices can either be 0 or 1: - - - 0 corresponds to a *sentence A* token, - - 1 corresponds to a *sentence B* token. - - Its data type should be `int64` and it has a shape of [batch_size, sequence_length]. - Defaults to `None`, which means we don't add segment embeddings. - - Returns: - tuple: Returns `last_hidden_state` (Tensor) - Sequence of hidden-states at the last layer of the model. - It's data type should be float32 and its shape is [batch_size, sequence_length, hidden_size]. - - Example: - .. code-block:: - - import paddle - from paddlenlp.transformers import GAUAlphaModel, GAUAlphaTokenizer - - tokenizer = GAUAlphaTokenizer.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - model = GAUAlphaModel.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - - inputs = tokenizer("欢迎使用百度飞桨!") - inputs = {k:paddle.to_tensor([v], dtype="int64") for (k, v) in inputs.items()} - last_hidden_state = model(**inputs) - - """ - - if attention_mask is None: - attention_mask = input_ids != self.pad_token_id - if attention_mask.ndim == 2: - attention_mask = attention_mask.unsqueeze(1) # bs, 1, seqlen - attention_mask = attention_mask.astype(paddle.get_default_dtype()) - attention_mask.stop_gradient = True - - embedding_output = self.embeddings( - input_ids=input_ids, - token_type_ids=token_type_ids, - ) - - last_hidden_state = self.encoder(embedding_output, attention_mask=attention_mask) - - return last_hidden_state - - -class GAUAlphaEmbeddings(Layer): - """ - Include embeddings from word and token_type embeddings - """ - - def __init__(self, config: GAUAlphaConfig): - super(GAUAlphaEmbeddings, self).__init__() - self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size) - self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size) - self.norm = Norm(config.norm_eps) - self.dropout = nn.Dropout(config.hidden_dropout_prob) - - def forward(self, input_ids, token_type_ids=None): - if token_type_ids is None: - token_type_ids = paddle.zeros_like(input_ids, dtype="int64") - - input_embedings = self.word_embeddings(input_ids) - token_type_embeddings = self.token_type_embeddings(token_type_ids) - - embeddings = input_embedings + token_type_embeddings - embeddings = self.norm(embeddings) - embeddings = self.dropout(embeddings) - return embeddings - - -class GAUAlphaEncoder(Layer): - def __init__(self, config: GAUAlphaConfig): - super().__init__() - self.layer = nn.LayerList([GAULayer(config) for _ in range(config.num_hidden_layers)]) - - def forward(self, hidden_states, attention_mask=None): - for layer_module in self.layer: - hidden_states = layer_module( - hidden_states, - attention_mask, - ) - return hidden_states - - -class GAUAlphaForQuestionAnswering(GAUAlphaPretrainedModel): - """ - GAUAlpha with a linear layer on top of the hidden-states output to compute `span_start_logits` - and `span_end_logits`, designed for question-answering tasks like SQuAD. - - Args: - gau_alpha (:class:`GAUAlphaModel`): - An instance of GAUAlphaModel. - dropout (float, optional): - The dropout probability for output of GAUAlpha. - If None, use the same value as `hidden_dropout_prob` of `GAUAlphaModel` - instance `gau_alpha`. Defaults to `None`. - """ - - def __init__(self, config: GAUAlphaConfig): - super(GAUAlphaForQuestionAnswering, self).__init__(config) - self.gau_alpha = GAUAlphaModel(config) - self.dropout = nn.Dropout( - config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob - ) - self.classifier = nn.Linear(config.hidden_size, 2) - - def forward(self, input_ids, token_type_ids=None, attention_mask=None): - r""" - The GAUAlphaForQuestionAnswering forward method, overrides the __call__() special method. - - Args: - input_ids (Tensor): - See :class:`GAUAlphaModel`. - token_type_ids (Tensor, optional): - See :class:`GAUAlphaModel`. - attention_mask (Tensor, optional): - See :class:`GAUAlphaModel`. - - Returns: - tuple: Returns tuple (`start_logits`, `end_logits`). - - With the fields: - - - `start_logits` (Tensor): - A tensor of the input token classification logits, indicates the start position of the labelled span. - Its data type should be float32 and its shape is [batch_size, sequence_length]. - - - `end_logits` (Tensor): - A tensor of the input token classification logits, indicates the end position of the labelled span. - Its data type should be float32 and its shape is [batch_size, sequence_length]. - - Example: - .. code-block:: - - import paddle - from paddlenlp.transformers import GAUAlphaForQuestionAnswering, GAUAlphaTokenizer - - tokenizer = GAUAlphaTokenizer.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - model = GAUAlphaForQuestionAnswering.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - - inputs = tokenizer("欢迎使用百度飞桨!") - inputs = {k:paddle.to_tensor([v], dtype="int64") for (k, v) in inputs.items()} - outputs = model(**inputs) - - start_logits = outputs[0] - end_logits = outputs[1] - """ - sequence_output = self.gau_alpha(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask) - - logits = self.classifier(sequence_output) - start_logits, end_logits = paddle.unstack(logits, axis=-1) - - return start_logits, end_logits - - -class GAUAlphaForSequenceClassification(GAUAlphaPretrainedModel): - """ - GAUAlpha Model with a linear layer on top of the output layer, - designed for sequence classification/regression tasks like GLUE tasks. - - Args: - gau_alpha (`GAUAlphaModel`): - An instance of `paddlenlp.transformers.GAUAlphaModel`. - num_labels (int, optional): - The number of classes. Default to `2`. - dropout (float, optional): - The dropout probability for output of GAUAlpha. - If None, use the same value as `hidden_dropout_prob` - of `paddlenlp.transformers.GAUAlphaModel` instance. Defaults to `None`. - """ - - def __init__(self, config: GAUAlphaConfig): - super(GAUAlphaForSequenceClassification, self).__init__(config) - self.num_labels = config.num_labels - self.gau_alpha = GAUAlphaModel(config) - self.dropout = nn.Dropout( - config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob - ) - self.classifier = nn.Linear(config.hidden_size, config.num_labels) - - def forward(self, input_ids, token_type_ids=None, attention_mask=None): - r""" - Args: - input_ids (Tensor): - See :class:`GAUAlphaModel`. - token_type_ids (Tensor, optional): - See :class:`GAUAlphaModel`. - attention_mask (Tensor, optional): - See :class:`GAUAlphaModel`. - - Returns: - Tensor: Returns tensor `logits`, a tensor of the input text classification logits. - Shape as `[batch_size, num_labels]` and dtype as float32. - - Example: - .. code-block:: - - import paddle - from paddlenlp.transformers import GAUAlphaForSequenceClassification, GAUAlphaTokenizer - - tokenizer = GAUAlphaTokenizer.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - model = GAUAlphaForSequenceClassification.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - - inputs = tokenizer("欢迎使用百度飞桨!") - inputs = {k:paddle.to_tensor([v], dtype="int64") for (k, v) in inputs.items()} - logits = model(**inputs) - - """ - sequence_output = self.gau_alpha(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask) - pooled_output = sequence_output[:, 0] - - pooled_output = self.dropout(pooled_output) - logits = self.classifier(pooled_output) - return logits - - -class GAUAlphaForTokenClassification(GAUAlphaPretrainedModel): - """ - GAUAlpha Model with a linear layer on top of the hidden-states output layer, - designed for token classification tasks like NER tasks. - - Args: - gau_alpha (`GAUAlphaModel`): - An instance of `paddlenlp.transformers.GAUAlphaModel`. - num_labels (int, optional): - The number of classes. Default to `2`. - dropout (float, optional): - The dropout probability for output of GAUAlpha. - If None, use the same value as `hidden_dropout_prob` - of `paddlenlp.transformers.GAUAlphaModel` instance. Defaults to `None`. - """ - - def __init__(self, config: GAUAlphaConfig): - super(GAUAlphaForTokenClassification, self).__init__(config) - self.num_labels = config.num_labels - self.gau_alpha = GAUAlphaModel(config) # allow gau_alpha to be config - self.dropout = nn.Dropout( - config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob - ) - self.classifier = nn.Linear(config.hidden_size, config.num_labels) - - def forward(self, input_ids, token_type_ids=None, attention_mask=None): - r""" - Args: - input_ids (Tensor): - See :class:`GAUAlphaModel`. - token_type_ids (Tensor, optional): - See :class:`GAUAlphaModel`. - attention_mask (Tensor, optional): - See :class:`GAUAlphaModel`. - - Returns: - Tensor: Returns tensor `logits`, a tensor of the input token classification logits. - Shape as `[batch_size, sequence_length, num_labels]` and dtype as `float32`. - - Example: - .. code-block:: - - import paddle - from paddlenlp.transformers import GAUAlphaForTokenClassification, GAUAlphaTokenizer - - tokenizer = GAUAlphaTokenizer.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - model = GAUAlphaForTokenClassification.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - - inputs = tokenizer("欢迎使用百度飞桨!") - inputs = {k:paddle.to_tensor([v], dtype="int64") for (k, v) in inputs.items()} - logits = model(**inputs) - - """ - sequence_output = self.gau_alpha(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask) - - sequence_output = self.dropout(sequence_output) - logits = self.classifier(sequence_output) - return logits - - -class GAUAlphaForMultipleChoice(GAUAlphaPretrainedModel): - """ - GAUAlpha Model with a linear layer on top of the hidden-states output layer, - designed for multiple choice tasks like RocStories/SWAG tasks. - - Args: - gau_alpha (:class:`GAUAlphaModel`): - An instance of GAUAlphaModel. - num_choices (int, optional): - The number of choices. Defaults to `2`. - dropout (float, optional): - The dropout probability for output of GAUAlpha. - If None, use the same value as `hidden_dropout_prob` of `GAUAlphaModel` - instance `gau_alpha`. Defaults to None. - """ - - def __init__(self, config: GAUAlphaConfig): - super(GAUAlphaForMultipleChoice, self).__init__(config) - self.num_choices = config.num_choices - self.gau_alpha = GAUAlphaModel(config) - self.dropout = nn.Dropout( - config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob - ) - self.classifier = nn.Linear(config.hidden_size, 1) - - def forward(self, input_ids, token_type_ids=None, attention_mask=None): - r""" - The GAUAlphaForMultipleChoice forward method, overrides the __call__() special method. - - Args: - input_ids (Tensor): - See :class:`GAUAlphaModel` and shape as [batch_size, num_choice, sequence_length]. - token_type_ids(Tensor, optional): - See :class:`GAUAlphaModel` and shape as [batch_size, num_choice, sequence_length]. - attention_mask (list, optional): - See :class:`GAUAlphaModel` and shape as [batch_size, num_choice, sequence_length]. - - Returns: - Tensor: Returns tensor `reshaped_logits`, a tensor of the multiple choice classification logits. - Shape as `[batch_size, num_choice]` and dtype as `float32`. - - Example: - .. code-block:: - - import paddle - from paddlenlp.transformers import GAUAlphaForMultipleChoice, GAUAlphaTokenizer - from paddlenlp.data import Pad - - tokenizer = GAUAlphaTokenizer.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - model = GAUAlphaForMultipleChoice.from_pretrained('chinese_GAU-alpha-char_L-24_H-768', num_choices=2) - - data = [ - { - "question": "如何打开ipad屏幕？", - "answer1": "按音量按钮。", - "answer2": "按下锁定按钮。", - "label": 1, - }, - { - "question": "如何缩进一些文本？", - "answer1": "在开始写之前留一些空格。", - "answer2": "按空格键。", - "label": 0, - }, - ] - - text = [] - text_pair = [] - for d in data: - text.append(d["question"]) - text_pair.append(d["answer1"]) - text.append(d["question"]) - text_pair.append(d["answer2"]) - - inputs = tokenizer(text, text_pair) - input_ids = Pad(axis=0, pad_val=tokenizer.pad_token_id)(inputs["input_ids"]) - token_type_ids = Pad(axis=0, pad_val=tokenizer.pad_token_type_id)(inputs["token_type_ids"]) - - reshaped_logits = model( - input_ids=paddle.to_tensor(input_ids, dtype="int64"), - token_type_ids=paddle.to_tensor(token_type_ids, dtype="int64"), - ) - print(reshaped_logits.shape) - # [2, 2] - - """ - # input_ids: [bs, num_choice, seq_l] - input_ids = input_ids.reshape(shape=(-1, input_ids.shape[-1])) # flat_input_ids: [bs*num_choice,seq_l] - - if token_type_ids is not None: - token_type_ids = token_type_ids.reshape(shape=(-1, token_type_ids.shape[-1])) - - if attention_mask is not None: - attention_mask = attention_mask.reshape(shape=(-1, attention_mask.shape[-1])) - - sequence_output = self.gau_alpha(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask) - - pooled_output = sequence_output[:, 0] - pooled_output = self.dropout(pooled_output) - - logits = self.classifier(pooled_output) # logits: (bs*num_choice,1) - reshaped_logits = logits.reshape(shape=(-1, self.num_choices)) # logits: (bs, num_choice) - - return reshaped_logits - - -class GAUAlphaLMPredictionHead(Layer): - def __init__( - self, - config: GAUAlphaConfig, - embedding_weights=None, - ): - super(GAUAlphaLMPredictionHead, self).__init__() - self.use_bias = config.use_bias - self.decoder_weight = ( - self.create_parameter(shape=[config.vocab_size, config.hidden_size], dtype=self.transform.weight.dtype) - if embedding_weights is None - else embedding_weights - ) - if self.use_bias: - self.decoder_bias = self.create_parameter( - shape=[config.vocab_size], dtype=self.decoder_weight.dtype, is_bias=True - ) - - def forward(self, hidden_states): - hidden_states = paddle.matmul(hidden_states, self.decoder_weight, transpose_y=True) - if self.use_bias: - hidden_states = hidden_states + self.decoder_bias - - return hidden_states - - -class GAUAlphaForMaskedLM(GAUAlphaPretrainedModel): - """ - GAUAlpha Model with a `masked language modeling` head on top. - - Args: - gau_alpha (:class:`GAUAlphaModel`): - An instance of :class:`GAUAlphaModel`. - - """ - - def __init__( - self, - config: GAUAlphaConfig, - ): - super(GAUAlphaForMaskedLM, self).__init__(config) - self.gau_alpha = GAUAlphaModel(config) - self.cls = GAUAlphaLMPredictionHead( - config=config, - embedding_weights=self.gau_alpha.embeddings.word_embeddings.weight, - ) - - def forward(self, input_ids, token_type_ids=None, attention_mask=None): - r""" - - Args: - input_ids (Tensor): - See :class:`GAUAlphaModel`. - token_type_ids (Tensor, optional): - See :class:`GAUAlphaModel`. - attention_mask (Tensor, optional): - See :class:`GAUAlphaModel`. - - Returns: - Tensor: Returns tensor `prediction_scores`, The scores of masked token prediction. - Its data type should be float32 and shape is [batch_size, sequence_length, vocab_size]. - - Example: - .. code-block:: - - import paddle - from paddlenlp.transformers import GAUAlphaForMaskedLM, GAUAlphaTokenizer - - tokenizer = GAUAlphaTokenizer.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - model = GAUAlphaForMaskedLM.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - - inputs = tokenizer("欢迎使用百度飞桨!") - inputs = {k:paddle.to_tensor([v], dtype="int64") for (k, v) in inputs.items()} - - logits = model(**inputs) - print(logits.shape) - # [1, 11, 12000] - - """ - sequence_output = self.gau_alpha(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask) - - prediction_scores = self.cls(sequence_output) - return prediction_scores diff --git a/paddlenlp/transformers/gau_alpha/tokenizer.py b/paddlenlp/transformers/gau_alpha/tokenizer.py deleted file mode 100644 index eca6b9aeb0..0000000000 --- a/paddlenlp/transformers/gau_alpha/tokenizer.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from ..bert.tokenizer import BasicTokenizer, WordpieceTokenizer -from ..tokenizer_utils import PretrainedTokenizer - -__all__ = ["GAUAlphaTokenizer"] - -PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"chinese_GAU-alpha-char_L-24_H-768": 512} - - -class GAUAlphaTokenizer(PretrainedTokenizer): - """ - Constructs a GAUAlpha tokenizer. It uses a basic tokenizer to do punctuation - splitting, lower casing and so on, and follows a WordPiece tokenizer to - tokenize as subwords. - - This tokenizer inherits from :class:`~paddlenlp.transformers.tokenizer_utils.PretrainedTokenizer` - which contains most of the main methods. For more information regarding those methods, - please refer to this superclass. - - Args: - vocab_file (str): - The vocabulary file path (ends with '.txt') required to instantiate - a `WordpieceTokenizer`. - do_lower_case (bool,optional): - Whether or not to lowercase the input when tokenizing. - If you use the GAUAlpha pretrained model, lower is set to - False when using the cased model, otherwise it is set to True. - Defaults to`True`. - unk_token (str,optional): - A special token representing the *unknown (out-of-vocabulary)* token. - An unknown token is set to be `unk_token` inorder to be converted to an ID. - Defaults to "[UNK]". - sep_token (str,optional): - A special token separating two different sentences in the same input. - Defaults to "[SEP]". - pad_token (str,optional): - A special token used to make arrays of tokens the same size for batching purposes. - Defaults to "[PAD]". - cls_token (str,optional): - A special token used for sequence classification. It is the last token - of the sequence when built with special tokens. Defaults to "[CLS]". - mask_token (str,optional): - A special token representing a masked token. This is the token used - in the masked language modeling task which the model tries to predict the original unmasked ones. - Defaults to "[MASK]". - - Examples: - .. code-block:: - - from paddlenlp.transformers import GAUAlphaTokenizer - tokenizer = GAUAlphaTokenizer.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - - tokens = tokenizer('欢迎使用百度飞桨') - ''' - {'input_ids': [101, 3223, 6500, 421, 4179, 4331, 2008, 7263, 3055, 102], - 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} - ''' - - """ - - resource_files_names = {"vocab_file": "vocab.txt"} # for save_pretrained - pretrained_resource_files_map = { - "vocab_file": { - "chinese_GAU-alpha-char_L-24_H-768": "https://bj.bcebos.com/paddlenlp/models/transformers/gau_alpha/chinese_GAU-alpha-char_L-24_H-768/vocab.txt", - } - } - pretrained_init_configuration = { - "chinese_GAU-alpha-char_L-24_H-768": {"do_lower_case": True}, - } - padding_side = "right" - - max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES - - def __init__( - self, - vocab_file, - do_lower_case=True, - unk_token="[UNK]", - sep_token="[SEP]", - pad_token="[PAD]", - cls_token="[CLS]", - mask_token="[MASK]", - **kwargs - ): - - if not os.path.isfile(vocab_file): - raise ValueError( - "Can't find a vocabulary file at path '{}'. To load the " - "vocabulary from a pretrained model please use " - "`tokenizer = GAUAlphaTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file) - ) - self.do_lower_case = do_lower_case - self.vocab = self.load_vocabulary(vocab_file, unk_token=unk_token) - self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) - self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=unk_token) - - @property - def vocab_size(self): - """ - Return the size of vocabulary. - - Returns: - int: The size of vocabulary. - """ - - return len(self.vocab) - - def _tokenize(self, text): - """ - End-to-end tokenization for GAUAlpha models. - - Args: - text (str): The text to be tokenized. - - Returns: - list: A list of string representing converted tokens. - """ - split_tokens = [] - for token in self.basic_tokenizer.tokenize(text): - for sub_token in self.wordpiece_tokenizer.tokenize(token): - split_tokens.append(sub_token) - return split_tokens - - def convert_tokens_to_string(self, tokens): - """ - Converts a sequence of tokens (list of string) to a single string. Since - the usage of WordPiece introducing `##` to concat subwords, also removes - `##` when converting. - - Args: - tokens (list): A list of string representing tokens to be converted. - - Returns: - str: Converted string from tokens. - - Examples: - .. code-block:: - - from paddlenlp.transformers import GAUAlphaTokenizer - - tokenizer = GAUAlphaTokenizer.from_pretrained('chinese_GAU-alpha-char_L-24_H-768') - tokens = tokenizer.tokenize('欢迎使用百度飞桨!') - ''' - ['欢', '迎', '使', '用', '百', '度', '飞', '桨', '!'] - ''' - strings = tokenizer.convert_tokens_to_string(tokens) - ''' - '欢 迎 使 用 百 度 飞 桨 !' - ''' - """ - - out_string = " ".join(tokens).replace(" ##", "").strip() - return out_string - - def num_special_tokens_to_add(self, pair=False): - """ - Returns the number of added tokens when encoding a sequence with special tokens. - - Args: - pair(bool): - Whether the input is a sequence pair or a single sequence. - Defaults to `False` and the input is a single sequence. - - Returns: - int: Number of tokens added to sequences. - """ - token_ids_0 = [] - token_ids_1 = [] - return len(self.build_inputs_with_special_tokens(token_ids_0, token_ids_1 if pair else None)) - - def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): - """ - Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and - adding special tokens. - - A GAUAlpha sequence has the following format: - - - single sequence: ``[CLS] X [SEP]`` - - pair of sequences: ``[CLS] A [SEP] B [SEP]`` - - Args: - token_ids_0 (List[int]): - List of IDs to which the special tokens will be added. - token_ids_1 (List[int], optional): - Optional second list of IDs for sequence pairs. Defaults to None. - - Returns: - List[int]: List of input_id with the appropriate special tokens. - """ - if token_ids_1 is None: - return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] - _cls = [self.cls_token_id] - _sep = [self.sep_token_id] - return _cls + token_ids_0 + _sep + token_ids_1 + _sep - - def build_offset_mapping_with_special_tokens(self, offset_mapping_0, offset_mapping_1=None): - """ - Build offset map from a pair of offset map by concatenating and adding offsets of special tokens. - - A GAUAlpha offset_mapping has the following format: - - - single sequence: ``(0,0) X (0,0)`` - - pair of sequences: ``(0,0) A (0,0) B (0,0)`` - - Args: - offset_mapping_ids_0 (List[tuple]): - List of wordpiece offsets to which the special tokens will be added. - offset_mapping_ids_1 (List[tuple], optional): - Optional second list of wordpiece offsets for offset mapping pairs. Defaults to None. - - Returns: - List[tuple]: A list of wordpiece offsets with the appropriate offsets of special tokens. - """ - if offset_mapping_1 is None: - return [(0, 0)] + offset_mapping_0 + [(0, 0)] - - return [(0, 0)] + offset_mapping_0 + [(0, 0)] + offset_mapping_1 + [(0, 0)] - - def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): - """ - Create a mask from the two sequences passed to be used in a sequence-pair classification task. - - A GAUAlpha sequence pair mask has the following format: - :: - - 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 - | first sequence | second sequence | - - If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s). - - Args: - token_ids_0 (List[int]): - A list of `inputs_ids` for the first sequence. - token_ids_1 (List[int], optional): - Optional second list of IDs for sequence pairs. Defaults to None. - - Returns: - List[int]: List of token_type_id according to the given sequence(s). - """ - _sep = [self.sep_token_id] - _cls = [self.cls_token_id] - if token_ids_1 is None: - return len(_cls + token_ids_0 + _sep) * [0] - return len(_cls + token_ids_0 + _sep) * [0] + len(token_ids_1 + _sep) * [1] - - def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): - """ - Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding - special tokens using the tokenizer ``encode`` methods. - - Args: - token_ids_0 (List[int]): - A list of `inputs_ids` for the first sequence. - token_ids_1 (List[int], optional): - Optional second list of IDs for sequence pairs. Defaults to None. - already_has_special_tokens (bool, optional): Whether or not the token list is already - formatted with special tokens for the model. Defaults to None. - - Returns: - List[int]: The list of integers either be 0 or 1: 1 for a special token, 0 for a sequence token. - """ - - if already_has_special_tokens: - if token_ids_1 is not None: - raise ValueError( - "You should not supply a second sequence if the provided sequence of " - "ids is already formatted with special tokens for the model." - ) - return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) - - if token_ids_1 is not None: - return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] - return [1] + ([0] * len(token_ids_0)) + [1] - - def get_vocab(self): - return dict(self.vocab.token_to_idx, **self.added_tokens_encoder) diff --git a/pyproject.toml b/pyproject.toml index 31ca3ed42a..93668e5843 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ testpaths = [ "tests/data", "tests/dataaug", "tests/datasets", - "tests/embeddings", "tests/experimental", "tests/generation", "tests/layers", diff --git a/slm/applications/question_answering/unsupervised_qa/README.md b/slm/applications/question_answering/unsupervised_qa/README.md index 9dcf996616..8651c2e3ab 100644 --- a/slm/applications/question_answering/unsupervised_qa/README.md +++ b/slm/applications/question_answering/unsupervised_qa/README.md @@ -28,7 +28,7 @@ ## 简介 问答（QA）系统中最关键的挑战之一是标记数据的稀缺性，这是因为对目标领域获取问答对或常见问答对（FAQ）的成本很高，需要消耗大量的人力和时间。由于上述制约，这导致检索式问答系统落地困难，解决此问题的一种方法是依据问题上下文或大量非结构化文本自动生成的 QA 问答对。 -在此背景下，无监督检索式问答系统（即问答对自动生成智能检索式问答），基于 PaddleNLP[问题生成](../../../examples/question_generation/README.md)、[UIE](../../../model_zoo/uie/README.md)、[检索式问答](https://github.com/PaddlePaddle/PaddleNLP/blob/release/2.8/applications/question_answering/supervised_qa/faq_finance/README.md)，支持以非结构化文本形式为上下文自动生成 QA 问答对，生成的问答对语料可以通过无监督的方式构建检索式问答系统。 +在此背景下，无监督检索式问答系统（即问答对自动生成智能检索式问答），基于 PaddleNLP[问题生成](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/question_generation/README.md)、[UIE](../../../model_zoo/uie/README.md)、[检索式问答](https://github.com/PaddlePaddle/PaddleNLP/blob/release/2.8/applications/question_answering/supervised_qa/faq_finance/README.md)，支持以非结构化文本形式为上下文自动生成 QA 问答对，生成的问答对语料可以通过无监督的方式构建检索式问答系统。 若开发者已有 FAQ 语料，请参考[supervised_qa](https://github.com/PaddlePaddle/PaddleNLP/blob/release/2.8/applications/question_answering/supervised_qa)。 diff --git a/slm/examples/README.md b/slm/examples/README.md index 040837387d..2f0d51f000 100644 --- a/slm/examples/README.md +++ b/slm/examples/README.md @@ -12,21 +12,21 @@ PaddleNLP provides rich application examples covering mainstream NLP task to hel | lexical_analysis | [词法分析 (Lexical Analysis)](./lexical_analysis/) | | dependency_parsing | [句法依存分析 (Dependency Parsing)](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/dependency_parsing/ddparser) | | text_to_sql | [语义解析 (Semantic Parsing/Text to SQL)](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/text_to_sql):star: | -| text_classification | [文本分类 (Text Classification)](./text_classification/) | +| text_classification | [文本分类 (Text Classification)](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/slm/applications/text_classification) | | text_matching | [文本匹配 (Text Matching)](./text_matching/) | -| text_generation | [文本生成 (Text Generation)](./text_generation/) | +| text_generation | [文本生成 (Text Generation)](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/text_generation/) | | text_summarization | [文本摘要 (Text Summarization)](./text_summarization/) | -| text_correction | [文本纠错 (Text Correction)](./text_correction/):star: | +| text_correction | [文本纠错 (Text Correction)](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/text_correction/):star: | | semantic_indexing | [语义索引 (Semantic Indexing)](./semantic_indexing/) | | information_extraction | [信息抽取 (Information Extraction)](./information_extraction/) | -| question_generation | [问题生成 (Question Generation)](./question_generation/) | +| question_generation | [问题生成 (Question Generation)](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/question_generation/) | ## NLP 系统应用 (NLP System Applications) | 目录 Folder | 任务 Task | |:------------------------------|------------------------------------------------------------------------------| | sentiment_analysis | [情感分析 (Sentiment Analysis)](./sentiment_analysis/):star2: | -| dialogue | [通用对话 (General Dialogue System)](./dialogue/) | +| dialogue | [通用对话 (General Dialogue System)](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/dialogue/) | | machine_translation | [文本翻译 (Machine Translation)](./machine_translation/) | | simultaneous_translation | [同声翻译 (Simultaneous Translation)](./simultaneous_translation/) | | machine_reading_comprehension | [阅读理解 (Machine Reading Comprehension)](./machine_reading_comprehension/) | @@ -39,4 +39,4 @@ PaddleNLP provides rich application examples covering mainstream NLP task to hel | text_to_knowledge | [解语知识关联框架 (Text Knowledge Mining)](./text_to_knowledge/):star2: | | model_compression | [模型压缩 (Model Compression）](./model_compression/) | | text_graph | [文本图学习 (Text Graph Learning)](./text_graph/erniesage/) | -| contrastive_learning | [对比学习 (Contrastive Learning)](./contrastive_learning/) | \ No newline at end of file +| contrastive_learning | [对比学习 (Contrastive Learning)](./contrastive_learning/) | diff --git a/slm/examples/RLHF/README.md b/slm/examples/RLHF/README.md deleted file mode 100644 index 8a4216ab59..0000000000 --- a/slm/examples/RLHF/README.md +++ /dev/null @@ -1,241 +0,0 @@ -# RLHF PPO - -提供了基于强化学习 PPO 算法对 LLM 进行人类偏好对齐的代码及完整使用示例，支持**3D 分布式并行训练以及 rollout 阶段使用预测优化进行生成加速**。其中 PPO 代码实现细节参考了 [PKU-Alignment/safe-rlhf](https://github.com/PKU-Alignment/safe-rlhf)（PKU Beaver） 中的 PPO 实现，支持 reward normalization、pretraining loss 等常用的 PPO 稳定训练策略；示例使用 PKU-Alignment/safe-rlhf 提供的部分数据集和模型。后续将持续完善扩展，支持更好效果、更低成本、更高性能、更大规模的 RLHF 能力。 - -## 快速开始 - -项目整体组织结构如下： - -``` -. -├── reward_main.py # reward model训练脚本 -├── reward_config.json # reward model训练配置文件 -├── reward_trainer.py # reward训练执行器py脚本 -├── ppo_main.py # RLHF训练脚本 -├── ppo_config.json # RLHF训练配置文件 -├── ppo_trainer.py # RLHF训练执行器py脚本 -├── ppo_config.json # RLHF训练配置文件 -├── trainer_utils.py # Trainer补丁及工具py脚本 -├── infer_utils.py # 生成加速工具py脚本 -├── data # 数据集相关目录 -│ └── base.py # 数据集基类及工具py文件 -│ └── alpaca.py # alpaca(raw)数据集py文件 -│ └── safe_rlhf.py # safe_rlhf(raw)数据集py文件 -│ └── preference.py # 偏好数据集py文件 -│ └── prompt_only.py # prompt only数据集py文件 -│ └── supervised.py # supervised数据集py文件 -├── models # 模型相关目录 -│ └── score_model_utils.py # score model基类及工具py文件 -│ └── score_model.py # score model模型定义py文件 -│ └── ppo_model_utils.py # PPO loss等模型策略py文件 -│ └── pp_model_utils.py # 流水线并行补丁及工具py文件 -│ └── model_pp.py # 流水线并行模型py文件 -│ └── infer_model_utils.py # 预测加速模型补丁及工具py文件 -└── README.md -``` - -### 环境准备 - -- Python >= 3.10 -- PaddlePaddle >= 2.6.0 -- PaddleNLP 最新版本 - -如需使用生成加速功能，需要安装 [paddlenlp_ops](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/csrc) ，请使用 `git clone https://github.com/PaddlePaddle/PaddleNLP.git` 克隆 PaddleNLP 代码库并且将 PaddleNLP/llm 目录的路径加入 PYTHONPATH（后续将进行完善）。安装 paddlenlp_ops 后训练时将直接开启生成加速（开启流水线并行时不支持生成加速），否则使用原生动态图进行生成。 - -### 数据准备 - -PPO 训练包括 Supervised Fine-Tuning、Reward Model Fine-Tuning、RLHF 三个阶段（可见下文训练部分），会涉及到多个数据集，下面分别介绍并给出自定义数据的方法。 - -#### Supervised Fine-Tuning 数据 -同[LLM 精调](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm#2-%E7%B2%BE%E8%B0%83)，可以直接参考对应内容进行数据准备。 - -#### Reward Model Fine-Tuning 数据 -Reward Model Fine-Tuning 阶段需要使用人类偏好数据。示例使用 PKU-Alignment/safe-rlhf 提供的 [PKU-Alignment/PKU-SafeRLHF-30K](https://huggingface.co/datasets/PKU-Alignment/PKU-SafeRLHF-30K) 数据集，下面是其中一条样本，这里使用其中的`prompt、response_0、response_1、better_response_id`字段来组织偏好数据（safe 字段，该数据集将 helpful 和 harmless 分开标注，示例这里使用其 helpful 标注）。 - -```json -{ - "prompt": "Is it possible to purchase a complete chess set at a pawn shop?", - "response_0": "Yes, it is possible to purchase a complete chess set at a pawn shop.", - "response_1": "Yes, it is possible to purchase a complete chess set at a pawn shop. Pawn shops may carry chess sets that have been either inherited or previously borrowed. There is also a possibility of finding individual chess pieces such as rooks, knights, bishops, queens, and kings. If you cannot find a complete set, you may be able to purchase the various chess pieces separately.", - "is_response_0_safe": true, - "is_response_1_safe": true, - "better_response_id": 1, - "safer_response_id": 1 -} -``` - -#### RLHF 数据 -RLHF 阶段使用 prompt only 数据，另外可以可选的提供额外的监督数据用于构建 LM 损失约束 RLHF 训练。示例使用 [PKU-Alignment/PKU-SafeRLHF](https://huggingface.co/datasets/PKU-Alignment/PKU-SafeRLHF) 数据集（同样是人类偏好数据集，这里只使用其 prompt 字段并对 prompt 去重）。此外还使用了 [tatsu-lab/alpaca](https://huggingface.co/datasets/tatsu-lab/alpaca)的数据来构建额外的损失项。 - -以上示例数据集在训练时将自动下载缓存使用。 - -#### 自定义数据 -数据定义围绕 `RawSample` 和 `RawDataset` 两个预置的类进行；`RawSample` 提供了数据样本级别接入协议规范，`RawDataset` 提供了数据集级别接入协议规范；按照下面介绍的规范接入，即可通过预置的 `SupervisedDataset`、`PreferenceDataset`、`PromptOnlyDataset` 三类 RLHF 训练所需数据接口来使用自定义数据。 - -自定义数据集需要： -- 继承 `RawDataset` ，并定义类属性 `NAME` 用于注册数据集。 -- 实现 `__init__` 方法（加载数据），`__getitem__` 方法（根据 index 获取样本并转换为 `RawSample` 对象返回）、`__len__` 方法（数据集大小）。 - -示例如下： - -```python -from datasets import load_dataset -from data import RawDataset, RawSample - -class MyRawDataset(RawDataset): - NAME = 'my-dataset-name' - - def __init__(self, path=None) -> None: - # Load a dataset from Hugging Face or any other data source - # self.data = load_dataset(path or 'my-organization/my-dataset')['train'] - self.data = [{ - 'col1': 'question', - 'col2': 'answer1', - 'col3': 'answer2', - 'col4': 1, # score of answer1 - 'col5': 2 # score of answer2 - }] * 10 # dummy data for example - - def __getitem__(self, index: int) -> RawSample: - data = self.data[index] - # Construct a `RawSample` dictionary from your custom dataset item - return RawSample( - input=data['col1'], - answer=data['col2'], - other_answer=data['col3'], - better=float(data['col4']) > float(data['col5']), - ) - - def __len__(self) -> int: - return len(self.data) # dataset size -``` - -其中 `RawSample` 是整个 RLHF 训练过程用到的几种数据类型的超集，如下所示，其可以桥接各训练阶段所需样本类型。在自定义数据时，对于 SFT 数据使用 `RawSample` 的 `(input, answer)` 字段；对于人类偏好数据使用 `RawSample` 的 `(input, answer, other_answer, better)` 字段；对于 prompt only 数据，使用 `RawSample` 的 `(input)`字段。 - -```python -class RawSample(TypedDict, total=False): - """Raw sample type. - - For SupervisedDataset, should provide (input, answer) or (dialogue). - For PreferenceDataset, should provide (input, answer, other_answer, better). - For SafetyPreferenceDataset, should provide (input, answer, other_answer, safer, is_safe, is_other_safe). - For PromptOnlyDataset, should provide (input). - - When input is a list, it would be processed as a dialogue. - """ - - # Texts - input: NotRequired[str | list[str]] # either `input` or `dialogue` should be provided - """User input text.""" - answer: NotRequired[str] - """Assistant answer text.""" - other_answer: NotRequired[str] - """Other assistant answer text via resampling.""" - dialogue: NotRequired[list[str]] # either `input` or `dialogue` should be provided - """Dialogue history.""" - - # Flags - better: NotRequired[bool] - """Whether ``answer`` is better than ``other_answer``.""" - safer: NotRequired[bool] - """Whether ``answer`` is safer than ``other_answer``.""" - is_safe: NotRequired[bool] - """Whether ``answer`` is safe.""" - is_other_safe: NotRequired[bool] - """Whether ``other_answer`` is safe.""" -``` - -如此定义的数据集将可以通过预置接口根据 `NAME` 来使用，当前内置支持`"PKU-SafeRLHF/train", "PKU-SafeRLHF/test", "PKU-SafeRLHF-30K/train", "PKU-SafeRLHF-30K/test", "PKU-SafeRLHF-10K/train", "alpaca"` 几个数据集。另外还支持使用多个数据集并指定数据比例，我们可以按照需要为每个阶段训练准备多份数据集。示例如下： - -```python -from paddlenlp.transformers import AutoTokenizer -from data import PreferenceDataset - -tokenizer = AutoTokenizer.from_pretrained('facebook/llama-7b') -dataset = PreferenceDataset({ - 'alpaca': 0.75, - 'my-dataset-name': 0.5 -}, tokenizer) -``` - -### 训练 - -PPO 完整的训练过程包括以下 3 个阶段，如下图所示（来自[DeepSpeed-Chat](https://github.com/microsoft/DeepSpeedExamples/blob/master/applications/DeepSpeed-Chat)）： - -

- -

- -1. Supervised Fine-Tuning (SFT) - -同[LLM 精调](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm#2-%E7%B2%BE%E8%B0%83)，可以直接参考对应内容进行训练并使用其产出模型。 - -2. Reward Model Fine-Tuning - -使用 `reward_main.py` 脚本根据 `reward_config.json` 训练奖励模型 - -``` -python -u -m paddle.distributed.launch reward_main.py ./reward_config.json -``` - -`reward_config.json` 中的绝大部分参数释义同[LLM 精调](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm#2-%E7%B2%BE%E8%B0%83)，不再赘述；稍有区别的是 `train_datasets`/`eval_datasets` 分别使用数据集定义注册时的`NAME`属性给出训练和验证集。另外对于奖励模型训练有以下特殊参数配置及释义（使用 PKU-Alignment/PKU-SafeRLHF 中的默认值）： - -- `normalize_score_during_training`：是否在训练过程中对奖励进行 normalize，默认为 `False`。 -- `normalizer_type`：使用 normalizer 时计算 mean、var 的方式，可选`"RunningMeanStd", "ExponentialMovingAverage"`。 -- `normalizer_momentum`：使用 `ExponentialMovingAverage` normalizer 时指定的 momentum ，默认为 `0.9`。 -- `loss_type`：使用 token 级或是 sequence 级 loss 进行奖励模型训练，可选`"token-wise", "sequence-wise"`，默认为 `"sequence-wise"`。 -- `regularization`：奖励模型训练目标中对奖励的正则化系数，默认为 `0.001`。 - -3. RLHF： - -RLHF 阶段需要 actor model、reference model、critic model、reward model 四个模型；actor-model/reference-model 使用 SFT 模型进行 initialize/frozen；critic-model/reward-model 使用 reward 模型进行 initialize/frozen (另外注意若 SFT 使用 LoRA 请先将 LoRA 权重合并）。这里使用 PKU-Alignment/PKU-SafeRLHF 提供的 SFT 模型（[PKU-Alignment/alpaca-7b-reproduced](https://huggingface.co/PKU-Alignment/alpaca-7b-reproduced)）和 reward 模型（[PKU-Alignment/beaver-7b-v1.0-reward](https://huggingface.co/PKU-Alignment/beaver-7b-v1.0-reward)，注意该模型只关注 helpful 未考量 harmless）作为示例，使用 `ppo_main.py` 脚本根据 `ppo_config.json` 进行 RLHF 训练。 - -``` -# 类型提升 warning 暂时通过 loglevel 屏蔽，待后续修复 -GLOG_minloglevel=2 python -u -m paddle.distributed.launch ppo_main.py ./ppo_config.json -``` - -`ppo_config.json` 中的绝大部分参数释义同[LLM 精调](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm#2-%E7%B2%BE%E8%B0%83)，不再赘述，重点给出以下参数配置及释义（使用 PKU-Alignment/PKU-SafeRLHF 中的默认值）： - -- `train_datasets`：使用数据集定义注册时的`NAME`属性给出训练集。 -- `eval_datasets`：使用数据集定义注册时的`NAME`属性给出验证集。 -- `ptx_datasets`：使用数据集定义注册时的`NAME`属性给出 ptx-loss 使用的数据集，未提供时将不使用 ptx-loss。 -- `actor_model_name_or_path`：actor-model/reference-model 用来 initialize/frozen 的模型名称或目录。 -- `reward_model_name_or_path`：reward-model 的模型名称或目录。 -- `reward_critic_model_name_or_path`：critic-model 的模型名称或目录，未提供时将使用`reward_model_name_or_path`进行 critic-model 的初始化。 -- `per_device_prompt_batch_size`：训练时 prompt only 数据集读取用于 rollout 生成的批次大小（每张卡）。 -- `per_device_train_batch_size`：根据 prompt 进行生成及训练使用的批次大小（每张卡）。 -- `num_return_sequences`：生成时每个 prompt 生成的回复个数，即 `GenerationConfig.num_return_sequences`，所有回复都将用来训练。 -- `temperature`：生成采样时使用的 `temperature` ，即 `GenerationConfig.temperature`。 -- `top_p`：生成采样时 top-p-filtering 阈值，即 `GenerationConfig.top_p`。 -- `repetition_penalty`：生成采样时长度惩罚系数，即 `GenerationConfig.repetition_penalty`。 -- `update_iters`：一次生成的数据被使用的次数。 -- `kl_coeff`：对 reward 进行 KL-Penalty 的系数。 -- `clip_range_score`：对 reward 进行裁剪的阈值。 -- `clip_range_value`：critic model（value function）对当前 sequence 的新值与 Experience Buffer 中旧值的差距超过该范围将进行裁剪。 -- `clip_range_ratio`：将当前 sequence 的新概率与 Experience Buffer 中旧概率比值裁剪到`(1-clip_range_ratio, 1+clip_range_ratio)`范围（PPO-Clip）。 -- `ptx_coeff`： 预训练损失项 ptx-loss 的系数。 - -另外所有 [`TrainingArguments` 支持参数配置](https://paddlenlp.readthedocs.io/zh/latest/trainer.html#trainingarguments)将为 actor-model 和 critic-model 的训练复用（如`sharding_stage`），除单独提供了 `critic_learning_rate/critic_weight_decay/critic_lr_scheduler_type/critic_warmup_ratio/critic_recompute` 这些参数支持为 critic-model 训练单独指定相应配置。actor-model 和 critic-model 的 checkpoints 将分别保存在 `outpt_dir` 所指定目录的 policy 和 value 文件夹下。 - -此外为了支持更高性、更大规模的 RLHF 训练提供了以下特殊参数配置，可以按需使用： -- `use_fusemt`：安装 paddlenlp_ops 后将在 rollout 生成时开启生成加速（开启流水线并行时不支持生成加速），通过此设置可以禁用生成加速。 -- `eval_mode`：支持为空或者设置为 "single"、"tensor_parallel"；通常可以在使用流水线并行训练时设置为"tensor_parallel"，以此在 rollout 生成阶段使用非流水线并行模型并进行生成加速。 -- `offload_level`：支持设置为"freeze_model"、"optimizer"、"train_model"或者同时使用(空格分隔），分别指示 reward+reference 两个冻结模型、actor+critic 两个训练模型的优化器状态和模型参数的 offload/reload，用于在不同阶段 model/optimizer 使用结束后及时 offload 并在下次使用时 reload 相应参数权重以节省显存。 - -另外注意，在使用流水线并行时（pipeline_parallel_degree 大于1）建议将 `dataloader_drop_last` 设置为 true, 以此避免不同 batch size 带来的问题。 - - - - -### 推理 - -训练完成后可以直接使用 `outpt_dir` 所指定目录中 policy 文件夹下的 checkpoints 按照[LLM 推理](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/llm#4-%E6%8E%A8%E7%90%86)部分的介绍来进行推理，请参考相应部分内容。 - -## Acknowledge - -我们借鉴了[PKU-Alignment/safe-rlhf](https://github.com/PKU-Alignment/safe-rlhf)（PKU Beaver）的优秀设计实现，在此对其作者表示感谢。 - -## 参考文献 -- Zheng R, Dou S, Gao S, et al. Secrets of rlhf in large language models part i: Ppo[J]. arXiv preprint arXiv:2307.04964, 2023. -- Dai J, Pan X, Sun R, et al. Safe rlhf: Safe reinforcement learning from human feedback[J]. arXiv preprint arXiv:2310.12773, 2023. diff --git a/slm/examples/RLHF/comm_utils.py b/slm/examples/RLHF/comm_utils.py deleted file mode 100644 index de077c65db..0000000000 --- a/slm/examples/RLHF/comm_utils.py +++ /dev/null @@ -1,403 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -import paddle -import paddle.distributed as dist - -from paddlenlp.trainer.plugins.unified_checkpoint import flatten_list -from paddlenlp.trainer.trainer import Trainer, logger -from paddlenlp.trainer.utils.helper import nested_broadcast_tensor_with_empty -from paddlenlp.utils.distributed import distributed_gather - -global_dev_id = 0 if paddle.get_device() == "cpu" else int(paddle.get_device().split(":")[1]) - - -def offload_tensor_to_cpu(tensors): - if isinstance(tensors, dict): - for _, v in tensors.items(): - offload_tensor_to_cpu(v) - elif isinstance(tensors, paddle.Tensor): - if tensors.place.is_gpu_place(): - cpu_tensor = tensors._copy_to(paddle.CUDAPinnedPlace(), False) - tensors.value().get_tensor()._share_data_with(cpu_tensor.value().get_tensor()) - else: - logger.warning(f"Can't parse for type {type(tensors)}") - return tensors - - -def reload_tensor_to_gpu(tensors): - if isinstance(tensors, dict): - for _, v in tensors.items(): - reload_tensor_to_gpu(v) - elif isinstance(tensors, paddle.Tensor): - if tensors._is_initialized() and not tensors.place.is_gpu_place(): - gpu_tensor = tensors._copy_to(paddle.CUDAPlace(global_dev_id), False) - tensors.value().get_tensor()._share_data_with(gpu_tensor.value().get_tensor()) - else: - logger.warning(f"Can't parse for type {type(tensors)}") - return tensors - - -def cleanup_tensor_space(tensors): - if isinstance(tensors, dict): - for _, v in tensors.items(): - cleanup_tensor_space(v) - elif isinstance(tensors, paddle.Tensor): - tensors._clear_data() - else: - logger.warning(f"Can't parse for type {type(tensors)}") - return tensors - - -def data_group_split(tensors, group): - if group is None: - return tensors - if isinstance(tensors, (list, tuple)): - return type(tensors)(data_group_split(t, group) for t in tensors) - elif isinstance(tensors, dict): - new_dict = {} - for k, v in tensors.items(): - new_dict[k] = data_group_split(v, group) - return new_dict - elif isinstance(tensors, paddle.Tensor): - return tensors.split(group.nranks)[group.rank] - else: - logger.warning(f"Can't parse for type {type(tensors)}") - return tensors - - -def data_group_merge(tensors, group): - if group is None: - return tensors - - if isinstance(tensors, (list, tuple)): - return type(tensors)(data_group_merge(t, group) for t in tensors) - elif isinstance(tensors, dict): - new_dict = {} - for k, v in tensors.items(): - new_dict[k] = data_group_merge(v, group) - return new_dict - elif isinstance(tensors, paddle.Tensor): - tensor_list = [] - all_gather_nd(tensor_list, tensors, group=group, padded=True) - return paddle.concat(tensor_list) - else: - logger.warning(f"Can't parse for type {type(tensors)}") - return tensors - - -def group_rank_guard(group, rank=0): - def decorator(func): - def wrapper_func(*args, **kwargs): - if group.rank == rank: - ret = func(*args, **kwargs) - dist.barrier() - else: - ret = None - dist.barrier() - ret = nested_broadcast_tensor_with_empty(ret, group=group) - return ret - - return wrapper_func - - return decorator - - -def repad_rl_batches(batches, input_lengths): - if batches.get("position_ids", None) is not None: - v = batches["position_ids"] - for x in range(v.shape[0]): - v[x, input_lengths[x] :] = 1 - batches["position_ids"] = v - for key in list(batches.keys()): - if batches[key].shape[0] != input_lengths.shape[0]: - batches[key] = batches[key].mean() - - return batches - - -# https://stackoverflow.com/questions/12594148/skipping-execution-of-with-block -class SkipWithBlock(Exception): - pass - - -class SkipContextManager: - def __init__(self, skip): - self.skip = skip - - def __enter__(self): - if self.skip: - sys.settrace(lambda *args, **keys: None) - frame = sys._getframe(1) - frame.f_trace = self.trace - - def trace(self, frame, event, arg): - raise SkipWithBlock() - - def __exit__(self, type, value, traceback): - if type is None: - return # No exception - if issubclass(type, SkipWithBlock): - return True # Suppress special SkipWithBlock exception - - -def all_gather_nd(tensor_list, tensor, group=None, padded=False): - """ - Gathers tensor arrays of different lengths in a list. - The length dimension is 0. This supports any number of extra dimensions in the tensors. - All the other dimensions should be equal between the tensors. - - Args: - tensor (Tensor): Tensor to be broadcast from current process. - - Returns: - (Tensor): output list of tensors that can be of different sizes - """ - if len(tensor.shape) == 0: - tensor = tensor.reshape([1]) - dist.all_gather(tensor_list, tensor, group=group) - return tensor_list - - world_size = group.nranks - local_size = paddle.to_tensor(tensor.shape, place=tensor.place) - all_sizes = [paddle.zeros_like(local_size) for _ in range(world_size)] - dist.all_gather(all_sizes, local_size, group=group) - - # max_length = max(size[0] for size in all_sizes) - - # length_diff = max_length.item() - local_size[0].item() - # if length_diff: - # pad_size = (length_diff, *tensor.size()[1:]) - # padding = paddle.zeros(pad_size, place=tensor.place(), dtype=tensor.dtype) - # tensor = padle.concat((tensor, padding)) - - max_length = max(size[-1] for size in all_sizes) - - length_diff = max_length.item() - local_size[-1].item() - if length_diff: - pad_size = (*tensor.shape[:-1], length_diff) - padding = paddle.zeros(pad_size, dtype=tensor.dtype) - tensor = paddle.concat([tensor, padding], axis=-1) - - all_tensors_padded = [] - dist.all_gather(all_tensors_padded, tensor, group=group) - # all_tensors = [] - if padded: - tensor_list.extend(all_tensors_padded) - return all_tensors_padded - - for tensor_, size in zip(all_tensors_padded, all_sizes): - tensor_list.append(tensor_[..., : size[-1]]) - return tensor_list - - -def export_evaluate_model(self: Trainer, train_model, eval_model, **kwargs): - if eval_model is None: - return None - - with_offload = kwargs.pop("with_offload", False) - train_tp_size = max(train_model.config.tensor_parallel_degree, 1) - eval_tp_size = max(eval_model.config.tensor_parallel_degree, 1) - eval_tp_rank = max(eval_model.config.tensor_parallel_rank, 0) - - hcg = dist.fleet.get_hybrid_communicate_group() - tp_group = hcg.get_model_parallel_group() - pp_group = hcg.get_pipe_parallel_group() - sd_group = hcg.get_sharding_parallel_group() - dp_group = hcg.get_data_parallel_group() - - global_rank = paddle.distributed.get_rank() - - train_state_dict = train_model.state_dict() - eval_state_dict = eval_model.state_dict() - - if dp_group.rank <= 0 and sd_group.rank <= 0: - train_pp_size = pp_group.nranks - if eval_tp_size > 1 and train_tp_size != eval_tp_size: - raise ValueError("Only support for the same tensor_parallel_degree for train and eval model for now.") - - # 单卡情况 - # tp->single - # tp+pp -> single - if eval_tp_size == 1: - if train_pp_size == 1 and train_tp_size > 1: - # tp ->single - logger.error("using tp to single eval model.") - # state = train_model.merge_tensor_parallel() - tp_actions = train_model.get_tensor_parallel_convert_actions( - train_model.config, - loaded_state_dict_keys=eval_state_dict.keys(), - is_split=False, - ignore_error=False, - ) - - is_dst = global_rank == 0 - for key in eval_state_dict.keys(): - tensor = train_state_dict[key] - if key in tp_actions: - ret = distributed_gather(tensor, dst=0, group=tp_group, offload=False) - action = tp_actions.pop(key) - tensor = action(ret) if is_dst else None - else: - tensor = tensor._copy_to(paddle.CPUPlace(), False) if is_dst else None - - if tensor is not None: - eval_state_dict[key].set_value(tensor) - - if not eval_state_dict[key]._is_initialized(): - v = eval_state_dict[key] - t = paddle._C_ops.full_like(v, 0, v.dtype, paddle.CUDAPlace(global_dev_id)) - v.get_tensor()._share_data_with(t.get_tensor()) - - if with_offload: - offload_tensor_to_cpu(train_state_dict[key]) - else: - # single to single - # tp+pp -> single - raise ValueError("Not support yet.") - - def create_send_recv_table(train_keys, eval_keys): - recv_table = [] - send_table = [] - if pp_group.rank == 0: - for key in eval_keys: - recv_table.append((key, global_rank)) - - for key in train_keys: - send_table.append((key, global_rank)) - - all_recv, all_send = [], [] - paddle.distributed.all_gather_object(all_recv, [recv_table], group=pp_group) - paddle.distributed.all_gather_object(all_send, [send_table], group=pp_group) - all_recv = flatten_list(all_recv) - all_send = flatten_list(all_send) - - send_dict = {} - for k, v in all_send: - send_dict[k] = v - - table = [] - for k, v in all_recv: - # key, send, recv - table.append([k, send_dict.pop(k), v]) - assert len(send_dict) == 0, f"Some key can't be recv {send_dict.keys()}" - return table - - # pp0tp0 -> pp0tp0 - # pp0tp1 -> pp0tp1 - # pp1tp0 -> pp0tp0 - # pp1tp1 -> pp0tp1 - - # tp情况 - # tp+pp->tp - self.timers and self.timers("export-merge-pp").start() - if eval_tp_size > 1 and train_pp_size > 1: - table = create_send_recv_table(train_state_dict.keys(), eval_state_dict.keys()) - - for key, src_rank, dst_rank in table: - # Init tensor for model is cleaned - if not eval_state_dict[key]._is_initialized(): - v = eval_state_dict[key] - t = paddle._C_ops.full_like(v, 0, v.dtype, paddle.CUDAPlace(global_dev_id)) - v.get_tensor()._share_data_with(t.get_tensor()) - - if src_rank == dst_rank and global_rank == src_rank: - eval_state_dict[key].copy_(train_state_dict[key], True) - else: - if global_rank == src_rank: - dist.stream.send(train_state_dict[key], dst=dst_rank) - - if global_rank == dst_rank: - dist.stream.recv(eval_state_dict[key], src=src_rank) - - # Offload train model if need - if global_rank == src_rank and with_offload: - offload_tensor_to_cpu(train_state_dict[key]) - - self.timers and self.timers("export-merge-pp").stop() - self.timers and self.timers("export-broadcast-pp").start() - if pp_group.nranks > 1: - paddle.distributed.parallel.sync_params_buffers( - eval_model, comm_group=pp_group, src_rank=pp_group.ranks[0], fuse_params=False - ) - self.timers and self.timers("export-broadcast-pp").stop() - else: - # 其他 DP rank 的state dict, 适配 offload 和初始化 - self.timers and self.timers("export-offload-and-init").start() - if with_offload: - for key in list(train_state_dict.keys()): - offload_tensor_to_cpu(train_state_dict[key]) - for k, v in eval_state_dict.items(): - if not v._is_initialized(): - t = paddle._C_ops.full_like(v, 0, v.dtype, paddle.CUDAPlace(global_dev_id)) - v.get_tensor()._share_data_with(t.get_tensor()) - self.timers and self.timers("export-offload-and-init").stop() - - paddle.distributed.barrier() - self.timers and self.timers("export-broadcast-sd-dp").start() - if eval_tp_size == 1: - for _, tensor in eval_state_dict.items(): - paddle.distributed.broadcast(tensor, src=0, group=None, sync_op=True) - else: - if sd_group.nranks > 1: - if dp_group.rank <= 0: - paddle.distributed.parallel.sync_params_buffers( - eval_model, comm_group=sd_group, src_rank=sd_group.ranks[0], fuse_params=False - ) - if dp_group.nranks > 1: - paddle.distributed.parallel.sync_params_buffers( - eval_model, comm_group=dp_group, src_rank=dp_group.ranks[0], fuse_params=False - ) - self.timers and self.timers("export-broadcast-sd-dp").stop() - # paddle.save(eval_state_dict, f"./tmp/eval_{sd_group.rank}_tp_{eval_tp_rank}_pp_{pp_group.rank}.pdparams") - # paddle.save(train_state_dict, f"./tmp/train_{sd_group.rank}_tp_{tp_group.rank}_pp_{pp_group.rank}.pdparams") - # paddle.distributed.barrier() - # exit(-1) - - old_dp_workers = self.args.world_size // (max(sd_group.nranks, 1) * max(dp_group.nranks, 1)) - group_nums = self.args.logical_process_index // old_dp_workers * eval_tp_size + eval_tp_rank - - if not hasattr(self, "_policy_model_eval_group") or self._policy_model_eval_group is None: - self._policy_model_eval_group = create_data_trans_group(global_rank, group_nums) - - return None - - -def create_data_trans_group(global_rank, group_nums): - all_split_table = [] - paddle.distributed.all_gather_object(all_split_table, [(global_rank, group_nums)]) - all_split_table = flatten_list(all_split_table) - split_dict = {} - for k, v in all_split_table: - split_dict[k] = v - - split_ranks = {} - for k, v in all_split_table: - if v in split_ranks: - split_ranks[v].append(k) - else: - split_ranks[v] = [k] - - group = None - for k, ranks in split_ranks.items(): - gp = paddle.distributed.new_group(ranks=ranks) - if global_rank in ranks: - group = gp - - return group - - -Trainer.export_evaluate_model = export_evaluate_model diff --git a/slm/examples/RLHF/data/__init__.py b/slm/examples/RLHF/data/__init__.py deleted file mode 100644 index 227b120a4b..0000000000 --- a/slm/examples/RLHF/data/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.io import Dataset - -from .alpaca import * -from .base import * -from .preference import * -from .prompt_only import * -from .safe_rlhf import * -from .supervised import * - - -class DummyDataset(Dataset): - def __init__(self, length: int) -> None: - self.length = length - - def __len__(self) -> int: - return self.length - - def __getitem__(self, index: int): - return {} diff --git a/slm/examples/RLHF/data/alpaca.py b/slm/examples/RLHF/data/alpaca.py deleted file mode 100644 index 2ce894b3df..0000000000 --- a/slm/examples/RLHF/data/alpaca.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Stanford Alpaca dataset for supervised instruction fine-tuning.""" - -from __future__ import annotations - -from datasets import load_dataset - -from .base import RawDataset, RawSample - -__all__ = ["AlpacaDataset"] - - -class AlpacaDataset(RawDataset): - NAME: str = "alpaca" - ALIASES: tuple[str, ...] = ("stanford-alpaca",) - - def __init__(self, path: str | None = None) -> None: - self.data = load_dataset(path or "tatsu-lab/alpaca", split="train") - - def __getitem__(self, index: int) -> RawSample: - data = self.data[index] - input = ( # pylint: disable=redefined-builtin - " ".join((data["instruction"], data["input"])) if data["input"] else data["instruction"] - ) - answer = data["output"] - return RawSample(input=input, answer=answer) - - def __len__(self) -> int: - return len(self.data) diff --git a/slm/examples/RLHF/data/base.py b/slm/examples/RLHF/data/base.py deleted file mode 100644 index 6d0f4717c4..0000000000 --- a/slm/examples/RLHF/data/base.py +++ /dev/null @@ -1,520 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Base dataset class.""" - -from __future__ import annotations - -import abc -import bisect -import copy -import os -import warnings -from fractions import Fraction -from typing import Any, Callable, ClassVar, Collection, Iterable, Iterator, List -from weakref import WeakValueDictionary - -import numpy as np -import paddle -from paddle.io import Dataset, IterableDataset, Subset -from paddle.io.dataloader.collate import default_collate_fn -from tqdm import tqdm -from typing_extensions import NotRequired # Python 3.11+ -from typing_extensions import TypedDict # Python 3.10+ - -from paddlenlp.transformers.tokenizer_utils import ( - PaddingStrategy, - PretrainedTokenizerBase, - TruncationStrategy, -) - -# from safe_rlhf.utils import is_main_process - - -__all__ = [ - "RawDataset", - "RawSample", - "TokenizedDataset", - "CollatorBase", - "parse_dataset", -] - -IGNORE_INDEX: int = -100 -PROMPT_BEGIN: str = "BEGINNING OF CONVERSATION: " -PROMPT_USER: str = "USER: {input} " -PROMPT_ASSISTANT: str = "ASSISTANT:" # should not have a space at the end -PROMPT_INPUT: str = PROMPT_BEGIN + PROMPT_USER + PROMPT_ASSISTANT - - -def format_prompt( - input: str | list[str], # pylint: disable=redefined-builtin - eos_token: str, -) -> str: - if isinstance(input, str): - input = [input] - elif not isinstance(input, list): - raise ValueError(f"Unsupported type of `input`: {type(input)}. Expected: str or list[str].") - - if len(input) % 2 != 1: - raise ValueError( - "The length of `input` must be odd, while `input` must end at the user question.", - ) - - buffer = [PROMPT_BEGIN] - for i, line in enumerate(input): - if i % 2 == 0: - # User input - buffer.extend((PROMPT_USER.format(input=line), PROMPT_ASSISTANT)) - else: - # Assistant response - buffer.extend((line, eos_token)) - - return "".join(buffer) - - -def left_padding(sequences, padding_value=0): - arrs = [np.asarray(seq) for seq in sequences] - max_length = max([len(seq) for seq in sequences]) - bs = len(sequences) - data = np.full([bs, max_length], padding_value, dtype=arrs[0].dtype) - for i, arr in enumerate(arrs): - data[i, -len(arr) :] = arr - return data - - -def right_padding(sequences, padding_value=0): - arrs = [np.asarray(seq) for seq in sequences] - max_length = max([len(seq) for seq in sequences]) - bs = len(sequences) - data = np.full([bs, max_length], padding_value, dtype=arrs[0].dtype) - for i, arr in enumerate(arrs): - data[i, : len(arr)] = arr - return data - - -class ConcatDataset(Dataset): - r"""Dataset as a concatenation of multiple datasets. - - This class is useful to assemble different existing datasets. - - Args: - datasets (sequence): List of datasets to be concatenated - """ - datasets: List[Dataset] - cumulative_sizes: List[int] - - @staticmethod - def cumsum(sequence): - r, s = [], 0 - for e in sequence: - l = len(e) - r.append(l + s) - s += l - return r - - def __init__(self, datasets: Iterable[Dataset]) -> None: - super(ConcatDataset, self).__init__() - self.datasets = list(datasets) - assert len(self.datasets) > 0, "datasets should not be an empty iterable" # type: ignore[arg-type] - for d in self.datasets: - assert not isinstance(d, IterableDataset), "ConcatDataset does not support IterableDataset" - self.cumulative_sizes = self.cumsum(self.datasets) - - def __len__(self): - return self.cumulative_sizes[-1] - - def __getitem__(self, idx): - if idx < 0: - if -idx > len(self): - raise ValueError("absolute value of index should not exceed dataset length") - idx = len(self) + idx - dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) - if dataset_idx == 0: - sample_idx = idx - else: - sample_idx = idx - self.cumulative_sizes[dataset_idx - 1] - return self.datasets[dataset_idx][sample_idx] - - @property - def cummulative_sizes(self): - warnings.warn( - "cummulative_sizes attribute is renamed to " "cumulative_sizes", DeprecationWarning, stacklevel=2 - ) - return self.cumulative_sizes - - -def parse_dataset(string: str) -> tuple[str, dict[str, Any]]: - """Parse dataset path and its proportion and optionally additional arguments from a string. - - Args: - string (str): Dataset string in the format of ``dataset_name[:proportion[:dataset_path]]``. - """ - if string.count(":") > 2: - raise ValueError( - f"Invalid dataset string `{string}`, " - "should be in the format of `dataset_name[:proportion[:dataset_path]]`.", - ) - name, colon, proportion = string.partition(":") - if not colon: - return name, {"proportion": 1.0} - proportion, colon, path = proportion.partition(":") - if "/" in proportion: - left, right = proportion.split("/") - left, right = left.strip(), right.strip() - if right == "-1": - proportion = Fraction(int(left), -1) - if proportion == 0: - proportion = 0.0 - else: - proportion = float(left) / float(right) - elif proportion != "": - proportion = float(proportion) - else: - proportion = 1.0 - if not colon: - return name, {"proportion": proportion} - if not path: - raise ValueError(f"Invalid dataset path `{path}`.") - path = os.path.expanduser(path) - return name, {"proportion": proportion, "path": path} - - -class RawSample(TypedDict, total=False): - """Raw sample type. - - For SupervisedDataset, should provide (input, answer) or (dialogue). - For PreferenceDataset, should provide (input, answer, other_answer, better). - For SafetyPreferenceDataset, should provide (input, answer, other_answer, safer, is_safe, is_other_safe). - For PromptOnlyDataset, should provide (input). - - When input is a list, it would be processed as a dialogue. - """ - - # Texts - input: NotRequired[str | list[str]] # either `input` or `dialogue` should be provided - """User input text.""" - answer: NotRequired[str] - """Assistant answer text.""" - other_answer: NotRequired[str] - """Other assistant answer text via resampling.""" - dialogue: NotRequired[list[str]] # either `input` or `dialogue` should be provided - """Dialogue history.""" - - # Flags - better: NotRequired[bool] - """Whether ``answer`` is better than ``other_answer``.""" - safer: NotRequired[bool] - """Whether ``answer`` is safer than ``other_answer``.""" - is_safe: NotRequired[bool] - """Whether ``answer`` is safe.""" - is_other_safe: NotRequired[bool] - """Whether ``other_answer`` is safe.""" - - -class RawDataset(Dataset): - """Dataset that provides raw text samples.""" - - NAME: ClassVar[str] - """Name of the dataset.""" - ALIASES: ClassVar[Collection[str]] - """Name aliases of the dataset.""" - __REGISTRY: ClassVar[WeakValueDictionary[str, RawDataset]] = WeakValueDictionary() - """Registry of all subclasses.""" - __ALIAS_NAME_MAPPING: ClassVar[dict[str, str]] = {} - """Mapping from aliases to names.""" - - def __init_subclass__(cls) -> None: - super().__init_subclass__() - name = getattr(cls, "NAME", None) - if name is None: - return - - if not isinstance(name, str): - raise ValueError("`NAME` should be a string.") - if not name: - raise ValueError("`NAME` should not be empty.") - if name in cls.__REGISTRY or name in cls.__ALIAS_NAME_MAPPING: - raise ValueError(f"Duplicate dataset name `{name}`.") - cls.__REGISTRY[name] = cls - - aliases = set(cls.__dict__.get("ALIASES", ())) # do not inherit aliases from parent class - for alias in aliases: - if not isinstance(alias, str): - raise ValueError("`ALIASES` should be a set of strings.") - if not alias: - raise ValueError("`ALIASES` should not contain empty strings.") - if alias in cls.__REGISTRY or alias in cls.__ALIAS_NAME_MAPPING: - raise ValueError(f"Duplicate dataset alias `{alias}`.") - cls.__ALIAS_NAME_MAPPING.update(dict.fromkeys(aliases, name)) - - @staticmethod - def load(name: str, /, *args: Any, **kwargs: Any) -> RawDataset: # noqa: E999 - """Load a raw dataset by name.""" - normalized_name = RawDataset.__ALIAS_NAME_MAPPING.get(name, name) - try: - cls = RawDataset.__REGISTRY[normalized_name] - except KeyError as ex: - raise ValueError( - f"Unknown dataset name `{name}`. " - "You should implement a raw dataset with that name and import it in your code. " - f'Available datasets are: {", ".join(sorted(RawDataset.__REGISTRY.keys()))}.', - ) from ex - return cls(*args, **kwargs) - - make = load # alias - - @abc.abstractmethod - def __getitem__(self, index: int) -> RawSample: - """Get a data sample by index.""" - raise NotImplementedError - - @abc.abstractmethod - def __len__(self) -> int: - """Get the number of samples in the dataset.""" - raise NotImplementedError - - def __iter__(self) -> Iterator[RawSample]: - """Iterate over the dataset.""" - return (self[index] for index in range(len(self))) - - def split_train_test( - self, - split_ratio: float = 0.2, - shuffle: bool = True, - seed: int = 42, - ) -> tuple[Dataset[RawSample], Dataset[RawSample]]: - """Split the dataset into train dataset and test dataset by split ratio.""" - num_samples = len(self) - indices = list(range(num_samples)) - num_test_samples = round(num_samples * split_ratio) - if shuffle: - rng = np.random.default_rng(seed) - rng.shuffle(indices) - - train_dataset = Subset(self, indices[:-num_test_samples]) - test_dataset = Subset(self, indices[-num_test_samples:]) - return train_dataset, test_dataset - - -class TokenizedDataset(Dataset): - """Dataset that provides tokenized samples.""" - - raw_datasets: list[RawDataset] - """Raw datasets.""" - tokenizer: PretrainedTokenizerBase - """Tokenizer.""" - rawdata: list[RawSample] - """Merged raw data samples.""" - data: list[dict[str, paddle.Tensor]] - """Tokenized data samples.""" - - _SENTINEL: Any = object() - - def __init__( # pylint: disable=too-many-branches - self, - dataset_names_and_attributes: dict[str, float | dict[str, Any]] | Iterable[tuple[str, float | dict[str, Any]]], - tokenizer: PretrainedTokenizerBase, - lazy_tokenization: bool = True, - seed: int = 42, - ) -> None: - if not isinstance(dataset_names_and_attributes, dict): - dataset_names_and_attributes = tuple(dataset_names_and_attributes) - dataset_names = [name for name, _ in dataset_names_and_attributes] - if len(dataset_names) != len(set(dataset_names)): - raise ValueError( - f"Dataset names should be unique, but got {dataset_names}.", - ) - - super().__init__() - self.dataset_names_and_proportion: dict[str, float | Fraction] = {} - self.raw_datasets = [] - for name, attributes in dict(dataset_names_and_attributes).items(): - if isinstance(attributes, float): - kwargs = {"proportion": attributes} - elif isinstance(attributes, dict): - kwargs = dict(attributes) # copy - else: - raise TypeError( - f"Dataset `{name}` attributes should be a float or a dict, " f"got {type(attributes).__name__}.", - ) - proportion = kwargs.pop("proportion", 1.0) - if isinstance(proportion, Fraction): - if not (proportion < 0 and proportion.denominator == 1): - raise ValueError( - f"Dataset `{name}` proportion should be a negative integer " - f"represents `num_samples / -1`, got {proportion}.", - ) - else: - proportion = float(proportion) - if proportion < 0.0: - raise ValueError( - f"Dataset `{name}` proportion should be no less than 0.0, " f"got {proportion}.", - ) - if proportion == 0.0: - continue - self.dataset_names_and_proportion[name] = proportion - self.raw_datasets.append(RawDataset.load(name, **kwargs)) - - self.tokenizer = tokenizer - self.seed = seed - - merged_rawdata = self._merge_raw_datasets(seed=seed) - self.rawdata = [merged_rawdata[i] for i in range(len(merged_rawdata))] - if lazy_tokenization: - self.data = [self._SENTINEL for _ in range(len(self.rawdata))] - else: - self.data = list( - map( - self.preprocess, - tqdm( - self.rawdata, - desc="Preprocessing raw dataset...", - disable=False, # not is_main_process(), - ), - ), - ) - - def __getitem__(self, index: int) -> dict[str, paddle.Tensor]: - """Get a tokenized data sample by index.""" - data = self.data[index] - if data is self._SENTINEL: - raw_sample = self.rawdata[index] - data = self.preprocess(raw_sample) - self.data[index] = data - return data - - def __len__(self) -> int: - """Get the number of samples in the dataset.""" - return len(self.data) - - @abc.abstractmethod - def preprocess(self, raw_sample: RawSample) -> dict[str, paddle.Tensor]: - """Pre-process a raw sample into a tokenized sample.""" - raise NotImplementedError - - def _merge_raw_datasets(self, seed: int | None = None) -> Dataset[RawSample]: - """Merge multiple raw datasets into one dataset.""" - if seed is None: - seed = self.seed - - num_datasets = len(self.raw_datasets) - datasets = [] - for raw_dataset, seed_seq in zip( - self.raw_datasets, - np.random.SeedSequence(seed).spawn(num_datasets), - ): - num_raw_samples = len(raw_dataset) - proportion = self.dataset_names_and_proportion[raw_dataset.NAME] - if isinstance(proportion, Fraction): - num_selected_samples = abs(int(proportion)) - else: - num_selected_samples = round(num_raw_samples * proportion) - - indices = [] - while num_selected_samples >= num_raw_samples > 0: - indices.extend(range(num_raw_samples)) - num_selected_samples -= num_raw_samples - if num_selected_samples > 0: - rng = np.random.default_rng(seed_seq) - indices.extend( - rng.choice( - len(raw_dataset), - size=num_selected_samples, - replace=False, - ).tolist(), - ) - datasets.append(Subset(raw_dataset, indices)) - - if num_datasets == 1: - return datasets[0] - return ConcatDataset(datasets) - - def tokenize( - self, - text: str, - add_special_tokens: bool = True, - padding: bool | str | PaddingStrategy = PaddingStrategy.DO_NOT_PAD, - truncation: bool | str | TruncationStrategy = TruncationStrategy.LONGEST_FIRST, - max_length: int | None = None, - ) -> paddle.Tensor: # size = (L,) - """Tokenize a text string into a tensor representation.""" - if max_length is None: - max_length = self.tokenizer.model_max_length - - return self.tokenizer( - text, - add_special_tokens=add_special_tokens, - padding=padding, - max_length=max_length, - truncation=truncation, - return_tensors="np", - )["input_ids"][0] - - def get_collator(self) -> Callable[[list[dict[str, paddle.Tensor]]], dict[str, paddle.Tensor]]: - """Get a collator function for the dataset.""" - return default_collate_fn - - def resize(self, size: int) -> None: - """Resize the dataset to the given size.""" - if size < 0: - raise ValueError(f"Invalid dataset size: {size}") - old_size = len(self) - if size == old_size: - return - if size > old_size: - num_replicates = (size + old_size - 1) // old_size - self.data = self.data * num_replicates - self.rawdata = self.rawdata * num_replicates - self.data = self.data[:size] - self.rawdata = self.rawdata[:size] - - def split_train_test( - self, - split_ratio: float = 0.2, - shuffle: bool = True, - seed: int | None = None, - ) -> tuple[Dataset[dict[str, paddle.Tensor]], Dataset[dict[str, paddle.Tensor]]]: - """Split the dataset into train dataset and test dataset by split ratio.""" - if seed is None: - seed = self.seed - - num_samples = len(self) - indices = list(range(num_samples)) - num_test_samples = round(num_samples * split_ratio) - if shuffle: - rng = np.random.default_rng(seed) - rng.shuffle(indices) - - memo = {id(attr): attr for attr in self.__dict__.values()} - train_dataset = copy.deepcopy(self, memo=memo.copy()) - test_dataset = copy.deepcopy(self, memo=memo.copy()) - train_dataset.data = [self.data[i] for i in indices[:-num_test_samples]] - train_dataset.rawdata = [self.rawdata[i] for i in indices[:-num_test_samples]] - test_dataset.data = [self.data[i] for i in indices[-num_test_samples:]] - test_dataset.rawdata = [self.rawdata[i] for i in indices[-num_test_samples:]] - - return train_dataset, test_dataset - - -class CollatorBase(metaclass=abc.ABCMeta): - pad_token_id: int # The id of the padding token for the tokenizer. - - def __init__(self, pad_token_id: int) -> None: - """Initialize a collator.""" - self.pad_token_id = pad_token_id - - @abc.abstractmethod - def __call__(self, samples: list[dict[str, paddle.Tensor]]) -> dict[str, paddle.Tensor]: - """Collate a list of samples into a batch.""" - raise NotImplementedError diff --git a/slm/examples/RLHF/data/preference.py b/slm/examples/RLHF/data/preference.py deleted file mode 100644 index 320d734747..0000000000 --- a/slm/examples/RLHF/data/preference.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Dataset class for preference training.""" - -from __future__ import annotations - -from typing import Callable - -import numpy as np -import paddle -from typing_extensions import TypedDict # Python 3.10+ - -from .base import ( - CollatorBase, - RawSample, - TokenizedDataset, - format_prompt, - right_padding, -) - -__all__ = [ - "PreferenceDataset", - "PreferenceCollator", - "PreferenceSample", - "PreferenceBatch", -] - - -class PreferenceSample(TypedDict, total=True): - better_input_ids: paddle.Tensor # size = (L,) - worse_input_ids: paddle.Tensor # size = (L,) - - -class PreferenceBatch(TypedDict, total=True): - better_input_ids: paddle.Tensor # size = (B, L) - better_attention_mask: paddle.Tensor # size = (B, L) - - worse_input_ids: paddle.Tensor # size = (B, L) - worse_attention_mask: paddle.Tensor # size = (B, L) - - -class PreferenceDataset(TokenizedDataset): - def preprocess(self, raw_sample: RawSample, index=0) -> PreferenceSample: - prompt = format_prompt(input=raw_sample["input"], eos_token=self.tokenizer.eos_token) - better_answer = raw_sample["answer"] - worse_answer = raw_sample["other_answer"] - better = raw_sample["better"] - if not better: - better_answer, worse_answer = worse_answer, better_answer - - better_input_ids = self.tokenize(prompt + better_answer + self.tokenizer.eos_token) - worse_input_ids = self.tokenize(prompt + worse_answer + self.tokenizer.eos_token) - - if ( - better_input_ids.shape == worse_input_ids.shape - and np.all(np.equal(better_input_ids, worse_input_ids)).item() - ): - # raise ValueError( - import warnings - - warnings.warn( - "Two responses get the same `input_ids` after tokenization.\n\n" - f"Prompt: {prompt}\n\n" - f"Better answer: {better_answer}\n\n" - f"Worse answer: {worse_answer}", - ) - return { - "better_input_ids": better_input_ids, # size = (L,) - "worse_input_ids": worse_input_ids, # size = (L,) - } - - def get_collator(self) -> Callable[[list[dict[str, paddle.Tensor]]], dict[str, paddle.Tensor]]: - return PreferenceCollator(self.tokenizer.pad_token_id) - - -class PreferenceCollator(CollatorBase): - def __call__(self, samples: list[PreferenceSample]) -> PreferenceBatch: - input_ids = [sample["better_input_ids"] for sample in samples] + [ - sample["worse_input_ids"] for sample in samples - ] # size = (2 * B, L) - attention_mask = [np.ones(input_id.shape, dtype=bool) for input_id in input_ids] # size = (2 * B, L) - - input_ids = right_padding(input_ids, padding_value=self.pad_token_id) # size = (2 * B, L) - attention_mask = right_padding(attention_mask, padding_value=0) # size = (2 * B, L) - - bs = len(samples) - (better_input_ids, worse_input_ids,) = ( # size = (B, L) # size = (B, L) - input_ids[:bs], - input_ids[bs:], - ) - (better_attention_mask, worse_attention_mask,) = ( # size = (B, L) # size = (B, L) - attention_mask[:bs], - attention_mask[bs:], - ) - - return { - "better_input_ids": better_input_ids, # size = (B, L) - "better_attention_mask": better_attention_mask, # size = (B, L) - "worse_input_ids": worse_input_ids, # size = (B, L) - "worse_attention_mask": worse_attention_mask, # size = (B, L) - } diff --git a/slm/examples/RLHF/data/prompt_only.py b/slm/examples/RLHF/data/prompt_only.py deleted file mode 100644 index 964a55a705..0000000000 --- a/slm/examples/RLHF/data/prompt_only.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Dataset class for preference training.""" - -from __future__ import annotations - -from typing import Callable, Hashable - -import numpy as np -import paddle -from paddle.io import Dataset, Subset -from typing_extensions import TypedDict # Python 3.10+ - -from .base import CollatorBase, RawSample, TokenizedDataset, format_prompt, left_padding - -__all__ = [ - "PromptOnlyDataset", - "PromptOnlyCollator", - "PromptOnlySample", - "PromptOnlyBatch", -] - - -class PromptOnlySample(TypedDict, total=True): - input_ids: paddle.Tensor # size = (L,) - - -class PromptOnlyBatch(TypedDict, total=True): - input_ids: paddle.Tensor # size = (B, L) - attention_mask: paddle.Tensor # size = (B, L) - - -class PromptOnlyDataset(TokenizedDataset): - def preprocess(self, raw_sample: RawSample) -> PromptOnlySample: - prompt = format_prompt(input=raw_sample["input"], eos_token=self.tokenizer.eos_token) - input_ids = self.tokenize(prompt) - return { - "input_ids": input_ids, # size = (L,) - } - - def get_collator(self) -> Callable[[list[dict[str, paddle.Tensor]]], dict[str, paddle.Tensor]]: - return PromptOnlyCollator(self.tokenizer.pad_token_id) - - def _merge_raw_datasets(self, seed: int | None = None) -> Dataset[RawSample]: - """Merge multiple raw datasets into one dataset and remove duplicates.""" - - def to_hashable(raw_sample: RawSample) -> Hashable: - input = raw_sample["input"] # pylint: disable=redefined-builtin - return input if isinstance(input, str) else tuple(input) - - merged = super()._merge_raw_datasets(seed) - inputs = {to_hashable(merged[i]): i for i in range(len(merged))} - return Subset(merged, sorted(inputs.values())) - - -class PromptOnlyCollator(CollatorBase): - def __call__(self, samples: list[PromptOnlySample]) -> PromptOnlyBatch: - input_ids = [sample["input_ids"] for sample in samples] - attention_mask = [np.ones(input_id.shape, dtype=bool) for input_id in input_ids] - - input_ids = left_padding(input_ids, padding_value=self.pad_token_id) - attention_mask = left_padding(attention_mask, padding_value=0) - return { - "input_ids": input_ids, # size = (B, L) - "attention_mask": attention_mask, # size = (B, L) - } diff --git a/slm/examples/RLHF/data/safe_rlhf.py b/slm/examples/RLHF/data/safe_rlhf.py deleted file mode 100644 index 427c2c7a69..0000000000 --- a/slm/examples/RLHF/data/safe_rlhf.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Safe-RLHF preference datasets.""" - -from __future__ import annotations - -from typing import ClassVar - -from datasets import load_dataset - -from .base import RawDataset, RawSample - -__all__ = [ - "SafeRLHFDataset", - "SafeRLHFTrainDataset", - "SafeRLHFTestDataset", - "SafeRLHF10KTrainDataset", -] - - -class SafeRLHFDataset(RawDataset): - SPLIT: ClassVar[str] - PATH: ClassVar[str] - - def __init__(self, path: str | None = None) -> None: - self.data = load_dataset(path or self.PATH, split=self.SPLIT) - - def __getitem__(self, index: int) -> RawSample: - data = self.data[index] - return RawSample( - input=data["prompt"], - answer=data["response_0"], - other_answer=data["response_1"], - better=int(data["better_response_id"]) == 0, - safer=int(data["safer_response_id"]) == 0, - is_safe=bool(data["is_response_0_safe"]), - is_other_safe=bool(data["is_response_1_safe"]), - ) - - def __len__(self) -> int: - return len(self.data) - - -class SafeRLHFTrainDataset(SafeRLHFDataset): - NAME: str = "PKU-SafeRLHF/train" - ALIASES: tuple[str, ...] = ("PKU-Alignment/PKU-SafeRLHF/train",) - PATH: str = "PKU-Alignment/PKU-SafeRLHF" - SPLIT: str = "train" - - -class SafeRLHFTestDataset(SafeRLHFDataset): - NAME: str = "PKU-SafeRLHF/test" - ALIASES: tuple[str, ...] = ("PKU-Alignment/PKU-SafeRLHF/test",) - PATH: str = "PKU-Alignment/PKU-SafeRLHF" - SPLIT: str = "test" - - -class SafeRLHF30KTrainDataset(SafeRLHFDataset): - NAME: str = "PKU-SafeRLHF-30K/train" - ALIASES: tuple[str, ...] = ("PKU-Alignment/PKU-SafeRLHF-30K/train",) - PATH: str = "PKU-Alignment/PKU-SafeRLHF-30K" - SPLIT: str = "train" - - -class SafeRLHF30KTestDataset(SafeRLHFDataset): - NAME: str = "PKU-SafeRLHF-30K/test" - ALIASES: tuple[str, ...] = ("PKU-Alignment/PKU-SafeRLHF-30K/test",) - PATH: str = "PKU-Alignment/PKU-SafeRLHF-30K" - SPLIT: str = "test" - - -class SafeRLHF10KTrainDataset(SafeRLHFDataset): - NAME: str = "PKU-SafeRLHF-10K/train" - ALIASES: tuple[str, ...] = ("PKU-Alignment/PKU-SafeRLHF-10K/train",) - PATH: str = "PKU-Alignment/PKU-SafeRLHF-10K" - SPLIT: str = "train" diff --git a/slm/examples/RLHF/data/supervised.py b/slm/examples/RLHF/data/supervised.py deleted file mode 100644 index 26aa97a143..0000000000 --- a/slm/examples/RLHF/data/supervised.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Dataset class for preference training.""" - -from __future__ import annotations - -from typing import Callable - -import paddle -from typing_extensions import TypedDict # Python 3.10+ - -from .base import ( - IGNORE_INDEX, - PROMPT_ASSISTANT, - PROMPT_BEGIN, - PROMPT_USER, - CollatorBase, - RawSample, - TokenizedDataset, - format_prompt, - right_padding, -) - -__all__ = [ - "SupervisedDataset", - "SupervisedCollator", - "SupervisedSample", - "SupervisedBatch", -] - - -class SupervisedSample(TypedDict, total=True): - input_ids: paddle.Tensor # size = (L,) - labels: paddle.Tensor # size = (L,) - - -class SupervisedBatch(TypedDict, total=True): - input_ids: paddle.Tensor # size = (B, L) - labels: paddle.Tensor # size = (B, L) - attention_mask: paddle.Tensor # size = (B, L) - - -class SupervisedDataset(TokenizedDataset): - def preprocess(self, raw_sample: RawSample) -> SupervisedSample: - if raw_sample.get("input") is None and raw_sample.get("dialogue") is None: - raise ValueError("Either `input` or `dialogue` must be provided.") - if raw_sample.get("input") is not None and raw_sample.get("dialogue") is not None: - raise ValueError("At most one of `input` and `dialogue` can be provided.") - - if raw_sample.get("input") is not None: - input = raw_sample["input"] # pylint: disable=redefined-builtin - if not isinstance(input, str): - raise ValueError(f"Unsupported type of `input`: {type(input)}. Expected: str.") - prompt = format_prompt(input=input, eos_token=self.tokenizer.eos_token) - answer = raw_sample["answer"] - text = prompt + answer + self.tokenizer.eos_token - - input_ids = self.tokenize(text) - labels = input_ids.copy() - # Mask non-assistant input - labels[: len(self.tokenize(prompt))] = IGNORE_INDEX - return {"input_ids": input_ids, "labels": labels} - - dialogue = raw_sample["dialogue"] # is not None - text = PROMPT_BEGIN - offsets = [0] - for i, line in enumerate(dialogue): - if i % 2 == 0: - # User input - text += PROMPT_USER.format(input=line) + PROMPT_ASSISTANT - else: - # Assistant input - text += line + self.tokenizer.eos_token - input_ids = self.tokenize(text) - offsets.append(len(input_ids)) - - labels = input_ids.copy() - # Mask non-assistant input - for begin, end in zip(offsets[::2], offsets[1::2]): - labels[begin:end] = IGNORE_INDEX - - return { - "input_ids": input_ids, # size = (L,) - "labels": labels, # size = (L,) - } - - def get_collator( - self, shift: bool = False - ) -> Callable[[list[dict[str, paddle.Tensor]]], dict[str, paddle.Tensor]]: - return SupervisedCollator(self.tokenizer.pad_token_id, shift) - - -class SupervisedCollator(CollatorBase): - def __init__(self, pad_token_id: int, shift: bool = False) -> None: - """Initialize a collator.""" - super().__init__(pad_token_id) - self._shift = shift - - def __call__(self, samples: list[SupervisedSample]) -> SupervisedBatch: - input_ids = right_padding( - [sample["input_ids"] for sample in samples], - padding_value=self.pad_token_id, - ) - labels = right_padding( - [sample["labels"] for sample in samples], - padding_value=IGNORE_INDEX, - ) - if self._shift: - input_ids = input_ids[:, :-1] - labels = labels[:, 1:] - attention_mask = input_ids != self.pad_token_id - return { - "input_ids": input_ids, # size = (B, L) - "labels": labels, # size = (B, L) - "attention_mask": attention_mask, # size = (B, L) - } diff --git a/slm/examples/RLHF/infer_utils.py b/slm/examples/RLHF/infer_utils.py deleted file mode 100644 index 62cb5a9b9f..0000000000 --- a/slm/examples/RLHF/infer_utils.py +++ /dev/null @@ -1,379 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import copy -import inspect -import types -from contextlib import contextmanager - -import paddle -import paddle.distributed as dist -from comm_utils import cleanup_tensor_space, offload_tensor_to_cpu, reload_tensor_to_gpu -from paddle.utils import try_import -from trainer_utils import guard_set_args - -import paddlenlp -from paddlenlp.trainer.trainer import Trainer, logger -from paddlenlp.transformers import PretrainedModel, PretrainedTokenizer -from paddlenlp.transformers.model_utils import dtype_guard - - -class Predictor: - def __init__(self, config, model: PretrainedModel = None, tokenizer: PretrainedTokenizer = None): - self.model_config = model.config - self.config = config - self.tokenizer = tokenizer - self.model = model - self.is_available = False - self._weights_mapping = None - # TODO(guosheng): Removde dependency on llm.Predictor - # 1. buffer_maker creates caches and other buffer inputs can be shared - # among multi time prediction. define caches and extra inputs creation - # method instead of using predictor.__init__ - # 2. inputs_processer creates caches and other inputs can be shared among - # multi time prediction. define caches and extra inputs creation method - # instead of using predictor.__init__ - from predictor import InferencePredictorMixin - - self._buffer_maker = types.MethodType(InferencePredictorMixin.__init__, self) - self._inputs_processer = types.MethodType(InferencePredictorMixin._preprocess, self) - - @staticmethod - def create_predictor(trainer): - from predictor import PdArgumentParser, PredictorArgument - - from paddlenlp.trl.llm_utils import get_model_max_position_embeddings - - # create infer model - # NOTE: infer model use static name param_attr to create and cannot be - # created multiple times. - def create_infer_model(model, dtype, set_state=False): - from models.infer_model_utils import patch_infer_generate - - # apply patches to make FuseMT adapt - patch_infer_generate( - eos_token_id=trainer.tokenizer.eos_token_id, pad_token_id=trainer.tokenizer.pad_token_id - ) - config = copy.deepcopy(model.config) - hcg = dist.fleet.get_hybrid_communicate_group() # may differ with training - config.tensor_parallel_degree = hcg.get_model_parallel_world_size() - config.tensor_parallel_rank = hcg.get_model_parallel_rank() - config.quant_type = None - config.single_card_ptq = True - infer_model_cls = getattr(paddlenlp.experimental.transformers, model.__class__.__name__ + "InferenceModel") - # ori_init_weights = infer_model_cls.init_weights - # infer_model_cls.init_weights = lambda self: None - with dtype_guard(dtype): - infer_model = infer_model_cls(config) - # infer_model_cls.init_weights = ori_init_weights - - if set_state: - state_dict = {} - for k, v in model.state_dict().items(): - # state_dict[k] = np.from_dlpack(paddle.utils.dlpack.to_dlpack(v)) - state_dict[k] = v.numpy() - infer_model.set_state_dict(state_dict) - return infer_model - - # to avoid oom, clear param of infer_model imediately - ori_creat_param = paddle.nn.Layer.create_parameter - - def _create_param(self, *args, **kwargs): - param = ori_creat_param(self, *args, **kwargs) - param._clear_data() - # param._clear() - return param - - paddle.nn.Layer.create_parameter = _create_param - # trainer might use an extra model instead of trainer.model for eval - eval_model = getattr(trainer, "_inner_eval_model", None) - infer_model = create_infer_model(trainer.model if eval_model is None else eval_model, dtype=trainer.amp_dtype) - paddle.nn.Layer.create_parameter = ori_creat_param - # for k, v in infer_model.state_dict().items(): - # v._clear() - - # create predictor - parser = PdArgumentParser((PredictorArgument,)) - predictor_args = parser.parse_dict( - { - "src_length": get_model_max_position_embeddings( # can be changed dynamically by predictor.input_length - trainer.model.config if eval_model is None else eval_model.config - ), - "max_length": trainer.args.max_length, - "dtype": trainer.amp_dtype, - "batch_size": trainer.args.per_device_train_batch_size, - # infer model do not support top_k, and differ with non-infer model - # generation which gets default top_K=50 using generation_config.top_k - "top_p": trainer.args.top_p, - "temperature": trainer.args.temperature, - "repetition_penalty": trainer.args.repetition_penalty, - } - )[0] - policy_predictor = Predictor(predictor_args, model=infer_model, tokenizer=trainer.tokenizer) - return policy_predictor - - def _create_caches(self): - """inputs can be reused among multiple predictions, such as cache""" - if hasattr(self, "cache_kvs_shape"): # has created cache - input_length = getattr(self, "input_length", 0) - # TODO(guosheng): better way to get history max cahce length, we can - # not get cahce length form cache tensor when not know cache layout - if input_length <= self.config.src_length: # reuse cahce - return - else: # create longer cache - self._clear_caches() - self.config.src_length = getattr(self, "input_length", self.config.src_length) - if not hasattr(self, "_buffer_attrs"): - pre_attrs = set(self.__dict__.keys()) - self.cache_k_shapes, self.cache_v_shapes = self.model.get_cache_kvs_shape( - self.model_config, self.config.batch_size, self.config.total_max_length - ) - self._buffer_maker(self.config, self.tokenizer) - if not hasattr(self, "_buffer_attrs"): - self._buffer_attrs = set(self.__dict__.keys()) - pre_attrs - - def _clear_caches(self): - # del or offload - for attr in self._buffer_attrs: - delattr(self, attr) - - def disable(self, model, onload_model=True): - # clear caches - self._clear_caches() - # clear params - for _, param in self.model.state_dict().items(): - param._clear_data() - # param._clear() - if onload_model: - model.to(paddle.device.get_device()) - self.is_available = False - - def enable(self, model, offload_model=True): - if self.is_available: - return - # set params - self.set_state_dict(model, offload_model) - self.is_available = True - - @paddle.no_grad() - def set_state_dict(self, model, offload_model=True): - offload_place = paddle.CUDAPinnedPlace() - state_dict = {} - for k, v in model.state_dict().items(): - state_dict[k] = v - - if getattr(self, "_weights_mapping", None) is None: - self._weights_mapping = self.model.get_weights_mapping() - - # non_share_params = [] - for k, v in self._weights_mapping.items(): - param, (convert_fun, args) = k, v - args = [state_dict[name] for name in args] - value = convert_fun(*args) - if offload_model: - for arg in args: - # shared params no need to offload - if value is not arg: - cpu_arg = arg._copy_to(offload_place, blocking=False) - cpu_arg._share_buffer_to(arg) - if not isinstance(value, paddle.Tensor): - param.set_value(value) - # elif isinstance(value.place, paddle.CUDAPlace): - elif value.place.is_gpu_place(): - # NOTE: _share_buffer_to seems do not work - # value._share_buffer_to(param) - # value._share_underline_tensor_to(param) - param.get_tensor()._share_data_with(value.get_tensor()) - else: - param.copy_(value, True) - - paddle.device.cuda.synchronize() - - def _preprocess(self, source): - # make cache when infer happens to get actual shape to save memory - self._create_caches() - with guard_set_args(self.config, {"src_length": getattr(self, "input_length", self.config.src_length)}): - inputs = self._inputs_processer(source) - # We want to use a defined input_length to create cache and input_ids. - # However predictor could not use a specified length to pad currently. - # Thus we use this way to let get the actual input length. - self.infer_input_length = inputs["input_ids"].shape[-1] - return inputs - - @paddle.no_grad() - def _infer(self, inputs): - for key in inputs.keys(): - if paddle.is_tensor(inputs[key]): - continue - if isinstance(inputs[key], list): - if paddle.is_tensor(inputs[key]): - continue - inputs[key] = [paddle.to_tensor(item) for item in inputs[key]] - else: - inputs[key] = paddle.to_tensor(inputs[key]) - - inputs["cache_kvs"] = self.cache_kvs - return self.model.generate(**inputs) - - def _postprocess(self, predictions): - return predictions - - @paddle.no_grad() - def predict(self, input_texts: str | list[str]): - tokenized_source = self._preprocess(input_texts) - predictions = self._infer(tokenized_source) - decoded_predictions = self._postprocess(predictions) - return decoded_predictions - - -policy_predictor: Predictor = None - - -@contextmanager -def infer_guard(trainer, offload_model=True): - # trainer might use an extra model instead of trainer.model for eval - eval_model = getattr(trainer, "_inner_eval_model", None) - model = trainer.model if eval_model is None else eval_model - - # PipelineParallel does not support inference speedup - if not getattr(trainer, "use_fusemt", False) or isinstance( - model, (dist.fleet.meta_parallel.PipelineLayer, dist.fleet.model.PipelineParallel) - ): - yield - return - - try: - try_import("paddlenlp_ops") - except: - logger.warning("paddlenlp_ops does not exist, please install paddlenlp_ops for generation speedup.") - yield - return - - global policy_predictor - if policy_predictor is None: - policy_predictor = Predictor.create_predictor(trainer) - if not policy_predictor.is_available: - policy_predictor.enable(model, offload_model=offload_model) - - # TODO(guosheng): patch for dist.all_recude to use tp group, fix it later - ori_all_reduce = dist.all_reduce - ori_broadcast = dist.broadcast - hcg = dist.fleet.get_hybrid_communicate_group() - dist.all_reduce = lambda x: ori_all_reduce(x, group=hcg.get_model_parallel_group()) - dist.broadcast = lambda x, rank: ori_broadcast( - x, src=hcg.get_model_parallel_group_src_rank(), group=hcg.get_model_parallel_group() - ) - yield - dist.all_reduce = ori_all_reduce - dist.broadcast = ori_broadcast - - policy_predictor.disable(model, onload_model=offload_model) - - -class InferEvalModel: - """For faster generation, not support PipelineParallel yet.""" - - def __init__(self, trainer: Trainer): - # trainer might use an extra model instead of trainer.model for eval - eval_model = getattr(trainer, "_inner_eval_model", None) - self.model: PretrainedModel = trainer.model if eval_model is None else eval_model - self.tokenizer: PretrainedTokenizer = trainer.tokenizer - self.trainer = trainer - - def enable(self): - trainer = self.trainer - if trainer.model is not self.model: - trainer.export_evaluate_model( - trainer.model, - self.model, - with_offload="train_model" in trainer.args.offload_level, - ) - else: - reload_tensor_to_gpu(self.model.state_dict()) - - def disable(self): - trainer = self.trainer - if trainer.model is not self.model: - cleanup_tensor_space(self.model.state_dict()) - else: - offload_tensor_to_cpu(self.model.state_dict()) - - def __getattr__(self, name): - try: - return super().__getattr__(name) - except AttributeError: - return getattr(self.model, name) - - def eval(self): - self.model.eval() - - def train(self): - self.model.train() - - def __call__(self, *args, **kwargs): - # assert model is on GPU - assert policy_predictor is None or not policy_predictor.is_available - return self.model(*args, **kwargs) - - def generate(self, *args, **kwargs): - if policy_predictor is None or not policy_predictor.is_available: - return self.model.generate(*args, **kwargs) - - arg_dict = inspect.signature(self.model.generate).bind(*args, **kwargs).arguments - input_ids = arg_dict["input_ids"] - generation_config = arg_dict["generation_config"] - # convert text and tokenize again to convert left padding to right padding - # remove this if inputs is right padding - # TODO(guosheng): allow to use right padding to infer directly - prompts = self.tokenizer.batch_decode(input_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False) - # decoded prompts has been applied with chat_template - # NOTE(guosheng): Whether to add special token should be checked, None - # chat_template would not add special token in predictor, since it assumes - # chat_template includes special tokens. While Beaver dataset tokenization - # does not use chat_template, it uses hard coded template which excludes - # special tokens. - with guard_set_args( - policy_predictor.tokenizer, - { - # predictor use right padding for infer model by default - # "padding_side": "right", - # "chat_template": None - }, - ): - # NOTE: right padding in predictor according to prompt might have a - # different length with input_ids, espically when input_ids has more - # paddings than the necessary. Thus pass input_length to predictor to: - # 1. use a consistent length to replace input_ids back to output to - # keep the same padding format. however predictor could not use a - # specified length to pad currently - # 2. allow to use a dynamic length for memory efficiency (by a smaller - # cache) - policy_predictor.input_length = input_ids.shape[-1] - outputs = policy_predictor.predict(prompts) - - if generation_config.trunc_input: - outputs = (outputs[0][:, policy_predictor.infer_input_length :],) - return outputs - - if policy_predictor.input_length != policy_predictor.infer_input_length: - outputs = (paddle.concat([input_ids, outputs[0][:, policy_predictor.infer_input_length :]], axis=-1),) - return outputs - - outputs = (outputs[0],) - if self.tokenizer.padding_side == "left": - # convert back to left padding inputs - outputs[0][:, : input_ids.shape[-1]] = input_ids - return outputs diff --git a/slm/examples/RLHF/models/__init__.py b/slm/examples/RLHF/models/__init__.py deleted file mode 100644 index e381a3b619..0000000000 --- a/slm/examples/RLHF/models/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utility functions for score models.""" - -from .score_model_utils import AutoModelForScore, ScoreModelOutput - -__all__ = ["AutoModelForScore", "ScoreModelOutput"] diff --git a/slm/examples/RLHF/models/infer_model_utils.py b/slm/examples/RLHF/models/infer_model_utils.py deleted file mode 100644 index 3d63fe52aa..0000000000 --- a/slm/examples/RLHF/models/infer_model_utils.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for inference model.""" - -import numpy as np -import paddle - - -def patch_paddlenlp_ops(eos_token_id, pad_token_id): - import paddlenlp_ops - - paddlenlp_ops.save_with_output = lambda *args, **kwargs: None - - # TODO(guosheng): update the custom op code directly. - ori_set_ends = paddlenlp_ops.set_stop_value_multi_ends - - def _set_ends(topk_ids, stop_flags, end_ids, mode): - # infer model uses eos_token_id to pad and discriminate ending, - # patch to use pad_token_id to pad to unify with non-infer model. - topk_ids_out, stop_flags_out = ori_set_ends(topk_ids, stop_flags, end_ids, mode) - if pad_token_id != eos_token_id: - topk_ids_out = paddle.where(stop_flags, pad_token_id, topk_ids_out) - return topk_ids_out, stop_flags_out - - paddlenlp_ops.set_stop_value_multi_ends = _set_ends - - -def patch_infer_generate(eos_token_id, pad_token_id): - """patches for inference model to make FuseMT adapt""" - # patch paddlenlp_ops, maybe update the custom op code directly later - # NOTE: should patch paddlenlp_ops before infer model import - patch_paddlenlp_ops(eos_token_id, pad_token_id) - - # patch get_weights_mapping for InferenceModel - patch_infer_model() - - # patch GenerationInferenceModel.sample - from paddlenlp.experimental.transformers.generation_utils import ( - GenerationInferenceModel, - ) - - ori_update_model_kwargs = GenerationInferenceModel.update_model_kwargs_for_generation - - def _update_model_kwargs(self, *args, **kwargs): - # update_model_kwargs_for_generation only returns , hack to. - model_kwargs = ori_update_model_kwargs(self, *args, **kwargs) - next_tokens = model_kwargs["next_tokens"] - all_input_ids = paddle.concat([model_kwargs["all_input_ids"], next_tokens], axis=1) - model_kwargs["next_tokens"] = all_input_ids - model_kwargs["all_input_ids"] = None - return model_kwargs - - GenerationInferenceModel.update_model_kwargs_for_generation = _update_model_kwargs - - -_model_weights_mapping_dict = {} - - -def register_model(model_cls_name): - def mark_cls_name(func): - # Do not register here although we can, otherwise infer model would import - # before paddlenlp_ops. - _model_weights_mapping_dict[model_cls_name] = func - return func - - return mark_cls_name - - -def patch_infer_model(): - import paddlenlp.experimental.transformers as infer_transformers - - for model_cls_name, get_weights_mapping in _model_weights_mapping_dict.items(): - model_cls = getattr(infer_transformers, model_cls_name) - model_cls.get_weights_mapping = get_weights_mapping - - -@register_model("LlamaForCausalLMInferenceModel") -def get_weights_mapping(self): - """model to infer model""" - head_size = self.config.hidden_size // self.config.num_attention_heads - - def _concat_qkv(q, k, v): - if isinstance(q, paddle.Tensor): - concated_qkv_weight = paddle.concat([q, k, v], axis=-1).T.reshape( - [ - 3 * (self.config.num_attention_heads // self.config.tensor_parallel_degree) * (head_size), - self.config.hidden_size, - ] - ) - else: - concated_qkv_weight = ( - np.concatenate( - [q, k, v], - axis=-1, - ) - .transpose(1, 0) - .reshape( - 3 * (self.config.num_attention_heads // self.config.tensor_parallel_degree) * (head_size), - self.config.hidden_size, - ) - ) - - return concated_qkv_weight - - def _concat_ffn1(w1, w2): - if isinstance(w1, paddle.Tensor): - concated_ffn1_weight = paddle.concat([w1, w2], axis=-1) - else: - concated_ffn1_weight = np.concatenate([w1, w2], axis=-1) - return concated_ffn1_weight - - identity = lambda x: x - - weight_mapping = {} - weight_mapping[self.lm_head.weight] = [ - identity, - [ - "lm_head.weight", - ], - ] - weight_mapping[self.llama.embed_tokens.weight] = [ - identity, - [ - "llama.embed_tokens.weight", - ], - ] - weight_mapping[self.llama.norm.weight] = [ - identity, - [ - "llama.norm.weight", - ], - ] - for idx in range(self.config.num_hidden_layers): - weight_mapping[self.llama.transformer_block.qkv_weights[idx]] = [ - _concat_qkv, - [ - f"llama.layers.{idx}.self_attn.q_proj.weight", - f"llama.layers.{idx}.self_attn.k_proj.weight", - f"llama.layers.{idx}.self_attn.v_proj.weight", - ], - ] - weight_mapping[self.llama.transformer_block.ffn1_weights[idx]] = [ - _concat_ffn1, - [ - f"llama.layers.{idx}.mlp.gate_proj.weight", - f"llama.layers.{idx}.mlp.up_proj.weight", - ], - ] - weight_mapping[self.llama.transformer_block.linear_weights[idx]] = [ - identity, - [ - f"llama.layers.{idx}.self_attn.o_proj.weight", - ], - ] - weight_mapping[self.llama.transformer_block.ffn2_weights[idx]] = [ - identity, - [ - f"llama.layers.{idx}.mlp.down_proj.weight", - ], - ] - weight_mapping[self.llama.transformer_block.ln_scales[idx]] = [ - identity, - [ - f"llama.layers.{idx}.input_layernorm.weight", - ], - ] - weight_mapping[self.llama.transformer_block.ffn_ln_scales[idx]] = [ - identity, - [ - f"llama.layers.{idx}.post_attention_layernorm.weight", - ], - ] - return weight_mapping diff --git a/slm/examples/RLHF/models/model_pp.py b/slm/examples/RLHF/models/model_pp.py deleted file mode 100644 index 57bcf9f465..0000000000 --- a/slm/examples/RLHF/models/model_pp.py +++ /dev/null @@ -1,261 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import paddle -import paddle.nn as nn -from paddle.distributed.fleet.meta_parallel import LayerDesc - -from paddlenlp.transformers import LlamaForCausalLM, LlamaForCausalLMPipe -from paddlenlp.transformers.llama.modeling import LlamaDecoderLayer -from paddlenlp.transformers.llama.modeling_pp import ( - LlamaRMSNormPipe, - parse_args, - return_args, -) - -from .pp_model_utils import fwd_args_to_dict, get_expected_keys, pad_batches_inputs -from .ppo_model_utils import ( - RLHFPPOMixedLoss, - RLHFValueLoss, - create_loss, - make_position_ids, -) -from .score_model_utils import ScoreModelMixin - -# patches for base pipe model -# non-pipe model class, can be used to parse and convert forward args -# mainly used for generation with PipelienParallel model -LlamaForCausalLMPipe._non_pipe_model_class = LlamaForCausalLM -LlamaForCausalLMPipe._non_pipe_decoder_layer_class = LlamaDecoderLayer - - -class LlamaPolicyPipe(LlamaForCausalLMPipe): - # TODO(guosheng): maybe make a Mixin is better - - @fwd_args_to_dict - def _prepare_pipeline_inputs_func(self, inputs): - # first_stage_keys = ["input_ids", "attention_mask"] - first_stage_keys = ["input_ids", "attention_mask", "position_ids"] - # last_stage_keys = [ - # "labels", "input_ids", "log_probs", "advantages", "sequence_mask" - # ] - # TODO(guosheng): make input keys same with model arg names, maybe we - # can use inspect and set as global var which can then be used here and - # in PPOTrainer. - last_stage_keys = ["labels", "input_ids", "old_log_probs", "reward_advantages", "sequence_mask"] - - if type(inputs) is dict: - # for left padding, position_ids is nececessary - if "position_ids" not in inputs: - inputs["position_ids"] = make_position_ids(inputs["attention_mask"]) - # ppo-loss and ptx-loss need different labels, and data iter provides - # corrensponding data, thus add the not provided fields here. - # policy trian and infer has different inputs, infer uses position_ids. - # for key in last_stage_keys: - for key in first_stage_keys + last_stage_keys: - if key not in inputs: - inputs[key] = None - return [ - get_expected_keys(inputs, first_stage_keys), - get_expected_keys(inputs, last_stage_keys), - ] - - for data in inputs: - # for key in last_stage_keys: - for key in first_stage_keys + last_stage_keys: - if key not in data: - if key == "position_ids": - data[key] = make_position_ids(data["attention_mask"]) - continue - data[key] = None - # keys = list(inputs[0].keys()) - inputs_batch = {key: [data.get(key) for data in inputs] for key in first_stage_keys + last_stage_keys} - # NOTE(guosheng): PipelineParallel requires send/recv tensors among - # micro-batches/accu-steps have the same shape. Thus pad here, maybe - # should make data collator do padding and pad optionally here, since - # padding strategy may not be clear here. - # 1. For input_ids/attention_mask/labels (prompt+target) padding: - # Some data fields, such as input_ids/attention_mask/labels, should - # have same shape after padding, and each of them cannot pad only - # according to its own max length which might be different since the - # filed value is None for different batches/tasks. - src_tgt_keys = ["input_ids", "attention_mask", "labels", "position_ids"] - max_len = max([x.shape[-1] for x in inputs_batch["input_ids"]]) - pad_len = [max_len - x.shape[-1] for x in inputs_batch["input_ids"]] - for key in src_tgt_keys: - # Do not pad position_ids with 0 since 0s in position_ids has special - # usage in reward model. We use 1 to pad. - padding_value = self._ignore_index if key == "labels" else 1 if key == "position_ids" else 0 - inputs_batch[key] = pad_batches_inputs(inputs_batch[key], padding_value, pad_len=pad_len) - # 2. For old_log_probs/reward_advantages/sequence_mask (target) padding: - # hard to pad acorss batches, think in some cases one batch might have the - # longest prompt+target length but the shortest target lengh, which might - # cause mismatch between inputs with prompt+target length and labels with - # target length. NOTE: however trick can be used here, label fields with - # target length such as old_log_probs/reward_advantages/sequence_mask do - # not need to join comm and thus there is no need to keep same shape among - # batches of accumulation steps, they just need to pad as prompt+target - # fields such as input_ids. - tgt_keys = ["old_log_probs", "reward_advantages", "sequence_mask"] - for key in tgt_keys: - padding_value = 0 - inputs_batch[key] = pad_batches_inputs(inputs_batch[key], padding_value, pad_len=pad_len) - # for key, value in inputs_batch.items(): - # padding_value = self._ignore_index if key == "labels" else 0 - # max_len = max_len if key in [ - # "input_ids", "attention_mask", "labels" - # ] else None - # inputs_batch[key] = pad_batches_inputs(value, padding_value, max_len) - return [ - get_expected_keys(inputs_batch, first_stage_keys), - get_expected_keys(inputs_batch, last_stage_keys), - ] - - def __init__(self, config, **kwargs): - # NOTE: make _sequential_layers/_single_to_pp_mapping/_pp_to_single_mapping - # instance attrs instead of class attrs to support more than one pipeline - # models. Maybe make all sequential_layers add once. - self._sequential_layers = [] - self._single_to_pp_mapping = None - self._pp_to_single_mapping = None - # To be consistent with score model init and allow hyper-param be passed - # using __init__/from_pretrained - self._init_kwargs = kwargs - super().__init__(config) - self._ignore_index = self._loss_fn.sft_criterion.ignore_index - - def get_loss_fn(self, config): - return create_loss(RLHFPPOMixedLoss, config, self._init_kwargs) - - @property - def head_out_meta(self): - """mainly for eval/generation with PipelineParallel""" - # None means to use actual data info - return paddle.static.InputSpec(shape=[None, None, self.config.vocab_size], dtype=None) - - -class _LlamaRMSNormPipe(LlamaRMSNormPipe): - """ - We need position_ids for reward model, so wrap LlamaRMSNormPipe to pass position_ids - """ - - def __init__(self, config): - super().__init__(config) - - def forward(self, args): - hidden_states, attention_mask, position_ids, alibi = parse_args(args) - return return_args(self.norm(hidden_states), attention_mask, position_ids) - - -# LayerDesc of PipelineParallel requires head to be a nn.Layer -class ValueHead(nn.Layer, ScoreModelMixin): - def __init__(self, config, **kwargs): - super().__init__() - self.config = config - self.init_score_head(config, hidden_size=config.hidden_size, **kwargs) - - def forward(self, args): - # attention_mask passed from pre-stage is shaped (bs, 1, seq_len, seq_len) - hidden_state, attention_mask, position_ids, alibi = parse_args(args) - outputs = self.get_score( - hidden_state, attention_mask=attention_mask, position_ids=position_ids, return_dict=True - ) - return outputs - - -class LlamaValuePipe(LlamaForCausalLMPipe): - # TODO(guosheng): maybe make a Mixin is better - - @fwd_args_to_dict - def _prepare_pipeline_inputs_func(self, inputs): - # ValueHead/get_score needs original attention_mask or position_ids, - # while attention_mask passed from pre-stage is not the original, thus - # hack for position_ids here. - # Maybe add position_ids into inputs later and use position_ids instead - # of attention_mask to get score not only for pipeline parallel. - first_stage_keys = ["input_ids", "attention_mask", "position_ids"] - # TODO(guosheng): make input keys same with model arg names, maybe we - # can use inspect and set as global var which can then be used here and - # in PPOTrainer. - last_stage_keys = ["old_reward_values", "reward_returns", "sequence_mask"] - - if type(inputs) is dict: - if "position_ids" not in inputs: - inputs["position_ids"] = make_position_ids(inputs["attention_mask"]) - - return [ - get_expected_keys(inputs, first_stage_keys), - get_expected_keys(inputs, last_stage_keys), - ] - - for data in inputs: - if "position_ids" not in data: - data["position_ids"] = make_position_ids(data["attention_mask"]) - # keys = list(inputs[0].keys()) - inputs_batch = {key: [data.get(key) for data in inputs] for key in first_stage_keys + last_stage_keys} - # 1. For input_ids/attention_mask (prompt+target) padding: - # src_tgt_keys = ["input_ids", "attention_mask"] - src_tgt_keys = ["input_ids", "attention_mask", "position_ids"] - max_len = max([x.shape[-1] for x in inputs_batch["input_ids"]]) - pad_len = [max_len - x.shape[-1] for x in inputs_batch["input_ids"]] - for key in src_tgt_keys: - # Do not pad position_ids with 0 since 0s in position_ids has special - # usage in reward model. We use 1 to pad. - padding_value = self._ignore_index if key == "labels" else 1 if key == "position_ids" else 0 - inputs_batch[key] = pad_batches_inputs(inputs_batch[key], padding_value, pad_len=pad_len) - # 2. For old_reward_values/reward_returns/sequence_mask (target) padding: - tgt_keys = ["old_reward_values", "reward_returns", "sequence_mask"] - for key in tgt_keys: - padding_value = 0 - inputs_batch[key] = pad_batches_inputs(inputs_batch[key], padding_value, pad_len=pad_len) - # for key, value in inputs_batch.items(): - # inputs_batch[key] = pad_batches_inputs(value, padding_value=0) - # if "position_ids" not in inputs[0]: - # inputs_batch["position_ids"] = [ - # make_position_ids(attention_mask) for attention_mask in inputs_batch["attention_mask"] - # ] - return [ - get_expected_keys(inputs_batch, first_stage_keys), - get_expected_keys(inputs_batch, last_stage_keys), - ] - - def __init__(self, config, **kwargs): - # NOTE: make _sequential_layers/_single_to_pp_mapping/_pp_to_single_mapping - # instance attrs instead of class attrs to support more than one pipeline - # models. Maybe make all sequential_layers add once. - self._sequential_layers = [] - self._single_to_pp_mapping = None - self._pp_to_single_mapping = None - # To be consistent with score model init and allow hyper-param be passed - # using __init__/from_pretrained - self._init_kwargs = kwargs - super().__init__(config) - - def add_head(self, config): - init_kwargs = self._init_kwargs - # hack to replace original RMSNormPipe to support ValueHead inputs - norm_prefix = self._sequential_layers.pop(-1)["name_prefix"] - self.add_sequential_layer(LayerDesc(_LlamaRMSNormPipe, config=config), norm_prefix) - self.add_sequential_layer(LayerDesc(ValueHead, config, **init_kwargs), "") - - def get_loss_fn(self, config): - return create_loss(RLHFValueLoss, config, self._init_kwargs) - - @property - def head_out_meta(self): - # None means to use actual data info - return ( - paddle.static.InputSpec(shape=[None, None, 1], dtype=None), - paddle.static.InputSpec(shape=[None, 1], dtype=None), - ) diff --git a/slm/examples/RLHF/models/pp_model_utils.py b/slm/examples/RLHF/models/pp_model_utils.py deleted file mode 100644 index 1444cdbdd2..0000000000 --- a/slm/examples/RLHF/models/pp_model_utils.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -import inspect - -import paddle - - -def fwd_step_patch(func, output, self, *args, **kwargs): - # training patch - if self.training and self.is_pipeline_last_stage(): - if getattr(self, "_step_losses", None): - self._step_losses.append(output.detach()) - else: - self._step_losses = [output.detach()] - - -def make_wrapper(func, pre_patch=None, post_patch=None): - def wrapper(*args, **kwargs): - if pre_patch is not None: - pre_patch(func, None, *args, **kwargs) - output = func(*args, **kwargs) - if post_patch is not None: - post_patch(func, output, *args, **kwargs) - return output - - return wrapper - - -funcs = [(paddle.distributed.fleet.model.PipelineParallel._forward_step, fwd_step_patch)] - -for func in funcs: - fun, patch = func - module = importlib.import_module(fun.__module__) - cls_name = fun.__qualname__[: -len(fun.__name__) - 1] - wrap_fun = make_wrapper(fun, post_patch=patch) - cls_obj = getattr(module, cls_name) - setattr(cls_obj, fun.__name__, wrap_fun) - - -@paddle.no_grad() -def pad_batches_inputs(inputs, padding_value=0, max_len=None, pad_len=None): - """Pad length for tensors shaped [bs, seq_len] to [bs, max(seq_lens)]""" - if pad_len is not None: - pad_len = [pad_len] * len(inputs) if isinstance(pad_len, int) else pad_len - elif max_len is None: - # max_len = max([x.shape[-1] for x in inputs if x is not None]) - max_len = max([x.shape[-1] if isinstance(x, paddle.Tensor) else 0 for x in inputs]) - pad_len = [max_len - x.shape[-1] if isinstance(x, paddle.Tensor) else 0 for x in inputs] - for i in range(len(inputs)): - x = inputs[i] - # if x is None or x.shape[-1] == max_len: - if not isinstance(x, paddle.Tensor) or x.shape[-1] == max_len: - continue - inputs[i] = paddle.concat([x, paddle.full([x.shape[0], pad_len[i]], padding_value, dtype=x.dtype)], -1) - return inputs - - -def get_expected_keys(inputs, keys): - ret = tuple([inputs.get(k, None) for k in keys if k in inputs]) - if len(ret) == 1: - ret = ret[0] - return ret - - -def fwd_args_to_dict(fun): - def _impl(self, *args, **kwargs): - try: - return fun(self, *args, **kwargs) - except TypeError: - # otherwise, inputs is any valid format of non_pipe_model forward args, - # convert to dict, to support more args format in prediction_pipeline_step - # assume no arg is inspect.Parameter.VAR_KEYWORD - arg_dict = ( - inspect.signature(self._non_pipe_model_class.forward).bind(*((self,) + args), **kwargs).arguments - ) - arg_dict.pop("self") - return fun(self, arg_dict) - - return _impl diff --git a/slm/examples/RLHF/models/ppo_model.py b/slm/examples/RLHF/models/ppo_model.py deleted file mode 100644 index 7200091610..0000000000 --- a/slm/examples/RLHF/models/ppo_model.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from paddlenlp.transformers import LlamaForCausalLM, PretrainedConfig - -from .ppo_model_utils import PolicyOutput, RLHFPPOMixedLoss, RLHFValueLoss, ValueOutput -from .score_model import LlamaModelForScore - - -# TODO(guosheng): create Mixin and make model classes using metaclass. -class LlamaPolicyModel(LlamaForCausalLM): - def __init__(self, config: PretrainedConfig, **kwargs): - super().__init__(config) - self.loss_fn = RLHFPPOMixedLoss(config, **kwargs) - - def forward( - self, - input_ids=None, - position_ids=None, - attention_mask=None, - inputs_embeds=None, - labels=None, - use_cache=False, - past_key_values=None, - log_probs=None, - advantages=None, - sequence_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): - outputs = super().forward( - input_ids=input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - inputs_embeds=inputs_embeds, - labels=None, - use_cache=use_cache, - past_key_values=past_key_values, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - logits = outputs[0] - loss = None - if labels is not None or advantages is not None: - loss = self.loss_fn(logits, (labels, input_ids, log_probs, advantages, sequence_mask)) - if not return_dict: - return (loss,) + outputs if loss is not None else outputs - - return PolicyOutput( - loss=loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) - - -class LlamaValueModel(LlamaModelForScore): - def __init__(self, config, **kwargs): - super().__init__(config, **kwargs) - self.loss_fn = RLHFValueLoss(config, **kwargs) - - def forward( - self, - input_ids=None, - position_ids=None, - attention_mask=None, - inputs_embeds=None, - use_cache=False, - past_key_values=None, - old_values=None, - returns=None, - sequence_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): - outputs = super().forward( - input_ids=input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - past_key_values=past_key_values, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - reward_values, rewards = outputs - loss = None - if returns is not None: - loss = self.loss_fn(reward_values, old_values, returns, sequence_mask) - if not return_dict: - return (loss,) + outputs if loss is not None else outputs - - return ValueOutput( - loss=loss, - value=reward_values, - reward=rewards, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) diff --git a/slm/examples/RLHF/models/ppo_model_utils.py b/slm/examples/RLHF/models/ppo_model_utils.py deleted file mode 100644 index da8972cc5c..0000000000 --- a/slm/examples/RLHF/models/ppo_model_utils.py +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for score models.""" - -from __future__ import annotations - -import inspect -from dataclasses import dataclass -from typing import Optional, Tuple - -import paddle -import paddle.nn as nn -import paddle.nn.functional as F - -# use LlamaPretrainingCriterion as common PretrainingCriterion -from paddlenlp.transformers import LlamaPretrainingCriterion as PretrainingCriterion -from paddlenlp.transformers.model_outputs import ModelOutput - - -@dataclass -class PolicyOutput(ModelOutput): - loss: Optional[paddle.Tensor] = None - logits: paddle.Tensor = None - # logits_entropy: Optional[paddle.Tensor] = None - past_key_values: Optional[Tuple[Tuple[paddle.Tensor]]] = None - hidden_states: Optional[Tuple[paddle.Tensor]] = None - attentions: Optional[Tuple[paddle.Tensor]] = None - cross_attentions: Optional[Tuple[paddle.Tensor]] = None - - -@dataclass -class ValueOutput(ModelOutput): - loss: Optional[paddle.Tensor] = None - value: paddle.Tensor = None - reward: paddle.Tensor = None - past_key_values: Optional[Tuple[Tuple[paddle.Tensor]]] = None - hidden_states: Optional[Tuple[paddle.Tensor]] = None - attentions: Optional[Tuple[paddle.Tensor]] = None - cross_attentions: Optional[Tuple[paddle.Tensor]] = None - - -def merge_fwd_labels(loss_cls): - """ - PipelineParallel and trainer.criterion both use labels as tuple, thus wrap. - """ - ori_fwd = loss_cls.forward - - def loss_fwd(self, predict, labels): - return ori_fwd(self, predict, *labels) - - fwd_params = inspect.signature(ori_fwd).parameters - # forward(self, predict, label1, label2, ...) - loss_cls.label_names = list(fwd_params.keys())[2:] - loss_cls.label_default_values = {} - for label_name in loss_cls.label_names: - if fwd_params[label_name].default is not inspect.Parameter.empty: - loss_cls.label_default_values[label_name] = fwd_params[label_name].default - loss_cls.forward = loss_fwd - return loss_cls - - -def create_loss(loss_cls, config, extra_args, merge_labels=None): - """ - loss_cls(paddle.nn.Layer): loss class - config(PratrainedConfig): model config, to be consistent with loss defined - in transformers - extra_args(dict): create loss with more args not in config - merge_labels: use a wrapped loss_cls whose label args are merged into one arg, - this is useful to PipelineParallel and trainer.criterion since they only - support loss format corresponding to this format. - """ - # TODO(guosheng): merge_labels if loss_cls not - ori_fwd = loss_cls.forward - if merge_labels: - fwd_params = inspect.signature(ori_fwd).parameters - if len(fwd_params.keys()) > 3: # merge_fwd_labels has not done - loss_cls = merge_fwd_labels(loss_cls) - # forward(self, predict, label1, label2, ...) - loss_arg_names = list(inspect.signature(loss_cls.__init__).parameters.keys())[2:] - if isinstance(extra_args, dict): - loss_kwargs = dict([(name, extra_args[name]) for name in loss_arg_names if name in extra_args]) - else: - # create from TrainingArguments - loss_kwargs = dict([(name, getattr(extra_args, name)) for name in loss_arg_names if hasattr(extra_args, name)]) - loss = loss_cls(config, **loss_kwargs) - return loss - - -@paddle.no_grad() -def make_position_ids(attention_mask, source=None): - if len(attention_mask.shape) == 4: # causal mask - position_ids_p1 = attention_mask.cast(paddle.int64).sum(-1) - position_ids = position_ids_p1 - 1 - position_ids = paddle.where(position_ids == -1, position_ids_p1, position_ids) - return position_ids[:, 0, :] - assert len(attention_mask.shape) == 2 # padding mask - attention_mask_bool = attention_mask - attention_mask = attention_mask.cast(paddle.int64) - position_ids = attention_mask.cumsum(-1) - 1 - # Make padding positions in source be 0, since reward model use position_ids - # plus with padding size (number of 0s) in source to calculate end offsets. - # It does not matter when source is left padding and target is right padding - # which is the output of non-FuseMT generation, while when using FuseMT whose - # output is right padding source and right padding target, we have to set - # padding positions in source be 0 to make compatible. - if source is not None: - src_len = position_ids[:, source.shape[-1] - 1].unsqueeze(-1) - position_ids = paddle.where( - paddle.logical_and(paddle.logical_not(attention_mask_bool), position_ids <= src_len), - attention_mask, - position_ids, - ) - return position_ids - position_ids = paddle.where(position_ids == -1, attention_mask, position_ids) - return position_ids - - -@paddle.no_grad() -def make_attention_mask(input_ids, pad_id, unk_id=None, past_key_values_length=0, causal_mask=True): - attention_mask = input_ids != pad_id - if unk_id is not None and pad_id != unk_id: - attention_mask = paddle.logical_and(attention_mask, input_ids != unk_id) - if not causal_mask: - return attention_mask - - batch_size, target_length = input_ids.shape # target_length: seq_len - mask = paddle.tril(paddle.ones((target_length, target_length), dtype="bool")) - if past_key_values_length > 0: - # [tgt_len, tgt_len + past_len] - mask = paddle.concat([paddle.ones([target_length, past_key_values_length], dtype="bool"), mask], axis=-1) - # [bs, 1, tgt_len, tgt_len + past_len] - causal_mask = mask[None, None, :, :].expand([batch_size, 1, target_length, target_length + past_key_values_length]) - - attention_mask = attention_mask[:, None, None, :] - expanded_attn_mask = attention_mask & causal_mask - return expanded_attn_mask - - -def gather_log_probabilities(logits: paddle.Tensor, labels: paddle.Tensor) -> paddle.Tensor: - """Gather log probabilities of the given labels from the logits.""" - log_probs = F.log_softmax(logits, axis=-1) - log_probs_labels = paddle.take_along_axis(log_probs, axis=-1, indices=labels.unsqueeze(axis=-1)) - return log_probs_labels.squeeze(axis=-1) - - -class RLHFPPOLoss(nn.Layer): - def __init__(self, config, clip_range_ratio=0.2): - super().__init__() - self.clip_range_ratio = clip_range_ratio - self.config = config - - def actor_loss_fn( - self, log_probs: paddle.Tensor, old_log_probs: paddle.Tensor, advantages: paddle.Tensor, mask: paddle.Tensor - ) -> paddle.Tensor: - # policy gradient loss - ratio = paddle.exp(log_probs - old_log_probs) - pg_loss1 = -advantages * ratio - pg_loss2 = -advantages * paddle.clip( - ratio, - 1.0 - self.clip_range_ratio, - 1.0 + self.clip_range_ratio, - ) - return paddle.sum(paddle.maximum(pg_loss1, pg_loss2) * mask) / mask.sum() - - def forward(self, logits, input_ids, old_log_probs, reward_advantages, sequence_mask): - log_probs = gather_log_probabilities(logits[:, :-1], input_ids[:, 1:]) - if log_probs.shape[1] == old_log_probs.shape[1]: - # labels (old_log_probs, reward_advantages, sequence_mask) has - # src+tgt-1 length, valid length is determined by sequence_mask - pass - elif log_probs.shape[1] < old_log_probs.shape[1]: - # labels (old_log_probs, reward_advantages, sequence_mask) has - # src+tgt length and the last one is a padding to be consistent - # with input_ids - assert log_probs.shape[1] == old_log_probs.shape[1] - 1 - log_probs = paddle.concat([log_probs, paddle.zeros([log_probs.shape[0], 1], dtype=log_probs.dtype)], -1) - else: - # labels (old_log_probs, reward_advantages, sequence_mask) has tgt length - log_probs = log_probs[:, -old_log_probs.shape[1] :] - actor_loss = self.actor_loss_fn( - log_probs, - old_log_probs, - reward_advantages, - sequence_mask, - ) - return actor_loss - - -@merge_fwd_labels -class RLHFPPOMixedLoss(nn.Layer): - """provide two losses, one for PPO loss, the other for SFT loss.""" - - def __init__(self, config, ptx_coeff=16, clip_range_ratio=0.2): - super(RLHFPPOMixedLoss, self).__init__() - self.ptx_coeff = ptx_coeff - self.ppo_criterion = RLHFPPOLoss(config, clip_range_ratio) - self.sft_criterion = PretrainingCriterion(config) - - def forward(self, logits, labels, input_ids, old_log_probs, reward_advantages, sequence_mask): - logits = logits if isinstance(logits, paddle.Tensor) else logits[0] - loss = None - # sft, pt loss - if labels is not None: - loss = self.ptx_coeff * self.sft_criterion(logits, labels) - # ppo loss - if reward_advantages is not None: - loss = self.ppo_criterion(logits, input_ids, old_log_probs, reward_advantages, sequence_mask) - - return loss - - -@merge_fwd_labels -class RLHFValueLoss(nn.Layer): - def __init__(self, config, clip_range_value=5.0): - super().__init__() - self.clip_range_value = clip_range_value - self.config = config - - def critic_loss_fn( - self, - values: paddle.Tensor, - old_values: paddle.Tensor, - returns: paddle.Tensor, - mask: paddle.Tensor, - ) -> paddle.Tensor: - """Compute critic loss.""" - # TODO(guosheng): use paddle.clip when its min/max can support more than - # 0D Tensor - values_clipped = paddle.minimum( - paddle.maximum(values, old_values - self.clip_range_value), old_values + self.clip_range_value - ) - vf_loss1 = paddle.square(values - returns) - vf_loss2 = paddle.square(values_clipped - returns) - return 0.5 * paddle.sum(paddle.maximum(vf_loss1, vf_loss2) * mask) / mask.sum() - - def forward(self, reward_values, old_reward_values, reward_returns, sequence_mask): - reward_values = reward_values if isinstance(reward_values, paddle.Tensor) else reward_values[0] - reward_values = reward_values.squeeze(axis=-1)[:, :-1] - if reward_values.shape[1] == old_reward_values.shape[1]: - # labels (old_reward_values, reward_returns, sequence_mask) has - # src+tgt-1 length, valid length is determined by sequence_mask - pass - elif reward_values.shape[1] < old_reward_values.shape[1]: - # labels (old_reward_values, reward_returns, sequence_mask) has - # src+tgt length and the last one is a padding to be consistent - # with input_ids - assert reward_values.shape[1] == old_reward_values.shape[1] - 1 - reward_values = paddle.concat( - [reward_values, paddle.zeros([reward_values.shape[0], 1], dtype=reward_values.dtype)], -1 - ) - else: - # labels (old_reward_values, reward_returns, sequence_mask) has - # tgt length - reward_values = reward_values[:, -old_reward_values.shape[1] :] - reward_critic_loss = self.critic_loss_fn( - reward_values, - old_reward_values, - reward_returns, - sequence_mask, - ) - - return reward_critic_loss diff --git a/slm/examples/RLHF/models/score_model.py b/slm/examples/RLHF/models/score_model.py deleted file mode 100644 index aa0f509779..0000000000 --- a/slm/examples/RLHF/models/score_model.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import Any - -import paddle -from paddle import nn - -import paddlenlp -from paddlenlp.transformers import ( - LlamaConfig, - LlamaModel, - LlamaPretrainedModel, - PretrainedConfig, - PretrainedModel, -) -from paddlenlp.transformers.conversion_utils import ( - StateDictNameMapping, - init_name_mappings, -) - -from .score_model_utils import ScoreModelMixin, ScoreModelOutput - - -class LlamaModelForScore(ScoreModelMixin, LlamaPretrainedModel): - _keys_to_ignore_on_load_missing = ["lm_head.weight"] - - def __init__(self, config: PretrainedConfig, **kwargs: Any) -> None: - super().__init__(config) - self.llama = LlamaModel(config) - - # config.architectures = [self.__class__.__name__] - self.init_score_head(config, hidden_size=config.hidden_size, **kwargs) - - def get_input_embeddings(self) -> nn.Embedding: - return self.llama.embed_tokens - - def set_input_embeddings(self, value: nn.Embedding) -> None: - self.llama.embed_tokens = value - - def get_decoder(self) -> PretrainedModel: - return self.llama - - def set_decoder(self, decoder: PretrainedModel) -> None: - self.llama = decoder - - def forward( # pylint: disable=too-many-arguments - self, - input_ids: paddle.Tensor, - attention_mask: paddle.Tensor, - position_ids: paddle.Tensor | None = None, - past_key_values: list[paddle.Tensor] | None = None, - inputs_embeds: paddle.Tensor | None = None, - use_cache: bool | None = None, - output_attentions: bool | None = None, - output_hidden_states: bool | None = None, - return_dict: bool | None = None, - ) -> tuple[paddle.Tensor, paddle.Tensor] | ScoreModelOutput: - assert attention_mask is not None - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - outputs = self.llama( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - hidden_states = outputs[0] # size = (B, L, E) - return self.get_score( - hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - return_dict=return_dict, - ) - - @classmethod - def _get_name_mappings(cls, config: LlamaConfig) -> list[StateDictNameMapping]: - mappings: list[StateDictNameMapping] = [] - model_mappings = [ - ["embed_tokens.weight"], - ["norm.weight"], - ] - for layer_index in range(config.num_hidden_layers): - layer_mappings = [ - [f"layers.{layer_index}.self_attn.q_proj.weight", None, "transpose"], - [f"layers.{layer_index}.self_attn.k_proj.weight", None, "transpose"], - [f"layers.{layer_index}.self_attn.v_proj.weight", None, "transpose"], - [f"layers.{layer_index}.self_attn.o_proj.weight", None, "transpose"], - [f"layers.{layer_index}.self_attn.rotary_emb.inv_freq"], - [f"layers.{layer_index}.mlp.gate_proj.weight", None, "transpose"], - [f"layers.{layer_index}.mlp.down_proj.weight", None, "transpose"], - [f"layers.{layer_index}.mlp.up_proj.weight", None, "transpose"], - [f"layers.{layer_index}.input_layernorm.weight"], - [f"layers.{layer_index}.post_attention_layernorm.weight"], - ] - model_mappings.extend(layer_mappings) - - init_name_mappings(mappings=model_mappings) - # base-model prefix "LlamaModel" - if "LlamaModel" not in config.architectures: - for mapping in model_mappings: - mapping[0] = "model." + mapping[0] - mapping[1] = "llama." + mapping[1] - model_mappings.append(["lm_head.weight", "lm_head.weight", "transpose"]) - model_mappings.extend( - [ - ["score_head.weight", "score_head.weight", "transpose"], - ["normalizer.var", "normalizer.var"], - ["normalizer.mean", "normalizer.mean"], - ["normalizer.count", "normalizer.count"], - ] - ) - - mappings = [StateDictNameMapping(*mapping, index=index) for index, mapping in enumerate(model_mappings)] - return mappings - - -paddlenlp.transformers.LlamaModelForScore = LlamaModelForScore diff --git a/slm/examples/RLHF/models/score_model_utils.py b/slm/examples/RLHF/models/score_model_utils.py deleted file mode 100644 index 5515d56fbc..0000000000 --- a/slm/examples/RLHF/models/score_model_utils.py +++ /dev/null @@ -1,417 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# Copyright 2023 PKU-Alignment Team. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for score models.""" - -from __future__ import annotations - -import importlib -import io -import json -from abc import abstractmethod -from collections import OrderedDict -from dataclasses import dataclass -from typing import Any, Literal - -import paddle -import paddle.distributed as dist -import paddle.nn as nn - -from paddlenlp.transformers import PretrainedConfig -from paddlenlp.transformers.auto.modeling import ( - MAPPING_NAMES, - _BaseAutoModelClass, - get_init_configurations, - get_name_mapping, - is_standard_config, -) -from paddlenlp.transformers.model_outputs import ModelOutput - - -class AutoModelForScore(_BaseAutoModelClass): - CONFIGURATION_MODEL_MAPPING = get_init_configurations() - _pretrained_model_dict = CONFIGURATION_MODEL_MAPPING - # "ForScore" might be more consistent with other tasks, while this suffix is - # consistent with Beaver now. - _task_suffix = "ModelForScore" - _name_mapping: OrderedDict[str, Any] = get_name_mapping(_task_suffix) - _score_module_name: str = "models.score_model" - - @classmethod - def _get_model_class_from_config(cls, pretrained_model_name_or_path, config_file_path, config=None): - if config is None: - with io.open(config_file_path, encoding="utf-8") as f: - config = json.load(f) - - # Get class name corresponds to this configuration - if is_standard_config(config): - architectures = config["architectures"] - init_class = architectures.pop() if len(architectures) > 0 else None - else: - init_class = config.pop("init_class", None) - - # Get class name corresponds to this task, since we might init Score - # model with CausalLM model. - init_class = init_class[:-5] if init_class is not None and init_class.endswith("Model") else init_class - model_name = None - if init_class: - for model_flag, name in MAPPING_NAMES.items(): - if model_flag in init_class: - model_name = model_flag + "Model" - break - if model_name is None: - raise AttributeError( - f"Unable to parse 'architectures' or 'init_class' from {config_file_path}. Also unable to infer model class from 'pretrained_model_name_or_path'" - ) - init_class = cls._name_mapping[model_name + "_Import_Class"] - # module_name = cls._name_mapping[init_class] - module_name = cls._score_module_name - - import_class = importlib.import_module(module_name) - model_class = getattr(import_class, init_class) - return model_class - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): - return cls._from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) - - -@dataclass -class ScoreModelOutput(ModelOutput): - """ - Output of the score model. - - Args: - scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, score_dim, sequence_length)`): - Prediction scores of the score model. - end_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, score_dim)`): - Prediction scores of the end of the sequence. - """ - - scores: paddle.Tensor | None = None # size = (B, L, D) - end_scores: paddle.Tensor | None = None # size = (B, D) - - -class ScoreModelMixin: - """Base class for score models.""" - - score_head: nn.Linear - normalizer: Normalizer - do_normalize: bool = False - normalize_function: NormalizeFunction = "affine" - _initialized: bool = False - - def init_score_head(self, config: PretrainedConfig, hidden_size: int, **kwargs: Any) -> None: - """Initialize the score head.""" - if self._initialized: - return - - config.score_dim = kwargs.pop("score_dim", getattr(config, "score_dim", 1)) - config.bias = kwargs.pop("bias", getattr(config, "bias", False)) - - config.score_type = kwargs.pop("score_type", getattr(config, "score_type", "reward")) - if config.score_type == "reward": - self.normalize_function = "affine" - elif config.score_type == "cost": - self.normalize_function = "scale" - elif config.score_type == "critic": - self.normalize_function = "identity" - else: - raise ValueError( - f"Invalid score type: {config.score_type}. Expected one of 'reward', 'cost', or 'critic'.", - ) - - config.do_normalize = kwargs.pop( - "do_normalize", - getattr(config, "do_normalize", False), - ) - self.do_normalize = config.do_normalize - - config.normalizer_type = kwargs.pop( - "normalizer_type", - getattr(config, "normalizer_type", None), - ) - if config.normalizer_type not in {"RunningMeanStd", "ExponentialMovingAverage", None}: - raise ValueError( - f"Invalid norm type: {config.normalizer_type}." - "Expected one of 'RunningMeadStd', 'ExponentialMovingAverage', or None.", - ) - if config.normalizer_type == "ExponentialMovingAverage": - config.momentum = kwargs.pop("momentum", getattr(config, "momentum", None)) - momentum = getattr(config, "momentum", None) - - self.score_head = nn.Linear(hidden_size, config.score_dim, bias_attr=config.bias) - self.normalizer = Normalizer.instantiate( - normalizer_type=config.normalizer_type, - normalize_function=self.normalize_function, - shape=(config.score_dim,), - momentum=momentum, - ) - - mean = getattr(config, "mean", None) - var = getattr(config, "var", None) - self.normalizer.set_mean_var(mean, var) - - self._initialized = True - - def get_score( - self, - hidden_state: paddle.Tensor, # size = (B, L, E) - attention_mask: paddle.Tensor | None = None, # size = (B, L) - position_ids: paddle.Tensor | None = None, # size = (B, L) - return_dict: bool | None = None, - ) -> ScoreModelOutput: - """Forward pass of the score model.""" - scores = self.score_head(hidden_state) # size = (B, L, D) - - if position_ids is not None: - first_pos = paddle.arange(hidden_state.shape[0]).unsqueeze(-1) - # Take left padding into account, which has 0s in left and max_len - # in right. - left_pad_mask = position_ids == 0 - # position_ids = paddle.where( - # left_pad_mask, position_ids, position_ids + left_pad_mask.sum(-1, keepdim=True) - 1 - # ) - # the above limits right padding must not be 0s, the following suits - # to both left and right padding with 0s - left_pad_num = ( - paddle.where(left_pad_mask, position_ids.shape[-1] + 100, position_ids).argmin(axis=-1, keepdim=True) - - 1 - ) - position_ids = left_pad_num + position_ids - second_pos = paddle.max(position_ids, axis=-1, keepdim=True) - end_pos = paddle.stack([first_pos, second_pos], axis=-1).squeeze(1) - end_score = scores.gather_nd(end_pos) - else: - # attention_mask passed from pipeline pre-stage is shaped (bs, 1, seq_len, seq_len) - assert attention_mask is not None and len(attention_mask.shape) == 2 - end_score = [] - end_pos = [] - for i in range(hidden_state.shape[0]): - end_index = attention_mask[i].nonzero()[-1].item() - end_pos.append((i, end_index)) - end_score.append(scores[i, end_index]) # size = (D,) - end_score = paddle.stack(end_score, axis=0) # size = (B, D) - - if self.training and self.do_normalize: - - if dist.is_initialized(): - gathered_end_score_list = [] - try: - # gather among data parallel group - hcg = dist.fleet.get_hybrid_communicate_group() - group = hcg.get_sharding_parallel_group() - dist.all_gather(gathered_end_score_list, end_score, group) - except: - dist.all_gather(gathered_end_score_list, end_score) - gathered_end_score = paddle.concat(gathered_end_score_list, axis=0) - self.normalizer.update(gathered_end_score) - else: - self.normalizer.update(end_score) - self.config.mean = self.normalizer.mean.tolist() - self.config.var = self.normalizer.var.tolist() - - if self.do_normalize: - scores = self.normalizer.normalize(scores) - end_score = self.normalizer.normalize(end_score) - - if not return_dict: - return scores, end_score - - return ScoreModelOutput( - scores=scores, # size = (B, L, D) - end_scores=end_score, # size = (B, D) - ) - - def set_normalize(self, mode: bool = True) -> None: - if self.do_normalize == mode: - return - - self.do_normalize = self.config.do_normalize = mode - - -NormalizeFunction = Literal["affine", "scale", "translate", "identity"] -NormalizerType = Literal["RunningMeanStd", "ExponentialMovingAverage"] - - -class Normalizer(nn.Layer): - """Normalize input to have zero mean and unit variance.""" - - mean: paddle.Tensor - var: paddle.Tensor - count: paddle.Tensor - normalize_function: NormalizeFunction - - def __init__( - self, - normalize_function: NormalizeFunction, - shape: tuple[int, ...], - device: str | None = None, - ) -> None: - """Initialize.""" - super().__init__() - if normalize_function not in {"affine", "scale", "translate", "identity"}: - raise ValueError( - f"Invalid normalization function type: {normalize_function}. ", - 'Expected one of "affine", "scale", "translate", "identity".', - ) - self.normalize_function = normalize_function - self.register_buffer("mean", paddle.zeros(shape, dtype=paddle.get_default_dtype())) # align torch zeros/ones - self.register_buffer("var", paddle.ones(shape, dtype=paddle.get_default_dtype())) # align torch zeros/ones - self.register_buffer("count", paddle.zeros(1, dtype=paddle.int64)) - - @abstractmethod - def update(self, data: paddle.Tensor) -> None: - """Update mean and variance.""" - raise NotImplementedError - - @property - def std(self) -> paddle.Tensor: - """Return standard deviation.""" - return self.var.sqrt() - - def set_mean_var( - self, - mean: paddle.Tensor | list[float] | tuple[float, ...] | None, - var: paddle.Tensor | list[float] | tuple[float, ...] | None, - ) -> None: - """Set mean and variance.""" - mean = paddle.to_tensor(mean, dtype=self.mean.dtype, place=self.mean.place) if mean is not None else self.mean - var = paddle.to_tensor(var, dtype=self.var.dtype, place=self.var.place) if var is not None else self.var - - assert mean.shape == self.mean.shape - assert var.shape == self.var.shape - - self.mean = mean - self.var = var - - def forward( - self, - data: paddle.Tensor, - epsilon=1e-8, - ) -> paddle.Tensor: - """Update and normalize input.""" - if self.training: - self.update(data) - return self.normalize(data, epsilon=epsilon) - - def normalize( - self, - data: paddle.Tensor, - epsilon=1e-8, - ) -> paddle.Tensor: - """Normalize input.""" - if self.normalize_function == "affine": - return (data - self.mean.detach()) / (self.std.detach() + epsilon) - if self.normalize_function == "scale": - return data / (self.std.detach() + epsilon) - if self.normalize_function == "translate": - return data - self.mean.detach() - if self.normalize_function == "identity": - return data - raise ValueError( - f"Invalid normalization function type: {self.normalize_function}. ", - 'Expected one of "affine", "scale", "translate", "identity".', - ) - - @classmethod - def instantiate( - cls, - normalizer_type: NormalizerType | None, - normalize_function: NormalizeFunction, - shape: tuple[int, ...], - device: str | None = None, - **kwargs: Any, - ) -> Normalizer: - """Get a normalizer.""" - if normalizer_type == "RunningMeanStd": - return RunningMeanStd( - normalize_function, - shape=shape, - device=device, - ) - if normalizer_type == "ExponentialMovingAverage": - return ExponentialMovingAverage( - normalize_function, - shape=shape, - device=device, - **kwargs, - ) - if normalizer_type is None: - return IdentityNormalizer( - normalize_function, - shape=shape, - device=device, - ) - raise ValueError( - f"Invalid normalization function type: {normalizer_type}. " - 'Expected one of "RunningMeanStd", "ExponentialMovingAverage".', - ) - - -class RunningMeanStd(Normalizer): - """Running mean and standard deviation.""" - - def update(self, data: paddle.Tensor) -> None: - """Update mean and variance.""" - batch_mean = data.mean(0) - batch_var = data.var(0) - batch_count = data.shape[0] - - delta = batch_mean - self.mean - total_count = self.count + batch_count - - new_mean = self.mean + delta * batch_count / total_count - m_a = self.var * self.count - m_b = batch_var * batch_count - m2 = ( # pylint: disable=invalid-name - m_a + m_b + paddle.square(delta) * (self.count * batch_count / total_count) - ) - new_var = m2 / total_count - - self.mean = new_mean - self.var = new_var - self.count = total_count - - -class ExponentialMovingAverage(Normalizer): - """Exponential moving average.""" - - def __init__( - self, - normalize_function: NormalizeFunction, - shape: tuple[int, ...], - device: str | None = None, - momentum: float = 0.9, - ) -> None: - super().__init__(normalize_function, shape=shape, device=device) - self.momentum = momentum - - def update(self, data: paddle.Tensor) -> None: - """Update mean and variance.""" - batch_mean = data.mean(0) - batch_var = data.var(0) - batch_count = data.shape[0] - - self.mean = self.momentum * self.mean + (1.0 - self.momentum) * batch_mean - self.var = self.momentum * self.var + (1.0 - self.momentum) * batch_var - self.count += batch_count # pylint: disable=no-member - - -class IdentityNormalizer(Normalizer): - """Identity normalizer.""" - - def update(self, data: paddle.Tensor) -> None: - """Update mean and variance.""" - self.count += data.shape[0] # pylint: disable=no-member diff --git a/slm/examples/RLHF/ppo_config.json b/slm/examples/RLHF/ppo_config.json deleted file mode 100644 index 7bc5f88e51..0000000000 --- a/slm/examples/RLHF/ppo_config.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "train_datasets": "PKU-SafeRLHF/train", - "eval_datasets": "PKU-SafeRLHF/test", - "ptx_datasets": "alpaca", - "actor_model_name_or_path": "PKU-Alignment/alpaca-7b-reproduced", - "reward_model_name_or_path": "PKU-Alignment/beaver-7b-v1.0-reward", - "output_dir": "/root/paddlejob/workspace/guosheng/ckpts/ppo-reshard-sd38", - "max_length": 512, - "top_p": 0.8, - "temperature": 1.0, - "num_return_sequences":1, - "repetition_penalty": 1.0, - "num_train_epochs": 1, - "update_iters": 1, - "per_device_prompt_batch_size": 16, - "per_device_train_batch_size": 16, - "gradient_accumulation_steps": 1, - "learning_rate": 1e-5, - "weight_decay": 0.01, - "lr_scheduler_type": "cosine", - "warmup_ratio": 0.03, - "recompute": true, - "critic_learning_rate": 5e-6, - "critic_weight_decay": 0.0, - "critic_lr_scheduler_type": "constant", - "critic_warmup_ratio": 0.03, - "critic_recompute": true, - "normalize_reward": false, - "kl_coeff": 0.02, - "clip_range_ratio": 0.2, - "clip_range_score": 50.0, - "clip_range_value": 5.0, - "ptx_coeff": 16.0, - "per_device_eval_batch_size": 16, - "logging_steps": 1, - "evaluation_strategy": "steps", - "eval_steps": 100, - "save_strategy": "epoch", - "save_steps": 100000, - "bf16": true, - "fp16_opt_level": "O2", - "do_train": true, - "do_eval": true, - "disable_tqdm": true, - "save_total_limit": 1, - "sharding_parallel_degree": 4, - "sharding": "stage1", - "tensor_parallel_degree": 2, - "pipeline_parallel_degree": 1, - "pipeline_parallel_config": "disable_p2p_cache_shape", - "max_grad_norm": 1.0, - "adam_beta1": 0.9, - "adam_beta2": 0.95, - "dataloader_drop_last": false, - "eval_mode": "", - "offload_level": "freeze_model" -} diff --git a/slm/examples/RLHF/ppo_main.py b/slm/examples/RLHF/ppo_main.py deleted file mode 100644 index ec10b1d6b3..0000000000 --- a/slm/examples/RLHF/ppo_main.py +++ /dev/null @@ -1,511 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import os -import sys -import types -from dataclasses import dataclass, field -from functools import partial -from typing import Any, Dict, Tuple - -import paddle -from data import PromptOnlyDataset, SupervisedDataset, parse_dataset -from models import AutoModelForScore -from models.score_model import LlamaModelForScore # noqa -from ppo_trainer import PPOTrainer, cleanup_tensor_space, offload_tensor_to_cpu - -from paddlenlp.trainer import PdArgumentParser, TrainingArguments, get_last_checkpoint -from paddlenlp.transformers import ( - AutoConfig, - AutoModelForCausalLM, - AutoTokenizer, - LlamaTokenizer, -) -from paddlenlp.utils.log import logger - - -@dataclass -class TrainingArguments(TrainingArguments): - kl_coeff: float = field( - default=0.02, - metadata={"help": "The coefficient for the KL divergence between the reference and actor policy."}, - ) - clip_range_ratio: float = field( - default=0.2, - metadata={ - "help": "The clipping range for ratio between the old and new policy. This is the epsilon parameter in the PPO algorithm." - }, - ) - clip_range_score: float = field( - default=50.0, - metadata={ - "help": "The clipping range for the output of the score model. The reward is clipped into [-clip_range_score, clip_range_score]." - }, - ) - clip_range_value: float = field( - default=5.0, - metadata={ - "help": "The clipping range for the value function. The value is clipped into [value_estimate - clip_range_value, value_estimate + clip_range_value] during training." - }, - ) - ptx_coeff: float = field( - default=0.0, - metadata={"help": "The coefficient for the ptx loss."}, - ) - update_iters: int = field( - default=1, - metadata={"help": "The number of repeated updates on a generated batch."}, - ) - critic_learning_rate: float = field( - default=None, - metadata={"help": "Initial learning rate (after the potential warmup period) for the critic model training."}, - ) - critic_weight_decay: float = field( - default=None, - metadata={"help": "Weight decay to for the critic model training."}, - ) - critic_lr_scheduler_type: str = field( - default=None, - metadata={"help": "The scheduler type for critic model."}, - ) - critic_warmup_ratio: float = field( - default=None, - metadata={"help": "Ratio of warm steps over total training steps for the critic lr scheduler."}, - ) - critic_recompute: bool = field( - default=None, - metadata={"help": "Enable gradient checkpointing for critic model."}, - ) - normalize_reward: bool = field( - default=None, - metadata={"help": "Whether to normalize the reward during RL training."}, - ) - temperature: float = field( - default=1.0, - metadata={"help": "The value used to module the next token probabilities."}, - ) - # top_k: int = field( - # default=1, - # metadata={"help": "top_k"}, - # ) - top_p: float = field( - default=0.8, - metadata={ - "help": "If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to`top_p` or higher are kept for generation." - }, - ) - num_return_sequences: int = field( - default=1, - metadata={"help": "The number of independently computed returned sequences for each element in the batch."}, - ) - repetition_penalty: float = field( - default=1.0, - metadata={"help": "The parameter for repetition penalty. 1.0 means no penalty."}, - ) - per_device_prompt_batch_size: int = field( - default=16, - metadata={"help": "Batch size (per device) for the training dataloader."}, - ) - eval_mode: str = field( - default=None, - metadata={ - "help": "eval mode for actor model and reward_critic_model, optional for: None, single, tensor_parallel." - }, - ) - - offload_level: str = field( - default="", - metadata={"help": "Offload model, optional for: eval, reward, optimizer, train_model"}, - ) - use_fusemt: bool = field( - default=True, - metadata={"help": "use inference model to speedup in rollout generation"}, - ) - - # save_generation_output: bool = field( - # default=False, - # metadata={"help": "Whether to save generated text to file when eval"}, - # ) - - -@dataclass -class ModelArgument: - actor_model_name_or_path: str = field( - default=None, metadata={"help": "Built-in pretrained model name or the path to local model."} - ) - reward_model_name_or_path: str = field( - default=None, metadata={"help": "Built-in pretrained model name or the path to local model."} - ) - reward_critic_model_name_or_path: str = field( - default=None, metadata={"help": "Built-in pretrained model name or the path to local model."} - ) - use_flash_attention: bool = field(default=False, metadata={"help": "Whether to use flash attention"}) - - # # LoRA related parameters - # lora: bool = field(default=False, metadata={"help": "Whether to use LoRA technique"}) - # lora_path: str = field(default=None, metadata={"help": "Initialize lora state dict."}) - # lora_rank: int = field(default=8, metadata={"help": "Lora attention dimension"}) - - # # prefix tuning related parameters - # prefix_tuning: bool = field(default=False, metadata={"help": "Whether to use Prefix technique"}) - # num_prefix_tokens: int = field(default=128, metadata={"help": "Number of prefix tokens"}) - - -@dataclass -class DataArgument: - train_datasets: str = field(default=None, metadata={"help": "Dataset name(s) registered in the raw dataset."}) - eval_datasets: str = field(default=None, metadata={"help": "Dataset name(s) registered in the raw dataset."}) - eval_split_ratio: float = field(default=None, metadata={"help": "Ratio of eval data to train data"}) - ptx_datasets: str = field(default=None, metadata={"help": "Dataset name(s) registered in the raw dataset."}) - max_length: int = field( - default=2048, - metadata={ - "help": "The maximum length that model input tokens can have. When intokens is set to True, it's also the maximum length for InTokens data stream" - }, - ) - - @property - def parsed_train_datasets(self) -> Tuple[str, Dict[str, Any]]: - """Parse dataset path and its proportion and optionally additional arguments from `train_datasets`.""" - return [parse_dataset(string) for string in self.train_datasets.split(",")] - - @property - def parsed_eval_datasets(self) -> Tuple[str, Dict[str, Any]]: - """Parse dataset path and its proportion and optionally additional arguments from `eval_datasets`.""" - if self.eval_datasets is None: - return None - return [parse_dataset(string) for string in self.eval_datasets.split(",")] - - @property - def parsed_ptx_datasets(self) -> Tuple[str, Dict[str, Any]]: - """Parse dataset path and its proportion and optionally additional arguments from `ptx_datasets`.""" - if self.ptx_datasets is None: - return None - return [parse_dataset(string) for string in self.ptx_datasets.split(",")] - - -def main(): - # Arguments - parser = PdArgumentParser((ModelArgument, DataArgument, TrainingArguments)) - if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): - model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) - else: - model_args, data_args, training_args = parser.parse_args_into_dataclasses() - training_args.print_config(model_args, "Model") - training_args.print_config(data_args, "Data") - if training_args.eval_mode is not None and len(training_args.eval_mode) == 0: - training_args.eval_mode = None - if training_args.eval_mode is None and training_args.offload_level is not None: - training_args.offload_level = training_args.offload_level.replace("eval", "") - - # Setup GPU & distributed training - paddle.set_device(training_args.device) - logger.warning( - f"Process rank: {training_args.local_rank}, device: {training_args.device}, world_size: {training_args.world_size}, " - + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16 or training_args.bf16}" - ) - - # Detecting last checkpoint. - last_checkpoint = None - if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: - last_checkpoint = get_last_checkpoint(training_args.output_dir) - if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 1: - raise ValueError( - f"Output directory ({training_args.output_dir}) already exists and is not empty. " - "Use --overwrite_output_dir to overcome." - ) - if last_checkpoint is not None and training_args.resume_from_checkpoint is None: - logger.info( - f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " - "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." - ) - - # Load model - if training_args.fp16_opt_level == "O2": - if training_args.fp16: - dtype = "float16" - elif training_args.bf16: - dtype = "bfloat16" - else: - raise ValueError("Please specific dtype: --fp16 or --bf16") - else: - dtype = "float32" - training_args.max_length = data_args.max_length - - model_class_lm, model_class_score = AutoModelForCausalLM, AutoModelForScore - if training_args.pipeline_parallel_degree > 1: - from models.model_pp import LlamaPolicyPipe, LlamaValuePipe - - model_class_lm = LlamaPolicyPipe - model_class_score = LlamaValuePipe - extra_args = { - "ptx_coeff": training_args.ptx_coeff, - "clip_range_ratio": training_args.clip_range_ratio, - } - else: - # non-pipe modelForCausalLM does not accept extra_args and use other ways - # (StepTrainer.create_criterion) to set hyper-parameters - extra_args = {} - - # actor model - model_config = AutoConfig.from_pretrained( - model_args.actor_model_name_or_path, - tensor_parallel_output=False, - tensor_parallel_degree=training_args.tensor_parallel_degree, - tensor_parallel_rank=training_args.tensor_parallel_rank, - dtype=dtype, - ) - if hasattr(model_config, "use_flash_attention"): - model_config.use_flash_attention = model_args.use_flash_attention - - # model_config.num_hidden_layers = 2 - - actor_model = model_class_lm.from_pretrained( - model_args.actor_model_name_or_path, - config=model_config, - **extra_args, - # ptx_coeff=training_args.ptx_coeff, - # clip_range_ratio=training_args.clip_range_ratio, - ) - if training_args.eval_mode is not None: - config = copy.deepcopy(actor_model.config) - if training_args.eval_mode == "single": - config.tensor_parallel_degree = -1 - config.tensor_parallel_rank = 0 - actor_eval_model = AutoModelForCausalLM.from_config(config) - # TODO(guosheng): AutoModel (in `_get_model_class_from_config`) pop out - # architecture which is necessary for infer predictor currently - config.architectures = actor_model.config.architectures - # actor_eval_model = AutoModelForCausalLM.from_pretrained(model_args.actor_model_name_or_path, config=config) - else: - actor_eval_model = None - - # todo reference model - if training_args.eval_mode is not None: - config = copy.deepcopy(model_config) - if training_args.eval_mode == "single": - config.tensor_parallel_degree = -1 - config.tensor_parallel_rank = 0 - actor_reference_model = AutoModelForCausalLM.from_pretrained( - model_args.actor_model_name_or_path, - config=config, - ) - else: - actor_reference_model = model_class_lm.from_pretrained( - model_args.actor_model_name_or_path, - config=model_config, - ) - - actor_tokenizer = AutoTokenizer.from_pretrained( - model_args.actor_model_name_or_path, model_max_length=data_args.max_length, padding_side="left" - ) - - # reward model - model_config = AutoConfig.from_pretrained( - model_args.reward_model_name_or_path, - tensor_parallel_output=False, - tensor_parallel_degree=training_args.tensor_parallel_degree, - tensor_parallel_rank=training_args.tensor_parallel_rank, - dtype=dtype, - ) - if hasattr(model_config, "use_flash_attention"): - model_config.use_flash_attention = model_args.use_flash_attention - # model_config.num_hidden_layers = 2 - # todo - if training_args.eval_mode is not None: - config = copy.deepcopy(model_config) - if training_args.eval_mode == "single": - config.tensor_parallel_degree = -1 - config.tensor_parallel_rank = 0 - reward_model = AutoModelForScore.from_pretrained( - model_args.reward_model_name_or_path, - config=config, - score_type="reward", - do_normalize=training_args.normalize_reward, - ) - else: - reward_model = model_class_score.from_pretrained( - model_args.reward_model_name_or_path, - config=model_config, - score_type="reward", - do_normalize=training_args.normalize_reward, - ) - reward_tokenizer = AutoTokenizer.from_pretrained( - model_args.reward_model_name_or_path, model_max_length=data_args.max_length, padding_side="right" - ) - # critic model - if model_args.reward_critic_model_name_or_path is None: - model_args.reward_critic_model_name_or_path = model_args.reward_model_name_or_path - reward_critic_model = model_class_score.from_pretrained( - model_args.reward_critic_model_name_or_path, - config=model_config, - score_type="critic", - do_normalize=False, - clip_range_value=training_args.clip_range_value, - ) - reward_critic_tokenizer = AutoTokenizer.from_pretrained( - model_args.reward_critic_model_name_or_path, model_max_length=data_args.max_length, padding_side="left" - ) - if training_args.eval_mode is not None: - config = copy.deepcopy(reward_critic_model.config) - if training_args.eval_mode == "single": - config.tensor_parallel_degree = -1 - config.tensor_parallel_rank = 0 - reward_critic_eval_model = AutoModelForScore.from_config(config) - # reward_critic_eval_model = AutoModelForScore.from_pretrained( - # model_args.reward_critic_model_name_or_path,config=model_config - # ) - else: - reward_critic_eval_model = None - - # # actor model - # model_config = AutoConfig.from_pretrained( - # model_args.actor_model_name_or_path, - # tensor_parallel_output=False, - # tensor_parallel_degree=training_args.tensor_parallel_degree, - # tensor_parallel_rank=training_args.tensor_parallel_rank, - # dtype=dtype, - # ) - # model_config.num_hidden_layers = 2 - # if hasattr(model_config, "use_flash_attention"): - # model_config.use_flash_attention = model_args.use_flash_attention - # actor_model = AutoModelForCausalLM.from_pretrained( - # model_args.actor_model_name_or_path, - # config=model_config, - # ) - # - # if training_args.eval_mode is not None: - # config = copy.deepcopy(actor_model.config) - # if training_args.eval_mode == "single": - # config.tensor_parallel_degree = -1 - # config.tensor_parallel_rank = 0 - # actor_eval_model = AutoModelForCausalLM.from_config(config) - # else: - # actor_eval_model = None - # - # # reference model - # actor_reference_model = AutoModelForCausalLM.from_pretrained( - # model_args.actor_model_name_or_path, - # config=model_config, - # ) - # actor_tokenizer = AutoTokenizer.from_pretrained( - # model_args.actor_model_name_or_path, model_max_length=data_args.max_length, padding_side="left" - # ) - # - # # reward model - # model_config = AutoConfig.from_pretrained( - # model_args.reward_model_name_or_path, - # tensor_parallel_output=False, - # tensor_parallel_degree=training_args.tensor_parallel_degree, - # tensor_parallel_rank=training_args.tensor_parallel_rank, - # dtype=dtype, - # ) - # model_config.num_hidden_layers = 2 - # if hasattr(model_config, "use_flash_attention"): - # model_config.use_flash_attention = model_args.use_flash_attention - # reward_model = AutoModelForScore.from_pretrained( - # model_args.reward_model_name_or_path, - # config=model_config, - # score_type="reward", - # do_normalize=training_args.normalize_reward, - # ) - # reward_tokenizer = AutoTokenizer.from_pretrained( - # model_args.reward_model_name_or_path, model_max_length=data_args.max_length, padding_side="right" - # ) - # - # # critic model - # if model_args.reward_critic_model_name_or_path is None: - # model_args.reward_critic_model_name_or_path = model_args.reward_model_name_or_path - # reward_critic_model = AutoModelForScore.from_pretrained( - # model_args.reward_critic_model_name_or_path, config=model_config, score_type="critic", do_normalize=False - # ) - # reward_critic_tokenizer = AutoTokenizer.from_pretrained( - # model_args.reward_critic_model_name_or_path, model_max_length=data_args.max_length, padding_side="left" - # ) - # - # if training_args.eval_mode is not None: - # config = copy.deepcopy(reward_critic_model.config) - # if training_args.eval_mode == "single": - # config.tensor_parallel_degree = -1 - # config.tensor_parallel_rank = 0 - # reward_critic_eval_model = AutoModelForScore.from_config(config) - # else: - # reward_critic_eval_model = None - - for tokenizer in [actor_tokenizer, reward_tokenizer, reward_critic_tokenizer]: - if isinstance(tokenizer, LlamaTokenizer) and tokenizer.pad_token_id is None: - tokenizer.pad_token_id = tokenizer.eos_token_id - - train_ds = PromptOnlyDataset(data_args.parsed_train_datasets, tokenizer=actor_tokenizer) - if data_args.eval_datasets is None and data_args.eval_split_ratio: - train_ds, dev_ds = train_ds.split_train_test(split_ratio=data_args.eval_split_ratio) - elif data_args.eval_datasets is not None: - dev_ds = PromptOnlyDataset(data_args.parsed_eval_datasets, tokenizer=actor_tokenizer) - else: - dev_ds = None - - ptx_ds = ( - SupervisedDataset(data_args.parsed_ptx_datasets, tokenizer=actor_tokenizer) - if data_args.ptx_datasets is not None - else None - ) - if ptx_ds is not None: - # PretrainingCriterion requires shifted inputs and labels - ptx_ds.get_collator = types.MethodType(partial(ptx_ds.get_collator.__func__, shift=True), ptx_ds) - - # offload - # cleanup actor_eval_model, reward_critic_eval_model - # offload actor_reference_model reward_model - - if training_args.offload_level is not None: - if "eval" in training_args.offload_level: - cleanup_tensor_space(actor_eval_model.state_dict()) - cleanup_tensor_space(reward_critic_eval_model.state_dict()) - if "reward" in training_args.offload_level: - # if pp mode, should lazy offload - offload_tensor_to_cpu(actor_reference_model.state_dict()) - offload_tensor_to_cpu(reward_model.state_dict()) - - trainer = PPOTrainer( - # (policy_model, reference_model, reward_model, value_model) - # policy_model, sft_model, reward_model, value_model - # (policy_model, reference_model, reward_model, value_model, - # (policy_model, reference_model, reward_model, value_model, policy_eval_model, value_eval_model - # (actor_model, actor_reference_model, reward_model, reward_critic_model, actor_eval_model, reward_critic_eval_model - model=( - actor_model, - actor_reference_model, - reward_model, - reward_critic_model, - actor_eval_model, - reward_critic_eval_model, - ), - args=training_args, - train_dataset=train_ds, - eval_dataset=dev_ds, - ptx_dataset=ptx_ds, - tokenizer=(actor_tokenizer, actor_tokenizer, reward_tokenizer, reward_critic_tokenizer), - data_collator=train_ds.get_collator(), - ) - checkpoint = None - if training_args.resume_from_checkpoint is not None: - checkpoint = training_args.resume_from_checkpoint - elif last_checkpoint is not None: - checkpoint = last_checkpoint - trainer.train(resume_from_checkpoint=checkpoint) - - -if __name__ == "__main__": - main() diff --git a/slm/examples/RLHF/ppo_trainer.py b/slm/examples/RLHF/ppo_trainer.py deleted file mode 100644 index bdec462411..0000000000 --- a/slm/examples/RLHF/ppo_trainer.py +++ /dev/null @@ -1,1803 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import itertools -import math -import os -import sys -import time -from typing import Any, Callable, Dict, List, Optional, Tuple, Union - -import paddle -import paddle.distributed as dist -import paddle.nn as nn -from comm_utils import ( # noqa - cleanup_tensor_space, - create_data_trans_group, - data_group_merge, - data_group_split, - offload_tensor_to_cpu, - reload_tensor_to_gpu, -) -from infer_utils import InferEvalModel, infer_guard -from models.ppo_model_utils import ( - RLHFPPOMixedLoss, - RLHFValueLoss, - create_loss, - gather_log_probabilities, - make_attention_mask, - make_position_ids, -) -from paddle.distributed import fleet -from paddle.io import DataLoader, Dataset, DistributedBatchSampler -from paddle.utils import map_structure -from rich.console import Console -from rich.table import Table -from trainer_utils import ( - MuteDefaultFlowCallback, - PipeEvalModel, - batch_retokenize, - guard_set_args, - is_same_tokenizer, -) - -from paddlenlp.data import DataCollator -from paddlenlp.generation import GenerationConfig -from paddlenlp.trainer.trainer import ( - EvalLoopOutput, - EvalPrediction, - ShardingOption, - Trainer, - TrainerCallback, - TrainingArguments, - logger, - speed_metrics, -) -from paddlenlp.transformers import PretrainedModel, PretrainedTokenizer -from paddlenlp.utils import empty_device_cache - - -class StepTrainer(Trainer): - """ - Features of StepTrainer: - 1. Trainer enhanced with step-level training combining with patches of - Trianer. We can use this to do training whose step is composed of multi - models via multiple instances of StepTrainer, such as PPO. - 2. Additionally, using a mixed loss and get the separated loss metrics is - supported, which is helpful to PipelienParallel with a mixed loss. - 3. EMA is supported. - """ - - # used to create criterion for trainer, please refer to `create_criterion` - # for details. - loss_cls: type - - def __init__( - self, - model: Union[PretrainedModel, nn.Layer] = None, - criterion: nn.Layer = None, - args: TrainingArguments = None, - data_collator: Optional[DataCollator] = None, - train_dataset: Optional[Dataset] = None, - eval_dataset: Union[Dataset, Dict[str, Dataset]] = None, - tokenizer: Optional[PretrainedTokenizer] = None, - compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None, - callbacks: Optional[List[TrainerCallback]] = None, - optimizers: Tuple[paddle.optimizer.Optimizer, paddle.optimizer.lr.LRScheduler] = (None, None), - preprocess_logits_for_metrics: Callable[[paddle.Tensor, paddle.Tensor], paddle.Tensor] = None, - ): - super().__init__( - model, - criterion, - args, - data_collator, - train_dataset, - eval_dataset, - tokenizer, - compute_metrics, - callbacks, - optimizers, - preprocess_logits_for_metrics, - ) - # criterion is only used for non-PipelineParallel models. criterion is - # included in model for PipelineParallel. - if getattr(self, "loss_cls", None) and self.criterion is None: - self.criterion = self.create_criterion() - - self.use_fusemt = getattr(args, "use_fusemt", False) - # ablout 4s slower than infer generation without ema - self.use_ema = getattr(args, "use_ema", False) - self.shard_ema = getattr(args, "shard_ema", False) - self.offload_ema = getattr(args, "offload_ema", True) - self.ema_beta = getattr(args, "ema_beta", 0.992) - - def create_criterion(self): - """ - create loss using `loss_cls` for trainer. It would use a wrapped loss_cls - whose label arguments are merged into one argument, this is useful to - PipelineParallel and trainer.criterion which limit loss format. - """ - criterion = create_loss(self.loss_cls, self.model.config, self.args, merge_labels=True) - return criterion - - def loss_identifier(self, inputs: Dict) -> str: - """ - Moreover, a model/StepTrainer instance may use a mixed loss which uses a - different loss for different step and inputs, while we often want to get - the separated loss metric. We use a callable discriminator using inputs - (dict) as arguments and returning corresponding loss name to identify - current loss. NOTE: please make the loss name ends with "_loss". `tr_loss` - is the default loss name used in trainer.train. - """ - return "tr_loss" - - def set_eval_model(self, model): - """ - To avoid eval/generation with PipelineParallel when training with PP, we - allow to use an extra eval model to do eval/generation, which would need - to reshard parameters and dispatch data according to model's distributed - topo. Currently, the eval model should cancel PP setting and keep the same - TP setting with training. - """ - if model is None: - logger.warning("use None to set eval model for trainer and it would be ignored") - return - else: - self._inner_eval_model = model - # bind a new comm group for eval model data dispatch - # param dispatch is binded in `InferEvalModel.enable` - hcg = fleet.get_hybrid_communicate_group() - sd_group = hcg.get_sharding_parallel_group() - dp_group = hcg.get_data_parallel_group() - global_rank = dist.get_rank() - eval_tp_size = max(model.config.tensor_parallel_degree, 1) - eval_tp_rank = max(model.config.tensor_parallel_rank, 0) - old_dp_workers = self.args.world_size // (max(sd_group.nranks, 1) * max(dp_group.nranks, 1)) - group_nums = self.args.logical_process_index // old_dp_workers * eval_tp_size + eval_tp_rank - self._data_trans_group = create_data_trans_group(global_rank, group_nums) - # just for compatiable with old code - self._policy_model_eval_group = self._data_trans_group - - def get_model(self, train=False): - """ - model visitor wrapps PipelineParalle and Inference model to do evaulation - and generation. - """ - if train: - return self.model_wrapped - model = getattr(self, "_eval_model", None) - if model is not None: - return model - inner_eval_model = getattr(self, "_inner_eval_model", None) - if (self.args.pipeline_parallel_degree > 1 and inner_eval_model is None) or isinstance( - inner_eval_model, fleet.model.PipelineParallel - ): - # Only accept wrapped model for pipeline_parallel mode - model = PipeEvalModel(self) - self._eval_model = model - else: - model = InferEvalModel(self) - self._eval_model = model - return model - - def get_train_step_vars(self, vars: Dict = None) -> Dict: - """ - NOTE: This is transparent to users. - When using multiple instances of StepTrainer collaborate to do one training - step, each should use its own vars such as loss/model/step_control which are - local vars in Trainer.train, we define these vars by `train_step_vars`. They - are vars needed by full_training_step for training control, as following: - tr_loss, model, epoch, step, step_control. - some vars such as `epoch` are meaningless, they are needed just because - full_training_step copies code from Trainer.train which is designed for - complete training process. - - return `train_step_vars` (dict). If not exists, create it first. If `vars` - is not None, update `train_step_vars` with it. - - TODO(guosheng): use namedtuple or dataclass to make it more readable. - """ - if not hasattr(self, "train_step_vars"): - # should be called after model is wrapped since the model field should - # use model_wrapped. - - assert self.model is not self.model_wrapped - self.train_step_vars = { - # meaningless vars can pass from outter, dummy value is enough - "epoch": 0, # meaningless for step training - "step": 0, # meaningless for step training - "steps_in_epoch": 100000, # meaningless for step training - "step_control": 0, # to control training process - "model": self.model_wrapped, - # "tr_loss": paddle.to_tensor(0.0), # lazy create - } - if vars: - self.train_step_vars.update(vars) - return self.train_step_vars - - @property - def loss_names(self): - if not hasattr(self, "_loss_names"): - self._loss_names = [var_name for var_name in self.get_train_step_vars() if var_name.endswith("_loss")] - assert len(self._loss_names) > 0 - return self._loss_names - - def full_training_step(self, **inputs) -> paddle.Tensor: - """ - Accept any valid key word arguments of model and loss as inputs, they - would be sent to model and then loss. Mostly it is similar to output from - data collator. - Return loss var. However when using PipelienParallel, the loss returned - is 0 when not reach accumulated step and the loss returned at accumulated - step is a mixed loss. We can use `get_step_loss` to get the actual loss. - """ - # if model has multi losses which are combined into one mixed criterion, - # loss statistic var may change for different training steps according - # to inputs. - train_step_vars = self.get_train_step_vars() - loss_name = self.loss_identifier(inputs) - loss_var = train_step_vars.get(loss_name, None) - # trainer.train use `tr_loss` as loss var to accumulate loss. - # NOTE: `tr_loss` in trainer.train not only accumulate mean loss for - # steps in one `gradient_accumulation_steps`, but also accumulate for - # one logging intervel which may contains more than one accumulated steps. - # However, in StepTrainer we only want to use `tr_loss` to accumulate - # mean loss for steps in a `gradient_accumulation_steps` range. As for - # logging intervel loss accumulation is not take into account here and - # should be considered in outter. - if loss_var is None: # the first step of current loss type - loss_var = paddle.to_tensor(0.0) - train_step_vars[loss_name] = loss_var - elif self.is_accumulation_step: # begin a new accumulation step intervel - for name in self.loss_names: - train_step_vars[name] = paddle.to_tensor(0.0) - loss_var = train_step_vars[loss_name] - - train_step_vars["tr_loss"] = loss_var - - new_train_step_vars = super().full_training_step(inputs, **train_step_vars) - - # minimally update - train_step_vars = self.get_train_step_vars( - {"step_control": new_train_step_vars["step_control"], loss_name: new_train_step_vars["tr_loss"]} - ) - if loss_name != "tr_loss": - train_step_vars.pop("tr_loss") - - self.mark_step_loss(loss_name) - - if self.use_ema and self.is_accumulation_step: - # TODO(guosheng): assume rollout next thus make ema weights on gpu, - # but may not, maybe need a way to specify it. - self.ema_update(beta=self.ema_beta, offload_ema=self.offload_ema, offload_model=not self.offload_ema) - - return train_step_vars[loss_name] - - def _prepare_inputs(self, inputs: Dict[str, Union[paddle.Tensor, Any]]) -> Dict[str, Union[paddle.Tensor, Any]]: - """ - trainer.criterion only support criterion(prediction, labels), so we need - to reorganize the inputs to extract label data into one argument. This is - only used in non-PipelineParallel model training since loss is included - in PipelineLayer. - """ - inputs = super()._prepare_input(inputs) - if self.criterion is None or getattr(self.criterion, "label_names", None) is None: - return inputs - # criterion created by create_loss has `label_names` and `label_default_values` - label_names = self.criterion.__class__.label_names - # some data fields are used both in model and loss - shared_fields = set(["input_ids", "attention_mask"]) - labels = [] - for name in label_names: - if name not in inputs: - label = self.criterion.__class__.label_default_values.get(name, None) - elif name in shared_fields: - label = inputs[name] - else: - label = inputs.pop(name) - labels.append(label) - # "labels" is the pre-defined label name in Trainer - inputs["labels"] = labels - # NOTE: TensorParallel model requires non-Tensor inputs to be lists and - # broadcast them, thus do not or optionally use these inputs. labels use - # in criterion not send to model can workaround this. - return inputs - - def mark_step_loss(self, loss_name): - """ - NOTE: This is transparent to users. - When using a mixed loss we often want to get the separated loss metrics, - thus we mark loss type of each training step to separate them. This is - not necessary since the loss would be returnd after each training step. - However when using PipelienParallel, the loss returned is 0 when not reach - accumulated step and the loss returned at accumulated step is a mixed loss. - To separate loss metrics in PipelienParallel: - 1. We hack PipelineParallel._forward_step to record actual loss for each - step in a list (only in training and not in evaluation currently). - 2. We mark the loss type only once for each step using `loss_step_indice` - (dict), then wen can check out the corresponding loss metrics from the - loss list. - We assume a static order of multi-losses and mark the loss indice only once. - """ - self.loss_step_indice = getattr(self, "loss_step_indice", {}) - if loss_name not in self.loss_step_indice: - self.loss_step_indice[loss_name] = len(self.loss_step_indice) - - @paddle.no_grad() - def get_step_loss(self, loss_prefix: str = "", loss_accumulator: Dict = {}) -> Dict[str, paddle.Tensor]: - """ - Return a dict mapping loss name to value of current training step. This - is mainly to get loss for metric logging, and it would not affect the - training. This is mostly helpful to PipelienParallel with a mixed loss - in which the loss returned is 0 when not reach accumulated step and the - loss returned at accumulated step is a mixed loss. - NOTE: 1. Only when reaching accumulated step the losses returned are - accurate, and each loss is a mean loss of steps among one accumulated - steps range. - """ - if not self.is_accumulation_step: - msg = "The loss returned may not be accurate when not reaching accumulated step." - logger.error(msg) - model = self.get_model(train=True) - loss_dict = loss_accumulator if loss_accumulator else {} - if isinstance(model, fleet.model.PipelineParallel) and len(self.loss_names) > 1: - # NOTE: PipelineParallel only returns a accumulated loss after - # accumulated steps, which is a mixed loss of ppo-loss and - # ptx-loss. We hack PipelineParallel._forward_step to record - # loss metrics and postprocess the recorded losses here. - # Maybe better to make the last_stage worker log to reduce - # comm and for simplicity. - with paddle.no_grad(): - if model.is_pipeline_last_stage(): - # loss is 0D tensor, use stack rather than concat - mix_loss = paddle.stack(model._step_losses) - model._step_losses = None - else: - # The tessor shape is not policy_model.accumulate_steps - # (args.accu_steps) but policy_trainer.args.accu_steps, - # since policy_model is created with global pp_config - # using global args.accu_steps which is only half of - # policy_trainer.args.accu_steps, and indeed trainer hack - # model.accumulate_steps in training_pipeline_step to use - # trainer.args.accu_steps. The dtype is fp32(to be check), - # thus no need to broadcast. - mix_loss = paddle.empty(shape=[self.args.gradient_accumulation_steps], dtype=paddle.float32) - paddle.distributed.broadcast(mix_loss, src=model.pp_group.ranks[-1], group=model.pp_group) - for loss_name in self.loss_names: - # We assume a static order of multi-losses and mark the loss - # indice only once. - value = mix_loss[self.loss_step_indice[loss_name] :: len(self.loss_names)].mean() - loss_name = loss_prefix + loss_name if loss_prefix else loss_name - loss_dict[loss_name] = loss_dict[loss_name].add_(value) if loss_name in loss_dict else value - return loss_dict - elif isinstance(model, fleet.model.PipelineParallel): - model._step_losses = None - - for loss_name in self.loss_names: - value = self.get_train_step_vars()[loss_name] - loss_name = loss_prefix + loss_name if loss_prefix else loss_name - loss_dict[loss_name] = loss_dict[loss_name].add_(value) if loss_name in loss_dict else value - return loss_dict - - @property - def is_accumulation_step(self): - """Indicate whether accumulation steps' training is done.""" - return self.get_train_step_vars()["step_control"] == 0 - - def get_sharding_master_weight_structured_names(self, model, optimizer): - rank_param_names = [p.name for p in optimizer._rank2params[optimizer._sharding_rank]] - structured_names = [] - # for pipeline model, use `model.state_dict()` would auto map param name - # for name, p in model.named_parameters(): - for name, p in model.state_dict().items(): - if p.name in rank_param_names: - structured_names.append(name) - return structured_names - - def get_master_weight_state_dict(self, model, optimizer): - if self.amp_dtype in ["float16", "bfloat16"] and hasattr(optimizer, "_master_weights"): - master_weights = dict(optimizer._master_weights) - result = {} - # for pipeline model, use `model.state_dict()` would auto map param name - # for name, p in model.named_parameters(): - for name, p in model.state_dict().items(): - if p.name in master_weights: - result[name] = master_weights[p.name] - return result - else: - return model.state_dict() - - def ema_init(self, offload_ema=True, offload_model=False, shard_ema=True): - """should be called after model and optimizer are created and wrapped""" - self.ema_state_dict = {} - self.bak_state_dict = {} - hcg = fleet.get_hybrid_communicate_group() - sharding_size = hcg.get_sharding_parallel_world_size() - # NOTE: use optimizer.master_weight instead of model.state_dict to set - # ema_state_dict would make ema coupled with master_weight reshard. - structured_names = ( - self.get_sharding_master_weight_structured_names(self.model, self.optimizer) - if sharding_size > 1 and shard_ema - else None - ) - # for pipeline model, use `model.state_dict()` would auto map param name - # for name, p in self.model.named_parameters(): - for name, p in self.model.state_dict().items(): - if structured_names is None or name in structured_names: - ema_p = p.detach().cast(dtype=paddle.float32) - if offload_ema: - ema_p = ema_p.pin_memory() - self.ema_state_dict[name] = ema_p - if offload_model: - cpu_p = p.pin_memory() - cpu_p._share_buffer_to(p) - self.bak_state_dict[name] = p - if getattr(self.model, "tie_word_embeddings", False): - raise NotImplementedError - - @paddle.no_grad() - def ema_update(self, beta=0.992, offload_ema=True, offload_model=False): - """ - This would be called automatically in `full_training_step` if `use_ema` - is True to update ema state when ending an accumulated step intervel. - """ - model_keys = list(self.ema_state_dict.keys()) - hcg = fleet.get_hybrid_communicate_group() - sharding_size = hcg.get_sharding_parallel_world_size() - trainer_state_dict = ( - self.get_master_weight_state_dict(self.model, self.optimizer) - if sharding_size > 1 and self.shard_ema - else self.model.state_dict() - ) - for key in model_keys: - if getattr(self.model, "tie_word_embeddings", False) and "lm_head" in key: - raise NotImplementedError - trainer_data = trainer_state_dict[key].cuda() - if trainer_data.dtype != paddle.float32: - # use model state dict instead of master weights - trainer_data = trainer_data.cast(dtype=paddle.float32) - ema_data = self.ema_state_dict[key].cuda() - # update ema & offload ema - ema_result = (beta * ema_data) + (1.0 - beta) * trainer_data - self.ema_state_dict[key] = ema_result.pin_memory() if offload_ema else ema_result - if offload_model: - cpu_p = trainer_data.pin_memory() - cpu_p._share_buffer_to(trainer_data) - if getattr(self.model, "tie_word_embeddings", False): - raise NotImplementedError - - def ema_apply(self): - """ - If use sharding and `shard_ema` is true, `ema_state_dict` only includes - sharded weights, thus we need the completed ema state to apply it to model - and ema would be coupled with reshard, then we need to reshard here. - """ - # TODO(guosheng): `bak_state_dict` is indeed trainer.model, allow to use - # a new model instead of trainer.model as target model. - # NOTE: if `shard_ema` is True, `ema_state_dict` is just a subset (sharded - # part) of model state_dict, and ema would coupled with reshard. - for k, v in self.bak_state_dict.items(): - # TODO(guosheng): reshard here - value = self.ema_state_dict[k].cuda().cast(dtype=v.dtype) - value._share_buffer_to(v) - - def ema_restore(self): - for k, v in self.bak_state_dict.items(): - value = v.cuda() - value._share_buffer_to(v) - if self.offload_ema: # ema weights always in pin_memory in fact - ema_v = self.ema_state_dict[k] - ema_value = ema_v.pin_memory() - ema_value._share_buffer_to(ema_v) - - -class ema(paddle.no_grad.__mro__[1]): - def __init__(self, trainer: StepTrainer): - self.trainer = trainer - - def __enter__(self): - trainer = self.trainer - if trainer.use_ema and not hasattr(trainer, "ema_state_dict"): - # call ema_init here since it should be called after model and - # optimizer are created and wrapped - trainer.ema_init( - offload_ema=trainer.offload_ema, offload_model=not trainer.offload_ema, shard_ema=trainer.shard_ema - ) - if self.trainer.use_ema: - self.trainer.ema_apply() - - def __exit__(self, *args): - if self.trainer.use_ema: - self.trainer.ema_restore() - - -class enable(paddle.no_grad.__mro__[1]): - """offload""" - - def __init__(self, *args): - self.objs = args - - def __enter__(self): - for obj in self.objs: - if hasattr(obj, "enable"): - obj.enable() - else: - reload_tensor_to_gpu(obj.state_dict()) - # offload_tensor_to_cpu/reload_tensor_to_gpu use non-blocking copy - # maybe overlap with compute later - if len(self.objs) > 0: - paddle.device.synchronize() - - def __exit__(self, *args): - for obj in self.objs: - if hasattr(obj, "disable"): - obj.disable() - else: - offload_tensor_to_cpu(obj.state_dict()) - # offload_tensor_to_cpu/reload_tensor_to_gpu use non-blocking copy - # maybe overlap with compute later - if len(self.objs) > 0: - paddle.device.synchronize() - - -class PolicyTrainer(StepTrainer): - loss_cls = RLHFPPOMixedLoss - - def loss_identifier(self, inputs: Dict) -> str: - labels = inputs.get("labels", None) - if labels is not None: # use ptx - loss_name = "ptx_loss" - else: - loss_name = "actor_loss" - return loss_name - - -class ValueTrainer(StepTrainer): - loss_cls = RLHFValueLoss - # define loss name for logging - loss_identifier = lambda self, inputs: "reward_critic_loss" - - -class PPOMetric: - def set_metric_meta(self, use_ptx=True): - self.metric_names = [ - "train/" + name - for name in [ - "actor_loss", - "ptx_loss", - "reward_critic_loss", - "reward", - "kl_divergence", - "mean_generated_length", - "max_generated_length", - ] - ] - - self.metric_ops = ["mean", "mean", "mean", "mean", "mean", "mean", "max"] - if not use_ptx: - self.metric_names.pop(1) - self.metric_ops.pop(1) - - def __init__(self, freq, use_stack=True, use_ptx=True): - self.set_metric_meta(use_ptx=use_ptx) - self.freq = freq - self.counter = 0 - self.use_stack = use_stack - if use_stack: - self.metrics = paddle.zeros([freq, len(self.metric_names)], dtype=paddle.float32) - else: - self.metrics = [None] * len(self.metric_names) - for i in range(len(self.metrics)): - self.metrics[i] = paddle.zeros([freq], dtype=paddle.float32) - - @paddle.no_grad() - def update(self, metrics: Dict[str, paddle.Tensor]) -> Union[None, Dict[str, float]]: - """ - If has updated for`freq` times then return metrics (results reduced from - all worker) and reset metric states, otherwise return `None`. - """ - for name in self.metric_names: - # PipelineParallel broadcast loss with shape [1] - if len(metrics[name].shape) != 0: - metrics[name] = metrics[name].squeeze() - if metrics[name].dtype != paddle.float32: - metrics[name] = metrics[name].cast(paddle.float32) - if self.use_stack: - self.metrics[self.counter] = paddle.stack([metrics[name] for name in self.metric_names]) - else: - for i, name in enumerate(self.metric_names): - self.metrics[i][self.counter] = metrics[name] - if self.counter + 1 == self.freq: - from paddlenlp.trainer.utils import distributed_concat - - metrics = distributed_concat(self.metrics) - out_metrics = {} - if self.use_stack: - mean_metric = metrics.mean(0) - max_metric = metrics.max(0) - for i, (name, op) in enumerate(zip(self.metric_names, self.metric_ops)): - if op == "max": - out_metrics[name] = max_metric[i].item() if self.use_stack else metrics[i].max().item() - else: - out_metrics[name] = mean_metric[i].item() if self.use_stack else metrics[i].mean().item() - - # reset - self.counter = 0 - if self.use_stack: - self.metrics.fill_(0.0) - else: - for i, name in enumerate(self.metric_names): - self.metrics[i].fill_(0.0) - return out_metrics - - -def data_dispatch(fun): - def _impl(self, data): - gp = getattr(self.policy_trainer, "_data_trans_group", None) - data = data_group_split(data, group=gp) - data = fun(self, data) - data = data_group_merge(data, group=gp) - return data - - return _impl - - -class PPOTrainer(Trainer): - def __init__( - self, - model: Union[PretrainedModel, nn.Layer] = None, - criterion: nn.Layer = None, - args: TrainingArguments = None, - data_collator: Optional[DataCollator] = None, - train_dataset: Optional[Dataset] = None, - ptx_dataset: Optional[Dataset] = None, - eval_dataset: Union[Dataset, Dict[str, Dataset]] = None, - tokenizer: Optional[PretrainedTokenizer] = None, - compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None, - callbacks: Optional[List[TrainerCallback]] = None, - optimizers: Tuple[paddle.optimizer.Optimizer, paddle.optimizer.lr.LRScheduler] = (None, None), - preprocess_logits_for_metrics: Callable[[paddle.Tensor, paddle.Tensor], paddle.Tensor] = None, - ): - with guard_set_args( - args, - { - "recompute": False, - "fp16_opt_level": "O1", - "pipeline_parallel_degree": 1, # workaround for pipeline parallel model check - }, - ): - # just used to create trival attrs might be used in the training - # process of trainer, while changing some args to avoid model usage - # in __init__ such as recompute and AMP-O2 - super().__init__( - model, - criterion, - args, - data_collator, - train_dataset, - eval_dataset, - tokenizer, - compute_metrics, - callbacks, - optimizers, - preprocess_logits_for_metrics, - ) - - self.train_dataset = train_dataset - self.ptx_dataset = ptx_dataset - self.eval_dataset = eval_dataset - - (policy_model, reference_model, reward_model, value_model, policy_model_eval, value_model_eval) = model - self._model_config = policy_model.config # use this to change flash attention dynamicly - self._policy_model_eval = policy_model_eval - self._value_model_eval = value_model_eval - - # policy_tokenizer and value_tokenizer should be same - (policy_tokenizer, reference_tokenizer, reward_tokenizer, value_tokenizer) = tokenizer - - policy_training_args = copy.deepcopy(args) - self.use_ptx = self.ptx_dataset is not None - if self.use_ptx: - policy_training_args.gradient_accumulation_steps *= 2 - self.policy_trainer = PolicyTrainer( - policy_model, - criterion, - policy_training_args, - data_collator, - train_dataset, - eval_dataset, - policy_tokenizer, - compute_metrics, - callbacks, - optimizers, - preprocess_logits_for_metrics, - ) - value_training_args = copy.deepcopy(args) - for attr_name in [ - "critic_learning_rate", - "critic_weight_decay", - "critic_lr_scheduler_type", - "critic_warmup_ratio", - "critic_recompute", - ]: - if getattr(value_training_args, attr_name, None) is not None: - setattr(value_training_args, attr_name[len("critic_") :], getattr(value_training_args, attr_name)) - self.value_trainer = ValueTrainer( - value_model, - criterion, - value_training_args, - data_collator, - train_dataset, - eval_dataset, - value_tokenizer, - compute_metrics, - callbacks, - optimizers, - preprocess_logits_for_metrics, - ) - self.policy_trainer.set_eval_model(policy_model_eval) - self.value_trainer.set_eval_model(value_model_eval) - # disable inner trainers' callback/state/control - self.policy_trainer.add_callback(MuteDefaultFlowCallback) - self.value_trainer.add_callback(MuteDefaultFlowCallback) - - # use trainer for reference_model/reward_model to enable sharding stage-3 - # and PipelineParallel. maybe we should allow models to use different dist - # strategies later - - from paddle.distributed.fleet.meta_parallel import PipelineLayer - - # allow reference_model/reward_model to use different dist strategy - with guard_set_args( - args, - { - "recompute": False, - # "fp16_opt_level": "O1", - "pipeline_parallel_degree": args.pipeline_parallel_degree - if isinstance(reference_model, PipelineLayer) - else 1, # workaround for pipeline parallel model check - }, - ): - - self.reference_trainer = StepTrainer( - reference_model, - criterion, - copy.deepcopy(args), - data_collator, - train_dataset, - eval_dataset, - reference_tokenizer, - compute_metrics, - callbacks, - optimizers, - preprocess_logits_for_metrics, - ) - self.reward_trainer = StepTrainer( - reward_model, - criterion, - copy.deepcopy(args), - data_collator, - train_dataset, - eval_dataset, - reward_tokenizer, - compute_metrics, - callbacks, - optimizers, - preprocess_logits_for_metrics, - ) - # TODO(guosheng): sharding stage3 should create master weight optionally - # instead of creation and clear. - from paddlenlp.trainer.trainer_utils import ShardingOption - - if args.pipeline_parallel_degree > 1 or ShardingOption.FULL_SHARD in args.sharding: - self.reference_trainer.init_train_model_opt(100, None, clear_master_weight=True) # dummy max_steps - self.reward_trainer.init_train_model_opt(100, None, clear_master_weight=True) # dummy max_steps - - self.reference_model.eval() - self.reward_model.eval() - - self.reward_tokenizer = reward_tokenizer - self.tokenizer = policy_tokenizer - if is_same_tokenizer(self.tokenizer, self.reward_tokenizer): - self.reward_tokenizer = self.tokenizer - - self.generation_config = GenerationConfig( - max_new_tokens=self.args.max_length, - num_return_sequences=self.args.num_return_sequences, - temperature=self.args.temperature, - top_p=self.args.top_p, - top_k=0, # to disable top_k sampling, default is 50 - repetition_penalty=self.args.repetition_penalty, - do_sample=True, - # allow generation output to contain input - trunc_input=False, - bos_token_id=self.tokenizer.bos_token_id, - eos_token_id=self.tokenizer.eos_token_id, - pad_token_id=self.tokenizer.pad_token_id, - ) - # Those value can be changed - self.kl_coeff = self.args.kl_coeff - self.clip_range_score = self.args.clip_range_score - self.ptx_coeff = self.args.ptx_coeff - self.gamma = 1.0 - self.gae_lambda = 0.95 - - # dummy class and object for model to be compaible with methods of - # Trainer, such as evaluation_loop - self.DummyPPOModel = type( - "DummyPPOModel", (object,), {"eval": lambda _: self.set_eval(), "train": lambda _: self.set_train()} - ) - self.model = self.model_wrapped = self.DummyPPOModel() - - @property - def reference_model(self): - return self.reference_trainer.get_model(train=False) - - @property - def reward_model(self): - return self.reward_trainer.get_model(train=False) - - @property - def actor_model(self): - return self.policy_trainer.get_model(train=self.training) - - @property - def reward_critic_model(self): - return self.value_trainer.get_model(train=self.training) - - def set_train(self, mode: bool = True) -> None: - """Set training mode for all models.""" - if mode: - self.training = True - self.actor_model.train() - self.reward_critic_model.train() - else: - self.training = False - self.actor_model.eval() - self.reward_critic_model.eval() - - def set_eval(self) -> None: - """Set model to evaluation mode.""" - self.set_train(mode=False) - - def prediction_step( - self, - model: nn.Layer, - inputs: Dict[str, Union[paddle.Tensor, Any]], - prediction_loss_only: bool, - ignore_keys: Optional[List[str]] = None, - ) -> Tuple[Optional[paddle.Tensor], Optional[paddle.Tensor], Optional[paddle.Tensor]]: - inputs = self._prepare_inputs(inputs) - - with paddle.no_grad(): - with self.autocast_smart_context_manager(): - seq = self.actor_model.generate( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - position_ids=inputs["position_ids"] - if "position_ids" in inputs - else make_position_ids(inputs["attention_mask"]), - generation_config=self.generation_config, - synced_gpus=ShardingOption.FULL_SHARD in self.policy_trainer.args.sharding, - )[0] - if self.reward_tokenizer is not self.tokenizer: - reward_tokenize_output = batch_retokenize( - input_ids=seq, - src_tokenizer=self.tokenizer, - dest_tokenizer=self.reward_tokenizer, - skip_special_tokens=True, - device=self.args.device, - ) - reward_input_ids = reward_tokenize_output["input_ids"] - else: - reward_input_ids = seq - reward_attention_mask = make_attention_mask( - seq, - pad_id=self.reward_tokenizer.pad_token_id, - unk_id=self.reward_tokenizer.unk_token_id, - causal_mask=False, - ) - reward_position_ids = make_position_ids(reward_attention_mask) - - # unify PP with others since PP always return tuple - reward_score = self.reward_model( - reward_input_ids, - attention_mask=reward_attention_mask, - position_ids=reward_position_ids, - # return_dict=True, - )[ - 1 - ] # .end_scores - reward_score = reward_score.squeeze(axis=-1).cast(paddle.float32) - - # keep the first batch of eval output sequence to print and check - prompt = self.tokenizer.batch_decode(inputs["input_ids"], skip_special_tokens=True) - generated = self.tokenizer.batch_decode(seq, skip_special_tokens=True) - for i, text in enumerate(generated): - self._eval_out_file.write(text + "\n") - if getattr(self, "_eval_seq", None) is None: - generated = [text[len(prompt[i]) :] for i, text in enumerate(generated)] - # prompts.extend(prompt) - # generateds.extend(generated) - self._eval_seq = (prompt, generated, reward_score.tolist()) - - return reward_score.mean(), None, None - - def evaluation_loop( - self, - dataloader: DataLoader, - description: str, - prediction_loss_only: Optional[bool] = None, - ignore_keys: Optional[List[str]] = None, - metric_key_prefix: str = "eval", - max_eval_iters: Optional[int] = -1, - ) -> EvalLoopOutput: - # to save eval generated sequence - eval_out_file = os.path.join( - self.args.output_dir, f"eval_out-step{self.state.global_step}-rank{self.args.local_rank}.txt" - ) - self._eval_out_file = open(eval_out_file, "w") - - # TODO(guosheng): use _inner_eval_model (if trainer has one) instead of - # original trainer model to eval, especially when using sharded EMA - # NOTE: use here rather than in prediction_step since actor_model would - # be set to eval out of prediction_step - with guard_set_args( - self.policy_trainer, # disable _inner_eval_model - { - "_eval_model": None, # otherwise would use cached _eval_model - "_inner_eval_model": None, # otherwise would use _inner_eval_model to create _eval_model - }, - ): - output = super().evaluation_loop( - dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix, max_eval_iters - ) - output.metrics[f"{metric_key_prefix}/reward"] = output.metrics.pop(f"{metric_key_prefix}_loss") - - columns = ["Prompt", "Generated", "Reward"] - rows = list(zip(*self._eval_seq)) - rows = [[str(item) for item in row] for row in rows] - max_num_rows = 5 - table = Table(title="Evaluating...", show_lines=True, title_justify="left") - for column in columns: - table.add_column(column) - for row in rows[:max_num_rows]: - table.add_row(*row) - Console(soft_wrap=True, markup=False, emoji=False).print(table) - self._eval_seq = None - - self._eval_out_file.close() - - return output - - def get_eval_dataloader(self, eval_dataset: Optional[Dataset] = None) -> DataLoader: - with guard_set_args(self, {"data_collator": self.eval_dataset.get_collator()}): - return super().get_eval_dataloader(eval_dataset) - - def _save_checkpoint(self, model, metrics=None): - # maybe change args.output_dir of policy_trainer/value_trainer directly - with guard_set_args(self.policy_trainer.args, {"output_dir": os.path.join(self.args.output_dir, "policy")}): - self.policy_trainer._save_checkpoint(model, metrics) - with guard_set_args(self.value_trainer.args, {"output_dir": os.path.join(self.args.output_dir, "value")}): - self.value_trainer._save_checkpoint(model, metrics) - - def init_train_model_opt( - self: Trainer, max_steps: int, resume_from_checkpoint: bool = False, clear_master_weight: bool = False - ) -> PretrainedModel: - # resume should be triggered here - # maybe change args.output_dir of policy_trainer/value_trainer directly - with guard_set_args(self.policy_trainer.args, {"output_dir": os.path.join(self.args.output_dir, "policy")}): - policy_model = self.policy_trainer.init_train_model_opt( - max_steps, - os.path.join(resume_from_checkpoint, "policy") - if isinstance(resume_from_checkpoint, str) - else resume_from_checkpoint, - ) - with guard_set_args(self.value_trainer.args, {"output_dir": os.path.join(self.args.output_dir, "value")}): - value_model = self.value_trainer.init_train_model_opt( - max_steps, - os.path.join(resume_from_checkpoint, "value") - if isinstance(resume_from_checkpoint, str) - else resume_from_checkpoint, - ) - return policy_model, value_model - - def get_epoch_iterator(self): - def gen_epoch_data(): - for prompt_only_batch, ptx_batch in zip( - self.prompt_only_dataloader, - itertools.cycle(self.ptx_dataloader), - ): - # generate batches - self.set_eval() - - with ema(self.policy_trainer), ema(self.value_trainer): - rl_batches = self.split_rl_micro_batches(prompt_only_batch) - - self.timers and self.timers("ptx-batch").start() - if self.use_ptx: - ptx_batches = self.split_ptx_micro_batches(ptx_batch) - else: - ptx_batches = [None for _ in range(len(rl_batches))] - self.timers and self.timers("ptx-batch").stop() - - empty_device_cache() - - self.set_train() - for _ in range(self.args.update_iters): - for rl_batch, ptx_batch in zip(rl_batches, ptx_batches): - yield rl_batch, ptx_batch - - class EpochIterator: - def __iter__(self): - return gen_epoch_data() - - def __len__(self): - return len(self.prompt_only_dataloader) * ( - self.args.update_iters - * self.args.per_device_prompt_batch_size - * self.args.num_return_sequences - // self.args.per_device_train_batch_size - ) - - return EpochIterator() - - def init_train_num(self: Trainer, train_dataloader: DataLoader): - args = self.args - - total_train_batch_size = args.train_batch_size * args.gradient_accumulation_steps * args.dataset_world_size - len_dataloader = None - if not self._is_iterable_dataset(self.train_dataset): - len_dataloader = len(train_dataloader) - num_train_sub_steps = ( - len_dataloader - * self.args.update_iters - * self.args.per_device_prompt_batch_size - * self.args.num_return_sequences - // self.args.per_device_train_batch_size - ) - num_update_steps_per_epoch = num_train_sub_steps // args.gradient_accumulation_steps - num_examples = len(self.train_dataset) - if args.max_steps > 0: - max_steps = args.max_steps - num_train_epochs = args.max_steps // num_update_steps_per_epoch + int( - args.max_steps % num_update_steps_per_epoch > 0 - ) - else: - max_steps = int(num_update_steps_per_epoch * args.num_train_epochs) - num_train_epochs = math.ceil(args.num_train_epochs) - num_train_samples = total_train_batch_size * max_steps - else: - assert args.max_steps > 0 - max_steps = args.max_steps - num_train_epochs = sys.maxsize - num_update_steps_per_epoch = args.max_steps - num_examples = total_train_batch_size * args.max_steps - num_train_samples = args.max_steps * total_train_batch_size - - return ( - total_train_batch_size, - len_dataloader, - max_steps, - num_train_epochs, - num_update_steps_per_epoch, - num_examples, - num_train_samples, - ) - - def is_step_end(self): - # reach accumulation_steps, value trainer has the same step_control and - # gradient_accumulation_steps as PPO trainer. - # if (step_control + 1) % args.gradient_accumulation_steps == 0 - return self.value_trainer.is_accumulation_step - - def get_step_loss(self, loss_prefix: str = "") -> Dict: - rl_loss = self.policy_trainer.get_step_loss(loss_prefix) - value_loss = self.value_trainer.get_step_loss(loss_prefix) - rl_loss.update(value_loss) - return rl_loss - - def train( - self, - resume_from_checkpoint: Optional[Union[str, bool]] = None, - ignore_keys_for_eval: Optional[List[str]] = None, - ) -> None: - # ##### The following code try to keep same as the Trainer.train ##### - args = self.args - self.is_in_train = True - - # ##### trainging data and related num setting ##### - # TODO(guosheng): remove the binding method get_collator of dataset - with guard_set_args( - args, {"per_device_train_batch_size": self.args.per_device_prompt_batch_size} - ), guard_set_args( - self, {"train_dataset": self.train_dataset, "data_collator": self.train_dataset.get_collator()} - ): - train_dataloader = self.prompt_only_dataloader = self.get_train_dataloader() - - if self.use_ptx: - with guard_set_args( - args, - { - "per_device_train_batch_size": 1 - if getattr(self.ptx_dataset, "is_intokens", False) - else self.args.per_device_prompt_batch_size * self.args.num_return_sequences - }, - ), guard_set_args( - self, {"train_dataset": self.ptx_dataset, "data_collator": self.ptx_dataset.get_collator()} - ): - self.ptx_dataloader = self.get_train_dataloader() - else: - self.ptx_dataloader = range(100) - ( - total_train_batch_size, - len_dataloader, - max_steps, - num_train_epochs, - num_update_steps_per_epoch, - num_examples, - num_train_samples, - ) = self.init_train_num(train_dataloader) - - # ##### model and optimizer related setting ##### - policy_model, value_model = self.init_train_model_opt(max_steps, resume_from_checkpoint) - empty_device_cache() - - # ##### traing statistic logging ##### - # Number of trainable parameters only account for policy_model - self.init_train_log( - num_examples, num_train_epochs, total_train_batch_size, max_steps, num_train_samples, policy_model - ) - - # ##### set training state and resume ##### - # consumed_samples used to set train_dataloader.batch_sampler may not be - # correct. Thus, data cannot be resumed perfectly when not breaking at epoch end. - epochs_trained, steps_trained_in_current_epoch, steps_trained_progress_bar = self.init_train_state( - resume_from_checkpoint, train_dataloader, max_steps, num_train_epochs, num_update_steps_per_epoch - ) - - epoch_iterator = self.get_epoch_iterator() - steps_in_epoch = num_update_steps_per_epoch * args.gradient_accumulation_steps - - # self.callback_handler.model = self.model - # self.callback_handler.optimizer = self.optimizer - # self.callback_handler.lr_scheduler = self.lr_scheduler - # self.callback_handler.train_dataloader = train_dataloader - self.state.max_steps = int(max_steps) - self.state.num_train_epochs = num_train_epochs - self.state.is_local_process_zero = self.is_local_process_zero() - self.state.is_world_process_zero = self.is_world_process_zero() - - self.control = self.callback_handler.on_train_begin(args, self.state, self.control) - - self._globalstep_last_logged = self.state.global_step - metric = PPOMetric(freq=self.args.logging_steps, use_ptx=self.use_ptx) - - start_time = time.time() - self._globalstep_last_start_time = start_time - # self.timers and self.timers("read-data").start() - - for epoch in range(epochs_trained, num_train_epochs): - if isinstance(train_dataloader, paddle.io.DataLoader) and isinstance( - train_dataloader.batch_sampler, DistributedBatchSampler - ): - train_dataloader.batch_sampler.set_epoch(epoch) - - self.control = self.callback_handler.on_epoch_begin(args, self.state, self.control) - - for step, inputs in enumerate(epoch_iterator): - # self.timers and self.timers("read-data").stop() - # os.environ["TRAINER_GLOBAL_STEP"] = str(self.state.global_step) - # self.callback_handler.on_load_data_end(args, self.state, self.control, inputs=inputs) - rl_batch, ptx_batch = inputs - # TODO(guosheng): make rl_step/ptx_step run with autocast_smart_context_manager - logger.info("Doing rl step...") - self.timers and self.timers("rl_step").start() - with self.enable(self.actor_model, self.policy_trainer.optimizer): - # with self.enable(self.value_trainer.optimizer): - with self.enable(): # put value optimizer guard in rl_step - rl_info = self.rl_step(rl_batch) - empty_device_cache() - self.timers and self.timers("rl_step").stop() - - if self.use_ptx: - logger.info("Doing ptx step...") - self.timers and self.timers("ptx_step").start() - with guard_set_args( - self._model_config, - { - # "set_attn_func": True, - # "use_flash_attention": True - }, - ): - ptx_info = self.ptx_step(ptx_batch) - rl_info.update(ptx_info) - self.timers and self.timers("ptx_step").stop() - empty_device_cache() - - self.state.global_step += 1 - self.state.epoch = epoch + (step + 1) / steps_in_epoch - if self.is_step_end(): - rl_info.update(self.get_step_loss(loss_prefix="train/")) - rl_info = metric.update(rl_info) - # on_step_end - self.control = self.callback_handler.on_step_end(args, self.state, self.control) - else: - # on_sub_step_end - self.control = self.callback_handler.on_substep_end(args, self.state, self.control) - self._maybe_log_save_evaluate(rl_info, None, epoch, ignore_keys_for_eval, inputs=inputs) - self._print_timer() - if self.control.should_epoch_stop or self.control.should_training_stop: - break - - if step < 0: - logger.warning( - f"There seems to be not a single sample in your epoch_iterator, stopping training at step" - f" {self.state.global_step}! This is expected if you're using an IterableDataset and set" - f" num_steps ({self.state.max_steps}) higher than the number of available samples." - ) - self.control.should_training_stop = True - - self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) - # argument model is not used in _maybe_log_save_evaluate, thus use None - self._maybe_log_save_evaluate(rl_info, None, epoch, ignore_keys_for_eval, inputs=inputs) - self._print_timer() - - if self.control.should_training_stop: - break - # TODO(guosheng): add epilogue of training - - def _maybe_log_save_evaluate(self, tr_loss, model, epoch, ignore_keys_for_eval, **kwargs): - if self.control.should_log: - - logs: Dict[str, float] = {} - # use_ptx would double the gradient_accumulation_steps which causes - # actor_loss and ptx_loss reduced by half. Moreover, ptx_loss should - # be divided by ptx_coeff for logging. - if "train/ptx_loss" in tr_loss: - tr_loss["train/actor_loss"] = tr_loss["train/actor_loss"] * 2 - tr_loss["train/ptx_loss"] = tr_loss["train/ptx_loss"] * 2 / self.ptx_coeff - logs.update(tr_loss) - logs["global_step"] = int(self.state.global_step) - logs["train/actor_lr"] = float("{0:.3e}".format(self.policy_trainer._get_learning_rate())) - logs["train/reward_critic_lr"] = float("{0:.3e}".format(self.value_trainer._get_learning_rate())) - - total_train_batch_size = ( - self.args.train_batch_size * self.args.gradient_accumulation_steps * self.args.dataset_world_size - ) - num_steps = self.state.global_step - self._globalstep_last_logged - logs.update( - speed_metrics( - "interval", - self._globalstep_last_start_time, - num_samples=total_train_batch_size * num_steps, - num_steps=num_steps, - ) - ) - - self._globalstep_last_logged = self.state.global_step - self._globalstep_last_start_time = time.time() - - self.log(logs, **kwargs) - - # To trigger evaluation and save but avoid log again - with guard_set_args(self.control, {"should_log": False}): - super()._maybe_log_save_evaluate(tr_loss, model, epoch, ignore_keys_for_eval) - - def add_kl_divergence_regularization( - self, - prompt: paddle.Tensor, # size = (B, S) # pylint: disable=unused-argument - log_probs: paddle.Tensor, # size = (B, L) - ref_log_probs: paddle.Tensor, # size = (B, L) - reward_score: paddle.Tensor, # size = (B,) - sequence_mask: paddle.Tensor, # size = (B, L) - ) -> paddle.Tensor: - kl_divergence_estimate = -self.kl_coeff * (log_probs - ref_log_probs) # size = (B, L) - rewards = kl_divergence_estimate # size = (B, L) - reward_clip = paddle.clip( # size = (B,) - reward_score, - min=-self.clip_range_score, - max=self.clip_range_score, - ) - # TODO(guosheng): use scatter_add/put_along_axis - index = paddle.cumsum(sequence_mask.cast(paddle.int64), axis=-1).argmax(-1, keepdim=True) - rewards = paddle.put_along_axis(rewards, index, reward_clip.unsqueeze(axis=-1), axis=-1, reduce="add") - # batch_size = log_probs.shape[0] - # for i in range(batch_size): - # # print("="*20, sequence_mask[i]) - # end_index = sequence_mask[i].nonzero()[-1] - # # rewards[i, end_index] += reward_clip[i] - # rewards[i, end_index] = rewards[i, end_index] + reward_clip[i] - - return rewards - - def get_advantages_and_returns( - self, - values: paddle.Tensor, - rewards: paddle.Tensor, - sequence_mask: paddle.Tensor, - start: int, - use_tgt_len_return: bool = True, - ) -> Tuple[paddle.Tensor, paddle.Tensor]: - """Compute advantages and returns using Generalized Advantage Estimation (GAE).""" - # Modified from https://github.com/CarperAI/trlx/blob/main/trlx/models/modeling_ppo.py - last_gae_lambda = 0.0 - advantages_reversed = [] - values = values * sequence_mask - rewards = rewards * sequence_mask - length = rewards.shape[-1] - if use_tgt_len_return and start > 0: - # consistent with Beaver - # values length is src+tgt-1, start is src-1, return length is tgt - pass - elif use_tgt_len_return: - # values length is tgt, start is 0, return length is tgt - assert start == 0 - else: - # values length is src+tgt-1, start is src-1, return length is src+tgt-1 - pass - for t in reversed(range(start, length)): # pylint: disable=invalid-name - next_values = values[:, t + 1] if t < length - 1 else 0.0 - delta = rewards[:, t] + self.gamma * next_values - values[:, t] - last_gae_lambda = delta + self.gamma * self.gae_lambda * last_gae_lambda - advantages_reversed.append(last_gae_lambda) - advantages = paddle.stack(advantages_reversed[::-1], axis=1) - returns = advantages + values[:, start:] - if not use_tgt_len_return: - advantages = paddle.concat( - [paddle.zeros([advantages.shape[0], start], dtype=advantages.dtype), advantages], -1 - ) - returns = paddle.concat([paddle.zeros([returns.shape[0], start], dtype=returns.dtype), returns], -1) - return advantages.detach(), returns - - def rl_step(self, rl_batch: Dict[str, paddle.Tensor]) -> Dict[str, Any]: - # inputs shared by policy and value trainer - input_ids = rl_batch["input_ids"] # length: src+tgt - attention_mask = rl_batch["attention_mask"] # length: src+tgt - position_ids = rl_batch["position_ids"] # length: src+tgt - sequence_mask = rl_batch["sequence_mask"] # length: src+tgt(-1) - # inputs used by policy trainer - old_log_probs = rl_batch["log_probs"] # length: src+tgt(-1) - reward_advantages = rl_batch["reward_advantages"] # length: src+tgt(-1) - # inputs used by value trainer - old_reward_values = rl_batch["reward_values"] # length: src+tgt(-1) - reward_returns = rl_batch["reward_returns"] # length: src+tgt(-1) - - value_trainer_inputs = { - "input_ids": input_ids, - "attention_mask": attention_mask, - "position_ids": position_ids, - "old_reward_values": old_reward_values, - "reward_returns": reward_returns, - "sequence_mask": sequence_mask, - } - with self.enable(self.reward_critic_model, self.value_trainer.optimizer): - reward_critic_loss = self.value_trainer.full_training_step(**value_trainer_inputs) - - policy_trainer_inputs = { - "input_ids": input_ids, - "attention_mask": attention_mask, - "position_ids": position_ids, - "old_log_probs": old_log_probs, - "reward_advantages": reward_advantages, - "sequence_mask": sequence_mask, - } - actor_loss = self.policy_trainer.full_training_step(**policy_trainer_inputs) - - # metric - with paddle.no_grad(): - rewards = rl_batch["rewards"] - rewards = rewards.mean() - ref_log_probs = rl_batch["ref_log_probs"] - kl_divergence = ((old_log_probs - ref_log_probs) * sequence_mask).sum(axis=-1).mean() - mean_generated_length = sequence_mask.cast(paddle.float32).sum(axis=-1).mean() - max_generated_length = sequence_mask.cast(paddle.float32).sum(axis=-1).max() - - return { - # when using PipelienParallel, the loss returned is 0 when not reach - # accumulated step and the loss returned at accumulated step is a - # mixed loss. - "train/actor_loss": actor_loss, - "train/reward_critic_loss": reward_critic_loss, - "train/reward": rewards, - "train/kl_divergence": kl_divergence, - "train/mean_generated_length": mean_generated_length, - "train/max_generated_length": max_generated_length, - } - - def ptx_step(self, ptx_batch: Dict[str, paddle.Tensor]) -> Dict[str, Any]: - """Perform a single update step with PTX loss.""" - # sft inputs use right padding, position_ids is optional - # ptx_batch["position_ids"] = ptx_batch.get( - # "position_ids", make_position_ids(ptx_batch["attention_mask"])) - ptx_loss = self.policy_trainer.full_training_step(**ptx_batch) - return { - "train/ptx_loss": ptx_loss, - } - - def enable(self, *args): - # note: must keep the same model since actor_model, reward_model etc. - # are property - enable_map = { - # maybe use `model: (pattern, enable_method, disable_method)`` - self.actor_model: "train_model", - self.reward_critic_model: "train_model", - self.reference_model: "freeze_model", - self.reward_model: "freeze_model", - self.policy_trainer.optimizer: "optimizer", - self.value_trainer.optimizer: "optimizer", - } - # if use an extra eval model to do eval/generation, switch on actor_model - # and reward_critic_model; otherwise no need to switch - if getattr(self.policy_trainer, "_inner_eval_model", None) is not None: - enable_map.pop(self.actor_model) - if getattr(self.value_trainer, "_inner_eval_model", None) is not None: - enable_map.pop(self.reward_critic_model) - objs = [arg for arg in args if enable_map.get(arg, "") in self.args.offload_level] - return enable(*objs) - - def split_ptx_micro_batches( - self, - ptx_batch: Dict[str, paddle.Tensor], - ) -> List[Dict[str, paddle.Tensor]]: - """Split a batch of PTX samples into micro-batches.""" - micro_batches = [] - total_batch_size = ptx_batch["input_ids"].shape[0] - micro_batch_size = self.args.per_device_train_batch_size - for i in range(0, total_batch_size, micro_batch_size): - micro_batch = map_structure( - # pylint: disable-next=cell-var-from-loop - lambda tensor: tensor[i : i + micro_batch_size], # noqa: B023 - ptx_batch, - ) - micro_batches.append(micro_batch) - return micro_batches - - # @staticmethod - # def data_dispatch(fun): - # def _impl(self, data): - # gp = getattr(self.policy_trainer, "_data_trans_group", None) - # data = data_group_split(data, group=gp) - # data = fun(self, data) - # data = data_group_merge(data, group=gp) - # return data - - # return _impl - - @paddle.no_grad() - @data_dispatch # 3.10 static methods are now callable as regular functions. - def split_rl_micro_batches( - self, - prompt_only_batch: Dict, - ) -> List[Dict]: - """Split a batch of RL samples into micro-batches.""" - total_batch_size = prompt_only_batch["input_ids"].shape[0] - micro_batch_size = self.args.per_device_train_batch_size - micro_batches = [] - - # TODO(guosheng): clean get_epoch_iterator: - # 1. scope guard for offload, we would split post_rollout into multiple - # sub-methods to offload in-time - # 2. decorate split_rl_micro_batches to automatically split/merge data - with self.enable(self.actor_model, self.reference_model): - # generate for multi batches and then disable FuseMT model - with infer_guard(self.policy_trainer): - # dist.barrier() - # print("="*20, "begin generate") - for i in range(0, total_batch_size, micro_batch_size): - micro_batch = {} - micro_batch = map_structure( - lambda tensor: tensor[i : i + micro_batch_size], - prompt_only_batch, - ) - micro_batches.extend(self.generate(micro_batch)) - # dist.barrier() - # paddle.device.cuda.synchronize() - # get log_probs for multi batches and then disable actor/refer rmodel - for micro_batch in micro_batches: - # position_ids is necessary for non-right padding - # If using right padding source + left padding target, make padding positions - # in source be 0, since reward model use position_ids plus with padding size - # (number of 0s) in source to calculate end offsets. - micro_batch["position_ids"] = make_position_ids(micro_batch["attention_mask"]) - micro_batch.update(self.rollout_logprob(**micro_batch)) - # print("="*20, "micro_batch", micro_batch) - - # get reward/value for multi batches and then disable reward/value model - with self.enable(self.reward_critic_model, self.reward_model): - for micro_batch in micro_batches: - micro_batch.update(self.rollout_reward_value(**micro_batch)) - - # - micro_batches = [self.normalize_data(micro_batch, use_tgt_len_value=False) for micro_batch in micro_batches] - # size of micro_batches (num of training batch) would be: - # per_device_prompt_batch_size * num_return_sequences // per_device_train_batch_size - # micro_batches = [self.post_rollout(**micro_batch) for micro_batch in micro_batches] - return micro_batches - - @paddle.no_grad() - def generate(self, prompt_only_batch: Dict) -> List[Dict[str, Any]]: - """Rollout a batch of experiences.""" - input_ids = prompt_only_batch["input_ids"] - attention_mask = prompt_only_batch["attention_mask"] - - self.timers and self.timers("actor-model-generate").start() - sequences = self.actor_model.generate( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=prompt_only_batch["position_ids"] - if "position_ids" in prompt_only_batch - else make_position_ids(attention_mask), - generation_config=self.generation_config, - synced_gpus=ShardingOption.FULL_SHARD in self.policy_trainer.args.sharding, - )[0] - - self.timers and self.timers("actor-model-generate").stop() - sequences = sequences.reshape([input_ids.shape[0], self.args.num_return_sequences, -1]).transpose([1, 0, 2]) - - # prompt, sequence, attention_mask - return [ - { - "prompt": input_ids, - "input_ids": seq, # "sequence": - "attention_mask": make_attention_mask( - seq, - pad_id=self.tokenizer.pad_token_id, - unk_id=self.tokenizer.unk_token_id, - causal_mask=False, - ), - # "sequence_mask": make_attention_mask( - # seq, - # pad_id=self.tokenizer.pad_token_id, - # unk_id=self.tokenizer.unk_token_id, - # causal_mask=False, - # ), - } - for seq in sequences - ] - - @paddle.no_grad() - def rollout_logprob( - self, input_ids: paddle.Tensor, attention_mask: paddle.Tensor, position_ids: paddle.Tensor = None, **kwargs - ) -> Dict[str, paddle.Tensor]: - # pipe model outputs a logits tensor with LMHead, while non-pipe model - # outputs a tuple with logits tensor as the only one element. - logits = self.actor_model( - input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - # return_dict=True, - ) # .logits - if not isinstance(logits, paddle.Tensor): - logits = logits[0] - ref_logits = self.reference_model( - input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - # return_dict=True, - ) # .logits - if not isinstance(ref_logits, paddle.Tensor): - ref_logits = ref_logits[0] - log_probs = gather_log_probabilities(logits[:, :-1], input_ids[:, 1:]) - ref_log_probs = gather_log_probabilities(ref_logits[:, :-1], input_ids[:, 1:]) - return {"log_probs": log_probs, "ref_log_probs": ref_log_probs} - - @paddle.no_grad() - def rollout_reward_value( - self, input_ids: paddle.Tensor, attention_mask: paddle.Tensor, position_ids: paddle.Tensor = None, **kwargs - ) -> Dict[str, paddle.Tensor]: - if self.reward_tokenizer is not self.tokenizer: - # right padding - reward_tokenize_output = batch_retokenize( - input_ids, - src_tokenizer=self.tokenizer, - dest_tokenizer=self.reward_tokenizer, - skip_special_tokens=True, - ) - reward_input_ids = reward_tokenize_output["input_ids"] - reward_attention_mask = make_attention_mask( - reward_input_ids, - pad_id=self.reward_tokenizer.pad_token_id, - unk_id=self.reward_tokenizer.unk_token_id, - causal_mask=False, - ) - reward_position_ids = make_position_ids(reward_attention_mask) - else: - # for text in self.tokenizer.batch_decode(input_ids, skip_special_tokens=False): - # print(text) - reward_input_ids = input_ids - reward_attention_mask = attention_mask - reward_position_ids = position_ids - reward_score = self.reward_model( - reward_input_ids, - attention_mask=reward_attention_mask, - position_ids=reward_position_ids, - # return_dict=True, - )[ - 1 - ] # .end_scores - - reward_value = self.reward_critic_model( - input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - # return_dict=True, - )[ - 0 - ] # .scores - reward_score = reward_score.squeeze(axis=-1) - reward_value = reward_value.squeeze(axis=-1) - - reward_value = reward_value[:, :-1] - return {"rewards": reward_score, "reward_values": reward_value} - - @paddle.no_grad() - def post_rollout( - self, - prompt: paddle.Tensor, - sequence: paddle.Tensor, - attention_mask: paddle.Tensor, - ) -> Dict[str, Any]: - if self.reward_tokenizer is not self.tokenizer: - # right padding - reward_tokenize_output = batch_retokenize( - sequence, - src_tokenizer=self.tokenizer, - dest_tokenizer=self.reward_tokenizer, - skip_special_tokens=True, - ) - reward_seq = reward_tokenize_output["input_ids"] - reward_attention_mask = reward_tokenize_output["attention_mask"] - else: - # actor_model_in_use gen - # for text in self.tokenizer.batch_decode(sequence, skip_special_tokens=True): - # print(text) - reward_seq = sequence - reward_attention_mask = attention_mask - # position_ids is necessary for non-right padding - # If using right padding source + left padding target, make padding positions - # in source be 0, since reward model use position_ids plus with padding size - # (number of 0s) in source to calculate end offsets. - position_ids = make_position_ids(attention_mask) - - # pipe model outputs a logits tensor with LMHead, while non-pipe model - # outputs a tuple with logits tensor as the only one element. - self.timers and self.timers("actor-model-logit").start() - logits = self.actor_model( - sequence, - attention_mask=attention_mask, - position_ids=position_ids, - # return_dict=True, - ) # .logits - self.timers and self.timers("actor-model-logit").stop() - if not isinstance(logits, paddle.Tensor): - logits = logits[0] - self.timers and self.timers("reference-model-logit").start() - ref_logits = self.reference_model( - sequence, - attention_mask=attention_mask, - position_ids=position_ids, - # return_dict=True, - ) # .logits - self.timers and self.timers("reference-model-logit").stop() - if not isinstance(ref_logits, paddle.Tensor): - ref_logits = ref_logits[0] - - self.timers and self.timers("reward-model-score").start() - reward_score = self.reward_model( - reward_seq, - attention_mask=reward_attention_mask, - position_ids=position_ids, - # return_dict=True, - )[ - 1 - ] # .end_scores - - reward_value = self.reward_critic_model( - sequence, - attention_mask=attention_mask, - position_ids=position_ids, - # return_dict=True, - )[ - 0 - ] # .scores - reward_score = reward_score.squeeze(axis=-1) - reward_value = reward_value.squeeze(axis=-1) - - self.timers and self.timers("reward-model-score").stop() - reward_value = reward_value[:, :-1] - log_probs = gather_log_probabilities(logits[:, :-1], sequence[:, 1:]) - ref_log_probs = gather_log_probabilities(ref_logits[:, :-1], sequence[:, 1:]) - rollout_data = { - "prompt": prompt, - "input_ids": sequence, - "position_ids": position_ids, - "attention_mask": attention_mask, - "rewards": reward_score, - "reward_values": reward_value, - "log_probs": log_probs, - "ref_log_probs": ref_log_probs, - } - rollout_data = self.normalize_data(rollout_data, use_tgt_len_value=False) - return rollout_data - - @paddle.no_grad() - def normalize_data( - self, - rl_batch: Dict[str, paddle.Tensor], - use_tgt_len_value: bool = False, - ) -> Dict[str, Any]: - """ - data dispatch comm among devices needs padding, while the lengths of - all data fields are different and related, and it's hard to pad. - """ - prompt = rl_batch["prompt"] # length: src - attention_mask = rl_batch["attention_mask"] # length: src + tgt - if len(attention_mask.shape) == 4: - # use padding mask instead of causal mask - attention_mask = rl_batch["sequence_mask"] # length: src + tgt - old_log_probs = rl_batch["log_probs"] # length: src + tgt -1 - ref_log_probs = rl_batch["ref_log_probs"] # length: src + tgt -1 - rewards = rl_batch["rewards"] # length: 1 - old_reward_values = rl_batch["reward_values"] # length: src + tgt -1 - - # Beaver uses label data with target length, while we do not slice from - # inputs and use label data with target length: - # 1. Sometimes we cannot use label data with target length, mostly because - # it is hard to pad acorss batches. Think in some cases one batch might - # have the longest prompt+target length but the shortest target lengh, which - # might cause mismatch between inputs with prompt+target length and labels - # with target length. Padding acorss batches is needed in PP and data comm. - # 2. Additionally, when using flash_attn with casual mask and right padding - # we cannot use label data with target length. - start = prompt.shape[-1] - 1 - # sequence_mask is for label masking, make source be masked out - # clone to avoid to change attention_mask - sequence_mask = attention_mask[:, 1:].clone() # length: src + tgt -1 - sequence_mask[:, :start] = False - if use_tgt_len_value: - ref_log_probs = ref_log_probs[:, start:] - old_log_probs = old_log_probs[:, start:] - old_reward_values = old_reward_values[:, start:] - sequence_mask = sequence_mask[:, start:] - old_rewards = self.add_kl_divergence_regularization( - None, # prompt, - old_log_probs, - ref_log_probs, - rewards, - sequence_mask, - ) # length: tgt if use_tgt_len_value src + tgt -1 - reward_advantages, reward_returns = self.get_advantages_and_returns( - old_reward_values, - old_rewards, - sequence_mask, - start=0 if use_tgt_len_value else start, - use_tgt_len_return=use_tgt_len_value, - ) # length: tgt if use_tgt_len_value src + tgt -1 - - rl_batch.update( - { - "log_probs": old_log_probs, - "reward_values": old_reward_values, - "reward_advantages": reward_advantages, - "reward_returns": reward_returns, - "sequence_mask": sequence_mask, - "ref_log_probs": ref_log_probs, - "rewards": rewards, - } - ) - # pop out to reduce data dispatch comm overhead - rl_batch.pop("prompt") - return rl_batch diff --git a/slm/examples/RLHF/reward_config.json b/slm/examples/RLHF/reward_config.json deleted file mode 100644 index bfe5b2d1b5..0000000000 --- a/slm/examples/RLHF/reward_config.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "model_name_or_path": "PKU-Alignment/alpaca-7b-reproduced", - "train_datasets": "PKU-SafeRLHF-30K/train", - "eval_datasets": "PKU-SafeRLHF-30K/test", - "output_dir": "/root/paddlejob/workspace/guosheng/checkpoints/llama_sft_ckpts-test", - "per_device_train_batch_size": 16, - "gradient_accumulation_steps": 1, - "per_device_eval_batch_size": 16, - "eval_accumulation_steps": 1, - "num_train_epochs": 2, - "logging_steps": 1, - "evaluation_strategy": "epoch", - "eval_steps": 1000, - "save_strategy": "epoch", - "save_steps": 1000, - "max_length": 512, - "bf16": true, - "fp16_opt_level": "O2", - "do_train": true, - "do_eval": true, - "disable_tqdm": true, - "load_best_model_at_end": false, - "metric_for_best_model": "accuracy", - "recompute": true, - "save_total_limit": 1, - "sharding_parallel_degree": 4, - "sharding": "stage2", - "weight_decay": 0.1, - "max_grad_norm": 0.0, - "adam_beta1": 0.9, - "adam_beta2": 0.95, - "lr_scheduler_type": "cosine", - "learning_rate": 2e-5, - "warmup_ratio": 0.03, - "normalize_score_during_training": false, - "normalizer_type": "ExponentialMovingAverage", - "normalizer_momentum": 0.9, - "loss_type": "sequence-wise", - "regularization": 0.001 -} diff --git a/slm/examples/RLHF/reward_main.py b/slm/examples/RLHF/reward_main.py deleted file mode 100644 index ea29182e33..0000000000 --- a/slm/examples/RLHF/reward_main.py +++ /dev/null @@ -1,202 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -from dataclasses import dataclass, field -from typing import Any, Dict, Tuple - -import paddle - -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal - -from data import PreferenceDataset, parse_dataset -from models import AutoModelForScore -from reward_trainer import RewardTrainer - -from paddlenlp.trainer import PdArgumentParser, TrainingArguments, get_last_checkpoint -from paddlenlp.transformers import AutoConfig, AutoTokenizer, LlamaTokenizer -from paddlenlp.utils.log import logger - - -@dataclass -class TrainingArguments(TrainingArguments): - loss_type: Literal["token-wise", "sequence-wise"] = field( - default="sequence-wise", - metadata={ - "help": "Calculate ranking loss with all token-wise reward outputs in the sequence or the " - "sequence-wise reward output only (the reward of the last token in each sequence)." - }, - ) - # regularization - regularization: float = field( - default=0.0, - metadata={"help": "The regularization strength for the L2 regularization for score outputs."}, - ) - - -@dataclass -class ModelArgument: - model_name_or_path: str = field( - default=None, metadata={"help": "Built-in pretrained model name or the path to local model."} - ) - normalize_score_during_training: bool = field( - default=False, metadata={"help": "Whether to normalize score during training."} - ) - normalizer_type: Literal["RunningMeanStd", "ExponentialMovingAverage"] = field( - default=None, metadata={"help": "The type of the reward normalizer."} - ) - normalizer_momentum: float = field( - default=None, - metadata={ - "help": "The momentum use in ExponentialMovingAverage, EMA_{t+1} = momentum * x + (1 - momentum) * EMA_t." - }, - ) - use_flash_attention: bool = field(default=False, metadata={"help": "Whether to use flash attention"}) - - @property - def extra_model_kwargs(self): - """Extra keyword arguments for initializing the model.""" - return { - "score_type": "reward", - "do_normalize": self.normalize_score_during_training, - "normalizer_type": self.normalizer_type, - "momentum": self.normalizer_momentum, - } - - -@dataclass -class DataArgument: - dataset_name_or_path: str = field(default=None, metadata={"help": "Name or path for dataset"}) - task_name: str = field(default=None, metadata={"help": "Additional name to select a more specific task."}) - train_datasets: str = field(default=None, metadata={"help": "Dataset name(s) registered in the raw dataset."}) - eval_datasets: str = field(default=None, metadata={"help": "Dataset name(s) registered in the raw dataset."}) - max_length: int = field( - default=2048, - metadata={"help": "The maximum length that model input tokens can ."}, - ) - eval_with_do_generation: bool = field(default=False, metadata={"help": "Whether to do generation for evaluation"}) - save_generation_output: bool = field( - default=False, - metadata={"help": "Whether to save generated text to file when eval_with_do_generation set to True."}, - ) - lazy: bool = field( - default=False, - metadata={ - "help": "Weather to return `MapDataset` or an `IterDataset`.True for `IterDataset`. False for `MapDataset`." - }, - ) - - @property - def parsed_train_datasets(self) -> Tuple[str, Dict[str, Any]]: - """Parse dataset path and its proportion and optionally additional arguments from `train_datasets`.""" - return [parse_dataset(string) for string in self.train_datasets.split(",")] - - @property - def parsed_eval_datasets(self) -> Tuple[str, Dict[str, Any]]: - """Parse dataset path and its proportion and optionally additional arguments from `eval_datasets`.""" - return [parse_dataset(string) for string in self.eval_datasets.split(",")] - - -def main(): - # Arguments - parser = PdArgumentParser((ModelArgument, DataArgument, TrainingArguments)) - if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): - model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) - else: - model_args, data_args, training_args = parser.parse_args_into_dataclasses() - training_args.print_config(model_args, "Model") - training_args.print_config(data_args, "Data") - - # Setup GPU & distributed training - paddle.set_device(training_args.device) - logger.warning( - f"Process rank: {training_args.local_rank}, device: {training_args.device}, world_size: {training_args.world_size}, " - + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16 or training_args.bf16}" - ) - - # Detecting last checkpoint. - last_checkpoint = None - if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: - last_checkpoint = get_last_checkpoint(training_args.output_dir) - if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 1: - raise ValueError( - f"Output directory ({training_args.output_dir}) already exists and is not empty. " - "Use --overwrite_output_dir to overcome." - ) - if last_checkpoint is not None and training_args.resume_from_checkpoint is None: - logger.info( - f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " - "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." - ) - - # Load model - if training_args.fp16_opt_level == "O2": - if training_args.fp16: - dtype = "float16" - elif training_args.bf16: - dtype = "bfloat16" - else: - raise ValueError("Please specific dtype: --fp16 or --bf16") - else: - dtype = "float32" - - if training_args.pipeline_parallel_degree > 1: - raise ValueError("Not support pipeline parallel mode.") - else: - model_config = AutoConfig.from_pretrained( - model_args.model_name_or_path, - tensor_parallel_output=False, - tensor_parallel_degree=training_args.tensor_parallel_degree, - tensor_parallel_rank=training_args.tensor_parallel_rank, - dtype=dtype, - ) - if hasattr(model_config, "use_flash_attention"): - model_config.use_flash_attention = model_args.use_flash_attention - model = AutoModelForScore.from_pretrained( - model_args.model_name_or_path, config=model_config, **model_args.extra_model_kwargs - ) - tokenizer = AutoTokenizer.from_pretrained( - model_args.model_name_or_path, model_max_length=data_args.max_length, padding_side="right" - ) - if isinstance(tokenizer, LlamaTokenizer) and tokenizer.pad_token_id is None: - # tokenizer.pad_token_id = tokenizer.eos_token_id - # to be consistent with PKU-Alignment/alpaca-7b-reproduced - tokenizer.pad_token_id = tokenizer.eos_token_id - - train_ds = PreferenceDataset(data_args.parsed_train_datasets, tokenizer=tokenizer) - - dev_ds = PreferenceDataset(data_args.parsed_eval_datasets, tokenizer=tokenizer) - - trainer = RewardTrainer( - model=model, - args=training_args, - train_dataset=train_ds, - eval_dataset=dev_ds, - tokenizer=tokenizer, - data_collator=train_ds.get_collator(), - ) - checkpoint = None - if training_args.resume_from_checkpoint is not None: - checkpoint = training_args.resume_from_checkpoint - elif last_checkpoint is not None: - checkpoint = last_checkpoint - trainer.train(resume_from_checkpoint=checkpoint) - - -if __name__ == "__main__": - main() diff --git a/slm/examples/RLHF/reward_trainer.py b/slm/examples/RLHF/reward_trainer.py deleted file mode 100644 index a542d55942..0000000000 --- a/slm/examples/RLHF/reward_trainer.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any, Callable, Dict, List, Optional, Tuple, Union - -import numpy as np -import paddle -import paddle.nn as nn -import paddle.nn.functional as F -from models import ScoreModelOutput -from paddle.io import Dataset - -import paddlenlp.trainer.trainer as trainer -from paddlenlp.data import DataCollator -from paddlenlp.trainer import ( - EvalPrediction, - Trainer, - TrainerCallback, - TrainingArguments, -) -from paddlenlp.trainer.utils import nested_detach -from paddlenlp.transformers import PretrainedModel, PretrainedTokenizer - -_tr_acc = None - -speed_metrics = trainer.speed_metrics - - -def patch_speed_metrics(split, start_time, num_samples=None, num_steps=None, seq_length=None): - # split: interval, train, eval, test - result = speed_metrics(split, start_time, num_samples, num_steps, seq_length) - if split not in ["train", "interval"]: - return result - # accuracy - global _tr_acc - tr_acc, total_acc_scalar, nested_gather = _tr_acc - tr_acc_scalar = nested_gather(tr_acc).mean().item() - total_acc_scalar += tr_acc_scalar - tr_acc.subtract_(tr_acc) - _tr_acc[1] = total_acc_scalar - result["accuracy"] = round(tr_acc_scalar / num_steps, 8) - if split == "train": - result["train_accuracy"] = round(total_acc_scalar / num_steps, 8) - return result - - -trainer.speed_metrics = patch_speed_metrics - - -def compute_accuracy(eval_pred) -> Dict[str, float]: - higher_end_rewards, lower_end_rewards = eval_pred - accuracy = (higher_end_rewards > lower_end_rewards).astype("float32").mean().item() - rewards = np.concatenate([higher_end_rewards, lower_end_rewards], axis=0) - reward_mean = rewards.mean().item() - reward_std = rewards.std().item() - return { - "accuracy": accuracy, - "rewards_mean": reward_mean, - "rewards_std": reward_std, - } - - -class RewardTrainer(Trainer): - def __init__( - self, - model: Union[PretrainedModel, nn.Layer] = None, - criterion: nn.Layer = None, - args: TrainingArguments = None, - data_collator: Optional[DataCollator] = None, - train_dataset: Optional[Dataset] = None, - eval_dataset: Union[Dataset, Dict[str, Dataset]] = None, - tokenizer: Optional[PretrainedTokenizer] = None, - compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None, - callbacks: Optional[List[TrainerCallback]] = None, - optimizers: Tuple[paddle.optimizer.Optimizer, paddle.optimizer.lr.LRScheduler] = (None, None), - preprocess_logits_for_metrics: Callable[[paddle.Tensor, paddle.Tensor], paddle.Tensor] = None, - ): - if compute_metrics is None: - compute_metrics = compute_accuracy - - super().__init__( - model, - criterion, - args, - data_collator, - train_dataset, - eval_dataset, - tokenizer, - compute_metrics, - callbacks, - optimizers, - preprocess_logits_for_metrics, - ) - - def compute_loss(self, model, inputs, return_outputs=False): - """ - How the loss is computed by Trainer. By default, all models return the loss in the first element. - Subclass and override for custom behavior. - """ - better_input_ids = inputs["better_input_ids"] - worse_input_ids = inputs["worse_input_ids"] - better_attention_mask = inputs["better_attention_mask"] - worse_attention_mask = inputs["worse_attention_mask"] - - assert better_input_ids.shape[0] == worse_input_ids.shape[0], "batch size mismatch!" - batch_size = better_input_ids.shape[0] - - output: ScoreModelOutput = model( - paddle.concat([better_input_ids, worse_input_ids], axis=0), - attention_mask=paddle.concat([better_attention_mask, worse_attention_mask], axis=0), - ) - if isinstance(output, dict): - scores = output.scores # size = (2 * B, L, 1) - end_scores = output.end_scores # size = (2 * B, 1) - else: - scores, end_scores = output - - # size = (B, L) - higher_rewards, lower_rewards = scores.squeeze(axis=-1).chunk(chunks=2, axis=0) - # size = (B,) - higher_end_rewards, lower_end_rewards = end_scores.squeeze(axis=-1).chunk(chunks=2, axis=0) - - if self.args.loss_type == "token-wise": - losses = [] - for i in range(batch_size): - assert not paddle.all( - paddle.equal(better_input_ids[i], worse_input_ids[i]), - ).item(), "The better and worse answers are the same!" - higher_end_index = better_attention_mask[i].nonzero()[-1] - lower_end_index = worse_attention_mask[i].nonzero()[-1] - end_index = max(higher_end_index, lower_end_index) - - diverge_index = (better_input_ids[i] != worse_input_ids[i]).nonzero()[0] - assert 0 <= diverge_index <= end_index, "diverge index is out of range!" - - # size = (L,) - higher_truncated_rewards = higher_rewards[i, diverge_index : end_index + 1] - lower_truncated_rewards = lower_rewards[i, diverge_index : end_index + 1] - - losses.append( - -F.log_sigmoid(higher_truncated_rewards - lower_truncated_rewards).mean(), - ) - - if self.args.regularization > 0.0: - losses[-1] = losses[-1] + self.args.regularization * ( - paddle.square(lower_truncated_rewards).mean() + paddle.square(higher_truncated_rewards).mean() - ) - - loss = paddle.stack(losses).mean() # size = () - elif self.args.loss_type == "sequence-wise": - loss = -F.log_sigmoid(higher_end_rewards - lower_end_rewards).mean() - - if self.args.regularization > 0.0: - loss = loss + self.args.regularization * ( - paddle.square(lower_end_rewards).mean() + paddle.square(higher_end_rewards).mean() - ) - else: - raise ValueError(f"Unknown loss type: {self.args.loss_type}") - - accuracy = (higher_end_rewards > lower_end_rewards).cast("float32").mean() # size = () - - # TODO(guosheng): use a formal way to replace this hack for accuracy track - # in training - global _tr_acc - if _tr_acc is None: - _tr_acc = [paddle.to_tensor(0.0), 0.0, self._nested_gather] - _tr_acc[0] = _tr_acc[0] + accuracy.detach() - - if return_outputs: - return loss, { - "higher_end_rewards": higher_end_rewards, - "lower_end_rewards": lower_end_rewards, - "accuracy": accuracy, - } - return loss - - def prediction_step( - self, - model: nn.Layer, - inputs: Dict[str, Union[paddle.Tensor, Any]], - prediction_loss_only: bool, - ignore_keys: Optional[List[str]] = None, - ): - if self.args.pipeline_parallel_degree > 1: - # hack for pipeline mode - inputs = self._prepare_inputs(inputs) - return self.prediction_pipeline_step(model, inputs, prediction_loss_only, ignore_keys) - else: - inputs = self._prepare_inputs(inputs) - - better_input_ids = inputs["better_input_ids"] - worse_input_ids = inputs["worse_input_ids"] - better_attention_mask = inputs["better_attention_mask"] - worse_attention_mask = inputs["worse_attention_mask"] - - with paddle.no_grad(): - with self.autocast_smart_context_manager(): - higher_rewards = self.model(better_input_ids, better_attention_mask) - lower_rewards = self.model(worse_input_ids, worse_attention_mask) - if isinstance(higher_rewards, dict): - higher_end_rewards = higher_rewards.end_scores.squeeze(axis=-1) - lower_end_rewards = lower_rewards.end_scores.squeeze(axis=-1) - else: - higher_end_rewards = higher_rewards[-1].squeeze(axis=-1) - lower_end_rewards = lower_rewards[-1].squeeze(axis=-1) - higher_end_rewards = nested_detach(higher_end_rewards) - lower_end_rewards = nested_detach(lower_end_rewards) - return None, higher_end_rewards.cast("float32"), lower_end_rewards.cast("float32") diff --git a/slm/examples/RLHF/run.sh b/slm/examples/RLHF/run.sh deleted file mode 100644 index a4bdca9e97..0000000000 --- a/slm/examples/RLHF/run.sh +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PYTHONPATH=../../ GLOG_minloglevel=2 python3.10 -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" ppo_main.py ppo_config.json diff --git a/slm/examples/RLHF/tests/run_model.py b/slm/examples/RLHF/tests/run_model.py deleted file mode 100644 index 391bc86e4f..0000000000 --- a/slm/examples/RLHF/tests/run_model.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -from dataclasses import dataclass, field - -import numpy -import paddle -from paddle.distributed import fleet -from ppo_trainer import Trainer, data_group_merge, data_group_split, group_rank_guard - -from paddlenlp.trainer import PdArgumentParser, TrainingArguments -from paddlenlp.transformers import ( - AutoConfig, - AutoModelForCausalLM, - AutoModelForCausalLMPipe, -) - - -@dataclass -class ModelArgument: - model_name_or_path: str = field( - default=None, metadata={"help": "Built-in pretrained model name or the path to local model."} - ) - test_mode: str = field(default="export", metadata={"help": "export data_split or rank_guard."}) - - -def test_group_rank_guard(group): - @group_rank_guard(group=group, rank=0) - def func(): - tensor = paddle.randn([4, 64]) - return tensor - - t = func() - ret = [] - paddle.distributed.stream.all_gather(ret, t, group=group) - - for x in ret: - assert x._md5sum() == t._md5sum(), f"{x} {t}" - - -def main(): - # Arguments - parser = PdArgumentParser((ModelArgument, TrainingArguments)) - model_args, training_args = parser.parse_args_into_dataclasses() - - hcg = fleet.get_hybrid_communicate_group() - pp_group = hcg.get_pipe_parallel_group() - tp_group = hcg.get_model_parallel_group() - - if model_args.test_mode == "rank_guard": - test_group_rank_guard(tp_group) - return 0 - - model_config = AutoConfig.from_pretrained( - model_args.model_name_or_path, - tensor_parallel_output=False, - tensor_parallel_degree=training_args.tensor_parallel_degree, - tensor_parallel_rank=training_args.tensor_parallel_rank, - dtype="float32", - ) - - model_class = AutoModelForCausalLM - if training_args.pipeline_parallel_degree > 1: - model_class = AutoModelForCausalLMPipe - - actor_model = model_class.from_pretrained( - model_args.model_name_or_path, - config=model_config, - ) - - if True: # test export_evaluate_model - # 随机初始化 - config = copy.deepcopy(model_config) - if training_args.pipeline_parallel_degree <= 1: - config.tensor_parallel_degree = -1 - config.tensor_parallel_rank = 0 - - actor_eval_model = AutoModelForCausalLM.from_config(config) - # ground truth模型 - actor_gt_model = AutoModelForCausalLM.from_pretrained(model_args.model_name_or_path, config=config) - - trainer = Trainer( - model=actor_model, - args=training_args, - ) - trainer.export_evaluate_model(actor_model, actor_eval_model) - - gp_state = actor_gt_model.state_dict() - export_state = actor_eval_model.state_dict() - - for k, v in gp_state.items(): - assert ( - v._md5sum() == export_state[k]._md5sum() - ), f"{k} groud_truth: {v.shape}, export: {export_state[k].shape}" - - split_group = tp_group - if training_args.pipeline_parallel_degree > 1: - split_group = pp_group - - input_ids = paddle.randint(low=1, high=50, shape=[8, 64]) - paddle.distributed.broadcast(input_ids, src=0) - - split_input_ids = data_group_split(input_ids, group=split_group) - ret = actor_eval_model(input_ids=split_input_ids, return_dict=True) - eval_loggits = data_group_merge(ret.logits, group=split_group) - - gt_ret = actor_gt_model(input_ids=input_ids, return_dict=True) - gt_loggits = gt_ret.logits - numpy.testing.assert_almost_equal(eval_loggits.numpy(), gt_loggits.numpy(), decimal=5) - - -if __name__ == "__main__": - main() diff --git a/slm/examples/RLHF/tests/test_export.py b/slm/examples/RLHF/tests/test_export.py deleted file mode 100644 index 6254427e89..0000000000 --- a/slm/examples/RLHF/tests/test_export.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from tests.parallel_launch import TestMultipleGpus - -environment_variables = { - "NCCL_ALGO": "Tree", - "NVIDIA_TF32_OVERRIDE": "0", - "NCCL_IB_TIMEOUT": "22", - "NCCL_DEBUG": "INFO", - "FLAGS_embedding_deterministic": "1", - "FLAGS_cudnn_deterministic": "1", - "Flags_mp_aysnc_allreduce": "1", - "Flags_skip_mp_c_identity": "1", - "FLAGS_shard_norm_align_dp": "0", - "FLAGS_shard_use_reduce": "1", - "test_ci_no_save_model": "1", -} - - -class TestExportEvalModel(TestMultipleGpus): - def setUp(self): - os.environ.update(environment_variables) - super().setUp() - - def test_pptp_to_tp(self): - config = { - "output_dir": "./tmp", - "model_name_or_path": "__internal_testing__/tiny-random-llama", - "tensor_parallel_degree": 2, - "pipeline_parallel_degree": 2, - } - scripts = "tests/run_model.py" - self.run_4gpu(scripts, **config) - - def test_tp_to_single(self): - config = { - "output_dir": "./tmp", - "model_name_or_path": "__internal_testing__/tiny-random-llama", - "tensor_parallel_degree": 2, - "pipeline_parallel_degree": 1, - } - scripts = "tests/run_model.py" - self.run_2gpu(scripts, **config) - - def test_group_rank_guard(self): - config = { - "output_dir": "./tmp", - "model_name_or_path": "__internal_testing__/tiny-random-llama", - "tensor_parallel_degree": 2, - "pipeline_parallel_degree": 1, - "test_mode": "rank_guard", - } - scripts = "tests/run_model.py" - self.run_2gpu(scripts, **config) diff --git a/slm/examples/RLHF/trainer_utils.py b/slm/examples/RLHF/trainer_utils.py deleted file mode 100644 index e10a339851..0000000000 --- a/slm/examples/RLHF/trainer_utils.py +++ /dev/null @@ -1,662 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import inspect -import os -import time -from contextlib import contextmanager -from typing import Dict - -import numpy as np -import paddle -import tqdm -from paddle.distributed import fleet -from paddle.io import DataLoader - -from paddlenlp.generation.utils import GenerationMixin -from paddlenlp.trainer.trainer import ( - TRAINER_STATE_NAME, - HybridParallelOptimizer, - NlpDistributedBatchSampler, - ShardingOption, - Trainer, - TrainerCallback, - TrainerControl, - TrainerState, - TrainingArguments, - _obtain_optimizer_parameters_list, - distributed_file, - distributed_isfile, - fused_allreduce_gradients, - logger, - reshard_util, - split_inputs_sequence_dim, -) -from paddlenlp.transformers import BatchEncoding, PretrainedModel, PretrainedTokenizer -from paddlenlp.transformers.configuration_utils import PretrainedConfig -from paddlenlp.transformers.model_outputs import ModelOutput -from paddlenlp.transformers.tokenizer_utils_base import ( - PaddingStrategy, - TruncationStrategy, -) - - -# ########## patches for Trianer ########## -def init_train_model_opt( - self: Trainer, max_steps: int, resume_from_checkpoint: bool = False, clear_master_weight: bool = False -) -> PretrainedModel: - # Copy of model/optimizer init and resuming related code in `Trainer.train`. - # NOTE: this `_load_from_checkpoint` is indeed to load model states in the - # following elif-else branches, though they are apart away in `Trainer.train`. - if not self.args.should_load_sharding_stage1_model: - self._load_from_checkpoint(resume_from_checkpoint) - - # delay_optimizer_creation = ( - # self.sharding is not None - # and ShardingOption.SHARD_OP in self.args.sharding - # ) - delay_optimizer_creation = False - - if not delay_optimizer_creation: - self.create_optimizer_and_scheduler(num_training_steps=max_steps) - - if self.args.should_load_sharding_stage1_model: - model = self._wrap_model_and_load_sharded_checkpoint(resume_from_checkpoint) - elif self.args.should_save_sharding_stage1_model: - # In the non-sharded mode, should invoke _load_from_checkpoint before _wrap_model. - # In this mode, the rank0 load all params and the _wrap_model implicitly broadcast params from rank0 to the other ranks. - model = self._wrap_model(self.model_wrapped) - if self.sharding_io is not None: - assert delay_optimizer_creation is False, "delay_optimizer_creation should be False" - # the self.optimizer should be wrapped and it is done in _wrap_model - self.sharding_io.set_optimizer(self.optimizer) - # for the rest of this function `model` is the outside model, whether it was wrapped or not - if model is not self.model: - self.model_wrapped = model - if delay_optimizer_creation: - self.create_optimizer_and_scheduler(num_training_steps=max_steps) - self._load_optimizer_and_scheduler(resume_from_checkpoint) - else: - model = self._wrap_model(self.model_wrapped) - # for the rest of this function `model` is the outside model, whether it was wrapped or not - if model is not self.model: - self.model_wrapped = model - if delay_optimizer_creation: - self.create_optimizer_and_scheduler(num_training_steps=max_steps) - self._load_optimizer_and_scheduler(resume_from_checkpoint) - - if ShardingOption.FULL_SHARD in self.args.sharding and clear_master_weight: - # for inference model to use Trainer sharding stage3, clear master_weight - # which is created in GroupShardedStage3.__init__ - self.optimizer._master_weights = None - - if self.args.device == "npu" and self.args.flatten_param_grads: - from .plugins.npu_plugin import npu_accelerate_plugin - - npu_accelerate_plugin(self.optimizer) - - return model - - -def init_train_state( - self: Trainer, - resume_from_checkpoint: bool, - train_dataloader: DataLoader, - max_steps: int, - num_train_epochs: int, - num_update_steps_per_epoch: int, -): - args = self.args - - self.state = TrainerState() - self.state.epoch = 0 - epochs_trained = 0 - steps_trained_in_current_epoch = 0 - steps_trained_progress_bar = None - - # Check if continuing training from a checkpoint - if resume_from_checkpoint is not None and distributed_isfile( - os.path.join(resume_from_checkpoint, TRAINER_STATE_NAME) - ): - self.state = TrainerState.load_from_json( - distributed_file(os.path.join(resume_from_checkpoint, TRAINER_STATE_NAME)) - ) - epochs_trained = self.state.global_step // num_update_steps_per_epoch - if not args.ignore_data_skip: - steps_trained_in_current_epoch = self.state.global_step % (num_update_steps_per_epoch) - steps_trained_in_current_epoch *= args.gradient_accumulation_steps - else: - steps_trained_in_current_epoch = 0 - - logger.info(" Continuing training from checkpoint, will skip to saved global_step") - logger.info(f" Continuing training from epoch {epochs_trained}") - logger.info(f" Continuing training from global step {self.state.global_step}") - if not args.ignore_data_skip: - logger.info( - f" Will skip the first {epochs_trained} epochs then the first {steps_trained_in_current_epoch} " - "batches in the first epoch. If this takes a lot of time, you can add the `--ignore_data_skip` " - "flag to your launch command, but you will resume the training on data already seen by your model." - ) - if self.is_local_process_zero() and not args.disable_tqdm: - steps_trained_progress_bar = tqdm(total=steps_trained_in_current_epoch) - steps_trained_progress_bar.set_description("Skipping the first batches") - if not args.ignore_data_skip: - if isinstance(train_dataloader, paddle.io.DataLoader) and isinstance( - train_dataloader.batch_sampler, NlpDistributedBatchSampler - ): - consumed_samples = ( - self.state.global_step - * args.train_batch_size - * args.gradient_accumulation_steps - * args.dataset_world_size - ) - train_dataloader.batch_sampler.set_epoch(consumed_samples=consumed_samples) - logger.info(f"Set DistributedBatchSampler consumed_samples to {consumed_samples}") - - self.state.max_steps = int(max_steps) - self.state.num_train_epochs = num_train_epochs - self.state.is_local_process_zero = self.is_local_process_zero() - self.state.is_world_process_zero = self.is_world_process_zero() - - return epochs_trained, steps_trained_in_current_epoch, steps_trained_progress_bar - - -def init_train_log( - self: Trainer, - num_examples: int, - num_train_epochs: int, - total_train_batch_size: int, - max_steps: int, - num_train_samples: int, - model: PretrainedModel, -): - args = self.args - - logger.info("***** Running training *****") - logger.info(f" Num examples = {num_examples:,}") - logger.info(f" Num Epochs = {num_train_epochs}") - logger.info(f" Instantaneous batch size per device = {args.per_device_train_batch_size}") - logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_train_batch_size}") - logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") - logger.info(f" Total optimization steps = {max_steps:,}") - logger.info(f" Total num train samples = {num_train_samples:,}") - # per_device_trainable_numel = sum(p.numel().item() for p in model.parameters() if not p.stop_gradient) - # TODO: Temporary fix since Tensor.numel() not supported in distributed mode - per_device_trainable_numel = sum(np.prod(p.shape) for p in model.parameters() if not p.stop_gradient) - logger.info(f" Number of trainable parameters = {per_device_trainable_numel:,} (per device)") - if self.args.use_hybrid_parallel: - # todo fix for pipeline_parallel_degree - parts_num = max(self.args.tensor_parallel_degree, 1) * max(self.args.pipeline_parallel_degree, 1) - if parts_num > 1: - all_reduce_dtype = "int64" - if paddle.get_device().split(":")[0] in ["npu", "xpu"]: - # TODO(duanyanhui): fix when NPU all_reduce supports int64 - all_reduce_dtype = "float32" - trainable_numel_tensor = paddle.to_tensor(per_device_trainable_numel, dtype=all_reduce_dtype) - paddle.distributed.all_reduce(trainable_numel_tensor) - trainable_numel = int(trainable_numel_tensor.item()) // self.args.dataset_world_size - # the numel is roughly, because the tensor parallel still hold own bias or layer_norm weight without splited - # so, the trainable numel is a little bigger than real. - logger.info(f" Number of trainable parameters = {trainable_numel:,} (all devices, roughly)") - - -def full_training_step(self: Trainer, inputs: Dict[str, paddle.Tensor], **kwargs): - """ - Just a copy of single training step complete code in Trainer.train while loop - which including forward+backward+step, while wraps the inputs and outputs to - make the complicated copied code no need to change. Maybe a better way is to - add fine-grained methods including these steps to Trainer which is similar to - DeepSpeed engine. - """ - - # TODO(guosheng): step, steps_trained_in_current_epoch and steps_trained_progress_bar - # should use reference since they would be overwrite. - # for state update - epoch = kwargs.get("epoch", 0) - step = kwargs.get("step", 0) - steps_in_epoch = kwargs.get("steps_in_epoch", 0) - step_control = kwargs.get("step_control", 0) - # for step and progress update when resuming data - train_dataloader = kwargs.get("train_dataloader", None) - resume_from_checkpoint = kwargs.get("resume_from_checkpoint", None) - steps_trained_in_current_epoch = kwargs.get("steps_trained_in_current_epoch", 0) - steps_trained_progress_bar = kwargs.get("steps_trained_progress_bar", None) - # for eval output ignore to gather - ignore_keys_for_eval = kwargs.get("ignore_keys_for_eval", None) - tr_loss = kwargs.get("tr_loss", 0.0) - model = kwargs.get("model", self.model_wrapped) - # needed in _maybe_log_save_evaluate - self._globalstep_last_logged = getattr(self, "_globalstep_last_logged", 0) - self._globalstep_last_start_time = getattr(self, "_globalstep_last_start_time", time.time()) - - args = self.args - - if self.args.use_hybrid_parallel and self.args.sep_parallel_degree > 1: - inputs = split_inputs_sequence_dim(inputs) - # self.timers and self.timers("read-data").stop() - os.environ["TRAINER_GLOBAL_STEP"] = str(self.state.global_step) - self.callback_handler.on_load_data_end(args, self.state, self.control, inputs=inputs) - - # Skip past any already trained steps if resuming training - # for paddlenlp.utils.batch_sampler.DistributedBatchSampler - # We use consumed_samples to reset the status - if isinstance(train_dataloader, paddle.io.DataLoader) and isinstance( - train_dataloader.batch_sampler, NlpDistributedBatchSampler - ): - if step == 0: - if steps_trained_progress_bar is not None: - steps_trained_progress_bar.update(steps_trained_in_current_epoch) - steps_trained_progress_bar.close() - steps_trained_progress_bar = None - self._load_rng_state(resume_from_checkpoint) - step += steps_trained_in_current_epoch - elif steps_trained_in_current_epoch > 0: - steps_trained_in_current_epoch -= 1 - if steps_trained_progress_bar is not None: - steps_trained_progress_bar.update(1) - if steps_trained_in_current_epoch == 0: - self._load_rng_state(resume_from_checkpoint) - # continue - final_local_vars = locals() - for k in kwargs.keys(): - if k in final_local_vars: - kwargs[k] = final_local_vars[k] - return kwargs - elif steps_trained_progress_bar is not None: - steps_trained_progress_bar.close() - steps_trained_progress_bar = None - - if step_control % args.gradient_accumulation_steps == 0: - self.control = self.callback_handler.on_step_begin(args, self.state, self.control) - self.timers and self.timers("forward-backward").start() - - dp_enabled = self.args.data_parallel_degree > 1 if self.args.use_hybrid_parallel else args.local_rank != -1 - forbidden_no_sync = False - # stage2 and stage3 should not no_sync, because the is no DDP wrapper and no_sync API - # hybrid_parallel (tp or pp or sharding stage 1) should not no_sync - if self.args.use_hybrid_parallel: - forbidden_no_sync = True - - availiable_no_sync = dp_enabled and not forbidden_no_sync - - is_no_sync = ( - ((step_control + 1) % args.gradient_accumulation_steps != 0) - and availiable_no_sync - and args._no_sync_in_gradient_accumulation - ) or (args.recompute and availiable_no_sync) - # sharding - # stage1. the same as ddp - # stage2. manualy collect gradient on dp group - - dp_master_grad = self.args.world_size > 1 and self.args.amp_master_grad and not self.args.use_hybrid_parallel - if dp_master_grad: - is_no_sync = True - - if is_no_sync: - # Avoid unnecessary DDP synchronization since there will be no backward pass on this example. - with model.no_sync(): - tr_loss_step = self.training_step(model, inputs) - else: - tr_loss_step = self.training_step(model, inputs) - - tr_loss += tr_loss_step - - if (step_control + 1) % args.gradient_accumulation_steps == 0 or ( - # last step in epoch but step is always smaller than gradient_accumulation_steps - steps_in_epoch <= args.gradient_accumulation_steps - and (step + 1) == steps_in_epoch - ): - if self.args.pipeline_parallel_degree <= 1 and self._enable_delay_scale_loss(): - tr_loss /= self.args.gradient_accumulation_steps - - self.timers and self.timers("forward-backward").stop() - # Maunally collect gradients - # Case 1: Use recompute and dp - # Case 2: Hack dp with master_grad - # Case 3: Pipeline or sharding overlap - # local_rank != -1 don't means dp in networks. - self.timers and self.timers("all-reduce").start() - - # Case 1: Use recompute and dp / sharding stage1, - # manualy collect gradient for dp. - if args.recompute and availiable_no_sync: - fused_allreduce_gradients(list(model.parameters()), None) - - # Case 2: hack dp with master_grad - if dp_master_grad and not (args.recompute and availiable_no_sync): - fused_allreduce_gradients(list(model.parameters()), None) - - # Pipeline parallel mode, handle gradient reduce here to overlap - enable_dp_comm_overlap = False - enable_release_grads = False - if args.pipeline_parallel_degree > 1: - enable_dp_comm_overlap = "enable_dp_comm_overlap" in args.pipeline_parallel_config - enable_release_grads = "enable_release_grads" in args.pipeline_parallel_config - - # Case 3: Pipeline parallel mode, overlap with dp - if isinstance(self.optimizer, HybridParallelOptimizer) and not self.do_grad_scaling: - parameters_list = _obtain_optimizer_parameters_list(self.optimizer._inner_opt) - - if not enable_dp_comm_overlap: - if self.optimizer._sharding_enable: - assert reshard_util.is_sharding_opt(self.optimizer) - self.optimizer._inner_opt.reduce_gradients(list(parameters_list), self.optimizer._hcg) - - if self.optimizer._dp_enable or getattr(self.optimizer, "_sep_enable", False): - fused_allreduce_gradients(list(parameters_list), self.optimizer._hcg) - - self.timers and self.timers("all-reduce").stop() - self.timers and self.timers("optimizer-step").start() - - if self.args.gradient_accumulation_steps > 1 and self._enable_delay_scale_loss(): - for p in model._layers.parameters(): - with paddle.no_grad(): - if hasattr(p, "main_grad") and p.main_grad is not None: - assert p.grad is None - p.main_grad.scale_(1.0 / self.args.gradient_accumulation_steps) - elif p.grad is not None: - p.grad.scale_(1.0 / self.args.gradient_accumulation_steps) - - # Optimizer step - self.callback_handler.on_optimizer_begin( - args, self.state, self.control, scaler=self.scaler if self.do_grad_scaling else None - ) - optimizer_was_run = True - if self.do_grad_scaling: - scale_before = paddle.assign(self.scaler._scale) - self.scaler.step(self.optimizer) - self.scaler.update() - scale_after = self.scaler._scale - # Compatible with paddlepaddle 2.6.0 using typo word. - if hasattr(self.scaler, "_cache_founf_inf"): - optimizer_was_run = not self.scaler._cache_founf_inf - else: - optimizer_was_run = not self.scaler._cache_found_inf - if not optimizer_was_run: - scale_before_value = scale_before.cpu().numpy() - scale_after_value = scale_after.cpu().numpy() - logger.warning( - f"optimizer not run, scale_before: {scale_before_value[0]}, scale_after: {scale_after_value[0]}" - ) - elif isinstance(self.optimizer, HybridParallelOptimizer): - self.optimizer._step(parameters_list) - else: - self.optimizer.step() - - self.timers and self.timers("optimizer-step").stop() - - if optimizer_was_run: - self.lr_scheduler.step() - - if enable_release_grads and args.pipeline_parallel_degree > 1: - self.optimizer.clear_grad(set_to_zero=False) - for _, buffers in model._chunk_2_comm_buffers.items(): - for buffer in buffers: - buffer._clear_grad_storage() - else: - self.optimizer.clear_grad(set_to_zero=False) - - self.callback_handler.on_optimizer_end( - args, self.state, self.control, scaler=self.scaler if self.do_grad_scaling else None - ) - - self.state.global_step += 1 - self.state.epoch = epoch + (step + 1) / steps_in_epoch - self.control = self.callback_handler.on_step_end(args, self.state, self.control) - self._maybe_log_save_evaluate(tr_loss, model, epoch, ignore_keys_for_eval, inputs=inputs) - self._print_timer() - step_control = 0 - else: - self.control = self.callback_handler.on_substep_end(args, self.state, self.control) - step_control += 1 - - if self.control.should_epoch_stop or self.control.should_training_stop: - # break - final_local_vars = locals() - for k in kwargs.keys(): - if k in final_local_vars: - kwargs[k] = final_local_vars[k] - return kwargs - # self.timers and self.timers("read-data").start() - - final_local_vars = locals() - for k in kwargs.keys(): - if k in final_local_vars: - kwargs[k] = final_local_vars[k] - return kwargs - - -Trainer.init_train_model_opt = init_train_model_opt -Trainer.init_train_log = init_train_log -Trainer.init_train_state = init_train_state -Trainer.full_training_step = full_training_step -# ########## patches for Trianer ########## - - -class MuteDefaultFlowCallback(TrainerCallback): - """ - Add this callback can cencel logging/evaluation/saving by DefaultFlowCallback. - Use this when having multi trainer. - """ - - def on_step_end(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs): - control.should_save = False - control.should_evaluate = False - control.should_log = False - return control - - -@contextmanager -def guard_set_args(args, arg_name_values): - for k, v in arg_name_values.items(): - old_value = getattr(args, k, None) - setattr(args, k, v) - arg_name_values[k] = old_value - yield - for k, v in arg_name_values.items(): - old_value = getattr(args, k) - setattr(args, k, v) - arg_name_values[k] = old_value - - -class PipeEvalModel(GenerationMixin): - """ - Wrapper for PipelineParallel to do evaluate and generate. Currently only - support . - """ - - def __init__(self, trainer: Trainer): - eval_model = getattr(trainer, "_inner_eval_model", None) - self.model: fleet.model.PipelineParallel = trainer.model_wrapped if eval_model is None else eval_model - self.config: PretrainedConfig = trainer.model.config - self._is_gen = False - self.update_model_kwargs_for_generation = ( - self.model._layers._non_pipe_model_class.update_model_kwargs_for_generation - ) - - @property - def pp_group(self): - return self.model.pp_group - - def eval(self): - self.model.eval() - - def train(self): - self.model.train() - - def __getattr__(self, name): - try: - return super().__getattr__(name) - except AttributeError: - return getattr(self.model, name) - - def _broadcast_outputs(self, outputs): - # outputs is PipelineParallel.eval_batch which is a list of batches. - out = [] - outputs = (outputs,) if isinstance(outputs, paddle.Tensor) else outputs - for tensors in outputs: - if not self.model.is_pipeline_last_stage(): - tensor = tensors if isinstance(tensors, paddle.Tensor) else tensors[0] - head_out_meta = ( - (self.model._layers.head_out_meta,) - if isinstance(self.model._layers.head_out_meta, paddle.static.InputSpec) - else self.model._layers.head_out_meta - ) - tensors = tuple( - paddle.empty( - shape=[ - tensor.shape[i] if (meta.shape[i] is None or meta.shape[i] < 0) else meta.shape[i] - for i in range(len(meta.shape)) - ], - dtype=tensor.dtype if meta.dtype is None else meta.dtype, - ) - for meta in head_out_meta - ) - else: - # Currently use tuple instead of ModelOutput and require the - # caller use the return result as tuple. - tensors = ( - (tensors,) - if isinstance(tensors, paddle.Tensor) - else tensors.to_tuple() - if isinstance(tensors, ModelOutput) - else tensors - ) - - # use map_structure seems hung - for tensor in tensors: - paddle.distributed.broadcast(tensor, src=self.model.pp_group.ranks[-1], group=self.model.pp_group) - out.append(tensors[0] if len(tensors) == 1 else tensors) - return out[0] if len(out) == 1 else out - - def __call__(self, *args, **kwargs): - model = self.model - assert self.model.training is False - if self._is_gen: - # inputs by `prepare_inputs_for_generation` is a dict with following keys: - # "input_ids", "position_ids", "past_key_values", "use_cache", "attention_mask" - # NOTE: 1. cache/past_key_values should be passed across decoding steps - # by using as model attr rather than input args to reduce comm overhead. - # Also, pipe model defined for training not support this cache input. - # 2. ignore use_cache since _check_data_vaild requires tensor if not None. - # 3. attention_mask can reuse _prepare_decoder_attention_mask in LlamaEmbeddingPipe. - # 4. position_ids pass through _prepare_pipeline_inputs_func and PipeLayer. - inputs, labels = model._prepare_pipeline_inputs_func(*args, **kwargs) - # currently, set accumulate_steps to 1 to avoid multi-batch eval/gen - with guard_set_args(model, {"_compute_loss": False, "accumulate_steps": 1}): - outputs = model.eval_batch([inputs, labels], compute_loss=False) - # TODO(guosheng): Broadcasted logits are used to get next_scores, remove - # it to reduce comm overhead. Also note that we still need broadcast - # next_tokens though logits are broadcasted since pp ranks' seeds differs. - # Currently, just slice the last token to reduce comm overhead. - outputs = [ - micro_batch_output[:, -1, :].unsqueeze(1) - if isinstance(micro_batch_output, paddle.Tensor) - else micro_batch_output[0][:, -1, :].unsqueeze(1) - for micro_batch_output in outputs - ] - outputs = self._broadcast_outputs(outputs) - else: - # use _prepare_pipeline_inputs_func to convert pipeline inputs - inputs, labels = model._prepare_pipeline_inputs_func(*args, **kwargs) - # NOTE(guosheng): bug seems exist. pp.eval_batch(compute_loss=False) - # will set pp._compute_loss to False and would not set it back. Thus - # hack here to set it back. - with guard_set_args(model, {"_compute_loss": False, "accumulate_steps": 1}): - outputs = model.eval_batch([inputs, labels], compute_loss=False) - outputs = self._broadcast_outputs(outputs) - return outputs - - def generate(self, *args, **kwargs): - self._is_gen = True - # patch DecoderLayerPipe to use cache, DecoderLayerPipe is subclass of - # DecoderLayer, and would call super().forward - ori_decoder_layer_forward = self.model._layers._non_pipe_decoder_layer_class.forward - - def decoder_layer_forward(layer_self, *args, **kwargs): - kwargs.update({"use_cache": True, "past_key_value": getattr(layer_self, "_cache", None)}) - outputs = ori_decoder_layer_forward(layer_self, *args, **kwargs) - output = outputs[0] - layer_self._cache = outputs[1] - self._has_cache = True - return output - - with guard_set_args(self.model._layers._non_pipe_decoder_layer_class, {"forward": decoder_layer_forward}): - outputs = super().generate(*args, **kwargs) - self._is_gen = False - # clear cache of decoder layers, sublayers is incursive thus suitable - # to both 1F1B and interleave - for layer in self.model._layers.sublayers(): - if isinstance(layer, self.model._layers._non_pipe_decoder_layer_class): - layer._cache = None - self._has_cache = False - return outputs - - def prepare_inputs_for_generation(self, *args, **kwargs): - arg_bind = inspect.signature(self.model._layers._non_pipe_model_class.prepare_inputs_for_generation).bind( - *((self,) + args), **kwargs - ) - arg_bind.apply_defaults() - arg_dict = arg_bind.arguments - last_arg_name, last_arg_value = arg_dict.popitem() - if arg_bind.signature.parameters[last_arg_name].kind == inspect.Parameter.VAR_KEYWORD: - arg_dict.update(last_arg_value) - else: - arg_dict[last_arg_name] = last_arg_value - arg_dict.pop("self") - past_key_values = arg_dict.get("past_key_values", None) - # prepare_inputs_for_generation use past_key_values to discrimate prefill - # or decode and slice inputs accordingly. - if getattr(self, "_has_cache", False): - arg_dict.update({"past_key_values": True}) - model_inputs = self.model._layers._non_pipe_model_class.prepare_inputs_for_generation(self, **arg_dict) - model_inputs.update({"past_key_values": past_key_values}) - return model_inputs - - -def is_same_tokenizer( - tokenizer: PretrainedTokenizer, - other_tokenizer: PretrainedTokenizer, -) -> bool: - """Check if two tokenizers are the same.""" - return tokenizer is other_tokenizer or ( - tokenizer.__class__ == other_tokenizer.__class__ and tokenizer.get_vocab() == other_tokenizer.get_vocab() - ) - - -def batch_retokenize( - input_ids: paddle.Tensor, - src_tokenizer: PretrainedTokenizer, - dest_tokenizer: PretrainedTokenizer, - *, - padding: bool | str | PaddingStrategy = PaddingStrategy.LONGEST, - truncation: bool | str | TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE, - skip_special_tokens: bool = True, -) -> BatchEncoding: - """Re-tokenize a batch of input ids from one tokenizer to another.""" - output = dest_tokenizer( - [ - text + dest_tokenizer.eos_token - for text in src_tokenizer.batch_decode( - input_ids, - skip_special_tokens=skip_special_tokens, - ) - ], - padding=padding, - truncation=truncation, - return_tensors="pd", - ) - return output diff --git a/slm/examples/dialogue/README_DGU.md b/slm/examples/dialogue/README_DGU.md deleted file mode 100644 index 7716e4f2a4..0000000000 --- a/slm/examples/dialogue/README_DGU.md +++ /dev/null @@ -1,145 +0,0 @@ -# 对话通用理解模型 (DGU, Dialogue General Understanding) - -> **注意** 部分内容在 PaddleNLP 3.0以后不再进行维护，更多历史内容请参考[PaddleNLP 2.8](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/dialogue)。 - -## 模型简介 - -对话系统 (Dialogue System) 常常需要根据应用场景的变化去解决多种多样的任务。任务的多样性（意图识别、槽填充、行为识别、状态追踪等等），以及领域训练数据的稀少，给 Dialogue System 的研究和应用带来了巨大的困难和挑战，要使得 Dialogue System 得到更好的发展，需要开发一个通用的对话理解模型。为此，我们给出了基于 BERT 的对话通用理解模型 (DGU: Dialogue General Understanding)，通过实验表明，使用 base-model (BERT)并结合常见的学习范式，就可以在几乎全部对话理解任务上取得比肩甚至超越各个领域业内最好的模型的效果，展现了学习一个通用对话理解模型的巨大潜力。 - -DGU 模型内共包含6个任务，全部基于公开数据集在 Paddle2.0上完成训练及评估，详细说明如下： - -``` -udc: 使用UDC (Ubuntu Corpus V1) 数据集完成对话匹配 (Dialogue Response Selection) 任务; -dstc2: 使用DSTC2 (Dialog State Tracking Challenge 2) 数据集完成对话状态追踪 (Dialogue State Tracking) 任务; -atis_slot: 使用ATIS (Airline Travel Information System) 数据集完成对话槽填充 (Dialogue Slot Filling) 任务； -atis_intent: 使用ATIS (Airline Travel Information System) 数据集完成对话意图识别 (Dialogue Intent Detection) 任务； -mrda: 使用MRDAC (Meeting Recorder Dialogue Act Corpus) 数据集完成对话行为识别 (Dialogue Act Detection) 任务； -swda: 使用SwDAC (Switchboard Dialogue Act Corpus) 数据集完成对话行为识别 (Dialogue Act Detection) 任务; -``` - -## 模型效果 - -DGU 模型中的6个任务，分别采用不同的评估指标在 test 集上进行评估，结果如下： - - - - - - - - - - - -
任务评估指标DGU
udcR1@1081.04%
R2@1089.85%
R5@1097.59%
dstc2Joint_Acc90.43%
atis_slotF1_Micro97.98%
atis_intentAcc97.42%
mrdaAcc90.94%
swdaAcc80.61%
- -**NOTE:** 以上结果均是采用默认配置在 GPU 单卡上训练和评估得到的，用户如需复现效果，可采用默认配置在单卡上进行训练评估。 - -## 快速开始 - -### 数据准备 - -下载数据集压缩包并解压后，DGU_datasets 目录下共存在6个目录，分别对应每个任务的训练集 train.txt、评估集 dev.txt 和测试集 test.txt。 - -```shell -wget https://bj.bcebos.com/paddlenlp/datasets/DGU_datasets.tar.gz -tar -zxf DGU_datasets.tar.gz -``` - -DGU_datasets 目录结构： - -```text -DGU_datasets/ -├── atis_intent -│   ├── dev.txt -│   ├── map_tag_intent_id.txt -│   ├── test.txt -│   └── train.txt -├── udc -│   ├── dev.txt -│   ├── dev.txt-small -│   ├── test.txt -│   └── train.txt -├── atis_slot -│   ├── dev.txt -│   ├── map_tag_slot_id.txt -│   ├── test.txt -│   └── train.txt -├── dstc2 -│   ├── dev.txt -│   ├── map_tag_id.txt -│   ├── test.txt -│   └── train.txt -├── mrda -│   ├── dev.txt -│   ├── map_tag_id.txt -│   ├── test.txt -│   └── train.txt -└── swda - ├── dev.txt - ├── map_tag_id.txt - ├── test.txt - └── train.txt -``` - -数据的每一行由多列组成，都以"\t"作为分割符，详细数据格式说明如下： - -``` -udc：由label、多轮对话conv和回应response组成 -格式：label \t conv1 \t conv2 \t conv3 \t ... \t response - -dstc2：由多轮对话id、当前轮QA对(使用\1拼接)和对话状态序列state_list(state_list中每个state由空格分割)组成 -格式：conversation_id \t question \1 answer \t state1 state2 state3 ... - -atis_slot：由对话内容conversation_content和标签序列label_list (label_list中每个label由空格分割) 组成, 其中标签序列和对话内容中word为一一对应关系 -格式：conversation_content \t label1 label2 label3 ... - -atis_intent：由标签label和对话内容conversation_content组成 -格式： label \t conversation_content - -mrda：由多轮对话id、标签label、发言人caller、对话内容conversation_content组成 -格式：conversation_id \t label \t caller \t conversation_content - -swda：由多轮对话id、标签label、发言人caller、对话内容conversation_content组成 -格式：conversation_id \t label \t caller \t conversation_content -``` - -**NOTE:** 上述数据集来自于 [Paddle1.8静态图版本](https://github.com/PaddlePaddle/models/tree/release/1.8/PaddleNLP/dialogue_system/dialogue_general_understanding)，是由相应的开源数据集经过数据格式转换而得来的，本项目中暂未包含数据格式转换脚本，细节请参考 [Paddle1.8静态图版本](https://github.com/PaddlePaddle/models/tree/release/1.8/PaddleNLP/dialogue_system/dialogue_general_understanding)。 - -### 模型训练 - -运行如下命令即可在训练集 (train.tsv) 上进行模型训练，并在开发集 (dev.tsv) 验证，训练结束后会在测试集 (test.txt) 上进行模型评估 - -```shell -# GPU启动，gpus指定训练所用的GPU卡号，可以是单卡，也可以多卡。默认会进行训练、验证和评估 -unset CUDA_VISIBLE_DEVICES -python -m paddle.distributed.launch --gpus "0" --log_dir ./log main.py --task_name=udc --data_dir=./DGU_datasets/udc --output_dir=./checkpoints/udc --device=gpu -# 若只需进行评估，do_train设为False，并且必须指定init_from_ckpt -# python -m paddle.distributed.launch --gpus "0" --log_dir ./log main.py --task_name=udc --data_dir=./DGU_datasets/udc --do_train=False --init_from_ckpt=./checkpoints/udc/best --device=gpu -``` - -以上参数表示： - -* `task_name`：任务名称，可以为 udc、dstc2、atis_slot、atis_intent、mrda 或 swda。 -* `data_dir`：训练数据路径。 -* `output_dir`：训练保存模型的文件路径。 -* `do_train：是否进行训练，默认为`True`。 -* `init_from_ckpt`：恢复模型参数的路径。 -* `device`：表示训练使用的设备。 - -其他可选参数和参数的默认值请参考`args.py`。 - -程序运行时将会自动进行训练，验证和评估。同时训练过程中会自动保存模型在指定的`output_dir`中。 -如： -```text -checkpoints/ -├── 1000.pdopt -├── 1000.pdparams -├── 2000.pdopt -├── 2000.pdparams -├── ... -├── best.pdopt -└── best.pdparams -``` - -**NOTE:** 如需恢复模型训练，则 init_from_ckpt 只需指定到文件名即可，不需要添加文件尾缀。如`--init_from_ckpt=checkpoints/1000`即可，程序会自动加载模型参数`checkpoints/1000.pdparams`，也会自动加载优化器状态`checkpoints/1000.pdopt`。 diff --git a/slm/examples/dialogue/README_LIC2021.md b/slm/examples/dialogue/README_LIC2021.md deleted file mode 100644 index 11531e91d0..0000000000 --- a/slm/examples/dialogue/README_LIC2021.md +++ /dev/null @@ -1,148 +0,0 @@ -# LIC 2021对话比赛 baseline - -> **注意** 部分内容在 PaddleNLP 3.0以后不再进行维护，更多历史内容请参考[PaddleNLP 2.8](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/dialogue)。 - -## 模型简介 - -近年来，人机对话系统受到了学术界和产业界的广泛关注并取得了不错的发展。开放域对话系统旨在建立一个开放域的多轮对话系统，使得机器可以流畅自然地与人进行语言交互，既可以进行日常问候类的闲聊，又可以完成特定功能，以使得开放域对话系统具有实际应用价值，例如进行对话式推荐，或围绕一个主题进行深入的知识对话等。具体的说，开放域对话可以继续拆分为支持不同功能的对话形式，例如对话式推荐，知识对话技术等，如何解决并有效融合以上多个技能面临诸多挑战。 - -LIC 2021对话比赛收集了一系列公开的开放域对话数据并提供了统一的评测方式，旨在为研究人员和开发者提供学术和技术交流的平台，进一步提升开放域对话的研究水平，推动自然语言理解和人工智能领域技术的应用和发展。 - -为了方便参赛者快速了解 LIC 2021对话比赛的流程，并快速地参与到比赛中，本项目基于 UnifiedTransformer 模型提供了一个基础 baseline，利用小规模样例数据在预训练模型上完成了微调及预测。参赛者可以针对赛题进行其他改进，例如修改数据预处理方法、修改网络结构、修改训练方式、修改预测的解码方式或对结果的后处理策略等方式提升模型效果。 - -UnifiedTransformer 模型的细节可以[参阅论文](https://arxiv.org/abs/2006.16779)。 - -## 快速开始 - -### 环境依赖 - -- sentencepiece - -安装方式：`pip install sentencepiece` - -### 数据准备 - -由于样例数据涉及 LIC 2021对话比赛，暂不开放。 -关于数据集及数据集的预处理过程，详见[2021语言与智能技术竞赛：多技能对话](https://aistudio.baidu.com/aistudio/competition/detail/67)及官方提供的基线系统 Baselines。 - -模型的输入由3部分组成：词向量 token_ids，句向量 token_type_ids 和位置向量 position_ids。本项目的数据集是样例文本经过数据预处理脚本得到的 id 化的数据集。数据的每一行由3列组成，以";"作为分割符，格式：token_ids;token_type_ids;position_ids。具体细节请参考`data.py`。 - -### 模型训练 - -运行如下命令即可在样例训练集上进行 finetune，并在样例验证集上进行验证 - -```shell -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -unset CUDA_VISIBLE_DEVICES -python -m paddle.distributed.launch --gpus "0" --log_dir ./log finetune.py \ - --model_name_or_path=unified_transformer-12L-cn \ - --train_data_path=./datasets/train.txt \ - --valid_data_path=./datasets/valid.txt \ - --save_dir=./checkpoints \ - --logging_steps=500 \ - --save_steps=8000 \ - --seed=2021 \ - --epochs=10 \ - --batch_size=8192 \ - --lr=1e-5 \ - --weight_decay=0.01 \ - --warmup_steps=4000 \ - --max_grad_norm=0.1 \ - --sort_pool_size=65536 \ - --device=gpu -``` - -其中参数释义如下： -- `gpus` 指示了训练所用的 GPU 卡号。 -- `model_name_or_path` 指示了 finetune 使用的具体预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的预训练模型。如果使用本地的预训练模型，可以配置本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 预训练模型 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - -- `train_data_path` 表示训练集文件路径。 -- `valid_data_path` 表示验证集文件路径。 -- `save_dir` 表示模型的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `save_steps` 表示模型保存及评估间隔。 -- `seed` 表示随机数生成器的种子。 -- `epochs` 表示训练轮数。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `lr` 表示基础学习率大小，将于 learning rate scheduler 产生的值相乘作为当前学习率。 -- `weight_decay` 表示 AdamW 优化器中使用的 weight_decay 的系数。 -- `warmup_steps` 表示学习率逐渐升高到基础学习率（即上面配置的 lr）所需要的迭代数，最早的使用可以参考[这篇论文](https://arxiv.org/pdf/1706.02677.pdf)。 -- `max_grad_norm` 表示梯度裁剪允许的最大梯度值。 -- `sort_pool_size` 表示在构建 batch 数据时，用来排序的 pool size。 -- `device` 表示训练使用的设备。 - -参数详情和参数的默认值请参考`args.py`。 - -程序运行时将会自动进行训练和验证，训练过程中会自动保存模型在指定的`save_dir`中。 -如： -```text -./checkpoints/ -├── model_8000 -│ ├── model_config.json -│ ├── model_state.pdparams -│ ├── spm.model -│ ├── tokenizer_config.json -│ └── vocab.txt -└── ... -``` - -**NOTE:** 如需恢复模型训练，`model_name_or_path`配置本地模型的目录地址即可。 - -### 模型预测 - -运行如下命令即可在样例测试集上进行测试 - -```shell -export CUDA_VISIBLE_DEVICES=0 -# GPU启动，预测仅支持单卡 -python infer.py \ - --model_name_or_path=./checkpoints/model_80000 \ - --test_data_path=./datasets/test.txt \ - --output_path=./predict.txt \ - --logging_steps=500 \ - --seed=2021 \ - --batch_size=4 \ - --min_dec_len=1 \ - --max_dec_len=64 \ - --num_samples=20 \ - --decode_strategy=sampling \ - --top_k=5 \ - --device=gpu -``` - -其中参数释义如下： -- `model_name_or_path` 指示了 finetune 使用的具体预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的预训练模型。如果使用本地的预训练模型，可以配置本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 预训练模型 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - -- `test_data_path` 表示预测集文件路径。 -- `output_path` 表示预测结果的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `seed` 表示随机数生成器的种子。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `min_dec_len` 表示预测生成的句子的最小长度。 -- `max_dec_len` 表示预测生成的句子的最大长度。 -- `num_samples` 表示每条样本生成的句子的数量。对于每条样本，模型会生成`num_samples`个句子，根据每个句子的概率得分进行排序，得分最高的句子作为最终的生成结果。 -- `decode_strategy` 表示预测解码时采取的策略，可选"sampling"、"greedy_search"和"beam_search"之一。 -- `top_k` 表示采用"sampling"解码策略时，token 的概率按从大到小排序，生成的 token 只从前`top_k`个中进行采样。 -- `device` 表示训练使用的设备。 - -参数详情和参数的默认值请参考`args.py`。 - -程序运行结束后会将预测结果保存在`output_path`中。将预测结果准备成比赛官网要求的格式，提交评估即可得评估结果。 - -采用不同的模型在样例测试集上有如下结果： - -| model_name_or_path | F1 | BLEU1 / BLEU2 | DISTINCT1 / DISTINCT2 | -| :-----------------------------: | :---: | :-----------: | :-------------------: | -| unified_transformer-12L-cn | 10.62 | 0.070 / 0.022 | 0.065 / 0.304 | -| unified_transformer-12L-cn-luge | 33.11 | 0.245 / 0.157 | 0.074 / 0.238 | -| ./checkpoints/model_80000 | 32.38 | 0.239 / 0.150 | 0.070 / 0.219 | diff --git a/slm/examples/dialogue/README_PLATO-2.md b/slm/examples/dialogue/README_PLATO-2.md deleted file mode 100644 index 961938c290..0000000000 --- a/slm/examples/dialogue/README_PLATO-2.md +++ /dev/null @@ -1,69 +0,0 @@ -# PLATO-2 - -> **注意** 部分内容在 PaddleNLP 3.0以后不再进行维护，更多历史内容请参考[PaddleNLP 2.8](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/dialogue)。 - -## 模型简介 - -构建高质量的开放领域（Open-Domain）的对话机器人，使得它能用自然语言与人自由地交流，这一直是自然语言处理领域终极目标之一。 - -为了能够简易地构建一个高质量的开放域聊天机器人，本项目在 Paddle2.0上实现了 PLATO-2的预测模型，并基于终端实现了简单的人机交互。用户可以通过下载预训练模型快速构建一个开放域聊天机器人。 - -PLATO-2的网络结构见下图： - -

- -PLATO-2的训练过程及其他细节详见 [Knover](https://github.com/PaddlePaddle/Knover) - -## 快速开始 - -### 环境依赖 - -- python 3.7+ -- sentencepiece -- termcolor - -安装方式：`pip install sentencepiece termcolor` - -### 数据准备 - -您可以从以下位置下载预训练模型文件： - -* PLATO-2, 24-layers, 16-heads, 1024-hidden, EN: [预训练模型](https://bj.bcebos.com/paddlenlp/models/transformers/plato2/24L.pdparams) -* PLATO-2, 32-layers, 32-heads, 2048-hidden, EN: [预训练模型](https://bj.bcebos.com/paddlenlp/models/transformers/plato2/32L.pdparams) - -以24层预训练模型为例： - -```shell -wget https://bj.bcebos.com/paddlenlp/models/transformers/plato2/24L.pdparams -``` - -**NOTE:** PLATO-2网络参数量较大，24层网络至少需要显存16G，32层网络至少需要显存22G，用户可选择合适的网络层数及预训练模型。 - -sentencepiece 分词预训练模型和词表文件下载： - -```shell -wget https://bj.bcebos.com/paddlenlp/models/transformers/plato2/data.tar.gz -tar -zxf data.tar.gz -``` - -### 人机交互 - -运行如下命令即可开始与聊天机器人用英语进行简单的对话 - -```shell -export CUDA_VISIBLE_DEVICES=0 -python interaction.py --vocab_path ./data/vocab.txt --spm_model_file ./data/spm.model --num_layers 24 --init_from_ckpt ./24L.pdparams -``` - -以上参数表示： - -* vocab_path：词表文件路径。 -* spm_model_file：sentencepiece 分词预训练模型路径。 -* num_layers：PLATO-2组网层数。 -* init_from_ckpt：PLATO-2预训练模型路径。 - -32层 PLATO-2网络交互示例： - -

- -**NOTE:** 输入"[EXIT]"退出交互程序，输入"[NEXT]"开启下一轮新的对话。 diff --git a/slm/examples/dialogue/README_PLATO-XL.md b/slm/examples/dialogue/README_PLATO-XL.md deleted file mode 100644 index bbc782684d..0000000000 --- a/slm/examples/dialogue/README_PLATO-XL.md +++ /dev/null @@ -1,150 +0,0 @@ -# PLATO-XL: Exploring the Large-scale Pre-training of Dialogue Generation - -> **注意** 部分内容在 PaddleNLP 3.0以后不再进行维护，更多历史内容请参考[PaddleNLP 2.8](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/dialogue)。 - -## 模型简介 - -构建高质量的开放领域（Open-Domain）的对话机器人，使得它能用自然语言与人自由地交流，这一直是自然语言处理领域终极目标之一。 - -PLATO-XL 是业界首个开源的百亿超大规模开放域对话预训练模型，其使用了参数高效(encoder-decoder 共享参数)的 UnifiedTransformer（prefix LM）模型架构，将模型参数量提升到了11B 量级，经过了十亿级样本对话数据的预训练，并引入 role embedding 区分多方对话中的对话角色提升预训练效果，最终模型闲聊测试效果超过了众多代表性的对话模型。可以直接使用 PLATO-XL 构建高质量的开放领域对话机器人。 - -PaddleNLP 内置了 PLATO-XL 英文预训练模型以供使用。由于 PLATO-XL 模型规模较大，这使得其在预测时生成对话回复的时间较长，并且 11B 的参数量也可能超出部分型号 GPU 显存容量，这是大模型推理与落地存在的普遍和关键问题。PaddleNLP FastGeneration 为 PLATO-XL 提供了 GPU 上的高性能生成加速能力，并且支持模型并行（张量并行）推理允许通过多张小显存容量的 GPU 使用百亿大模型，相比单卡代码中也只增加了`enable_ft_para()`一行，此外模型并行能进一步提升预测速度。 - -本项目提供了 PLATO-XL 英文模型使用 PaddleNLP FastGeneration 进行高性能预测的使用示例。PLATO-XL 的训练及更多内容请参考 [PaddlePaddle/Knover](https://github.com/PaddlePaddle/Knover/tree/develop/projects/PLATO-XL)。 - -## 开始运行 -### 单卡高性能推理 - -`infer.py` 是 PLATO-XL 高性能预测使用示例脚本，可以使用如下命令运行： - -```shell -python infer.py --topk 4 --max_out_len 64 --use_faster --use_fp16 -``` - -该脚本各个参数含义如下： - -- `topk` 用于 Top-K 采样策略，采样时将只从概率最高 K 个 token 中采样，默认为1，即 greedy search。 -- `topp` 用于 Top-P 采样策略，采样时将只从概率最高且累加概率不超过该值的 token 中采样，默认为1.0。 -- `max_out_len` 指定生成的最大长度，默认为64。 -- `min_out_len` 指定生成的最小长度，默认为1。 -- `temperature` 用于调整预测概率分布，默认为1.0，即保持模型原有的预测概率。 -- `use_faster` 使用 FastGeneration -- `use_fp16` 使用 FP16，只在使用 FastGeneration 时生效 - -脚本中使用了一条如下的多轮对话的样本数据， 由`List[str]`表示，其中每个`str`表示一句话，将根据历史对话内容生成回复。 - -```python - history = [ - "hi , Mary ! What do you usually like to do in your spare time ?", - "well , I spend a lot of time watching movies .", - "what a confidence ! I always watch a lot of movies , too ." - "oh really , Frank ? What kind of movies do you like ?" - ] -``` - -**注意** 由于 PLATO-XL 模型较大，单卡预测至少需要22G 显存（使用 FP16时），且模型下载需要一定时间（FP32的权重文件约41G）。 - -### 多卡并行推理 - -多卡并行推理当前依赖 MPI（[MPICH](https://www.mpich.org)、[OpenMPI](https://www.open-mpi.org)均可）和[NCCL](https://developer.nvidia.com/nccl)，如需使用还请先安装依赖。安装完成后仍然使用 `infer.py` 来进行预测，相比单卡时不同的只是通过 mpi 来启动运行，如下： - -```shell -mpirun -n 4 python infer.py --topk 4 --max_out_len 64 --use_faster --use_fp16 -``` - -其中`-n 4`指明使用的进程和 GPU 数，在`n`设置为1时仍将进行单卡推理。由于多卡并行推理使用和单卡使用不同的依赖库，第一次运行时将重新进行 JIT 编译。 - -### 性能测试 - -`infer.py` 中同时提供了性能测试的支持，在上面预测命令的基础上加上 `--profile` 即可，如下： - -```shell -mpirun -n 4 python infer.py --batch_size 8 --min_out_len 20 --max_out_len 20 --topk 1 --use_faster --use_fp16 --profile -``` - -此外还指定了`batch_size`和`min_out_len`来得到特定输入输出大小下的性能，性能测试将给出循环运行多次的平均时延。以下为单卡高性能推理和4卡张量并行推理性能数据（V100，CUDA 10.2，输入长度60、输出长度20），可以看出4卡并行速度为单卡的2倍左右。 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PLATO-XL 高性能推理速度  (in ms/batch)
batch sizeKFastGeneration
1卡
FP16		FastGeneration
4卡
FP16		多卡并行
SpeedUp
11706.937348.6532.027
110707.514348.6992.029
41768.597384.7301.997
410770.008385.2441.998
81862.017418.3132.060
810866.490418.9652.068
1611016.362486.9742.087
16101060.472488.1562.172
3211325.700606.7702.184
32101326.222608.4792.179
- -## Reference - -1. Bao S, He H, Wang F, et al. PLATO-XL: Exploring the Large-scale Pre-training of Dialogue Generation[J]. arXiv preprint arXiv:2109.09519, 2021. diff --git a/slm/examples/dialogue/README_UnifiedTransformer.md b/slm/examples/dialogue/README_UnifiedTransformer.md deleted file mode 100644 index 0c5caaf789..0000000000 --- a/slm/examples/dialogue/README_UnifiedTransformer.md +++ /dev/null @@ -1,219 +0,0 @@ -# UnifiedTransformer - -> **注意** 部分内容在 PaddleNLP 3.0以后不再进行维护，更多历史内容请参考[PaddleNLP 2.8](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/dialogue)。 - -## 模型简介 - -近年来，人机对话系统受到了学术界和产业界的广泛关注并取得了不错的发展。开放域对话系统旨在建立一个开放域的多轮对话系统，使得机器可以流畅自然地与人进行语言交互，既可以进行日常问候类的闲聊，又可以完成特定功能，以使得开放域对话系统具有实际应用价值。具体的说，开放域对话可以继续拆分为支持不同功能的对话形式，例如对话式推荐，知识对话技术等，如何解决并有效融合以上多个技能面临诸多挑战。 - -[UnifiedTransformer](https://arxiv.org/abs/2006.16779)以[Transformer](https://arxiv.org/abs/1706.03762) 编码器为网络基本组件，采用灵活的注意力机制，十分适合对话生成任务。 - -本项目是 UnifiedTransformer 在 Paddle 2.0上的开源实现，介绍了如何使用 UnifiedTransformer 在 DuConv 任务型对话数据集上进行微调，并给出了一个搭建简单中文聊天机器人的例子。 - -## 快速开始 - -### 环境依赖 - -- sentencepiece -- termcolor - -安装方式：`pip install sentencepiece termcolor` - -### 代码结构说明 - -以下是本项目主要代码结构及说明： - -```text -. -├── finetune.py # 模型finetune主程序入口 -├── infer.py # 模型预测主程序入口 -├── utils.py # 定义参数及一些工具函数 -└── README.md # 文档说明 -``` - -### 数据准备 - -**DuConv**是百度发布的基于知识图谱的主动聊天任务数据集，让机器根据构建的知识图谱进行主动聊天，使机器具备模拟人类用语言进行信息传递的能力。数据集的创新性是：强调了 bot 的主动性，并且在闲聊对话中引入了明确的对话目标，即将对话引导到特定实体上。数据中的知识信息来源于电影和娱乐人物领域有聊天价值的知识信息，如票房、导演、评价等，以三元组 SPO 的形式组织，对话目标中的话题为电影或娱乐人物实体。数据集中共有3万 session，约12万轮对话，划分为训练集、开发集、测试集1和测试集2，其中测试集1中包含对话的 response，而测试集2中只有对话历史。 - -为了方便用户快速测试，PaddleNLP Dataset API 内置了 DuConv 数据集，一键即可完成数据集加载，示例代码如下： - -```python -from paddlenlp.datasets import load_dataset -train_ds, dev_ds, test1_ds, test2_ds = load_dataset('duconv', splits=('train', 'dev', 'test_1', 'test_2')) -``` - -### 预训练模型 - -以下是 PaddleNLP 支持的对话类预训练模型： - -|模型名称| 模型参数 | 模型特点 | -|:-----:|:------:|:-------:| -|unified_transformer-12L-cn| 12-layers, 12-heads, 768-hidden| 在千万级别的中文会话数据上进行预训练。| -|unified_transformer-12L-cn-luge| 12-layers, 12-heads, 768-hidden|由 unified_transformer-12L-cn 预训练模型在千言对话数据集上进行微调。并且模型输入中加入了标识不同对话技能的 special token，使得模型能同时支持闲聊对话、推荐对话和知识对话。| -|plato-mini| 6-layers, 12-heads, 768-hidden|在十亿级别的中文对话数据上进行预训练。参数量更小，但效果更好。只支持闲聊型对话。| - -### 模型训练 - -运行如下命令即可在训练集上进行 finetune，并在验证集上进行验证 - -```shell -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -# 例如使用1号和2号卡，则：`--gpu '1,2'` -unset CUDA_VISIBLE_DEVICES -python -m paddle.distributed.launch --gpus '0' --log_dir ./log finetune.py \ - --model_name_or_path=unified_transformer-12L-cn-luge \ - --save_dir=./checkpoints \ - --logging_steps=100 \ - --save_steps=1000 \ - --seed=2021 \ - --epochs=3 \ - --batch_size=16 \ - --lr=5e-5 \ - --weight_decay=0.01 \ - --warmup_steps=2500 \ - --max_grad_norm=0.1 \ - --max_seq_len=512 \ - --max_response_len=128 \ - --max_knowledge_len=256 \ - --device=gpu -``` - -其中参数释义如下： -- `gpus` 指示了训练所用的 GPU -- `log_dir` 指示了日志保存目录 -- `model_name_or_path` 指示了 finetune 使用的预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的模型。如果使用本地的模型，则配置为本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 模型参数 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - -- `save_dir` 表示模型的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `save_steps` 表示模型保存及评估间隔。 -- `seed` 表示随机数生成器的种子。 -- `epochs` 表示训练轮数。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `lr` 表示基础学习率大小，将于 learning rate scheduler 产生的值相乘作为当前学习率。 -- `weight_decay` 表示 AdamW 优化器中使用的 weight_decay 的系数。 -- `warmup_steps` 表示学习率逐渐升高到基础学习率（即上面配置的 lr）所需要的迭代数，最早的使用可以参考[这篇论文](https://arxiv.org/pdf/1706.02677.pdf)。 -- `max_grad_norm` 表示梯度裁剪允许的最大梯度值。 -- `max_seq_len` 表示输入序列的最大长度。 -- `max_response_len` 表示输入 response 的最大长度。 -- `max_knowledge_len` 表示输入 knowledge 序列的最大长度。 -- `device` 表示使用的设备。 - -程序运行时将会自动进行训练和验证，训练过程中会自动保存模型在指定的`save_dir`中，其中 loss 最小的模型会被保存在`save_dir/model_best`中。如： - -```text -./checkpoints/ -├── model_1000 -│ ├── model_config.json -│ ├── model_state.pdparams -│ ├── spm.model -│ ├── tokenizer_config.json -│ └── vocab.txt -└── ... -``` - -**NOTE:** 如需恢复模型训练，只需指定`model_name_or_path`为本地微调模型的路径即可。 - -### 模型预测 - -运行如下命令即可在测试集上进行测试。 - -```shell -# GPU启动，预测仅支持单卡 -export CUDA_VISIBLE_DEVICES=0 -python infer.py \ - --model_name_or_path=./checkpoints/model_best \ - --output_path=./predict.txt \ - --logging_steps=10 \ - --seed=2021 \ - --max_seq_len=512 \ - --max_knowledge_len=256 \ - --batch_size=4 \ - --min_dec_len=1 \ - --max_dec_len=64 \ - --num_return_sequences=20 \ - --decode_strategy=sampling \ - --top_k=5 \ - --device=gpu -``` - -其中参数释义如下： -- `model_name_or_path` 指示了预测使用的模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的模型。如果使用本地的模型，则配置为本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 模型参数 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - -- `output_path` 表示预测结果的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `seed` 表示随机数生成器的种子。 -- `max_seq_len` 表示输入序列的最大长度。 -- `max_knowledge_len` 表示输入 knowledge 序列的最大长度。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `min_dec_len` 表示预测生成的句子的最小长度。 -- `max_dec_len` 表示预测生成的句子的最大长度。 -- `num_return_sequences` 表示每条样本生成的句子的数量。对于每条样本，模型会生成`num_return_sequences`个句子，根据每个句子的概率得分进行排序，得分最高的句子作为最终的生成结果。 -- `decode_strategy` 表示预测解码时采取的策略，可选"sampling"、"greedy_search"和"beam_search"之一。 -- `top_k` 表示采用"sampling"解码策略时，token 的概率按从大到小排序，生成的 token 只从前`top_k`个中进行采样。 -- `device` 表示使用的设备。 - -同时，我们提供了基于 FastGeneration 的高性能预测的选项，可以选择性开启是否需要采用高性能预测，PaddleNLP 提供了 JIT 的方式，可以自动完成对所需的自定义 op 的动态库编译： -- `faster` 表示是否开启高性能预测。设置 `--faster` 即表示开启。 -- `use_fp16_decoding` 表示在开启高性能预测的时候，是否使用 fp16 来完成预测过程。设置 `--use_fp16_decoding` 即表示使用 fp16 进行预测，否则使用 fp32。 - -程序运行结束后会将预测生成的 response 保存在`output_path`中。同时终端中会输出评估结果。 - -采用预训练模型及微调模型在测试集上有如下结果： - -| model_name_or_path | BLEU1 / BLEU2 | DISTINCT1 / DISTINCT2 | -| :-----------------------------: | :-------------: | :-------------------: | -| unified_transformer-12L-cn-luge | 0.2606 / 0.1576 | 0.1168 / 0.2977 | -| ./checkpoints/model_best | 0.2808 / 0.1744 | 0.1124 / 0.2899 | - -**NOTE:** `./checkpoints/model_best`是按本项目中的超参在单卡上 finetune 得到的结果。 - -### 人机交互 - -运行如下命令即可开始与聊天机器人用中文进行简单的对话。 - -```shell -# GPU启动，仅支持单卡 -export CUDA_VISIBLE_DEVICES=0 -python interaction.py \ - --model_name_or_path=plato-mini \ - --min_dec_len=0 \ - --max_dec_len=64 \ - --num_return_sequences=20 \ - --decode_strategy=sampling \ - --top_k=5 \ - --device=gpu -``` - -其中参数释义如下： -- `model_name_or_path` 指示了预测使用的模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的模型。如果使用本地的模型，则配置为本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 模型参数 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - | plato-mini | - -- `min_dec_len` 表示预测生成的句子的最小长度。 -- `max_dec_len` 表示预测生成的句子的最大长度。 -- `num_return_sequences` 表示每条样本生成的句子的数量。对于每条样本，模型会生成`num_return_sequences`个句子，根据每个句子的概率得分进行排序，得分最高的句子作为最终的生成结果。 -- `decode_strategy` 表示预测解码时采取的策略，可选"sampling"、"greedy_search"和"beam_search"之一。 -- `top_k` 表示采用"sampling"解码策略时，token 的概率按从大到小排序，生成的 token 只从前`top_k`个中进行采样。 -- `device` 表示使用的设备。 - -**NOTE:** 输入"[EXIT]"退出交互程序，输入"[NEXT]"开启下一轮新的对话。需要注意使用退格会导致错误。 - -## Reference - -- [UnifiedTransformer](https://arxiv.org/abs/2006.16779) -- [Knover/luge-dialogue](https://github.com/PaddlePaddle/Knover/tree/luge-dialogue/luge-dialogue) -- [DuConv](https://www.aclweb.org/anthology/P19-1369/) diff --git a/slm/examples/dialogue/dgu/README.md b/slm/examples/dialogue/dgu/README.md deleted file mode 100644 index 3b56f9447e..0000000000 --- a/slm/examples/dialogue/dgu/README.md +++ /dev/null @@ -1,143 +0,0 @@ -# 对话通用理解模型 (DGU, Dialogue General Understanding) - -## 模型简介 - -对话系统 (Dialogue System) 常常需要根据应用场景的变化去解决多种多样的任务。任务的多样性（意图识别、槽填充、行为识别、状态追踪等等），以及领域训练数据的稀少，给 Dialogue System 的研究和应用带来了巨大的困难和挑战，要使得 Dialogue System 得到更好的发展，需要开发一个通用的对话理解模型。为此，我们给出了基于 BERT 的对话通用理解模型 (DGU: Dialogue General Understanding)，通过实验表明，使用 base-model (BERT)并结合常见的学习范式，就可以在几乎全部对话理解任务上取得比肩甚至超越各个领域业内最好的模型的效果，展现了学习一个通用对话理解模型的巨大潜力。 - -DGU 模型内共包含6个任务，全部基于公开数据集在 Paddle2.0上完成训练及评估，详细说明如下： - -``` -udc: 使用UDC (Ubuntu Corpus V1) 数据集完成对话匹配 (Dialogue Response Selection) 任务; -dstc2: 使用DSTC2 (Dialog State Tracking Challenge 2) 数据集完成对话状态追踪 (Dialogue State Tracking) 任务; -atis_slot: 使用ATIS (Airline Travel Information System) 数据集完成对话槽填充 (Dialogue Slot Filling) 任务； -atis_intent: 使用ATIS (Airline Travel Information System) 数据集完成对话意图识别 (Dialogue Intent Detection) 任务； -mrda: 使用MRDAC (Meeting Recorder Dialogue Act Corpus) 数据集完成对话行为识别 (Dialogue Act Detection) 任务； -swda: 使用SwDAC (Switchboard Dialogue Act Corpus) 数据集完成对话行为识别 (Dialogue Act Detection) 任务; -``` - -## 模型效果 - -DGU 模型中的6个任务，分别采用不同的评估指标在 test 集上进行评估，结果如下： - - - - - - - - - - - -
任务评估指标DGU
udcR1@1081.04%
R2@1089.85%
R5@1097.59%
dstc2Joint_Acc90.43%
atis_slotF1_Micro97.98%
atis_intentAcc97.42%
mrdaAcc90.94%
swdaAcc80.61%
- -**NOTE:** 以上结果均是采用默认配置在 GPU 单卡上训练和评估得到的，用户如需复现效果，可采用默认配置在单卡上进行训练评估。 - -## 快速开始 - -### 数据准备 - -下载数据集压缩包并解压后，DGU_datasets 目录下共存在6个目录，分别对应每个任务的训练集 train.txt、评估集 dev.txt 和测试集 test.txt。 - -```shell -wget https://bj.bcebos.com/paddlenlp/datasets/DGU_datasets.tar.gz -tar -zxf DGU_datasets.tar.gz -``` - -DGU_datasets 目录结构： - -```text -DGU_datasets/ -├── atis_intent -│   ├── dev.txt -│   ├── map_tag_intent_id.txt -│   ├── test.txt -│   └── train.txt -├── udc -│   ├── dev.txt -│   ├── dev.txt-small -│   ├── test.txt -│   └── train.txt -├── atis_slot -│   ├── dev.txt -│   ├── map_tag_slot_id.txt -│   ├── test.txt -│   └── train.txt -├── dstc2 -│   ├── dev.txt -│   ├── map_tag_id.txt -│   ├── test.txt -│   └── train.txt -├── mrda -│   ├── dev.txt -│   ├── map_tag_id.txt -│   ├── test.txt -│   └── train.txt -└── swda - ├── dev.txt - ├── map_tag_id.txt - ├── test.txt - └── train.txt -``` - -数据的每一行由多列组成，都以"\t"作为分割符，详细数据格式说明如下： - -``` -udc：由label、多轮对话conv和回应response组成 -格式：label \t conv1 \t conv2 \t conv3 \t ... \t response - -dstc2：由多轮对话id、当前轮QA对(使用\1拼接)和对话状态序列state_list(state_list中每个state由空格分割)组成 -格式：conversation_id \t question \1 answer \t state1 state2 state3 ... - -atis_slot：由对话内容conversation_content和标签序列label_list (label_list中每个label由空格分割) 组成, 其中标签序列和对话内容中word为一一对应关系 -格式：conversation_content \t label1 label2 label3 ... - -atis_intent：由标签label和对话内容conversation_content组成 -格式： label \t conversation_content - -mrda：由多轮对话id、标签label、发言人caller、对话内容conversation_content组成 -格式：conversation_id \t label \t caller \t conversation_content - -swda：由多轮对话id、标签label、发言人caller、对话内容conversation_content组成 -格式：conversation_id \t label \t caller \t conversation_content -``` - -**NOTE:** 上述数据集来自于 [Paddle1.8静态图版本](https://github.com/PaddlePaddle/models/tree/release/1.8/PaddleNLP/dialogue_system/dialogue_general_understanding)，是由相应的开源数据集经过数据格式转换而得来的，本项目中暂未包含数据格式转换脚本，细节请参考 [Paddle1.8静态图版本](https://github.com/PaddlePaddle/models/tree/release/1.8/PaddleNLP/dialogue_system/dialogue_general_understanding)。 - -### 模型训练 - -运行如下命令即可在训练集 (train.tsv) 上进行模型训练，并在开发集 (dev.tsv) 验证，训练结束后会在测试集 (test.txt) 上进行模型评估 - -```shell -# GPU启动，gpus指定训练所用的GPU卡号，可以是单卡，也可以多卡。默认会进行训练、验证和评估 -unset CUDA_VISIBLE_DEVICES -python -m paddle.distributed.launch --gpus "0" --log_dir ./log main.py --task_name=udc --data_dir=./DGU_datasets/udc --output_dir=./checkpoints/udc --device=gpu -# 若只需进行评估，do_train设为False，并且必须指定init_from_ckpt -# python -m paddle.distributed.launch --gpus "0" --log_dir ./log main.py --task_name=udc --data_dir=./DGU_datasets/udc --do_train=False --init_from_ckpt=./checkpoints/udc/best --device=gpu -``` - -以上参数表示： - -* `task_name`：任务名称，可以为 udc、dstc2、atis_slot、atis_intent、mrda 或 swda。 -* `data_dir`：训练数据路径。 -* `output_dir`：训练保存模型的文件路径。 -* `do_train：是否进行训练，默认为`True`。 -* `init_from_ckpt`：恢复模型参数的路径。 -* `device`：表示训练使用的设备。 - -其他可选参数和参数的默认值请参考`args.py`。 - -程序运行时将会自动进行训练，验证和评估。同时训练过程中会自动保存模型在指定的`output_dir`中。 -如： -```text -checkpoints/ -├── 1000.pdopt -├── 1000.pdparams -├── 2000.pdopt -├── 2000.pdparams -├── ... -├── best.pdopt -└── best.pdparams -``` - -**NOTE:** 如需恢复模型训练，则 init_from_ckpt 只需指定到文件名即可，不需要添加文件尾缀。如`--init_from_ckpt=checkpoints/1000`即可，程序会自动加载模型参数`checkpoints/1000.pdparams`，也会自动加载优化器状态`checkpoints/1000.pdopt`。 diff --git a/slm/examples/dialogue/dgu/args.py b/slm/examples/dialogue/dgu/args.py deleted file mode 100644 index 4139474c90..0000000000 --- a/slm/examples/dialogue/dgu/args.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser(__doc__) - parser.add_argument("--task_name", default=None, type=str, required=True, help="The name of the task to train.") - parser.add_argument("--model_name_or_path", default='bert-base-uncased', type=str, help="Path to pre-trained bert model or shortcut name.") - parser.add_argument("--output_dir", default=None, type=str, help="The output directory where the checkpoints will be saved.") - parser.add_argument("--data_dir", default=None, type=str, help="The directory where the dataset will be load.") - parser.add_argument("--init_from_ckpt", default=None, type=str, help="The path of checkpoint to be loaded.") - parser.add_argument("--max_seq_len", default=None, type=int, help="The maximum total input sequence length after tokenization for trainng. Sequences longer than this will be truncated, sequences shorter will be padded.") - parser.add_argument("--test_max_seq_len", default=None, type=int, help="The maximum total input sequence length after tokenization for testing. Sequences longer than this will be truncated, sequences shorter will be padded.") - parser.add_argument("--batch_size", default=None, type=int, help="Batch size per GPU/CPU for training.") - parser.add_argument("--test_batch_size", default=None, type=int, help="Batch size per GPU/CPU for testing.") - parser.add_argument("--learning_rate", default=None, type=float, help="The initial learning rate for Adam.") - parser.add_argument("--weight_decay", default=0.01, type=float, help="Weight decay if we apply some.") - parser.add_argument("--epochs", default=None, type=int, help="Total number of training epochs to perform.") - parser.add_argument("--logging_steps", default=None, type=int, help="Log every X updates steps.") - parser.add_argument("--save_steps", default=None, type=int, help="Save checkpoint every X updates steps.") - parser.add_argument("--seed", default=42, type=int, help="Random seed for initialization.") - parser.add_argument("--warmup_proportion", default=0.1, type=float, help="The proportion of warmup.") - parser.add_argument("--max_grad_norm", default=1.0, type=float, help="The max value of grad norm.") - parser.add_argument("--do_train", default=True, type=eval, help="Whether training.") - parser.add_argument("--do_eval", default=True, type=eval, help="Whether evaluation.") - parser.add_argument("--do_test", default=True, type=eval, help="Whether testing.") - parser.add_argument("--device", type=str, default="gpu", help="Device for selecting for the training.") - - args = parser.parse_args() - return args -# yapf: enable - - -def set_default_args(args): - args.task_name = args.task_name.lower() - if args.task_name == "udc": - if not args.save_steps: - args.save_steps = 1000 - if not args.logging_steps: - args.logging_steps = 100 - if not args.epochs: - args.epochs = 2 - if not args.max_seq_len: - args.max_seq_len = 210 - if not args.test_batch_size: - args.test_batch_size = 100 - elif args.task_name == "dstc2": - if not args.save_steps: - args.save_steps = 400 - if not args.logging_steps: - args.logging_steps = 20 - if not args.epochs: - args.epochs = 40 - if not args.learning_rate: - args.learning_rate = 5e-5 - if not args.max_seq_len: - args.max_seq_len = 256 - if not args.test_max_seq_len: - args.test_max_seq_len = 512 - elif args.task_name == "atis_slot": - if not args.save_steps: - args.save_steps = 100 - if not args.logging_steps: - args.logging_steps = 10 - if not args.epochs: - args.epochs = 50 - elif args.task_name == "atis_intent": - if not args.save_steps: - args.save_steps = 100 - if not args.logging_steps: - args.logging_steps = 10 - if not args.epochs: - args.epochs = 20 - elif args.task_name == "mrda": - if not args.save_steps: - args.save_steps = 500 - if not args.logging_steps: - args.logging_steps = 200 - if not args.epochs: - args.epochs = 7 - elif args.task_name == "swda": - if not args.save_steps: - args.save_steps = 500 - if not args.logging_steps: - args.logging_steps = 200 - if not args.epochs: - args.epochs = 3 - else: - raise ValueError("Not support task: %s." % args.task_name) - - if not args.data_dir: - args.data_dir = "./DGU_datasets/" + args.task_name - if not args.output_dir: - args.output_dir = "./checkpoints/" + args.task_name - if not args.learning_rate: - args.learning_rate = 2e-5 - if not args.batch_size: - args.batch_size = 32 - if not args.test_batch_size: - args.test_batch_size = args.batch_size - if not args.max_seq_len: - args.max_seq_len = 128 - if not args.test_max_seq_len: - args.test_max_seq_len = args.max_seq_len diff --git a/slm/examples/dialogue/dgu/data.py b/slm/examples/dialogue/dgu/data.py deleted file mode 100644 index 795d849ac2..0000000000 --- a/slm/examples/dialogue/dgu/data.py +++ /dev/null @@ -1,509 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from typing import List - -import numpy as np -from paddle.io import Dataset - -# The input data bigin with '[CLS]', using '[SEP]' split conversation content( -# Previous part, current part, following part, etc.). If there are multiple -# conversation in split part, using 'INNER_SEP' to further split. -INNER_SEP = "[unused0]" - - -def get_label_map(label_list): - """Create label maps""" - label_map = {} - for (i, l) in enumerate(label_list): - label_map[l] = i - return label_map - - -class UDCv1(Dataset): - """ - The UDCv1 dataset is using in task Dialogue Response Selection. - The source dataset is UDCv1(Ubuntu Dialogue Corpus v1.0). See detail at - http://dataset.cs.mcgill.ca/ubuntu-corpus-1.0/ - """ - - MAX_LEN_OF_RESPONSE = 60 - LABEL_MAP = get_label_map(["0", "1"]) - - def __init__(self, data_dir, mode="train"): - super(UDCv1, self).__init__() - self._data_dir = data_dir - self._mode = mode - self.read_data() - - def read_data(self): - if self._mode == "train": - data_path = os.path.join(self._data_dir, "train.txt") - elif self._mode == "dev": - data_path = os.path.join(self._data_dir, "dev.txt-small") - elif self._mode == "test": - data_path = os.path.join(self._data_dir, "test.txt") - self.data = [] - with open(data_path, "r", encoding="utf8") as fin: - for line in fin: - if not line: - continue - arr = line.rstrip("\n").split("\t") - if len(arr) < 3: - print("Data format error: %s" % "\t".join(arr)) - print("Data row contains at least three parts: label\tconversation1\t.....\tresponse.") - continue - label = arr[0] - text_a = arr[1:-1] - text_b = arr[-1] - self.data.append([label, text_a, text_b]) - - @classmethod - def get_label(cls, label): - return cls.LABEL_MAP[label] - - @classmethod - def num_classes(cls): - return len(cls.LABEL_MAP) - - @classmethod - def convert_example(cls, example, tokenizer, max_seq_length=512): - """Convert a glue example into necessary features.""" - - def _truncate_and_concat(text_a: List[str], text_b: str, tokenizer, max_seq_length): - tokens_b = tokenizer.tokenize(text_b) - tokens_b = tokens_b[: min(cls.MAX_LEN_OF_RESPONSE, len(tokens_b))] - tokens_a = [] - for text in text_a: - tokens_a.extend(tokenizer.tokenize(text)) - tokens_a.append(INNER_SEP) - tokens_a = tokens_a[:-1] - if len(tokens_a) > max_seq_length - len(tokens_b) - 3: - tokens_a = tokens_a[len(tokens_a) - max_seq_length + len(tokens_b) + 3 :] - tokens, segment_ids = [], [] - tokens.extend([tokenizer.cls_token] + tokens_a + [tokenizer.sep_token]) - segment_ids.extend([0] * len(tokens)) - tokens.extend(tokens_b + [tokenizer.sep_token]) - segment_ids.extend([1] * (len(tokens_b) + 1)) - input_ids = tokenizer.convert_tokens_to_ids(tokens) - return input_ids, segment_ids - - label, text_a, text_b = example - label = np.array([cls.get_label(label)], dtype="int64") - input_ids, segment_ids = _truncate_and_concat(text_a, text_b, tokenizer, max_seq_length) - return input_ids, segment_ids, label - - def __getitem__(self, index): - return self.data[index] - - def __len__(self): - return len(self.data) - - -class DSTC2(Dataset): - """ - The dataset DSTC2 is using in task Dialogue State Tracking. - The source dataset is DSTC2(Dialog State Tracking Challenges 2). See detail at - https://github.com/matthen/dstc - """ - - LABEL_MAP = get_label_map([str(i) for i in range(217)]) - - def __init__(self, data_dir, mode="train"): - super(DSTC2, self).__init__() - self._data_dir = data_dir - self._mode = mode - self.read_data() - - def read_data(self): - def _concat_dialogues(examples): - """concat multi turns dialogues""" - new_examples = [] - max_turns = 20 - for i in range(len(examples)): - multi_turns = examples[max(i - max_turns, 0) : i + 1] - new_qa = "\1".join([example[0] for example in multi_turns]) - new_examples.append((new_qa.split("\1"), examples[i][1])) - return new_examples - - if self._mode == "train": - data_path = os.path.join(self._data_dir, "train.txt") - elif self._mode == "dev": - data_path = os.path.join(self._data_dir, "dev.txt") - elif self._mode == "test": - data_path = os.path.join(self._data_dir, "test.txt") - self.data = [] - with open(data_path, "r", encoding="utf8") as fin: - pre_idx = -1 - examples = [] - for line in fin: - if not line: - continue - arr = line.rstrip("\n").split("\t") - if len(arr) != 3: - print("Data format error: %s" % "\t".join(arr)) - print("Data row should contains three parts: id\tquestion\1answer\tlabel1 label2 ...") - continue - idx = arr[0] - qa = arr[1] - label_list = arr[2].split() - if idx != pre_idx: - if idx != 0: - examples = _concat_dialogues(examples) - self.data.extend(examples) - examples = [] - pre_idx = idx - examples.append((qa, label_list)) - if examples: - examples = _concat_dialogues(examples) - self.data.extend(examples) - - @classmethod - def get_label(cls, label): - return cls.LABEL_MAP[label] - - @classmethod - def num_classes(cls): - return len(cls.LABEL_MAP) - - @classmethod - def convert_example(cls, example, tokenizer, max_seq_length=512): - """Convert a glue example into necessary features.""" - - def _truncate_and_concat(texts: List[str], tokenizer, max_seq_length): - tokens = [] - for text in texts: - tokens.extend(tokenizer.tokenize(text)) - tokens.append(INNER_SEP) - tokens = tokens[:-1] - if len(tokens) > max_seq_length - 2: - tokens = tokens[len(tokens) - max_seq_length + 2 :] - tokens = [tokenizer.cls_token] + tokens + [tokenizer.sep_token] - segment_ids = [0] * len(tokens) - input_ids = tokenizer.convert_tokens_to_ids(tokens) - return input_ids, segment_ids - - texts, labels = example - input_ids, segment_ids = _truncate_and_concat(texts, tokenizer, max_seq_length) - labels = [cls.get_label(l) for l in labels] - label = np.zeros(cls.num_classes(), dtype="int64") - for l in labels: - label[l] = 1 - return input_ids, segment_ids, label - - def __getitem__(self, index): - return self.data[index] - - def __len__(self): - return len(self.data) - - -class ATIS_DSF(Dataset): - """ - The dataset ATIS_DSF is using in task Dialogue Slot Filling. - The source dataset is ATIS(Airline Travel Information System). See detail at - https://www.kaggle.com/siddhadev/ms-cntk-atis - """ - - LABEL_MAP = get_label_map([str(i) for i in range(130)]) - - def __init__(self, data_dir, mode="train"): - super(ATIS_DSF, self).__init__() - self._data_dir = data_dir - self._mode = mode - self.read_data() - - def read_data(self): - if self._mode == "train": - data_path = os.path.join(self._data_dir, "train.txt") - elif self._mode == "dev": - data_path = os.path.join(self._data_dir, "dev.txt") - elif self._mode == "test": - data_path = os.path.join(self._data_dir, "test.txt") - self.data = [] - with open(data_path, "r", encoding="utf8") as fin: - for line in fin: - if not line: - continue - arr = line.rstrip("\n").split("\t") - if len(arr) != 2: - print("Data format error: %s" % "\t".join(arr)) - print("Data row should contains two parts: conversation_content\tlabel1 label2 label3.") - continue - text = arr[0] - label_list = arr[1].split() - self.data.append([text, label_list]) - - @classmethod - def get_label(cls, label): - return cls.LABEL_MAP[label] - - @classmethod - def num_classes(cls): - return len(cls.LABEL_MAP) - - @classmethod - def convert_example(cls, example, tokenizer, max_seq_length=512): - """Convert a glue example into necessary features.""" - text, labels = example - tokens, label_list = [], [] - words = text.split() - assert len(words) == len(labels) - for word, label in zip(words, labels): - piece_words = tokenizer.tokenize(word) - tokens.extend(piece_words) - label = cls.get_label(label) - label_list.extend([label] * len(piece_words)) - if len(tokens) > max_seq_length - 2: - tokens = tokens[len(tokens) - max_seq_length + 2 :] - label_list = label_list[len(tokens) - max_seq_length + 2 :] - tokens = [tokenizer.cls_token] + tokens + [tokenizer.sep_token] - label_list = [0] + label_list + [0] - segment_ids = [0] * len(tokens) - input_ids = tokenizer.convert_tokens_to_ids(tokens) - label = np.array(label_list, dtype="int64") - return input_ids, segment_ids, label - - def __getitem__(self, index): - return self.data[index] - - def __len__(self): - return len(self.data) - - -class ATIS_DID(Dataset): - """ - The dataset ATIS_ID is using in task Dialogue Intent Detection. - The source dataset is ATIS(Airline Travel Information System). See detail at - https://www.kaggle.com/siddhadev/ms-cntk-atis - """ - - LABEL_MAP = get_label_map([str(i) for i in range(26)]) - - def __init__(self, data_dir, mode="train"): - super(ATIS_DID, self).__init__() - self._data_dir = data_dir - self._mode = mode - self.read_data() - - def read_data(self): - if self._mode == "train": - data_path = os.path.join(self._data_dir, "train.txt") - elif self._mode == "dev": - data_path = os.path.join(self._data_dir, "dev.txt") - elif self._mode == "test": - data_path = os.path.join(self._data_dir, "test.txt") - self.data = [] - with open(data_path, "r", encoding="utf8") as fin: - for line in fin: - if not line: - continue - arr = line.rstrip("\n").split("\t") - if len(arr) != 2: - print("Data format error: %s" % "\t".join(arr)) - print("Data row should contains two parts: label\tconversation_content.") - continue - label = arr[0] - text = arr[1] - self.data.append([label, text]) - - @classmethod - def get_label(cls, label): - return cls.LABEL_MAP[label] - - @classmethod - def num_classes(cls): - return len(cls.LABEL_MAP) - - @classmethod - def convert_example(cls, example, tokenizer, max_seq_length=512): - """Convert a glue example into necessary features.""" - label, text = example - tokens = tokenizer.tokenize(text) - if len(tokens) > max_seq_length - 2: - tokens = tokens[len(tokens) - max_seq_length + 2 :] - tokens = [tokenizer.cls_token] + tokens + [tokenizer.sep_token] - segment_ids = [0] * len(tokens) - input_ids = tokenizer.convert_tokens_to_ids(tokens) - label = np.array([cls.get_label(label)], dtype="int64") - return input_ids, segment_ids, label - - def __getitem__(self, index): - return self.data[index] - - def __len__(self): - return len(self.data) - - -def read_da_data(data_dir, mode): - def _concat_dialogues(examples): - """concat multi turns dialogues""" - new_examples = [] - for i in range(len(examples)): - label, caller, text = examples[i] - cur_txt = "%s : %s" % (caller, text) - pre_txt = ["%s : %s" % (item[1], item[2]) for item in examples[max(0, i - 5) : i]] - suf_txt = ["%s : %s" % (item[1], item[2]) for item in examples[i + 1 : min(len(examples), i + 3)]] - sample = [label, pre_txt, cur_txt, suf_txt] - new_examples.append(sample) - return new_examples - - if mode == "train": - data_path = os.path.join(data_dir, "train.txt") - elif mode == "dev": - data_path = os.path.join(data_dir, "dev.txt") - elif mode == "test": - data_path = os.path.join(data_dir, "test.txt") - data = [] - with open(data_path, "r", encoding="utf8") as fin: - pre_idx = -1 - examples = [] - for line in fin: - if not line: - continue - arr = line.rstrip("\n").split("\t") - if len(arr) != 4: - print("Data format error: %s" % "\t".join(arr)) - print("Data row should contains four parts: id\tlabel\tcaller\tconversation_content.") - continue - idx, label, caller, text = arr - if idx != pre_idx: - if idx != 0: - examples = _concat_dialogues(examples) - data.extend(examples) - examples = [] - pre_idx = idx - examples.append((label, caller, text)) - if examples: - examples = _concat_dialogues(examples) - data.extend(examples) - return data - - -def truncate_and_concat( - pre_txt: List[str], cur_txt: str, suf_txt: List[str], tokenizer, max_seq_length, max_len_of_cur_text -): - cur_tokens = tokenizer.tokenize(cur_txt) - cur_tokens = cur_tokens[: min(max_len_of_cur_text, len(cur_tokens))] - pre_tokens = [] - for text in pre_txt: - pre_tokens.extend(tokenizer.tokenize(text)) - pre_tokens.append(INNER_SEP) - pre_tokens = pre_tokens[:-1] - suf_tokens = [] - for text in suf_txt: - suf_tokens.extend(tokenizer.tokenize(text)) - suf_tokens.append(INNER_SEP) - suf_tokens = suf_tokens[:-1] - if len(cur_tokens) + len(pre_tokens) + len(suf_tokens) > max_seq_length - 4: - left_num = max_seq_length - 4 - len(cur_tokens) - if len(pre_tokens) > len(suf_tokens): - suf_num = int(left_num / 2) - suf_tokens = suf_tokens[:suf_num] - pre_num = left_num - len(suf_tokens) - pre_tokens = pre_tokens[max(0, len(pre_tokens) - pre_num) :] - else: - pre_num = int(left_num / 2) - pre_tokens = pre_tokens[max(0, len(pre_tokens) - pre_num) :] - suf_num = left_num - len(pre_tokens) - suf_tokens = suf_tokens[:suf_num] - tokens, segment_ids = [], [] - tokens.extend([tokenizer.cls_token] + pre_tokens + [tokenizer.sep_token]) - segment_ids.extend([0] * len(tokens)) - tokens.extend(cur_tokens + [tokenizer.sep_token]) - segment_ids.extend([1] * (len(cur_tokens) + 1)) - if suf_tokens: - tokens.extend(suf_tokens + [tokenizer.sep_token]) - segment_ids.extend([0] * (len(suf_tokens) + 1)) - input_ids = tokenizer.convert_tokens_to_ids(tokens) - return input_ids, segment_ids - - -class MRDA(Dataset): - """ - The dataset MRDA is using in task Dialogue Act. - The source dataset is MRDA(Meeting Recorder Dialogue Act). See detail at - https://www.aclweb.org/anthology/W04-2319.pdf - """ - - MAX_LEN_OF_CUR_TEXT = 50 - LABEL_MAP = get_label_map([str(i) for i in range(5)]) - - def __init__(self, data_dir, mode="train"): - super(MRDA, self).__init__() - self.data = read_da_data(data_dir, mode) - - @classmethod - def get_label(cls, label): - return cls.LABEL_MAP[label] - - @classmethod - def num_classes(cls): - return len(cls.LABEL_MAP) - - @classmethod - def convert_example(cls, example, tokenizer, max_seq_length=512): - """Convert a glue example into necessary features.""" - label, pre_txt, cur_txt, suf_txt = example - label = np.array([cls.get_label(label)], dtype="int64") - input_ids, segment_ids = truncate_and_concat( - pre_txt, cur_txt, suf_txt, tokenizer, max_seq_length, cls.MAX_LEN_OF_CUR_TEXT - ) - return input_ids, segment_ids, label - - def __getitem__(self, index): - return self.data[index] - - def __len__(self): - return len(self.data) - - -class SwDA(Dataset): - """ - The dataset SwDA is using in task Dialogue Act. - The source dataset is SwDA(Switchboard Dialog Act). See detail at - http://compprag.christopherpotts.net/swda.html - """ - - MAX_LEN_OF_CUR_TEXT = 50 - LABEL_MAP = get_label_map([str(i) for i in range(42)]) - - def __init__(self, data_dir, mode="train"): - super(SwDA, self).__init__() - self.data = read_da_data(data_dir, mode) - - @classmethod - def get_label(cls, label): - return cls.LABEL_MAP[label] - - @classmethod - def num_classes(cls): - return len(cls.LABEL_MAP) - - @classmethod - def convert_example(cls, example, tokenizer, max_seq_length=512): - """Convert a glue example into necessary features.""" - label, pre_txt, cur_txt, suf_txt = example - label = np.array([cls.get_label(label)], dtype="int64") - input_ids, segment_ids = truncate_and_concat( - pre_txt, cur_txt, suf_txt, tokenizer, max_seq_length, cls.MAX_LEN_OF_CUR_TEXT - ) - return input_ids, segment_ids, label - - def __getitem__(self, index): - return self.data[index] - - def __len__(self): - return len(self.data) diff --git a/slm/examples/dialogue/dgu/main.py b/slm/examples/dialogue/dgu/main.py deleted file mode 100644 index 8c8d3467be..0000000000 --- a/slm/examples/dialogue/dgu/main.py +++ /dev/null @@ -1,293 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import random -import time -from functools import partial - -import data -import metric -import numpy as np -import paddle -import paddle.distributed as dist -import paddle.nn as nn -import paddle.nn.functional as F -from args import parse_args, set_default_args -from paddle.io import BatchSampler, DataLoader, DistributedBatchSampler -from paddle.metric import Accuracy -from paddle.optimizer import AdamW - -from paddlenlp.data import Pad, Stack, Tuple -from paddlenlp.datasets import MapDataset -from paddlenlp.transformers import ( - BertForSequenceClassification, - BertForTokenClassification, - BertTokenizer, - LinearDecayWithWarmup, -) - -TASK_CLASSES = { - "udc": (data.UDCv1, metric.RecallAtK), - "dstc2": (data.DSTC2, metric.JointAccuracy), - "atis_slot": (data.ATIS_DSF, metric.F1Score), - "atis_intent": (data.ATIS_DID, Accuracy), - "mrda": (data.MRDA, Accuracy), - "swda": (data.SwDA, Accuracy), -} - - -def set_seed(seed): - random.seed(seed) - np.random.seed(seed) - paddle.seed(seed) - - -def load_ckpt(args, model, optimizer=None): - if args.init_from_ckpt: - params_state_dict = paddle.load(args.init_from_ckpt + ".pdparams") - model.set_state_dict(params_state_dict) - if optimizer: - opt_state_dict = paddle.load(args.init_from_ckpt + ".pdopt") - optimizer.set_state_dict(opt_state_dict) - print("Loaded checkpoint from %s" % args.init_from_ckpt) - - -def save_ckpt(model, optimizer, output_dir, name): - params_path = os.path.join(output_dir, "{}.pdparams".format(name)) - opt_path = os.path.join(output_dir, "{}.pdopt".format(name)) - paddle.save(model.state_dict(), params_path) - paddle.save(optimizer.state_dict(), opt_path) - - -class DGULossFunction(nn.Layer): - def __init__(self, task_name): - super(DGULossFunction, self).__init__() - - self.task_name = task_name - self.loss_fn = self.get_loss_fn() - - def get_loss_fn(self): - if self.task_name in ["udc", "atis_slot", "atis_intent", "mrda", "swda"]: - return F.cross_entropy - elif self.task_name == "dstc2": - return nn.BCEWithLogitsLoss(reduction="sum") - - def forward(self, logits, labels): - if self.task_name in ["udc", "atis_intent", "mrda", "swda"]: - loss = self.loss_fn(logits, labels) - elif self.task_name == "dstc2": - loss = self.loss_fn(logits, paddle.cast(labels, dtype=logits.dtype)) - elif self.task_name == "atis_slot": - labels = paddle.unsqueeze(labels, axis=-1) - loss = self.loss_fn(logits, labels) - return loss - - -def print_logs(args, step, logits, labels, loss, total_time, metric): - if args.task_name in ["udc", "atis_intent", "mrda", "swda"]: - if args.task_name == "udc": - metric = Accuracy() - metric.reset() - correct = metric.compute(logits, labels) - metric.update(correct) - acc = metric.accumulate() - print("step %d - loss: %.4f - acc: %.4f - %.3fs/step" % (step, loss, acc, total_time / args.logging_steps)) - elif args.task_name == "dstc2": - metric.reset() - metric.update(logits, labels) - joint_acc = metric.accumulate() - print( - "step %d - loss: %.4f - joint_acc: %.4f - %.3fs/step" - % (step, loss, joint_acc, total_time / args.logging_steps) - ) - elif args.task_name == "atis_slot": - metric.reset() - metric.update(logits, labels) - f1_micro = metric.accumulate() - print( - "step %d - loss: %.4f - f1_micro: %.4f - %.3fs/step" - % (step, loss, f1_micro, total_time / args.logging_steps) - ) - - -def train(args, model, train_data_loader, dev_data_loader, metric, n_procs, rank): - num_examples = len(train_data_loader) * args.batch_size * n_procs - max_train_steps = args.epochs * len(train_data_loader) - print("\nNum train examples: %d" % num_examples) - print("Max train steps: %d" % max_train_steps) - print("Warmup proportion: %.2f" % args.warmup_proportion) - - lr_scheduler = LinearDecayWithWarmup(args.learning_rate, max_train_steps, args.warmup_proportion) - - # Generate parameter names needed to perform weight decay. - # All bias and LayerNorm parameters are excluded. - decay_params = [p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])] - optimizer = AdamW( - learning_rate=lr_scheduler, - parameters=model.parameters(), - weight_decay=args.weight_decay, - apply_decay_param_fun=lambda x: x in decay_params, - grad_clip=nn.ClipGradByGlobalNorm(args.max_grad_norm), - ) - loss_fn = DGULossFunction(args.task_name) - - load_ckpt(args, model, optimizer) - - step = 0 - best_metric = 0.0 - total_time = 0.0 - for epoch in range(args.epochs): - print("\nEpoch %d/%d" % (epoch + 1, args.epochs)) - batch_start_time = time.time() - for batch in train_data_loader: - step += 1 - input_ids, segment_ids, labels = batch - logits = model(input_ids, segment_ids) - loss = loss_fn(logits, labels) - loss.backward() - optimizer.step() - lr_scheduler.step() - optimizer.clear_grad() - total_time += time.time() - batch_start_time - if step % args.logging_steps == 0: - print_logs(args, step, logits, labels, loss, total_time, metric) - total_time = 0.0 - if step % args.save_steps == 0 or step == max_train_steps: - if rank == 0: - save_ckpt(model, optimizer, args.output_dir, step) - if args.do_eval: - print("\nEval begin...") - metric_out = evaluation(args, model, dev_data_loader, metric) - if rank == 0 and metric_out > best_metric: - best_metric = metric_out - save_ckpt(model, optimizer, args.output_dir, "best") - print("Best model, step: %d\n" % step) - batch_start_time = time.time() - - -@paddle.no_grad() -def evaluation(args, model, data_loader, metric): - model.eval() - metric.reset() - for batch in data_loader: - input_ids, segment_ids, labels = batch - logits = model(input_ids, segment_ids) - if args.task_name in ["atis_intent", "mrda", "swda"]: - correct = metric.compute(logits, labels) - metric.update(correct) - else: - metric.update(logits, labels) - model.train() - metric_out = metric.accumulate() - print("Total samples: %d" % (len(data_loader) * args.test_batch_size)) - if args.task_name == "udc": - print("R1@10: %.4f - R2@10: %.4f - R5@10: %.4f\n" % (metric_out[0], metric_out[1], metric_out[2])) - return metric_out[0] - elif args.task_name == "dstc2": - print("Joint_acc: %.4f\n" % metric_out) - return metric_out - elif args.task_name == "atis_slot": - print("F1_micro: %.4f\n" % metric_out) - return metric_out - elif args.task_name in ["atis_intent", "mrda", "swda"]: - print("Acc: %.4f\n" % metric_out) - return metric_out - - -def create_data_loader(args, dataset_class, trans_func, batchify_fn, mode): - dataset = dataset_class(args.data_dir, mode) - dataset = MapDataset(dataset).map(trans_func, lazy=True) - if mode == "train": - batch_sampler = DistributedBatchSampler(dataset, batch_size=args.batch_size, shuffle=True) - else: - batch_sampler = BatchSampler(dataset, batch_size=args.test_batch_size, shuffle=False) - data_loader = DataLoader(dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn, return_list=True) - return data_loader - - -def main(args): - paddle.set_device(args.device) - world_size = dist.get_world_size() - rank = dist.get_rank() - if world_size > 1 and args.do_train: - dist.init_parallel_env() - - set_seed(args.seed) - - dataset_class, metric_class = TASK_CLASSES[args.task_name] - tokenizer = BertTokenizer.from_pretrained(args.model_name_or_path) - trans_func = partial(dataset_class.convert_example, tokenizer=tokenizer, max_seq_length=args.max_seq_len) - test_trans_func = partial(dataset_class.convert_example, tokenizer=tokenizer, max_seq_length=args.test_max_seq_len) - metric = metric_class() - - if args.task_name in ("udc", "dstc2", "atis_intent", "mrda", "swda"): - batchify_fn = lambda samples, fn=Tuple( - Pad(axis=0, pad_val=tokenizer.pad_token_id), # input - Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # segment - Stack(dtype="int64"), # label - ): fn(samples) - model = BertForSequenceClassification.from_pretrained( - args.model_name_or_path, num_classes=dataset_class.num_classes() - ) - elif args.task_name == "atis_slot": - batchify_fn = lambda samples, fn=Tuple( - Pad(axis=0, pad_val=tokenizer.pad_token_id), # input - Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # segment - Pad(axis=0, pad_val=0, dtype="int64"), # label - ): fn(samples) - model = BertForTokenClassification.from_pretrained( - args.model_name_or_path, num_classes=dataset_class.num_classes(), dropout=0.0 - ) - if world_size > 1 and args.do_train: - model = paddle.DataParallel(model) - - if args.do_train: - train_data_loader = create_data_loader(args, dataset_class, trans_func, batchify_fn, "train") - if args.do_eval: - dev_data_loader = create_data_loader(args, dataset_class, test_trans_func, batchify_fn, "dev") - else: - dev_data_loader = None - train(args, model, train_data_loader, dev_data_loader, metric, world_size, rank) - - if args.do_test: - if rank == 0: - test_data_loader = create_data_loader(args, dataset_class, test_trans_func, batchify_fn, "test") - if args.do_train: - # If do_eval=True, use best model to evaluate the test data. - # Otherwise, use final model to evaluate the test data. - if args.do_eval: - args.init_from_ckpt = os.path.join(args.output_dir, "best") - load_ckpt(args, model) - else: - if not args.init_from_ckpt: - raise ValueError('"init_from_ckpt" should be set.') - load_ckpt(args, model) - print("\nTest begin...") - evaluation(args, model, test_data_loader, metric) - - -def print_args(args): - print("----------- Configuration Arguments -----------") - for arg, value in sorted(vars(args).items()): - print("%s: %s" % (arg, value)) - print("------------------------------------------------") - - -if __name__ == "__main__": - args = parse_args() - set_default_args(args) - print_args(args) - - main(args) diff --git a/slm/examples/dialogue/dgu/metric.py b/slm/examples/dialogue/dgu/metric.py deleted file mode 100644 index 50eec50ed6..0000000000 --- a/slm/examples/dialogue/dgu/metric.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np -import paddle -import paddle.nn as nn -from paddle.metric import Metric - - -class RecallAtK(Metric): - """ - Recall@K is the fraction of relevant results among the retrieved Top K - results, using to evaluate the performance of Dialogue Response Selection. - - Noted that this class manages the Recall@K score only for binary - classification task. - """ - - def __init__(self, name="Recall@K", *args, **kwargs): - super(RecallAtK, self).__init__(*args, **kwargs) - self._name = name - self.softmax = nn.Softmax() - self.reset() - - def reset(self): - """ - Resets all of the metric state. - """ - self.num_sampls = 0 - self.p_at_1_in_10 = 0.0 - self.p_at_2_in_10 = 0.0 - self.p_at_5_in_10 = 0.0 - - def get_p_at_n_in_m(self, data, n, m, idx): - """ - calculate precision in recall n - """ - pos_score = data[idx][0] - curr = data[idx : idx + m] - curr = sorted(curr, key=lambda x: x[0], reverse=True) - if curr[n - 1][0] <= pos_score: - return 1 - return 0 - - def update(self, logits, labels): - """ - Update the states based on the current mini-batch prediction results. - - Args: - logits (Tensor): The predicted value is a Tensor with - shape [batch_size, 2] and type float32 or float64. - labels (Tensor): The ground truth value is a 2D Tensor, - its shape is [batch_size, 1] and type is int64. - """ - probs = self.softmax(logits) - probs = probs.numpy() - labels = labels.numpy() - assert probs.shape[0] == labels.shape[0] - data = [] - for prob, label in zip(probs, labels): - data.append((prob[1], label)) - assert len(data) % 10 == 0 - - length = int(len(data) / 10) - self.num_sampls += length - for i in range(length): - idx = i * 10 - assert data[idx][1] == 1 - self.p_at_1_in_10 += self.get_p_at_n_in_m(data, 1, 10, idx) - self.p_at_2_in_10 += self.get_p_at_n_in_m(data, 2, 10, idx) - self.p_at_5_in_10 += self.get_p_at_n_in_m(data, 5, 10, idx) - - def accumulate(self): - """ - Calculate the final Recall@K. - - Returns: - A list with scaler float: results of the calculated R1@K, R2@K, R5@K. - """ - metrics_out = [ - self.p_at_1_in_10 / self.num_sampls, - self.p_at_2_in_10 / self.num_sampls, - self.p_at_5_in_10 / self.num_sampls, - ] - return metrics_out - - def name(self): - """ - Returns metric name - """ - return self._name - - -class JointAccuracy(Metric): - """ - The joint accuracy rate is used to evaluate the performance of multi-turn - Dialogue State Tracking. For each turn, if and only if all state in - state_list are correctly predicted, the dialog state prediction is - considered correct. And the joint accuracy rate is equal to 1, otherwise - it is equal to 0. - """ - - def __init__(self, name="JointAccuracy", *args, **kwargs): - super(JointAccuracy, self).__init__(*args, **kwargs) - self._name = name - self.sigmoid = nn.Sigmoid() - self.reset() - - def reset(self): - """ - Resets all of the metric state. - """ - self.num_samples = 0 - self.correct_joint = 0.0 - - def update(self, logits, labels): - """ - Update the states based on the current mini-batch prediction results. - - Args: - logits (Tensor): The predicted value is a Tensor with - shape [batch_size, num_classes] and type float32 or float64. - labels (Tensor): The ground truth value is a 2D Tensor, - its shape is [batch_size, num_classes] and type is int64. - """ - probs = self.sigmoid(logits) - probs = probs.numpy() - labels = labels.numpy() - assert probs.shape[0] == labels.shape[0] - assert probs.shape[1] == labels.shape[1] - for i in range(probs.shape[0]): - pred, refer = [], [] - for j in range(probs.shape[1]): - if probs[i][j] >= 0.5: - pred.append(j) - if labels[i][j] == 1: - refer.append(j) - if not pred: - pred = [np.argmax(probs[i])] - if pred == refer: - self.correct_joint += 1 - self.num_samples += probs.shape[0] - - def accumulate(self): - """ - Calculate the final JointAccuracy. - - Returns: - A scaler float: results of the calculated JointAccuracy. - """ - joint_acc = self.correct_joint / self.num_samples - return joint_acc - - def name(self): - """ - Returns metric name - """ - return self._name - - -class F1Score(Metric): - """ - F1-score is the harmonic mean of precision and recall. Micro-averaging is - to create a global confusion matrix for all examples, and then calculate - the F1-score. This class is using to evaluate the performance of Dialogue - Slot Filling. - """ - - def __init__(self, name="F1Score", *args, **kwargs): - super(F1Score, self).__init__(*args, **kwargs) - self._name = name - self.reset() - - def reset(self): - """ - Resets all of the metric state. - """ - self.tp = {} - self.fn = {} - self.fp = {} - - def update(self, logits, labels): - """ - Update the states based on the current mini-batch prediction results. - - Args: - logits (Tensor): The predicted value is a Tensor with - shape [batch_size, seq_len, num_classes] and type float32 or - float64. - labels (Tensor): The ground truth value is a 2D Tensor, - its shape is [batch_size, seq_len] and type is int64. - """ - probs = paddle.argmax(logits, axis=-1) - probs = probs.numpy() - labels = labels.numpy() - assert probs.shape[0] == labels.shape[0] - assert probs.shape[1] == labels.shape[1] - for i in range(probs.shape[0]): - start, end = 1, probs.shape[1] - while end > start: - if labels[i][end - 1] != 0: - break - end -= 1 - prob, label = probs[i][start:end], labels[i][start:end] - for y_pred, y in zip(prob, label): - if y_pred == y: - self.tp[y] = self.tp.get(y, 0) + 1 - else: - self.fp[y_pred] = self.fp.get(y_pred, 0) + 1 - self.fn[y] = self.fn.get(y, 0) + 1 - - def accumulate(self): - """ - Calculate the final micro F1 score. - - Returns: - A scaler float: results of the calculated micro F1 score. - """ - tp_total = sum(self.tp.values()) - fn_total = sum(self.fn.values()) - fp_total = sum(self.fp.values()) - p_total = float(tp_total) / (tp_total + fp_total) - r_total = float(tp_total) / (tp_total + fn_total) - if p_total + r_total == 0: - return 0 - f1_micro = 2 * p_total * r_total / (p_total + r_total) - return f1_micro - - def name(self): - """ - Returns metric name - """ - return self._name diff --git a/slm/examples/dialogue/lic2021_baseline/README.md b/slm/examples/dialogue/lic2021_baseline/README.md deleted file mode 100644 index 31430a23ec..0000000000 --- a/slm/examples/dialogue/lic2021_baseline/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# LIC 2021对话比赛 baseline - -## 模型简介 - -近年来，人机对话系统受到了学术界和产业界的广泛关注并取得了不错的发展。开放域对话系统旨在建立一个开放域的多轮对话系统，使得机器可以流畅自然地与人进行语言交互，既可以进行日常问候类的闲聊，又可以完成特定功能，以使得开放域对话系统具有实际应用价值，例如进行对话式推荐，或围绕一个主题进行深入的知识对话等。具体的说，开放域对话可以继续拆分为支持不同功能的对话形式，例如对话式推荐，知识对话技术等，如何解决并有效融合以上多个技能面临诸多挑战。 - -LIC 2021对话比赛收集了一系列公开的开放域对话数据并提供了统一的评测方式，旨在为研究人员和开发者提供学术和技术交流的平台，进一步提升开放域对话的研究水平，推动自然语言理解和人工智能领域技术的应用和发展。 - -为了方便参赛者快速了解 LIC 2021对话比赛的流程，并快速地参与到比赛中，本项目基于 UnifiedTransformer 模型提供了一个基础 baseline，利用小规模样例数据在预训练模型上完成了微调及预测。参赛者可以针对赛题进行其他改进，例如修改数据预处理方法、修改网络结构、修改训练方式、修改预测的解码方式或对结果的后处理策略等方式提升模型效果。 - -UnifiedTransformer 模型的细节可以[参阅论文](https://arxiv.org/abs/2006.16779)。 - -## 快速开始 - -### 环境依赖 - -- sentencepiece - -安装方式：`pip install sentencepiece` - -### 数据准备 - -由于样例数据涉及 LIC 2021对话比赛，暂不开放。 -关于数据集及数据集的预处理过程，详见[2021语言与智能技术竞赛：多技能对话](https://aistudio.baidu.com/aistudio/competition/detail/67)及官方提供的基线系统 Baselines。 - -模型的输入由3部分组成：词向量 token_ids，句向量 token_type_ids 和位置向量 position_ids。本项目的数据集是样例文本经过数据预处理脚本得到的 id 化的数据集。数据的每一行由3列组成，以";"作为分割符，格式：token_ids;token_type_ids;position_ids。具体细节请参考`data.py`。 - -### 模型训练 - -运行如下命令即可在样例训练集上进行 finetune，并在样例验证集上进行验证 - -```shell -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -unset CUDA_VISIBLE_DEVICES -python -m paddle.distributed.launch --gpus "0" --log_dir ./log finetune.py \ - --model_name_or_path=unified_transformer-12L-cn \ - --train_data_path=./datasets/train.txt \ - --valid_data_path=./datasets/valid.txt \ - --save_dir=./checkpoints \ - --logging_steps=500 \ - --save_steps=8000 \ - --seed=2021 \ - --epochs=10 \ - --batch_size=8192 \ - --lr=1e-5 \ - --weight_decay=0.01 \ - --warmup_steps=4000 \ - --max_grad_norm=0.1 \ - --sort_pool_size=65536 \ - --device=gpu -``` - -其中参数释义如下： -- `gpus` 指示了训练所用的 GPU 卡号。 -- `model_name_or_path` 指示了 finetune 使用的具体预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的预训练模型。如果使用本地的预训练模型，可以配置本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 预训练模型 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - -- `train_data_path` 表示训练集文件路径。 -- `valid_data_path` 表示验证集文件路径。 -- `save_dir` 表示模型的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `save_steps` 表示模型保存及评估间隔。 -- `seed` 表示随机数生成器的种子。 -- `epochs` 表示训练轮数。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `lr` 表示基础学习率大小，将于 learning rate scheduler 产生的值相乘作为当前学习率。 -- `weight_decay` 表示 AdamW 优化器中使用的 weight_decay 的系数。 -- `warmup_steps` 表示学习率逐渐升高到基础学习率（即上面配置的 lr）所需要的迭代数，最早的使用可以参考[这篇论文](https://arxiv.org/pdf/1706.02677.pdf)。 -- `max_grad_norm` 表示梯度裁剪允许的最大梯度值。 -- `sort_pool_size` 表示在构建 batch 数据时，用来排序的 pool size。 -- `device` 表示训练使用的设备。 - -参数详情和参数的默认值请参考`args.py`。 - -程序运行时将会自动进行训练和验证，训练过程中会自动保存模型在指定的`save_dir`中。 -如： -```text -./checkpoints/ -├── model_8000 -│ ├── model_config.json -│ ├── model_state.pdparams -│ ├── spm.model -│ ├── tokenizer_config.json -│ └── vocab.txt -└── ... -``` - -**NOTE:** 如需恢复模型训练，`model_name_or_path`配置本地模型的目录地址即可。 - -### 模型预测 - -运行如下命令即可在样例测试集上进行测试 - -```shell -export CUDA_VISIBLE_DEVICES=0 -# GPU启动，预测仅支持单卡 -python infer.py \ - --model_name_or_path=./checkpoints/model_80000 \ - --test_data_path=./datasets/test.txt \ - --output_path=./predict.txt \ - --logging_steps=500 \ - --seed=2021 \ - --batch_size=4 \ - --min_dec_len=1 \ - --max_dec_len=64 \ - --num_samples=20 \ - --decode_strategy=sampling \ - --top_k=5 \ - --device=gpu -``` - -其中参数释义如下： -- `model_name_or_path` 指示了 finetune 使用的具体预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的预训练模型。如果使用本地的预训练模型，可以配置本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 预训练模型 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - -- `test_data_path` 表示预测集文件路径。 -- `output_path` 表示预测结果的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `seed` 表示随机数生成器的种子。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `min_dec_len` 表示预测生成的句子的最小长度。 -- `max_dec_len` 表示预测生成的句子的最大长度。 -- `num_samples` 表示每条样本生成的句子的数量。对于每条样本，模型会生成`num_samples`个句子，根据每个句子的概率得分进行排序，得分最高的句子作为最终的生成结果。 -- `decode_strategy` 表示预测解码时采取的策略，可选"sampling"、"greedy_search"和"beam_search"之一。 -- `top_k` 表示采用"sampling"解码策略时，token 的概率按从大到小排序，生成的 token 只从前`top_k`个中进行采样。 -- `device` 表示训练使用的设备。 - -参数详情和参数的默认值请参考`args.py`。 - -程序运行结束后会将预测结果保存在`output_path`中。将预测结果准备成比赛官网要求的格式，提交评估即可得评估结果。 - -采用不同的模型在样例测试集上有如下结果： - -| model_name_or_path | F1 | BLEU1 / BLEU2 | DISTINCT1 / DISTINCT2 | -| :-----------------------------: | :---: | :-----------: | :-------------------: | -| unified_transformer-12L-cn | 10.62 | 0.070 / 0.022 | 0.065 / 0.304 | -| unified_transformer-12L-cn-luge | 33.11 | 0.245 / 0.157 | 0.074 / 0.238 | -| ./checkpoints/model_80000 | 32.38 | 0.239 / 0.150 | 0.070 / 0.219 | diff --git a/slm/examples/dialogue/lic2021_baseline/args.py b/slm/examples/dialogue/lic2021_baseline/args.py deleted file mode 100644 index cc08c7d618..0000000000 --- a/slm/examples/dialogue/lic2021_baseline/args.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser(__doc__) - parser.add_argument('--model_name_or_path', type=str, default='unified_transformer-12L-cn', help='The path or shortcut name of the pre-trained model.') - parser.add_argument('--save_dir', type=str, default='./checkpoints', help='The directory where the checkpoints will be saved.') - parser.add_argument('--output_path', type=str, default='./predict.txt', help='The file path where the infer result will be saved.') - parser.add_argument('--train_data_path', type=str, default='./datasets/train.txt', help='Specify the path to load train data.') - parser.add_argument('--valid_data_path', type=str, default='./datasets/valid.txt', help='Specify the path to load valid data.') - parser.add_argument('--test_data_path', type=str, default='./datasets/test.txt', help='Specify the path to load test data.') - parser.add_argument('--logging_steps', type=int, default=500, help='Log every X updates steps.') - parser.add_argument('--save_steps', type=int, default=8000, help='Save checkpoint every X updates steps.') - parser.add_argument('--seed', type=int, default=2021, help='Random seed for initialization.') - parser.add_argument('--batch_size', type=int, default=8192, required=True, help='Batch size per GPU/CPU for training.') - parser.add_argument('--lr', type=float, default=1e-5, help='The initial learning rate.') - parser.add_argument('--weight_decay', type=float, default=0.01, help='The weight decay for optimizer.') - parser.add_argument('--epochs', type=int, default=10, help='Total number of training epochs to perform.') - parser.add_argument('--warmup_steps', type=int, default=4000, help='The number of warmup steps.') - parser.add_argument('--max_grad_norm', type=float, default=0.1, help='The max value of grad norm.') - parser.add_argument('--sort_pool_size', type=int, default=65536, help='The pool size for sort in build batch data.') - parser.add_argument('--min_dec_len', type=int, default=1, help='The minimum sequence length of generation.') - parser.add_argument('--max_dec_len', type=int, default=64, help='The maximum sequence length of generation.') - parser.add_argument('--num_samples', type=int, default=1, help='The decode numbers in generation.') - parser.add_argument('--decode_strategy', type=str, default='sampling', help='The decode strategy in generation.') - parser.add_argument('--top_k', type=int, default=0, help='The number of highest probability vocabulary tokens to keep for top-k sampling.') - parser.add_argument('--temperature', type=float, default=1.0, help='The value used to module the next token probabilities.') - parser.add_argument('--top_p', type=float, default=1.0, help='The cumulative probability for top-p sampling.') - parser.add_argument('--num_beams', type=int, default=0, help='The number of beams for beam search.') - parser.add_argument('--length_penalty', type=float, default=1.0, help='The exponential penalty to the sequence length for beam search.') - parser.add_argument('--early_stopping', type=eval, default=False, help='Whether to stop the beam search when at least `num_beams` sentences are finished per batch or not.') - parser.add_argument('--device', type=str, default='gpu', help='Device for selecting for the training.') - - args = parser.parse_args() - return args -# yapf: enable - - -def print_args(args): - print("----------- Configuration Arguments -----------") - for arg, value in sorted(vars(args).items()): - print("%s: %s" % (arg, value)) - print("------------------------------------------------") diff --git a/slm/examples/dialogue/lic2021_baseline/data.py b/slm/examples/dialogue/lic2021_baseline/data.py deleted file mode 100644 index 6bc938d1b7..0000000000 --- a/slm/examples/dialogue/lic2021_baseline/data.py +++ /dev/null @@ -1,258 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from glob import glob - -import numpy as np -import paddle.distributed as dist -from paddle.io import IterableDataset - -from paddlenlp.transformers.tokenizer_utils import convert_to_unicode - - -class DialogueDataset(IterableDataset): - def __init__( - self, - filepattern, - batch_size, - pad_token_id, - bos_token_id, - sort_pool_size=2**16, - seed=1, - n_procs=None, - rank=None, - mode="test", - ): - super(DialogueDataset, self).__init__() - - self.file_list = glob(filepattern) - self.sort_pool_size = 0 if mode == "test" else sort_pool_size - self.n_procs = n_procs if n_procs else dist.get_world_size() - self.rank = rank if rank else dist.get_rank() - self.batch_size = batch_size * self.n_procs - self.shuffle = True if mode == "train" else False - self.mode = mode - self.pad_id = pad_token_id - self.bos_id = bos_token_id - self.global_rng = np.random.RandomState(seed) - - assert len(self.file_list) > 0, "There is no files in %s." % filepattern - - def load_file(self, file_path): - with open(file_path, "r", encoding="utf-8") as fin: - for i, line in enumerate(fin): - cols = convert_to_unicode(line).strip().split(";") - cols = list(map(lambda x: list(map(int, x.split(" "))), cols)) - if len(cols) > 3: - cols = cols[:3] - token_ids, type_ids, pos_ids = cols - if self.mode == "test": - tgt_start_idx = len(cols[0]) - else: - tgt_start_idx = token_ids.index(self.bos_id, 1) - sample = [token_ids, type_ids, pos_ids, tgt_start_idx] - yield sample - - def get_sorted_batch(self, pool): - """Generate sorted batches from pool.""" - pool = sorted(pool, key=lambda sample: len(sample[0])) - batches = [] - batch, max_len = [], 0 - for sample in pool: - max_len = max(max_len, len(sample[0])) - if self.mode == "test": - to_append = len(batch) < self.batch_size - else: - to_append = (len(batch) + 1) * max_len <= self.batch_size - if to_append: - batch.append(sample) - else: - batches.append(batch) - batch, max_len = [sample], len(sample[0]) - if len(batch) > 0: - batches.append(batch) - if self.shuffle: - self.global_rng.shuffle(batches) - for batch in batches: - yield batch - - @property - def get_batch(self): - all_files = list(self.file_list) - if self.shuffle: - self.global_rng.shuffle(all_files) - if self.sort_pool_size > 0: - pool = [] - for file_path in all_files: - for sample in self.load_file(file_path): - pool.append(sample) - if len(pool) == self.sort_pool_size: - for batch in self.get_sorted_batch(pool): - yield batch - pool = [] - if len(pool) > 0: - for batch in self.get_sorted_batch(pool): - yield batch - else: - batch, max_len = [], 0 - for file_path in all_files: - for sample in self.load_file(file_path): - max_len = max(max_len, len(sample[0])) - if self.mode == "test": - to_append = len(batch) < self.batch_size - else: - to_append = (len(batch) + 1) * max_len <= self.batch_size - if to_append: - batch.append(sample) - else: - yield batch - batch, max_len = [sample], len(sample[0]) - if len(batch) > 0: - yield batch - - def pad_batch_data(self, batch): - """Pad the instances to the max sequence length in batch.""" - max_len = max(map(len, batch)) - batch_data = np.array([list(data) + [self.pad_id] * (max_len - len(data)) for data in batch], dtype="int64") - return batch_data - - def gen_tgt_label_and_pos(self, batch_token_ids, batch_tgt_start_idx): - max_len = max(map(len, batch_token_ids)) - tgt_label = [] - tgt_pos = [] - for sent_index, sent in enumerate(batch_token_ids): - sent_b_index = batch_tgt_start_idx[sent_index] - tgt_label.extend(sent[sent_b_index + 1 :]) - tgt_pos.extend([sent_index * max_len + i for i in range(sent_b_index, len(sent) - 1)]) - tgt_label = np.array(tgt_label).astype("int64") - tgt_pos = np.array(tgt_pos).astype("int64") - - return tgt_label, tgt_pos - - def gen_self_attn_mask(self, batch_token_ids, batch_tgt_start_idx): - max_len = max(map(len, batch_token_ids)) - input_mask_data = np.zeros((len(batch_token_ids), max_len, max_len)) - for index, mask_data in enumerate(input_mask_data): - start = batch_tgt_start_idx[index] - end = len(batch_token_ids[index]) - mask_data[:end, :start] = 1.0 - # Generate the lower triangular matrix using the slice of matrix - b = np.tril(np.ones([end - start, end - start]), 0) - mask_data[start:end, start:end] = b - return input_mask_data.astype("float32") - - def __iter__(self): - for batch_data in self.get_batch: - # sample [token_ids, type_ids, pos_ids, tgt_start_idx] - # raw_batch [sample0, sample1, ...] - if self.n_procs > 1: - batch_data = batch_data[self.rank :: self.n_procs] - batch_data = zip(*batch_data) - token_ids, type_ids, pos_ids, tgt_start_idx = batch_data - - pad_token_ids = self.pad_batch_data(token_ids) - pad_type_ids = self.pad_batch_data(type_ids) - pad_pos_ids = self.pad_batch_data(pos_ids) - - generation_mask = self.gen_self_attn_mask(token_ids, tgt_start_idx) - - if self.mode == "test": - # [batch_size, 1] - tgt_ids = np.array([[self.bos_id]] * len(token_ids), dtype="int64") - tgt_type = np.ones((len(token_ids), 1), dtype="int64") - tgt_pos = np.array(tgt_start_idx, dtype="int64").reshape(-1, 1) - tgt_generation_mask = generation_mask[:, 0:1, :].astype("float32") - - pad_token_ids = np.concatenate((pad_token_ids, tgt_ids), axis=1) - pad_type_ids = np.concatenate((pad_type_ids, tgt_type), axis=1) - pad_pos_ids = np.concatenate((pad_pos_ids, tgt_pos), axis=1) - generation_mask = np.concatenate((generation_mask, tgt_generation_mask), axis=1) - - append_mask = np.zeros((generation_mask.shape[0], generation_mask.shape[1], 1), dtype="float32") - append_mask[:, -1, :] = 1.0 - generation_mask = np.concatenate((generation_mask, append_mask), axis=2) - generation_mask = (generation_mask - 1.0) * 1e9 - generation_mask = np.expand_dims(generation_mask, axis=1) - yield (pad_token_ids, pad_type_ids, pad_pos_ids, generation_mask) - else: - tgt_label, tgt_pos = self.gen_tgt_label_and_pos(token_ids, tgt_start_idx) - generation_mask = (generation_mask - 1.0) * 1e9 - generation_mask = np.expand_dims(generation_mask, axis=1) - yield (pad_token_ids, pad_type_ids, pad_pos_ids, generation_mask, tgt_label, tgt_pos) - - -def post_process_response(token_ids, tokenizer): - """ - Post-process the decoded sequence. Truncate from the first - and remove the and tokens currently. - """ - eos_pos = len(token_ids) - for i, tok_id in enumerate(token_ids): - if tok_id == tokenizer.sep_token_id: - eos_pos = i - break - token_ids = token_ids[:eos_pos] - tokens = tokenizer.convert_ids_to_tokens(token_ids) - response = tokenizer.merge_subword(tokens) - return token_ids, response - - -def get_in_turn_repetition(pred, is_cn=False): - """Get in-turn repetition.""" - if len(pred) == 0: - return 1.0 - if isinstance(pred[0], str): - pred = [tok.lower() for tok in pred] - if is_cn: - pred = "".join(pred) - tri_grams = set() - for i in range(len(pred) - 2): - tri_gram = tuple(pred[i : i + 3]) - if tri_gram in tri_grams: - return True - tri_grams.add(tri_gram) - return False - - -def select_response(ids, scores, tokenizer, max_dec_len=None, num_samples=1): - ids = ids.numpy().tolist() - scores = scores.numpy() - - if len(ids) != len(scores) or (len(ids) % num_samples) != 0: - raise ValueError("the length of `ids` is {}, but the `num_samples` is {}".format(len(ids), num_samples)) - - group = [] - tmp = [] - for pred, score in zip(ids, scores): - pred_token_ids, pred_tokens = post_process_response(pred, tokenizer) - num_token = len(pred_token_ids) - response = " ".join(pred_tokens) - - in_turn_repetition = get_in_turn_repetition(pred_tokens, True) or get_in_turn_repetition(pred_token_ids) - # not ending - if max_dec_len is not None and num_token >= max_dec_len: - score -= 1e3 - elif in_turn_repetition: - score -= 1e3 - - tmp.append([response, score]) - if len(tmp) == num_samples: - group.append(tmp) - tmp = [] - - results = [] - for preds in group: - preds = sorted(preds, key=lambda x: -x[1]) - results.append(preds[0][0]) - return results diff --git a/slm/examples/dialogue/lic2021_baseline/finetune.py b/slm/examples/dialogue/lic2021_baseline/finetune.py deleted file mode 100644 index 8f74d75ef8..0000000000 --- a/slm/examples/dialogue/lic2021_baseline/finetune.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import math -import os -import time - -import paddle -import paddle.distributed as dist -import paddle.nn as nn -import paddle.nn.functional as F -from args import parse_args, print_args -from data import DialogueDataset -from paddle.io import DataLoader -from paddle.optimizer import AdamW -from paddle.optimizer.lr import NoamDecay - -from paddlenlp.transformers import ( - UnifiedTransformerLMHeadModel, - UnifiedTransformerTokenizer, -) - - -def save_ckpt(model, tokenizer, save_dir, name): - output_dir = os.path.join(save_dir, "model_{}".format(name)) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Need better way to get inner model of DataParallel - model_to_save = model._layers if isinstance(model, paddle.DataParallel) else model - model_to_save.save_pretrained(output_dir) - tokenizer.save_pretrained(output_dir) - - -def main(args): - paddle.set_device(args.device) - paddle.seed(args.seed) - world_size = dist.get_world_size() - if world_size > 1: - dist.init_parallel_env() - - model = UnifiedTransformerLMHeadModel.from_pretrained(args.model_name_or_path) - tokenizer = UnifiedTransformerTokenizer.from_pretrained(args.model_name_or_path) - if world_size > 1: - model = paddle.DataParallel(model) - - train_dataset = DialogueDataset( - args.train_data_path, - args.batch_size, - tokenizer.pad_token_id, - tokenizer.cls_token_id, - args.sort_pool_size, - args.seed, - mode="train", - ) - train_dataloader = DataLoader(train_dataset, return_list=True, batch_size=None) - valid_dataset = DialogueDataset( - args.valid_data_path, - args.batch_size, - tokenizer.pad_token_id, - tokenizer.cls_token_id, - args.sort_pool_size, - mode="valid", - ) - valid_dataloader = DataLoader(valid_dataset, return_list=True, batch_size=None) - - lr_scheduler = NoamDecay(1 / (args.warmup_steps * (args.lr**2)), args.warmup_steps) - # Generate parameter names needed to perform weight decay. - # All bias and LayerNorm parameters are excluded. - decay_params = [p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])] - optimizer = AdamW( - learning_rate=lr_scheduler, - parameters=model.parameters(), - weight_decay=args.weight_decay, - apply_decay_param_fun=lambda x: x in decay_params, - grad_clip=nn.ClipGradByGlobalNorm(args.max_grad_norm), - ) - - step = 0 - total_time = 0.0 - for epoch in range(args.epochs): - print("\nEpoch %d/%d" % (epoch + 1, args.epochs)) - batch_start_time = time.time() - for inputs in train_dataloader: - step += 1 - token_ids, type_ids, pos_ids, generation_mask, tgt_label, tgt_pos = inputs - - logits = model(token_ids, type_ids, pos_ids, generation_mask, tgt_pos) - loss = F.cross_entropy(logits, tgt_label) - loss.backward() - optimizer.step() - lr_scheduler.step() - optimizer.clear_grad() - - total_time += time.time() - batch_start_time - if step % args.logging_steps == 0: - ppl = paddle.exp(loss) - print( - "step %d - loss: %.4f - ppl: %.4f - lr: %.7f - %.3fs/step" - % (step, loss, ppl, optimizer.get_lr(), total_time / args.logging_steps) - ) - total_time = 0.0 - if step % args.save_steps == 0: - evaluation(model, valid_dataloader) - if dist.get_rank() == 0: - save_ckpt(model, tokenizer, args.save_dir, step) - batch_start_time = time.time() - - -@paddle.no_grad() -def evaluation(model, data_loader): - print("\nEval begin...") - model.eval() - total_tokens = 0 - total_loss = 0.0 - start_time = time.time() - step = 0 - for inputs in data_loader: - step += 1 - token_ids, type_ids, pos_ids, generation_mask, tgt_label, tgt_pos = inputs - - logits = model(token_ids, type_ids, pos_ids, generation_mask, tgt_pos) - loss = F.cross_entropy(logits, tgt_label, reduction="sum") - - total_loss += float(loss.numpy()) - total_tokens += tgt_label.shape[0] - - avg_loss = total_loss / total_tokens - ppl = math.exp(avg_loss) - avg_speed = (time.time() - start_time) / step - print("loss: %.4f - ppl: %.4f - %.3fs/step\n" % (avg_loss, ppl, avg_speed)) - model.train() - - -if __name__ == "__main__": - args = parse_args() - print_args(args) - - main(args) diff --git a/slm/examples/dialogue/lic2021_baseline/infer.py b/slm/examples/dialogue/lic2021_baseline/infer.py deleted file mode 100644 index b41cb6fcf2..0000000000 --- a/slm/examples/dialogue/lic2021_baseline/infer.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -import paddle -from args import parse_args, print_args -from data import DialogueDataset, select_response -from paddle.io import DataLoader - -from paddlenlp.transformers import ( - UnifiedTransformerLMHeadModel, - UnifiedTransformerTokenizer, -) - - -def main(args): - paddle.set_device(args.device) - paddle.seed(args.seed) - - model = UnifiedTransformerLMHeadModel.from_pretrained(args.model_name_or_path) - tokenizer = UnifiedTransformerTokenizer.from_pretrained(args.model_name_or_path) - - test_dataset = DialogueDataset( - args.test_data_path, args.batch_size, tokenizer.pad_token_id, tokenizer.cls_token_id, mode="test" - ) - test_dataloader = DataLoader(test_dataset, return_list=True, batch_size=None) - - infer(model, test_dataloader, tokenizer) - - -@paddle.no_grad() -def infer(model, data_loader, tokenizer): - print("\nInfer begin...") - model.eval() - total_time = 0.0 - start_time = time.time() - responses = [] - for step, inputs in enumerate(data_loader, 1): - token_ids, type_ids, pos_ids, generation_mask = inputs - ids, scores = model.generate( - input_ids=token_ids, - token_type_ids=type_ids, - position_ids=pos_ids, - attention_mask=generation_mask, - max_length=args.max_dec_len, - min_length=args.min_dec_len, - decode_strategy=args.decode_strategy, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - num_return_sequences=args.num_samples, - use_fast=False, - ) - - total_time += time.time() - start_time - if step % args.logging_steps == 0: - print("step %d - %.3fs/step" % (step, total_time / args.logging_steps)) - total_time = 0.0 - results = select_response(ids, scores, tokenizer, args.max_dec_len, args.num_samples) - responses.extend(results) - - start_time = time.time() - - with open(args.output_path, "w", encoding="utf-8") as fout: - for response in responses: - fout.write(response + "\n") - print("\nSave inference result into: %s" % args.output_path) - - -if __name__ == "__main__": - args = parse_args() - print_args(args) - main(args) diff --git a/slm/examples/dialogue/plato-2/README.md b/slm/examples/dialogue/plato-2/README.md deleted file mode 100644 index 706f080c4a..0000000000 --- a/slm/examples/dialogue/plato-2/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# PLATO-2 - -## 模型简介 - -构建高质量的开放领域（Open-Domain）的对话机器人，使得它能用自然语言与人自由地交流，这一直是自然语言处理领域终极目标之一。 - -为了能够简易地构建一个高质量的开放域聊天机器人，本项目在 Paddle2.0上实现了 PLATO-2的预测模型，并基于终端实现了简单的人机交互。用户可以通过下载预训练模型快速构建一个开放域聊天机器人。 - -PLATO-2的网络结构见下图： - -

- -PLATO-2的训练过程及其他细节详见 [Knover](https://github.com/PaddlePaddle/Knover) - -## 快速开始 - -### 环境依赖 - -- python 3.7+ -- sentencepiece -- termcolor - -安装方式：`pip install sentencepiece termcolor` - -### 数据准备 - -您可以从以下位置下载预训练模型文件： - -* PLATO-2, 24-layers, 16-heads, 1024-hidden, EN: [预训练模型](https://bj.bcebos.com/paddlenlp/models/transformers/plato2/24L.pdparams) -* PLATO-2, 32-layers, 32-heads, 2048-hidden, EN: [预训练模型](https://bj.bcebos.com/paddlenlp/models/transformers/plato2/32L.pdparams) - -以24层预训练模型为例： - -```shell -wget https://bj.bcebos.com/paddlenlp/models/transformers/plato2/24L.pdparams -``` - -**NOTE:** PLATO-2网络参数量较大，24层网络至少需要显存16G，32层网络至少需要显存22G，用户可选择合适的网络层数及预训练模型。 - -sentencepiece 分词预训练模型和词表文件下载： - -```shell -wget https://bj.bcebos.com/paddlenlp/models/transformers/plato2/data.tar.gz -tar -zxf data.tar.gz -``` - -### 人机交互 - -运行如下命令即可开始与聊天机器人用英语进行简单的对话 - -```shell -export CUDA_VISIBLE_DEVICES=0 -python interaction.py --vocab_path ./data/vocab.txt --spm_model_file ./data/spm.model --num_layers 24 --init_from_ckpt ./24L.pdparams -``` - -以上参数表示： - -* vocab_path：词表文件路径。 -* spm_model_file：sentencepiece 分词预训练模型路径。 -* num_layers：PLATO-2组网层数。 -* init_from_ckpt：PLATO-2预训练模型路径。 - -32层 PLATO-2网络交互示例： - -

- -**NOTE:** 输入"[EXIT]"退出交互程序，输入"[NEXT]"开启下一轮新的对话。 diff --git a/slm/examples/dialogue/plato-2/imgs/case.jpg b/slm/examples/dialogue/plato-2/imgs/case.jpg deleted file mode 100644 index e3378e4164a3d0fe0a43334414bd4b0a13605458..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 101382 zcmb5WbyOT*vo1VnNN^3V0fGj10txQ!?!g%b8!SKq0fM``Gq`&Q?#|!@oxm`-OLBS7 zd5`?=y5Aq)?q0jQ*IwPvGgZ}ls&>`P-=)950CKGI5)l&{`}-4skAW12{2K)cAApRHgo2Osw;w?D%o7#m zUv&SwA)%qaKt@5uc=~mKelB z8e(!g`)+$OV4}g$U_rr-GoJh#q(yn~f-V;1Y_+B(dx8cE99vt1kzISM{w_bR8q2n@ z1Qs;ptCFR;bWH5Mja4;nV*|}(3pWT@+GI0`#$da-q;Y!;5gk)3&zAaBW5QO&g=%ue z=Dv$CS2|{xT4XjwK)Bk<_>52ba&$?7E=LRhCW{ix?;VJQGR09kuYO3v9^=UMz1G(wZEKSvfsEpaiM?Apy`W00t}@JV z5uKQ&BpLfQzD9<=o>3>wpBba;@oJt2$}*Q0v_RC4hBq#WLdZbKLE4X1tHD(OC}EOi zGXFOw@?AjE)X)=!TIweC*7{<8j<2xqW3zX|uJa|>PC{wNXUY*1*6j^VrRLaKnMRCu z!2(O*lTBNT*bOE13v13Wj~e+@ne)qi=vd)qVQ&88lh^>LL6Bp(8v%*Pac*7fCzAYx zg`!B7#|_ZJm()PR1E8S227F5R1VEDfHwOUVxAW(NbSL-2t=~WEa-sXaXTD4R-QBTz z5sT@cED~cDHC3{hTJM!!i=@n$&*&p`rYK5a-FV}W`-_x|gBp;RZB#X8G|Dlbv$Om( zH^qE>82N<>f29#fHj^nuo}$DuoI(sV(_(SUrrGBf;Z8 zd0-RpA@wW|=0ClL{rYUStLPty=oD4zG=2I*-GCc=&R4WtDBf`P0bg+IlUCP*#?QWbecOD^1TSsY`rv+|@4fYAZ}t~rL~OlWwRR%vI6!vzF927@ z(?7l3L}f{59C9^pmas+8nX~3w;W|jD!jCuJ##H({S>P|g&gBz;Ht+e9Abl0#A^P|CWn{OEd%_2HszQYN&%y^ob5UEmXi33#`#rOOnb}}l#3Gz za2&x^An}TM%5gd{apb2<(!u23USQw>Put3aV?oz{kJXC71>)Dz(2E%~OO$P{T zuXU((QCb8B&g5i;A6C0@CLeZ%QH!&w##ag&KsgN+IXPVdI!D@a`dpxu1^%giPKqcI zPgXeEa@5J)fq^lrY}vo`G-lrczY05h9@V)Y=p5S_SS6vezcL{BvM4_3SkX2ymwD$M z3=WKnElvNHTD;nV{wKiC_rmeQJI+r`)ijpTN(9j$G*MG7F>W&&li=f+JSJ=lF-qx+<}Lmkk(&7(QQ>gpi|4xWs|D)3ESO!m}~Z> z|4Nz;C|6B|O88!u^{a?2a5dKHrWx1xNux;%2=9S>fLTlDw2gs1L%d;}d$Wz)%QJ6p zls$Gm;kC=a4DF8j-9;m-5bwx71YI8#yF;qTlUX_$Q0! zvX#3Ct=)!}N44WBE=uP(Z#Xe3rmj(KM?2+1?anvC_R z21DDm>v~7)y;}!JCi|;k>A4<@8zh^Xi7lLQtkKYz29=j_E}b^)d!E|+-vb>Y`HvYV zLbI7J_~uN`t(w&2>-k^ot#JFVdwba`2xh9$Na30qZ0!_{K8pY<2v@pY@RKDOWy_?$ zXN0Zzx!2PS<{M^Q!#H6&Z}hOo3t1{kgAx|yOE&8=|8uSpaC?4}nU9a>Zd{o~OYZkZ z7ho$g$9L~cYZ&o`{sLM@jOZ5fG6b_n55dD(#hD$m$BW4dY`Jz6)%P910T3O`*)x;b zec2V#Jl{%+S@dIuwMd?<*m6JZZ4_7DJoX%|Mj_zS52OOwuqYIzuNIBgk!TKdAO1Bx z000zXIiuhApW4u*wWbgOa9GUgm3%-Tdm`(N|4^=7NPN}|D9MR)Bfq@t?lRPAIe<~f z+0lrp(=WSbzX~Tlm96RllzP!Z-9W#mv%hOZB{KuenvbsNpWq9V?3F@7eNCEfuMA9y z2F%L$Dbl|B56J)NUdZ`W|LGZ9?KBa~Ii~K^)5Oh09H0B>x z&Xko$d`&ghQtM)%_L}iuE(H`wGAmetwfzr`|2TV|?O~;V0Uyp*yG&o4 z2zt{ZEq&233-3>+8e7Rxsz}8{WKYGC)}~IYJ4F1fAQQL>wGbB@)_PcRFlhI za93n(H*m>FBeid1>~Li$DqhVEjkyp;nRV}jb=CQl_k3sJ2 ztp{D8*3TKW${iAfy{H~ z#7@Ws3oyu4PA4>ie)HXzX3uC=%Utyhuy^zZb&R$vb^USHz0ft}U8b&8V3pFC+yrzo zXG)^bKc|P=`(mV|z%PD(MMg(pre3wvBhy4Mr^FxLuVoxywdluh_D-7VPGE3N9LxiL z(ou6EX;RS=r_^vaoZ5j-W?RWB7k5WQ83>3_o0>T@oPAbY-M{hN#BSYQ zJwyyz9o!vv_h6N7o{No^a73gyUxAm2_YbC|FEf1Yy|X-hfuFQbFAgZ$zVjO`7*ytn zczmA-nAHHFVbhvpzcoh!r~{tO4~-C0>YrH8lTzFD7q{!-lgXLB*`3u?vW0}1zE%3` zXZ!2C1yYSK%leN|sJAIq($5)sQ^!G*&Xi$=svrEuq7yk9G-9O<{ShvQQUH6j zAJ4N6ZQgT-R1{wfKtlTmmf-#ETyDXwv*~6w$K79ml(gJ~-Uh04>jX@&sl;Pyy>uUV!ObZ+B;kL{39t9sr^2Pc(ity` zN+LBse&{Q1Pm+meW#Ac4N`(*7mjOlUh;;(>o^jog=! znYs7zU%Skl6EvQcfCoTB0z3x*>UbJx&r121Av{OP8i40r{eM3Zz;o{Fko#gSeqOrs zPs`YaT-*IFbr0E;^`F0hlneOX8Ozeuo0o3d!IA&C>;LoOg`5j<&&pl8E3*W1LoCD@ z^7-!bhq;l%i~aG9y?ugJZ34f{eaL1y-&%S7o+l^7b|U52f}du6u7Pg~SXs$_rPxmL$)B}+F;T52#tBhg=jt|$a}UQ8?D0BQ zQLuNVW2T>{=E8#JV6c&jI@Yzz1f4jI=h&=%L5K zn+(cloCR3s?;a>K=OZDDA5i{);`CuJ@Gl@a)l~h+@eSC>L3!zqb^SD_AAjqz=(vqe zic34JgUSHnkKpu$i#FA>d^2n|u5WRXrQjLaGcB^~!0o?K3i^MeJ*)Tp!G1IL{ymLE zwZzc$*<7X|&SduqC4BC}uPcWowPyz`DiAQ2Elfq7xU3LpbEf!F#IR|wHuDR1qf#bw ztXxjKl>^GpgUaz`frHoYI5!lRPJVl(yvYPj&vbv9|8z9aV_>ggrLKwsW^A)T43lRO?cy`%&eIZ$CcRFOP2sR*1 zx28|{Ju+OgDmJ3~Pf@sD1Iv1(PV;-Tw#C5WPyV7~kAX*z8xg7f_qjs8D$8Utb_O!; z-V?rR^`6@?0bS9nu{k~_Ml->sr4*R4er7GCxzv|cN*_FIfO>34-$97|htvSriB82~SKcA8$H4uXbOson_YZX;-##V94nB#hWiIEyw1wcM zdZl9`SC4;A)TUA&{z!yUF;V$U-FbvM=~GnUI|YC@URJkxf4|yub`m&;Yr8lOsTnkb zd8CCBr7$s3u(%$?>#kY&XW-yBFsUP!xx=Gc=2 z+4GrZgg#BrTCby^VIc=$AP0r~k0L4VyhAUphTFkLw_b>Ysf2ZI<-pIzvC9`XW*Bhp z3ZFhrVNgwA|7S1XluzB6*|YoFs{0yKVBZVycX@$TQ#D-=*VaX7^Nn(?>FG{Y)`iJ% zA+gV@^mu&dk>iSTRq9lU2uNFel0uee)|`9Im}$0lLOOa?>zlY@@tU%wUD1y@*)7JI z|K_uKw4?mz+pmTIn#^~1IgdOyXP1iay}NGq9{himMN7{b)&#sxSK6za6x1Go2Y5}S zcY~ip;hbE4i~UP5wB?-~x6gIKu_!S|wV#qzuA1l)E*g!?waS z{dZd~mzo?!ct)3%@j*2exC4}73#Dvt@r3=5IFXP+sIRpTI%YKp;&fI8Qdf-L$?9LZ zPfS4wZXYC4UGY64(7U{ow||S-uR^Xt)e54szYhrBd;FKSU&s|^p&UGJEJJQL?wbDs zGE&#M_Y?5 zi=svyOg5QF{JdeAmf(Bmj@R0Z1I@zPwynn3vU{px(0+?#&H{75e>M~eiT3o{zW|V( zgw|$FZ-pR(&i=U;;!Z7RGkbreQ!s0YqoUwuv@EkK(m{k52}7zgO&T z&b?Vnnvy9DGguid+2nlE1O|u-Qn8@5DZ2}7l~uXVbRzTP!FGv5Qg3{bzG)@ORx+!`N<=Y%9wwo0O z@n7c3UGy+Fe@JfuH+y!l@@!0;n_u9(hll9$VzJNY%TNCo@;{E~ClUp17XflmW?Z>c ztWJ0ia`dT3&^wSg3z69X$cWQ!wQa#&jhX|hz!v@ z?Nrr98}HqH`$K0!kKUxum5)uX)b31MYPMZELy_YC=vwfAuh;F2e*k6novID$cMH{+ zWhimLyx`dZY_h@rT&g-uaDJupFl{aAYD)9$K+Z+xT~wH_96Ugjcl+4H=1&YarwNl+ z;!dW8^?C;bq}?-D{H=wwLy-fZUHNUs8+*xr5d7VUxeRl!CU{5F5?g~Ft0s&)d3APr z#qYU8Zy7#dB|BGb?MUD|L#s=9&u8mpyd%0k9T#OioJ}vE;^L~sxzhm}S0qy)=&1WX z@?JN}d0Dq2s(;v|&~0?cfGa*PN>^dK@O2CqRB&@M)r>lhGhRIUtvS&@7(CCJT7lnx z^#|q-xnKDCJ?2EnWkNe9ey2UEmG5Rw8J%xtiMmf_nYnM;iCffApSOMk$4>%VW?PcJ zamJS@*JM({WnGEcJHF3NYz~WqM;sWIXFZLdSf*;C4?PF@z5Gsc9{6R#%DeMA-&(u0p5JkQ~vM((kD~7n%qOHi>DWV0cu~U_*L7EoZ79jz>5ZH z3vyfZv$~wbWl32kYs4v|Y+^vG`r7firj9p>%u=1g%^m7GSxajJI_%@*j9v+h4heN2 zEi}tvMdxbyHx3(l=4*UhE6o288rr$H8TnBA=Q%ewoMaTsn;;l-^+6;yw1sScci1L*!d-;+WDj`CE=%k(cN$Z{lkKEJW?3D;%f;hv@v`H%6W%^5 z&AT@9uBL>)fT5PmY8ua#lu_TBoI93Mg@93b7PLmbVbs9gFf7LV_he_;hKXtQn;Q;e zQEl(nGzwqt?i=ofA??i0nOWdVNHb%X?aJQjgwx@{)HGO?FgsxX#npSqZxmy>Ds9f) zxxAZ`H^Sf!Z9XupbFsn80^F%GgjL+7MHUG)pzO;IuVzaHL+8&X_%2hAeP{e)Im-P& z4TsGaYoBI>P9l+Ei+9RS(G!Aj7r21TKnF3|mT+5?oUBoLbDth32*`XNEqocjh5;0Bk$J2QucGD!Usu79D{T&L%fBg{X^f5Sjdmtq2I zccQ*Us3hNy*KqwTuKD6J9bG#%=ks%-QV_%$ui&84Je@}IMq7|*#+BrFu5=f0s)17#5gIwj%M^STY& z=e=AF^Q==wLi%ZzZ5$?#X8fl3eoGS3SC%BkV5>RIfgk?>7Q^|%TaDJYPY%|2N^M1O&*nK!7jvr%>49{ze z=9w}TE4#eTSjfWir?4jEMtk3tkCXz8QdS00*UBJKCY@o`H@y>U3wM*Z$+E(JNm(DI z__O~_Lk!WD+d1!KiBGHv3R-SbUK4LWNW$Vj1XJb?%k_7B8zUY_6`VsRt3I7L^mde- zKp#eVjU#MBdW}jgEUP&kv8_sPlgAj9B%LUUNhTB_s)TOAc<35LOQVoz{_cVZCnAp~ zt$?4p?lCz&5gjXFG2$ziuNx!%htVu6qPL4oZPfHtRz`ZvgcZr{)2&=lSH^{HXHyj# zwA%@4B8RS@KT~gqf!*9{>^W^`V;`K9y$Xb{=2A??%)jy~Yh0&)+=`2XtK)7MHyj_~ zg?$9vS8~1JAfWhld3COa;?A(#`;F&);nx-IHdKbYw0gJ71FOOc8|O1_C5k*HL>G; zS%|ktIMWm1Z)jGlz*VZGwHW{KaiF}c4nM-~oc%Ik_NceSP>Po7OBU|?WB$R`TTF)G zICy_IwPt%nx}VS@{*=&=$hO_RV~pp}PloWz7>0~xR`ye<6aCF#3P=|6d!Cs0CB zJ$%X$Jt1qq=!~p%%wM<)(&gJ9c`JSfO1?HFua4;^~s`5ZEwl?vsGh$v@xDb?NA zW<}qz6#I_rV%Wv#Thx8C-U0s=U7he9f$`6!@ zhavvvJ`|N~B6TWZ%~uGpPeUA@>b|hYK9gs!(89KQZ$(oH>w9eWGof>@$jX7N6}4}s z_X`9~N0sU#LrAY|lCYBq#zB!0L95inn)c02zU^LD{3r^JnCVc+kLW61L53vrc7XMg z!BaxcSQT)L>9e84ghoq-0)WAU3xo?R{#TM{9^>p24ttaqt5dQM1(TBb`B1#7((uDMFq`2NM$itqlB^vML@QSo`TzrQj=1DXT#0HD% zBGPjh;%jRZh6&=TJ4u9M+G01Mw_?ds?S8Tpr(s{~_0Y)RX|eD>bd>DYkwBs)leiJ( zFx*q(#xMEO(_xR`7#|o^`O}*>OX^6cHbuoD=+h{FG;-aiKQ|u`2Je!RF(xP*V}@TT z(kk!NyZ;3oPX@KsE=}czRJ|4aNyEmyFfsmf*^@uMWU1jwm7|`IFb)gu-oDg*`F%pm zWd_A5VvWzf#eng>OhPGV>!7TFsgBjBzW~R|sx#BWXl8B;+d@T=jmejIdWD6$W;1w;=ljdHT#2@DXY(HrPY>OLs%R}pHJ-rEl&J`!= zwrj}Qqf5jkc6omL^BO6yEjC;`9}~Xo-Y_WEyzu{iLp-sXug67vICwa!+DbUm>n-I( z1OD>PMJ~u~5yg4R!`aL5!ZdkE-7+A)edW+0b!lLfQ@=5~20l!DZBRs<%$Yz{9__4Z zs19o5f>*gBrx8BK#-x-gjBl$O@$M*4`}NmiG}@au_t~Hhq~Imyu%kKIl?wxN59c%I z+{jGg-zMtNHAS*<=$a9`PhWUQVHQqhc7knlrLAsz+BW#vXty)$IxNTyiS&>9pNooLw4~of8DiekXBjca6CGmN`h>)|(cBL8k^TilIfg$K{(35Q zwNk4q%-0VOKciHZe%fZ%!k6%YZQc9@bXidX`+@yxmy6sFvQ=6X7Qy%!(m+b!LACJn zAdsffG4_65T5g4&zhCY${MHKpsg686{6Si_fd2gqKE&!e8c2E34=U0k;#!Wc%aRj2 zque2UV%X^qoT-+88_OZcw}l?$$hVnK@I|&IK0zt}0yer=S~NM)QEll^@Py6~+d@0x zJETvATNZS_<(x{(D9Bt> zx`;pVHCVRuV&PFe$*wnt4A>Rc0re`M8f~qpsQO6FRuvmJX1)v&t?{_Y?GvroE54~4 zBK>~W4=m$3#{g6EdD44J{sjmIe@|=|q1)YNcfVy)BHB8 zJg!SP#gRi&w%$AA%Ii)0$RThcdwA&w6Vp?oN{w2j=}^MHMSlSkGeYdp>TTNMx*?I5 zQi^Jf4|29ni-SLpM}=%MP^KdUDX7rhk+bZx2)@_MNAiFG!)ifh_Xesh`qf1`m3r|2 z!UGXUC%wS!1<$KvzeI=h%lm=v-%O$-!qb|3M>nVIc4yuDgYkYDMK{w15x&1y%j73o zj$w9d_8g%lL@H|lRTQb>!ATH0hCd2k{_)#Rk>x3EU(cjaiWSl*GDZ>KXYw|cSGE}R z2)=MQy$%+46?tHG1H5eB%P4dpW8-CuE$2M>aixh!oNBU(W-@YOj6O^T4VE8scrFC{ zC95I|Bcr^{jD;7eT3Uzcd|8iMT4~(lCM0tw0;bWh+dkR$4w)&@{`nMRj=`~9oB9{< zNzcB^`&}ImJ|_DjW2G(vyWY_EOn3KV6aVd1VMCGV=ZPktiXKW3e(M7hrUJX&M6HrC zu9+k)Kkw?Mp15MY>1U+~XD&xaSR`aJ?kD!bhhu`%0uaTsn#4c0`@@M#BL%i@)J4i}G1d*p8A$1)r zD?m0w?x;1|m|UcSCppaSc1 zy}Mz3r9Yyd3MF=ieA|$Feugvoi9Y-68?=^1 zzMnQI^GP8uh`$2TFp(v9=%~7Dq0bqginn@^=i%~~F`N2C4qs8(?FV$mwb0&F=#qQ( z(p@XQHU{cTzQn>CSA*!TR%!{qDIt*|dnsFn2rsA`uv&?(ud=-^$3yRGugF}%K(f}= z2H@bwq$}XKH7=me<{Lx2M?adBSJ#L2e^nNBsj|LsKpmcL9#;5Dv{Y!iNwA0cMiyXh zHOsCk(?-3A$=4Ja0${*j^E&8MdRHn^D(T2#_5M7-a|MeWulLT=m8P&nB&0w$LA}hv z^ROb%Jnn~Kp6`egDOYrRZCUb5QZ(l#%?5ycI8S|ciUnG^IzEhnKw-5~HzAn`MG~(T zpY)eUc6XDlZ4m}3RR2{0GCj(3k;Qc+z@QB23&YO(`DntQ18J`b8@#k(Iqa3guKGF2 zM2e69A?teewxjF8Fz)d1%ZTcFv0Vm+;`FkBcq7^5+8@J~^4Is{l0^wTt&sfCeolG} z)LmPvbj=UkLO9%BStbDF9sHtT=>h_AwpDnsb*;9f0|9algrU0Y)fLa7HhSf(i~@^d zsP8M{c_%LU2xb=<1qMi-k%R8p14hZqywIvB^RFKca@}X&XOjuR-Yma;2PR=KKPOE- ztEh718D$8fbfF;+VX=@UM5*vXv;gZ1^(~A*4xpo#mQ^l)7<20JcE(yRES1^c@Y{L( zZkq`@cnJPZ%_8R{dw|B?FWw^{Oh*iz477iT7kM~7*G?SJ9+lgf6!4wka`7S#acrI* zyy5t|dE{GljqzE0OS7spMf;_u(Ml(?Yqwzk?e?p&P7nMCJ0q{1*L!_GdX4IK03^cp z9ivj|xD{w^8GiZ0yR=&+O>es-yqdY1DRk$IW(MH$~(6Z9E0nefcPDZBsxc z1QIrgm=EQX=B{h)EQrWsfbhgcZKLM;vl;Yu|7Ze#`#45IiG8f(!1OYW(A{+ak;M&$|3t4xFuvIi{t?3I<;g22{4 z)yPc!J0=a(iOt8@bR{ip3hNKfk=&*VYYzoAm^TX`To2!60uj?w*$)%`+J3#>zg{BW z8U}jAHq?AQA~F`1vSHe~(mh~&wiq)T zy_C80QzI+oo{BgIowm^D$V5}Kfq8j9nCC}(BVzWhj9TrkXFGSpBQ!O3zIRAUTkhnI zuZe|$^tnu)H2ZGe_aMS^`CT7;ZA|$mF!U?8i_^h<6a^0VjL)ic$!#~)!6Md z%fNzXA}PS{v$IP??IVLT7Qq`QC)vfanwKMK#U>0xRuq+<$lL8xX_1X#m6UEy7Ap`z zJVoI&(iJv2@1ipb%{mv^#dzltp53-=HV4hj7fg#24jM-1ZT-Y-XskbiV5o~k+=F80 zUD<5>NfbU*&FGu)8+W#`b_(MUtH|r=oGA}$41AN5a1wnyz=}DX1*~9KNukKWA0>7k z-KN2O*ChHeAd5murn$nQn2huX4iZXI4rE%VnSFU2zQ=a@_{lL;rsc!J?@Zjjy|Znf zr<0#mpUTpV${Lc#*i} z=iIc+geu8v%5@hgSM9u#uwy;KgUT9m#62@3Up6Rwgg@m;>TOvQsBeOc;QTx+kj$U_qC)%WvkL zAx#@GQ~x@Ju4VdT8S7ixT;@yTtO%0sPifMP`N8>Xnu-=l+=iuYRr>yl6H)Ec*xZbGclERB?>97s_(5hMl|l?sr@;2j<5{WpSUmy)&jz8K05QTG9#@gb zEJ_kX)y?@{7NKl3Wefb2!gaqar6|mHB?*^YnMJiQF-Ex-t`14FXj|vDYrort=os?j z;lpfcAU=73AjSCnt+vr}XH40bn8vw7J`(@K2|J3#T|QmcWwc6V>8K><{>B0#^OE+; zw6|~F2Ev@yn@$BjWWUNG;M(%-(RI;-f9cb?jtQ0j5R?1SQ0=WPqi!X^VqH~Ob`h9A ztr`d&n>gKMthF(+4V5x)Z!K)Rbz->}f#g!xCr=vc*Xl1bS<_TKzo(^?(3Bhzot&S8 zAEl%$Y^Gii7OfcVTQbj+27W-Y>ZeqYzc`*Zsd;yBj5E4gk!AgYzQl1<%^=tOm$!Oh zkRV2;3*Cz^7haq`&NDmvMzOAq1<|)CXjRe>#`h>a==kfwhSyblr zVO2kURpt%SLxohWJhV8g+l!3KbLj1~z4=Yh9=;mCcC73AxWz!RFcOD%I$3g9Nk}iv zX1ylaQ)0+9*BUB7q=Ii@BrnCJ&yC&wH4CX#@(-Ywmy2w-)LU{XI1wjh{iC$}8Usm- z;gxTj5C=!d^p}^z##d0v3^#lu53NeVNg2sfls^6}(##L;Trwz(_H@gMvV4M-869+e zblAui)vw+R=b_7&hRgo0BuT4{PcGSZ+MAuRUTjyc_grZ;`uxFet;y74-i;uFw#=-y zQc+>QTnPUw`@9{cEUl%t8bQQ#R9{pprPZuQRo!;-LYkR2!Z}IVjqLXvNsFas3{F>7 zIi$hTYNMmk)awnrAIYFyzD@@Pd2aFWJ4wEI@z-z#y#BOpqk0;L7DY)r6`6=8)A_d; zJ5KHntWvlvs~Q6iZsj+K1=|W+CZX(~oy)RZcoRM(qxBcG0*`IubsLr_ybXL62~UNh zo>EGNi@0c=QxqoIIy532&?$iimw;s!C(iE97t8SoV(5aQuI$7|^S8(WvR2yHaHOr^ z$RETl#^;U{MRwW(x0*4O08I+#*z+Pir1P+wdghJd#Q;Coc(f*O0l3^ga=trBb179O zA@x<3mcX{-gD6MP1sq01RhU1NgPgT@yd;2k|I^O$YK; zCLz0d)IxbncD=n0iay5uWuq9V=q`k=x;9sYhB!BB?%Hxr1Yj3^iyl*R3xc~t^jNs1 z231#`*4Q-N8V|d?iGSBm8+x{ApAi{DkaF`HW|fp0Mj4BQa6iFNrfMXpJ#ljWwBfvG zY3iaU~p84-&NS85u6}zP`Kf-j9Hm*{&!{QC~ zwo?evB;Z5p5>dkN+~B3R>nNw!)}HNj$BpBTY64!z4;;b@Bo+Mw+P%Icm#N}Q(O@At zuF$=^nx8Nb&dPWbW}eLXX)W16MzHbRgsow@y^|(uO`&j=j%uZbIqHYNQW;~ssKs_{ zk-X+Nt){EO07cBQPhzZiA(MRVkUQrYJeNcTetqk>K6$4$b@vA4zkuH_ZnAZgeh@+2 z@XUG7lV1O1&TS6;X-5Z7$){yzM(O=bNcoDEklpUrwjy&|7S_R(iH#DuKHB_(WkVwa zn`SUuu>Ox1y_2uDGBNSAKEO%X>v4CL2P9>+!XtX4@TM64Ae(leys7^Z8~J=p7{x&g zu}Dk8gEnUxU-95xG6jr%__JCxVl%AFl_*^2#$j=og;6`iGbyn+8S#8#!`r8M&mQ4pXhUkTSNZiI@VAD=*6S+dqnUSazEaTy}yE5SD$nYphf!)5IJ7fG=%+a~) z)=2aEfwd#|5$x81mjaHy1MRsZoX1DkgN@QHtDfsxfDc!Vh5{R(Y^5s)=oEH&do{om z$xxeS`jDe!G{f(zR`&YzHmV(*Ac2k-E_?ovdj4jXYh)^c^>H)d)^rpnd#Adf+(b(| zQ7Z7AF5iRh<4}NU$(6{)mOuS!B39!|Url3J)$&^V_R1o=U+XU#i>74h439a^=inVw z@8=gX)S48KysRUD5O$CmtlT@QoeOoU5>)@fL!Z4jpoZ}h8R`q;MF=tRB2CIq`k%E4w4o4fWF*p<)+hX)&F2~_CJ)&ew-ITq?7o<8 zCSjeh>nJkmNEb382KG=ADqd4XOUuvDm^|1B-7s*5wqPoh@YZ&-VOM~FkilV2%W)wN zBOyF3A}wO^K(%At(2$!K9|a%(dac+0O$mzlq%9F6I_a@%#&sp1T!Sv64aLAbAXKKx z(}x*{8N8pS9*jYlH6QYNgCNkcfYIbTLn+$(&)c$UBm74rmbGC#>yg3kSiPmq!i%J{ zOxJIeRfP#cVxh}$#-c*ht8^(jq9xtNg_{%yKeWS6+!x}BX?kb8)5h)EQYK&bi)vPhEcR)(S?+2D@%cMs*d`KA8 zS))S_5+Y*f>PP42n^uG;eXUYNMaLvXvh3}p?fpVSEG{noVPU(pw6sj5ll95HJMu={Wb2Mz*N%oPE6)#m^Oq#hjea? zQwS-j-QU*%qxc(v7*3_JXY-Vy+|XSE_wnyo;><$1rrPj~shF@DDOoo5c#;q4qaN^`FR5=`2$4v!tnS7nxmoaK~CrSd|Q3oiFHtG0^=y9p)GL z{J^*wl(BW{hIjd~wR3o@qU8+Z_v(faXgUvdnr@aYW&+@dBR}fYcGW#W_1yK`T(`l( zrH?W#MnUJsD@dH4ywG=`Rbk>2(Tro{`j&NoO7%cXd%s}KtJO;b-yID2;HS_0(WOb#aRfkf6hTf(%5l}r6LE#JuTrOu$ZLa zahH-45`r}PizBjf22=X+6W;0PVlD1_a!9)1it*9*Iw zY&w~pD;|1UR)~cAEFAfJ+P<|qt-D6jC%|I}Pk4AJMs19QKuAc+-6tg!GuE2bT(n+i z&xv}_i&E$BpsjxB{^)n-WU&H2zNd<=MEkT!vv+EY2O=CLZB~EGe>d+Ydoph#qXbD_ zFjgk%`2+2FUl#Dx;lRmjOgwckwcSJ$$XpupP<^ty7!BmN%>(sgBFuU;8lj<$;*b9H zA(p8T{iBYE!;0lkV;D2d(CQL)sio+qx9o+yNqXd019aoa0s~P0M#DR{T_Wc%4qYrCnh|1>*VM8S;*W z!1f}8(+EC&5-17{jLKx~Y4NT)uetC#tf{tR7YTWorCt8NZMKuI9_B?x6ZZ=o;h|# zZ#gXRK%5d(YB1wlPVUb)58=rDDR>S=^f-!V)s+%!u!PdHeP~=vCJRc@2y+r!J}N75 z{3{a4DNN{?L__&{qW}|g=<=!I$N{GpPZ(hwFgFNlQeUOEUS4ey8=2dxh)4x2W112S-A40 z7WIV?w*XAx)lkKDn3K>2ciO@ud0G&Z*)Vt8VJ`Ref9WE)0fY;Kz8EM8xFocfWQB%C*?&Rq7|JE=*y=Z8wvL4y zV_smDYbsG)+#8%kL=t!A^AhV2OVO6L!0zqkRW#FzsWdH+7qS!M9>YsnH_9_A;WJlt zdlb%}Yjzm@=Lus3;S*a$eT!v}28<9^z70;{ zq|xmvhAnmCPM#|5hMEC2a>w#J4Flb{SIGrUgG1jAirF%E!?!i~mWAI9+%FP6Iu$u8 z$VATAo+<<@{sm}1XM?L1;q+)%03nJyRm?3ywCk(s_LpBNl~wa485sdwhL{zijHXO} zKDYZ^!#Dx=o}s#w8J4ZL!0I@rK9NC#>i5__Oj5!-w@Yrt@u_*iQUK-8@5oU|Y@+t^ ze7GUTYCXRiL#&9aktKpp$j2sfz6|F7QcLQ8QakhX-oq5lX<%+|s)!V0j+eOjLV3z%=Ycw`#G1QKBI$D7JcK~FBtshPkXndo5s;oJ+% zqhuGcb2ah7E>D*V&;_yFs?l~%03_){u@+BV0wtygaN1`HM^O;ZS!P~4b-H$r+@p6> zQ-d?Wxl#W)Cyc+X0XSDhaj>qBQ--8g?TU(V?^~&ys5m+*=Nc0R>R8HjhK^R1%8oBs zVKfV~q)u6r+pkcRg@va*|LK6JbCj)lwmnp#Eev1Tws0oa9`0lza*Eqd*icT4Skm4X zkg;TvK_Szyj)BE8+j`(_^6p(Q2B(jyt^Eb0o75_qTt2BDR0;}m{A8>$cJzlZ=tE@H z*T)Md8c1ID;cjyT3K%lWwe+cd7GGTrtVoSInBZAu$;{^FQ&r6f4_XjiZSQ4Z5IVB$ zE4&hh`3vdKk9^Hzu?rQqo>4g*Q0%;_oBiTc9liabr5~9l)dfpS93dC1+b5la;HZkL zrPac8%is;N@=v9T*1L@;Y z60gQS{$hv=*=@_+G*CN%^qd&Ovq6I$AIZY3mQO{#+XMsMWv0mYJtw<2sXfP^e_4Zd zo*vbf`;E7_)hOi!TD9%x>b6A(zv?{O&bgo8;?2D- zcVXu`=UnS3%N}Dcc-J%g{5j*`+Ti(Ampt?Evs^AO2Jl{FFR5o z4w-{SL$sT=7DwFpFrjg-uxd6J6<<9T{%*R*Y?3qBQwW3Ly7kG<2x|YsL#f{CGgz_y{aD zid%_#v3!&leyj=Fy=%*OE11{+0L?aRHu0rT=@WaU=A*V1G%jQ8EwcEgOStPDB2S8% zX|}?sIIMMRuuzC5Iwn6%XRMZzj-_EG*f_^L@#_f){=to4KeTET13rK;tOhvt#-TIc z*(c7pj(OwPLs>%3$WTF@WVEt10Th7AK}{y+8UJ{{Y3~`SyLwNElrSA&f~bB^M{30_NlpP#UOB zJhGUW01&!(j7EftT5&NL=r_gO(;r|IpfPUdYw{O_c^WHEZPuc= zuTQ-G%P@dmKQeWUyk&!>Tbi z(z0*v7K{roTAISB0fJ&Iwz2L+(_U-bpK7(p4zO)o4_?&|d8=D?;&$HDoDVji6>Bxq z9Ktt-kh3+$t6#F#g3J^%v$oTLgyRDWH?7=KUDCd~`2lAV$g_fYsHEwu|;{_*W8tH9s4f>IS2DW74PvkABT3K5t~imbbZ) z+vgv<_ea&iu>60DGDA`$>{=YLXY_)`_7P(6m_#60HB&t zH?YIr{kFZ9W?}Ar+giayn)Lm$y_|Ci;k1(V0?cw9TcVrM!=g$Ok*>T0oNUxpxE|5x zUn;B9)e7J%*IiJ-V~kj=yFr6TD2=A9EW@wbiYcwxfM|BQ!h=o|Dz8N~OpRkXkim?J zMO8zBPCn-HxuDeQ=wlWCY zYYJGdzioO5sz)mhCbcBf=c*)PVhScEIfU)i>%l+=qBh#~r;uN&cB-}5CJUG(ZI$ag zv>>xy(^X6p74TXOuCm+N3-^4z9o?`q1)|#P8>OVaNt=2a*<_68Vv=nTlLm6JNmj7z z8JTqiJfMhYD6F`2Hy?e9s_(?Q~H*UX|4!Zi>bOOs^W1c^+TO9n;?LTI?>Z-yMpO?$AI zSf0Kmdyg`3#F})y(oLAnVhshjGOrlUk-7`4^vGc<-8-C+isyoNg*n0z9^E%^Gc@%q zp;NiUPdx|GqQ3zTR?Jw(h|vLunj@wIcsx)4+B zvAD%xDJ?*vX=hRV#75~(|D znDi8%GasWj`?QryDfF0<^i|->CT4Zue(f^UR2;d=be+6ZK{?FmYCyt$)2MXOFx zt;U94+ovkP3NuAo%#o~bo*QzET$Rpp1UpCyAzg*RZGdERnQCZH4Z-BS4IUJbHxo6u zQ6p2Zs#H=7o{q(C1XvpTm{P8eb8&9<9V%zESyowumK zio+YK`#{w&F>I9WwCq=U<`p5T`%uiJm)|(XSp~xc3stVBNd0 z*~8ei*xMofyg1Sk97IN~Q#&RT7D-WJn=E1Yq762HJce3Qa{ar%u1T}ng>yAsQX#F>FOiYkR}`6BBE}GX1_ttB3}P`H^ZGgc=SgY!veKNT7W+$y z$APr7pGln})IMf0D9{o4BobK+G)TxIecBD$shkWP>UUbg$q6~+_d3AYT%7V7y2bWE z(~h0ECb>);lxlZiP1{O$pjx!l*{O_yC(BeZp_c@v3zjS&%YNsGlk_a+=q)KD$fZ|O z<5+?7KQTWoV#Uo)-K+u6WZw!2VWb2@W^x`~V|K^6kcOombL z+$s!y0i5!8(!TynTb1&n0@x);ex%R2YQNQ`*Y=U8 zGx_9WOF`zjG94vGqc2#H0?y&gbDt*aHLP-hl}*NmsDxTuTNt^?f!oA$GsHySspi(} zg20pHU|qz#>Vtt14`emB>&?1tOiT>e+NR03BZwAlueFg`V)CW5ooisyWFCu|8RBu8 z`#XZlTE>aZZLz`PgPIbgZH@u1qi9M8Yk2gBYWiZUmwk=`4!sIIt>Q>B@!PeyqRN4x z742;-l(8s8Wg^tT6v6YbbueYzTJ2cBFQ9zqk1uY26lNiytYatV02{G$T z#4&&p-NL#aDPurQ_Mkf82}DhD(HyaYxHqj|oq!sNf{tQ&kJB6d`doKym3N%{|&B&1w!N;D*v%f_X3VHxO$o5Tsya~tY1^e}tMrB%9kIC)_;MlRIyVU=qk+}ru~BE5;!$-LdAzl!uy-B3%(CKuHm?0v zlcH3P!?=R)uJNokHGUd>9dz@X3ulrq@gl@k+pu+B)rk#Ibv>{K6`OxdKg~Kz0#_#i z)+?~K{!-`QB?ZqD&V|`DgAy6k0L1f*bKX8jN3MUtEeLEcFMguH6@o|q04MOZkd+aa zO=64nNk;gZ)c=58$>6_n9CELZk1dRAwf4yE~HLMfCMNnS4$! z$$&BQdZLGPQ;jA|kG8JV@4sy1PgeVwq(LQ;T-mOQwqO`-xlyPZYWEU0(J-W_VM!MB zgL6Hw%`ts+Ev$jlHa`=#qB(F*Cf%nJ23OHVliIihlHwNlrrdiaTSDM2UyxC$Hjx%k zVKJ7Lk@*Xj323}hbsb_BwwP_IBf3IMF-FFKt}+s%3l7hvoqy(3S!Y+Rb_KL6F(tMi z4bFmW`6`yli}WRLs4GgD)8ZJ7hd?$RWj0Flxsw)LtimnDur1G9gf%wJOXpV+jf(=s zyukc~5%z;j_UI_v>p^pt@sT-W(nK`tGSThvVjGC9r?k@T?Stmf_-3RM+I~yxLweXF zD0j=y9JHfwqIIg{T3IXVaMCV2B^x+k`?4xEF2`R?_Q4YNu<|ytuFz(C62VnOdd+X# zhEH#>BN-KCG7Ipa;MUZJcMej(*4s#v=8*QoHuaa2OAvUsXraw?xaFYwNwMZCStc%v z6E?{`M-lwl{pVG>aG6y5dH8}v4oV~4p>Puwy zxtuU(GN!OxIW|fw+aq{;%0&%t;fR|hss8|8I)nUK4`;!iAI<}wjLBlX-JO@3uZ5cc zjl~?>$6KALMUH%|7ptz~@d4eMihZG)6v^fQ?bhjoP%AzaV)F)_T-hSuF|FF`GHgQ1 zFGe&O-7t~1%PR>}<|6sNg;-$Ue!-Te9Vp9l@Auzo%}=Hzm^P-H+S#zH0VBq-;SP!z}v zloW``YH9&l?i`0{ z-~>W73J~TX1au(_j6`5#P_T%YfEbP=o_O^Yh<;EJ*I4J4Bc8Ar(x1I=aEFf#6{BLIM9(xjwc*+8;g!X7>AYfNA*pdRnXnht4)&NInm9(oWKiL!|&LF zxhjOf7UK5z%3(k#cX|}h%kbQa#9OBeU|9irsvUA`bnj ztRaa2kZ$%HT~YwP-IaoIT0C&lTSs-NX)%9ZJG*)c+qv00Ff;p&m zjG|mB>$zjZs@HLrg^eBDP2{d>?rwddPt%=j=U{=v)`10W*^Rh`Pg#blZw|8MeJ`S| z(#+syDe9{>=XACt)l5aZu63A5B!#<^R&P!r->bW;m1U|=URS#Y*5995#^Jg)wS!Xn z;#Hqe_Hmx|n4k-VlI~it8-Q0&R<`R?a*AA00$1(LX;6U#61Q6x6HlyF4BM?8v=JN( z9`l#r@Z&$3zk&=^-g}$V*ar%ewYoJke;5hXVzrrfit)c|!l-S5%2|5FV|v@tXGzxG zfwUFWQ!q0z9L;Ufzf;J86Gi(O)9r-FLj;J13M90DnskK7C#?7=3hQ6x9W4t~@*~8; zKu*e+O zy1MuQsE|aa7adDjrD6dXi;xxr1CBZv6X0mR!x*{an+VKlbG zL$fm+QAst6C5AG(X;X=y7?{#Xyqy8V6}Hz)djWhH(nt!11Bgb-)fjzfh|CL({}pr$2Iez7y*G8;9pHP zqCL&`_?p7!^c8Q9*%>t|k{zDN!s}5CgZ@6!k!T zGOCl7%!E#cf&oUj4{Sy;5g%e0F_9?bRkqel%!*@_?e*)N8rO*pB!h{f92&OT$Y7W% zAhZ+^w|0v7IHZUX96b)2Som~@)mlMBWHH2zDJ%X{WM*}1Xk+X>g3*m0*oRC@M>Dmw z8(wAxF$iBuKqRE~VuCHtEe!M(6a)nW>Xe0b3f4G->MGVfftHs=3hTjeI!XZ%FtNrM zkaii5Iw~kJV3Oivz^~9r>W;YEYX}%Zqa7N>p&)StE8FX+3`J4E1S}v%I2nwhVtrv0 zQC#u`@JQS1);yG`5(?{6SiuN`*Z;%-Ll6J~0|EmE0tf>J1O@{E000310ucieAu&M^ z6CzPzaWa7fAR|Ijk)g2^P+~AMKyty+LtuiEbF#wW@bM&5W77ZH00;pA01ZC_3MO2J zc& zVo6lQ_fd8T3GSVO)H`8Z_URg~YnmH+Z0Lq~qn6G~lH|FrTRARE?3{V7zrqlUB5npy zrw6*mDgv3U!V_`Xc1XOC`+}IVebfcBa6zOMep>(_iG=pXC&GY*Y+`ajg}K5%s9}l* zZ6$JAAo(qklHOhf=AH2ne#?^6h$6t%Y+pI6Eyck#u&p5C6Mgy3IL-2$Bg7M5cJgyu z>%5dq&RYu=^4xO}tOLdZW-aPacyolKv96dv5NXI$Ff>5U4uZ5(h>)D{a|yx_5uD2E zh}43JTbn1U47bD~XP!r9KGIVkcJgyu+mB^-^UW7saBxo|5soXc`6HANK53wWB{jj* za86`FcZD{Uv(*g2Kf(1xNcCcF=e4HNdhG4{HaxKRT#+6ZW<5t_b{3a;f8p9Rdt!Tf z+~d{j`HhIuJ}nbymgTAr{{W?9bOZI~i8mgLcl}j^A9rT?$8tO;kFvkm)Dr+{jzLL~ z2a4c;gc0hVQZWt{lC>kV*)+Xf*=$FcXqfny=78h z+{gHz-3kqRTHG#lWB&l9_+a>E7rXxeG*%&1puxetQvUEOLKQfNFsbgEY3Lntrq+<`E)ut4z>0y#=LileMEDb0P6_^w z=R@8dkCNOtzLt%Ff~wQLrhUvliN)fu)o(Cko#Vq@g004<{{Wg5>>i7yy}zmzz+luh zH`251Cbq+V?k9=Q`@r z8hQy=7;rNoQhd~JHtMU^@%#+YQ?9_;=TWMr=5Eg{LOEZUC=3spZ7}Clc1>~>>OX@M z9M|EUP+aih50b6u=ApkJQE4*Qhco`$8s^+A9C#{~t*hwzHM-!b`a_#$mh#xP$eK3cCzVBCI}3ptK(}(gY-=h7hhA9Z0x<}vkSYfU$E@DdaupzKSj-U^?Q2ywTH#sWebw+vHGy= z?7M8~>wQ-=dFs0+_!@jH?fgBOAxNC&QKfnArVq+f^|lp4MhLN6J$4os#pWjj7}IT< zNmPM|%%L21UGwF#l`#G>Jkx15ql?7Msn{II3-Savv z-PF6~A7sV{IJTAYf1)BwPKpbN6z?#cB|dn|3**Qt&T!&vK#5Or0o2)c9g(SEnnqpO zs++BGwh8{qb8Br+dug`pv0NNsxDO--Ktpp(r@U>W=!Grlut$>Jmxr1~f_W;84u4~RsGhCMhW#vWiT=Dos`cL`nsNh;x+sCD?pT&*-Cy5vOn(&Df{h;P(mP2-bBMA9oExWqS*#fq7^G1oM5I~$Xi#j>xDs)&-9 zWxWv93<7RahCn7y4OW=?CK6hH8RWSYGjCMv*`k?*?7>dV4O9F>ntN?MLbl%|;RX{# z0kfD?yd4zH!(*Tm*-h7Dnt8e1N}1+`N@s<#44s3!tc3Eb*9rJ$q$LvkD8=J z*z1~Q7qyw?D#GiBc!bJZ=78x0E!OyPP7Cnvscq7pCAPj#nyEZN(y6{6JX*{(=BvE3 zJeK14an|jEux9q`PCSrwe)o^ODo+dzpB&}2jty_>;g$GdS9V_ksqDYn)wHVa9w%aY zEkH82ci-liOKuN?15lez@LGByO8c5|m_>k+t^wT>Y6>jJ#`Q|x5a|RSs+(HtxylW| zg}UB@bjbebw((=`@+dT_)a9?c6AANem2SjyRf!I7!5N^yZMs^WS95*Zt@h&@oKxFi zWW_pwU5;zf%v62Y^@PTU=b=YJsZuU;%c9&9_jf!$5?1edH$tsXd5fYIU`$kfp|RCh zC9!ryCb|fxpOWjJ?cy_0Zx!Dn?s* zA0*dY!*(p}xzsYIL29cP8isO5R_3a-G`4b%N~1Sr?{tvfi*>n7Y%kt!!?+KX_dQdpmm0FzkCg1P z&XB4ZeKxy=$>21NFxgSIyi;^t-BJKFG>tXMZue4k3X=^nJe3)?qg};{ph*SCIp(?3 zFn@rF;#L-)-oZ(dNL4Ah!K`Gsng0OVso`ZdOOxA-ch;2sNQ$(gV^Sx!fJi=(Z+cMla zsJL$#7Y<=OOI-FqcjwV>5J8q8%LWSR`a*9^)gwL7WJwAl-GSE+XU06$S6t>>#n%+; z^!S1xPW6+Jj1W~BW2Lk5OlI!aiH@VHrAu|)HKn{wl}>+n*#=c9g4Xck(NZ&Ym>(o6 z8j|MVXQG+*Vgi}ydK79lrwd?Up5{HKJSD26WFixUN_{=o17gsg%O;y3t8E30f|%Q7 z6TsN;{(BP*QFdr-i4cZkWW}&YEL=2yikVr4TI8OaVvYETk!(F{x!F~Fw+OM%N{hwiRl45DBOSC$4{n)qcOc2zSD z=#GT#XNYn|mx{M_PQ+uHsBn&4(a5f=cvuYPxrFl#omD>`IVp*qQ#?bG8kELgy_7Ts zHMelO;UAIP3d?v*%BCX?Q8O}wBM#`q9dO*#BWl*MB!0h&VJBzMqlI2dN#NYM^)-^};Ux?otvyOQu*51zYK8m$V?s3JcyMwNC zu?oCTw6+la<=O1vc*i%54@BQZ-G8Yvm97ES8;c7Cgp!rdE6Go)apq zLsHgc5I0lm;J4q1dtH${l~*ysm^g6tRbm4SqY*vTT0xBsGZ0fL*c!t!oZ?Nm&*XWc zC)J4tcK-k&kJ$6s-E;Lh{{WE-GDeXcM^xHHxQ$SE{VN-|VO(!3cB~w8hQ_qA@lS_l z7QM8tpY=`%)pk6^n0-}(Wm&@UO8kC&D$z2xUIsSpRXT6KP%?m1T1M@n_;m7DJ;H;9 z@ZBUP);rNo1Ya@~akhe5HmZ#5*zrI^R)g=h>laJTs*>Z#Qh0C4##?+>R^d<^%# z-(8d%!p5=iCKb)zvi=y3NYc5Er0Z@y*6LdQ?(R=``YYKk=Z?>CG4XaO*c2$#f3tK+ z5ecuryvfNq@hp9xGMe(%m%8ma>Y5wdZ?`c;?He48Vls-0+Sw}$BMho!lX zt1{-XQ7wHZmmV*Pv#3uoNt#DBtHFp@yAe>|6NV@bI!K=Or zU>{K7E;$ZE*6O*q`ud}hZeF8=6wwnipR6&OW*1#o=rEcf56SF8 zk%%&lE01+BaOo-k0Hzf}L_*#F099<;uF9U*u+^%Q>=eT=K5~H>Ht(M#avb&-*m$E2 zf~z6NJk`Qj=7(L145|GwRV43Nz+uf5@N7=u(0pz#BkgphJIV;Duw0u05 z0KjnKc`8AruC_Bm14*Z3mDBu#{HS2NZ0wttMGyomr({I0h=uWyilvKL8u!_hxNIK~ zOc2mUNrD@OOFLr(G-^5(@J0*Zj1l)^1WdpQ@h^##TWm6&foB8!gKWyO?k!Wt4k}74 zpQ;%tv{yFC+*E;%>4KLL=cKEm!?l}g5y9DGr#728F?95K~Udt07zgjz$U zS1E*8P4z19oOx)eQ{D>+hbGbAIQPgZtS@D*oU|&VX5j3TY{L0&Nwo+#_GKeE4f)s`Q7JhTN~Wjrey6nt<_P!X+Iq(%X<@UO4^N1%QV;uQft2 zYp}&rjyg`b5{?)ub>7juc7v*ABUBa>7-*_e-nQ3CPGP8Re3tiKlXU)=Pd0;1#KL>a zV*WOPsR+K!ZtkQS*E`wU{AO4j)JGYd5Wrk!5z=r`XvI4{h1V7}=E8Zxy8Ye!7juN? zURi$uJmCzYtXtVr@Z++egNk)AM?ZBno4yy3b9og@z~;IfJkZs~NnZ;e0QW=R-y6PY zC+7Db4^>JmEigw^U48d)-Og1)9O-w%MrruvI)k%G&{``>h;;Xn~F>=XF`T zx^a@w{sMa`o!;=-^4Qmx+z&kbm;7A1c2!@zb~>kIF)#!xZt+UbWJtjb#=`Q@z@{UwDNW7Lc2<};-IC?Yza%up z5hF8)l6D}X%dW;{KNgS)l=j@%)GMu+IFxcBRPgmw-DA7*PIZGn7bQ;z4oDH=?y2~W zP?_SKBj%}zb_W24IBewvY<5Ew>;T0~NBz{>0i@!tvN6p}Mn81i2!e)q6yYI6#wZJo z&w_uII8Hy9+X~BgHB`h4bwteSgCQKb>>icAJ&~)uzuwMaI-!RCl4R_nOkC{cdQ@Cr zn{fK7v;`)7m*Lq>!x6mck}VtuMY$~VOkr!<9$SxfK$tpXn)GoIacudixZ@}|I*0;g z%9Ho5zb826omsW5_i)ixAn<8|FmdjxwXFe|feNE6o#A?Y9n((qxiW_9%|-{3YfDcV za)m^fi19gZ9Q8)7@H}{m{6TP`h$F5_wQIsIxx8d~LUF8r-IBVg{{V9!C(~;lH-)(m zs{A;k)lwwE(+xi|iNMMt?^Yxk+Y`|imj~!${{U;cJjy)EIP4T;&~C zIbAk6!GpO?ZN+puS|r+00NbcTJW)+H?B=;n!-n_WpUsOhMwt>$skDm_cl{_##318( z-K#*pc5kM}yN08yakWY+Ov$uf(WY8c`qDr1hMjx0W^fj2yn z$frK$RcH_dY^MGG9L|e-f8yk2KR%G3TvXmjh>%l6jxHIVUz=x?dV`+BnT>!<5$X{k zyE2pN3OIzu{<}n!;Ui;BrZg&_i*9hy9<0V&18yUy@I&1q59AgVyNR|fD&K1| zq~VTBcl}fh^4Y(Kg6EePidC| z?-!QFv<4mKT+ArNvoN01O5fG)pu>BdUyJ(Moj%fPHN7$uDAjWvj2K(VKN5Yo6EVz%qg#1{{-CBI;tFWEM?@&TpabDP6#oE3 zE^}sNs;qE$z31jmy^yN=T5jj}KG6u*4r~MA z!xYCh%T8EqxuQdlHE8@c>Ty0=%6FqT!8}j$R`E(LBsjU@v&)B723q^!{%uj=sAPoW z;<>@2y||{m&BvFT-*Gv-p&v?a>BeT%>VVV}C9T)0xtp@-=f&0*7Sj)NiwBx>F=hDA zIY)L|VSmM?%4)O3OaSZ`w`)V4ISv)NJJx$bXRL^oZxAN}sS{0&h_rq|-_$(L!Kbc$ z*~e>qAe;x6?1q!-G{d7qwB^+|65d-6jF;T+S357^p4)kP)ZKeyp4pH=#1!n__gwGE z>_U;Y)wwP`)zC$wl&Aba1yGS}E#LK4_#+Xs@ei@h@lDE`?F@!md(JAM8ml=D!Zwr-U4F=A79mg6;;i{{YOY({r9f0m)06OhmzjQt7D^ zmYAbh)8E)Qgf0H)R%=_7e7YxB6vN)uoDVQCRNNuZ2AvUeTY1T{b6(B6*8#)w!`Vfl5lA#U!68I@3Q!nmTzG2IM~&Hm)KOf~r- zQ+JWOza>VcryNRcMzWiJE3*E(D1csSb>zJrvir_`lPNZ1+ITbqW9xO;X z>9VmgenJ&rNPA_+geotv+mTgXDYkM!>>AsWq3=NgAgcimGxbbpbXoO5C@PgUxEFO!ZU-B@(*&*E;k3>wts_yE9I(}TO*c8+Fif<>lB!gewWl9F zj`WjHJ}gQVH*smZzb%ZfjmIA)NPt8ccVuao2D*N|Q<~x$XX=>CogkU$AylKi_L?J` zN!R#HxKu_aS9E%)feG2{A}&*Q4lzBILvxcGZ7|Qd*_4M6vM@-V>Mh|UVaYT=0ptLf zWJ%e`IEB>8ZXkmME3d)F0;YSZl=2DUIfYZlB$-alrNpNQB)FMGK?&`&%gMo0WsS!~ z?D&j)lYAhRSmV3MDwsLk_H*7j4MT|8GZz^7Y^)*p=7Vj)#7OFy;k3jdg3E=(CScnQ zu!cY;WJ8FA!!5Lsero;ah|H>)-b^M4xT30fq>@jXc!x=uM@xwarwd3fBtnaLi8yji zkPXKL%+w_?%Xp3(ChS~ddn>NN#wWJIZahXOvN_t~C%SG$t{|p(NO1%xiED^KwgWmQ zy5ZmWOJ|D}VR4ZO%)Jv5QR)hQM54)pP!o2WIO7a3bS71UP((0bzAMMsSw10-IqYk1 zHk}jc??t#^i*GFO+N!j;Hf?}2Rp!J&ykq6C)(qa^1zl$QPFZZG;M;yF$x_0j+oGb|`{6qsR58rZ=8MN09SG8Kux95- zgz+{SfJd25Prre>;ERhlZg3cms{P%!3=e#lR}2v92_03?$GNuLImhJ@2sYJigR+_W zLT*?ENXS94CsagRrpMIYPcDkHuRSvU%BCb*KIx*_02v;89Zjv^1B*1;TcF;OqS9Ic zk>;teju6+Go=#MQgft9_W?UiPLW62Hf0;Udq7_(^XgM|&Cx?S*8EC4#rOm@m`K&As zV_JInSHuLuf9V1hYJ0M54UdxCnqj#oFp8JcV@c%}rZCfJFfj5|+CxRHZ`pr}UdBXp zBNW`2X}A7pos53*w6K6x*X}f&^UgL?5ykJF3agDuZH_POs6vkfbgkjAF-NeE$=BoI17MC{m7TwuD^qB1_#v1EQu00B<{o8QWHZk3w zuPv0_W3a|kaB;N5dzC^j!zo|=PQmW&`J_f6S!V*KFtrmsP-G*Jjf2u0_=9=HRja&( zQm*jf$Ju&A?sj(s=(6(e4=k6YhPBf&ATLKD&Aw!!+bfVbZSS${hK9CfGL^2TT;M03 z3V|iB(;U?y1%!|v@>dffQ8F*cr`Hvoh8Sp$YD0nQwB5`tTm&bGlAl0Q^INUkYz8}& z8cF_hnffTtl4Qx366x-}9FhYjN}%hT1Bj|s_rn8mRbt@jpDm3dqvS-?881k6&XL49 zuSgA>!I27^V;uayG>L{{i*B73d9H;G0bGGmAcDDU<^YcQ>}#zTJH(xkbFnGDEOJlq z%d_pU{7!^SIoHL%-F~AB<1WE`Mc6Nn>^^IFX|&zr?1p_1)U2)!!3@hxsfi8<=$n~q zrVz|E$+?EQVLE2lcw=C7v2e$-GwkliG|`C#DB7PO;=U{G=x^GS*v%%?8FQ*pY0{o`^NKLxvEWadTgdr$oZySlHLP4=!_E zZNs*Sm0aUWMl58J#*APb*8y@?cS+T={ZF}Bzj8lb|$oZpE=RMFt>NTB@ zs2{&4A%+XmAcx(S5UMl|3vl_X19y%fk1&6tYfE0tI&)1S^CO3%G_2i{I0~l6bqyq@ z&jj`SyF9jx5a+X+w*0j9NRT!)w;x%eN+M9iNXu;~yU%1@Asv}duE&-h<7Z#nQZVhG zs6sgu(IpW(8eov#5PSYZVOzMDh=QZqT%^VZA#TUrK+i3Xrw7@(FS}>7I_G{0rAE1> ztt3VZhAEc2=;Y=);=oTAmci)!HqLACTA7t8%-9-I1dT0P2gzNNMwvR^LNss%go{3x5WV(!dlii z5Zz*NPH?k$A!-^4KSU(^Kt{M?faATSBEtY7=)^eHkuTFVyNZIx411k>E%WTX#t&2{ z@dl*AYtM-3$!`t(4#aoL5Nz)~)BJB!>K?lWn|WQe{vN)g{2=T5L5V)IaU*74slwua zG_h`sS{CG&cUMGtN`J&h{smBwLfxx>cNuK7%*5S-a~edv3$ZwCG95|J5HL+_*-h;Z z`5zX+L^}cmnJBrA;=t#cdkgPnRu{UoqF zd@!9-H;y5$YkQm!bTsc8^BqV*v%#t^V0BkGJ9ITo0N!zf%2Qh*Zb>+f92Vfz#0(Ea z>P6GNYlt#AWPxMb7z` zDm25+_GIBIqeG)(pOFWaxUI{ixEp>-q|0n`^5g=pTpe`4g-)~Otkwd2O@NTm^Th&W zv=)K$M^L|5kc{9bn!Hn*gPBfaoGhIk35+gz`{ucT$6}t#rI~`G zB+6!{U}_g<1P-ORN&G0SdPdXWMV%XJuUVEoz zIjNYHSI5;q2VQxi$uvdBRoykVgV79vZc3RhZ$gQ?%`n_Ms*b(g)3GC(e~Yfem?`2N)FFTfsEIh? zvZsji+5f}dA#y zamzV0tPW-TbGL)&HDFc0%2f2rcjyq@YSStdx;rLY4KRc7ojFSE?zhcoHv()K)lM40I{NgOzt%P#__ozrt4dbb%>e4v?TD-jmq5S!jEBW&W z1$wq;oE4NvX8Y!6!c{LYlJ1i&3-Vy|itj&^DpLx>9m-thtvW}%yhH*vJpTaF%xVG! zd*MICrFRR1s*3}iV{p)NGMB(#+%h9Qv+0Q2^3J9I09sZ(t~%dD{JWRXyOQ$?rgXkt zOk&-Zboy%CguU}Ct{A8K_KNF7m_~^44}7k>pyIc404B@ixtLN#3$HN&_h{GD00**N z!lIk-mC&&eDf~-h!Kj9xo`wC)kbC52C48s9w53}_6#8an3I(zHKXZfPUg45e3#srB zk8;GnUBa}jAYrU~2;{5_~6ENq{8f zul{idLk0}MG%8TL&!O`z%~{pLfV>#5o7BLOE(N%cO}z-d@iO;c56j%E&^dOzI)mN? zi({La0KHx@D3w}0%u97M9Ag*;WiR3yw$LW7xQ7vHI)PbxQTv;;GbrLp{d!>HI8J=l z?i(|pzteLD!8M$vPLDh3UiptbHX!#B;O1~0RL{F23p}j-OSyrHeX|dMtA{n5&Vleg zB?m^V%lY{9?f|ZAoE857(pI|600_J0cZuL+c$Vr|;g&d*oJ%uM#}8CC?v{f|_Xd|u zcN8f&yLuXl_=C`2Vhzx~oH%0!A$3~)r8&l>R0l%i=H}*b+&D-srdZ~(pQ&OEB1dr^ z^Fs7o#=^QbM_^BQKLI$!5PwNmFR5;&C%kkX`+qaH08qlE006W=OTX#zSX5#gcsP7d$U*T%cP#Pe&=hK9 zDDcb9=0;DaK7q;aE{RW|LVG{BM`$1q_WOiW=6lt6%lm|zTA!V=D@?%WBWVwnGiMX1 z{wjU3+Hxt+Mohr9>}iU4hR+j~ZV)%MMO&PHlmdy0U3@9Wk?DxvYX!75=j< zZq&4Jsm^(<#G~v#>z4*tzHTKs9@d^07rk~tD4?go4|6clU^r`S=gh$+@ULTSyv(kc z{{W<_e8!1-@j~9$-}k<5rsKCQ2j*Y>>F77j&V2+?`}FKtT7BamC3@ej`U*4)Ul7HP z^pbS>B0Y}t3_|Y{I%ToR2vwA|a6<4KRuWt#JB6x&yk>ctK|WtV1UrVwxKy&WVSCG_ z#vCClUVHQCB7jE0*Yu*LOCC>DQZO3_Poq-7crD-jg?u~!Mwh--m)G>}nx;?a;2g5g zrstyYLno&obl3f2+YBS1#ZTt8O{sp>0pCuED&<7oe%W_+#+!u>geK z*M4BDRio1}seeR>^SO(H`k1g+OA*SclK3sH)Otm)$Nqf$%Ai!Nv4imHx|~G|ZcDef z@J#rhhT5@5@dGUmSDG}bU4J2_PKv|^=vW(mCRl@<2Z3JF=hT$G4mF%hYNC~R0MW#z zUIWt?F!t4W^A(>@Jj>k7{w0iF0NvqkGY*@q_7B9+eHD0p2cJueRJPr4Y=88=^6@HL z)p|UO?II8KqgNI%vik~D*7@AeaA!|}`j@>h3kCjWo_80>%02m%pTJ&?rr1>o4L&}b zgrvPcTdVGDgj6o8$afQsFVSqT5{U$$I~wD}bfF{05}RDuP03gjEe!IE)7RCVjMmGT}+70<`4Z#Rr@sIkw;*S|mV-Okv>_80x?I^pV z`-IsnDM9t2VAQ)sQHlqoN=w0n$Zm zZp_dA$S22p|9oJRnhw9YhL1c_P^)@s*Dbrk5B zfkhZRNAp0rRbZ`SQ38-1CBOv$9w1)7iuHsD2ntsfKTsH$YZJviA8w?b_bZa6RTJ=J zAbK7#6vi0VC9YF+uRsgHUi`qQ@+&X{KzkmFTN?vW`hd*Bz;IIGRjl@i+w|EoZj~tA z7HR(gtNNeRA(mNjZeb@Q7Mqn0ZWRb_{9>f3p(q#AxvE-5O6FUID$%^e;vfOv&!cVG zK=t$-FOYI?g@ftXRiQJ&!~d9Yxq6CxbfomIyM>;cPp=PPegqs$avk*!FOZ zw7rw$nabL=IiV=?0cH3jQ?pS*M zv4pveTi9;?H8Eqb-c{xkjQtd@epvhvIYo}VBg!lLq1WZc;O~^_3w*9x&AA}=VVk|z zSaBVD%r{lj-gC8$cmDvCr-i83F!{*#nYxIeP(4m2gWCpP$j3l+O3EnZoLix_MhU=4 zUZWgCKDBk74?gmvEE|l=$k$c=bEq#^Xj$4<7AuJGf6QguwxoL^nOmjbdi&f;*uj5p z5LXnpD{k3_Sl1yd;HXxY2h0!vy92L7L%dBml(4hTV&#T~dg@u{g!qBDM=v#Bs0_PF z2?7e?c+tA9yuckcvzmHYblbK4&9i2)KD_`2RxjUvi^*=EgYv-%-XQ92B2Vemu9kHJ z0RTR3W9o2`rjKnzHNv=FCSR4;h$>H7Ni0z9%`doGc`4i6ppj5wnv_U;NId5=B?Y&V z+Wo{TOUQt|D;<)+(4()H!8q2>v?RqtzWdd{0JnqwQ%e8_N5@c7Uosi|zMkpH$Qa?f zocHhN(_bKaYxO>>ILnR*c}H+yEI(u?X@Ss0g?!kF>Aw&^W zC?eDvo?pRi+I<%1R-I?lDK;M@>=MV(jC<@A@%x!g8Q@F&ur!)2FP2~W`7k(v%d_JxlMEEaM8;yokTJI6>0Mu6SJHs2(7Sp7Ya+U^PdEzv4swUsYxcUb=RuMwks{> zT(>qh6>p>bZ&6%5l+(pRHK;1vHjXd_y7W=Th|-*5EQ(e6S-^B5-1XEpjuzBn#yQsf zdT9ID_ZZVM@tN0k+YU+viweE zOgWY(*5@ZOf={90)4`0zF}W}0Mg;IbkwJk0bcH_=aRZuUtB&A0u1#mm%RJC)W!4zV z$YR>{dS?@2=MG$MAT9N!Tpk!01EwK6=(4Un{5;qL_)QMRbl!gxI}px-;ppjHzNH%J zD1NL441b*J%HYWRtB*x4b3;F-X03_?D*M65mIK-jSbh8%VCRf`#*HUHK49vtTKg)# zNpl6hG7_*$L2tCdFklo?qhZS#JnFq%Tnz0U*yF%=Gr&xe)s&uFkq0Ahv}Rl_Dy=Qw zQETG&$DW2d%MhGmGeK3|NUCvi_v5aCbyE_qqjmoPS^UMgjxn!#Ga9d6Pr??s#gEC3 zy$vl_9*pl^H!kl{IB;`QIM1bTpzp-W3a^lz?xe|8+sstrV$3v^l2G-lD*>75&}&@W zE(dfh8~17*V|_=w>&(sShc@VRv4}x3zKg}zHU9uV(uUI4JapI8SiW|Z@d(jvzR@04 z1?O-k(+1pQ(_8JHl`20OyZGv3*MIl5E%8Xq*G-S2rx4Zg~rW=)w=|G4nnY%1jyyarO`tSIvwe!Vp}!belIVIICk-0R8(vI{n8)#k|zB zqzqCgGj34n%DD3@sktaqc%Ito1uMb8${k_1Nrdye4j=P`7p*n8)TJ^@r}g`aE3(eq ztNh35=WJhzX9>+$bX3lr0PBHvcN*Hb3$!j-qUsfHlEKn-s*@Uhq@%0^ zxC$jv+%6HX_NwyO=4#wVHK_?Cdes|tN#GIT@)2AU~DQrpMAoE~aZE767yT_%*9hd*sW>%eQycf z@uV5BlwQ6_aFX>lUAWv@X3{oqIDo8yMHQopexqy%HA0xV+z%G>{{T9r_6}1@Q9D!D&R|T_+jP^s^W>qBOSt|I|mk`iO@rr#k69|23zUB6PccBGFVZu`_ zKG~Offq9b4@rWQ1vB=D^9TrwRV3~5R$=#HH_4zB_Q+8vZHN`>EVN}7pIH{bbsZGBj zPU2{|Ul6qED_<&@?vW_`MvzJneD`z{8GAr>t}7T>0RGM)?5=k1Eu}e2gDuY%{GwHx zdZoIiP%PfQ`!+)k+}`j%-*Wh|73JoLO;rkx`IIPH1B`AH!zN8_oxRL__T4!|)CW2O z+yQB1S5JO~@DB2BiD*9_6S*L*5$~kbM z&?d*-P966%90naap4?M)H9f|z&%3Xwl76<&?z4nTL6^4UA#iyhk}E=$lh60eW@Xb2 zraom_2CE5Q6Gr8<%pi-7&gW$8TdU>=i|&V~+7vISSz~)Ocj%J$4~NtCO1@6hI)YoY z+4aEgxg%chXyJj!(8-p8Sj;=dx7>XqSESuT`{7iy5ht=F3FU%3HuX>Q-MWTsXkdKA z>=dt1A{yNR67mSswAJg-9m2WRIJ<`A=4U=4gTHHU)a+dSi#fgXNVN` z9m_e`z3Un-(sKHk$VF{7#Cs4>+ukZ3r-n^~{{RB+80t2@-?-&s7%=8o51S*z{{SwS z0Zq=|ywLHt4U}yAdDi_h6H*&#P)I)rK4UvOLV?w0VohXAcpt=SYw>~1_?n>&g0C*|F^?!SwTv#>mLA}j&bIYfoWum2J zRyhZ}YHuZ*jMeK$Q(;yk<*0m+lEC!eKUj?@u&_^O5m%INoTYt!mu5vOJb8=ho6Rzw z@W1(2?kcn5H*@?HaaAdQ8nyzdDY9St7f!KWmAYiZRQ4A#-aGmK0G-U+W!;|eZ1D%> zZlbuj-_k&Q%h+j9(?%dzoMF}Bfa8;q41OZ9%Kf7Nem-S#w#S>#*vG`OkIOdZn`cWU zRYg2n;b=7dOe69#vpAk>V`FPZJVUggzD#BQ#I1FiZsu~kB^);UN_I;gNF$`O0p$CP zZ0@T)cOh0A-FXC^nDL9ibtAdS*BhM!J9{0HiPQt7dyZ03qchffe^fA&%tr|JIy)B+1al^E{x zDy@2CDct;r?rJU$XNvUB61ME7Z?8U>dasWTdA%&`%8>3d{iz1 z(R^IG?FclvFX&6}x?HnLQr<7bXo9Wc&Qf3LG0*@kHP648=)#oHr~d%J^Yu9iYAAfm zOm~~mFJY`=5rdV%=v?-@{{UF&r{Am8QmTW#TzUv^vr}<pNxmOCb6oxXTae)AAUvrKyK)XmQ$CB5Q(Gx5)7`Lh0W{;-#!-<%~TpTK@o8 ziuH4Vb2`gg!-2aVIG3EW&pis*d23J_Uk+oq?qsz!F9z`iWuJk=4aTl+WR^;Z+XZIZ zQL20`5~X%u+XCSf%)IQag*kTyCgaU;cm6Ii7;Mv+Dc&!q#$t~_D{kTS6T>7@H7|=* zsdVjezyx0y>&8!*misk@_;ijH&LMmQM~F|?PLbHZ%+Kz9&%9nafy2w=XBV|`JC|X~ zk?m1~ffj5l&F9z3y`YU3y2F?EfT5m^g zqDUn9U$>7@{L~nyBivM9Ahfk^W9`qd;}L_35ZF$q1`)J%SqV&Tuf~x`2<0Jm`0~Lb`LZ~?pO9N)c&Tp zrN+YV+MxZbAyBCG8%)>|kjxO>aT}SA- zhi&hFNx$iieFy&l=&oFD5flm!5g7XkLwaR@dJMSEbyDVPogLl{%9+%-2arEAA)K*P zyi-Twgi0ZY6`+jD}&8qw6%HcWk>0-dhwola17kSPpzkKryxaJ8e}^C@-Q zwR2vV3?&uKbt(Ms_DbnmA0@jSt z_JeeIsGGpi*w6Q_@jqRE!gNBqZCZhcj;N-X2N>f~uue}9^m}!8cUIfxW**?yfBXsz znoG)CxXQVBsOv2>`BM>hlDo;`R~!ZJX<1ZcfVLso?sB#crdiKlOnbp@{{XWiN*zUA zx>tI`q`oY-k4C3qKfTvd)$+LH$C&upeIfz-mr_>hx4{~H0d^j04%(JO3b77>qTO}6 z7c%3o_(ov9Q*QqNluBK(R2x4Sob{-DC<-W9g!5r4VSkcT?bL_!np^!atuhyKWE+^l z8#y=-cm2F&uv*s3ugTOWrnP0cSEw|nq23z9Br@JFcn%d`Fx=ikUv506%nJP=@3J*+ zOB-5ffXK_dg6Km}Xk0IM=NA1S4e->fSJLJW6cuytnaz?lGbyh5Q1-tYmOq*q4Q$-r z>XA1h?cilbFx)+FsFyqzuzB^U!b$MVA-WQ;E80`zs;RNQ8jkgDi*$_DFDY=gb8RY? zXkfepWz+D=nV#FCtw(DtW!Yi)h=TgyIhR)m#&OY^s7}rc#J3rvUz}Ie%S)r=I1>=x z)8WFZzGmRZthb@(9Dm!+U{pqT)A)*CzO?BIRT&z5bs4>e{QIblb<5s&2xkG8qQEMP zvexI=BL4t0DLBa9TU&wB>v&s!5{Ly6hJ1Q8DBP`5Hgw>WC^m;ir(2({hUa3GVQa0g zoEMm6o?imFjLepY+2uN3>$lxw`r)hJLzZ)KMCSzNUrz~NgI;+}TiS99eGcDmmwa_#Ez*RReaT1qd z9Q@+T=RDYOtX>u1%J(J|n@!K?$QZ?Z2n&911S=?)279W{39qfA6Q^%5tx#ngtP1v; zj%yh9W?f?G4*vkjEEmEZxbZhq1z1IUrM#J|7K))FGsLJ|AnP&6yEPZCkW9=y8{@SJZW1q~rg$P*y+3y2xPx>1`A+(Da6JnyFSw8%#A#{voYbkN*H5E%v{tw=GZViZ+&=-!MsP z{)G%1RygHet8EeYJi)j&dx^A%TMK+Z?QHZO{RjhAiF{0aCH)+%WAC;VEd*2)`SJe% z(#95%muH;L1;|hx0$Al@UyTSy8H|A zaBfC4RMU#MHWN%!Edaw-&lB197vvAh9!?gE4xAeE7%$Zay1~Sww7^R~W0Ec6$)F*P z93gjtROns3;nc6i$3WN>Z^xl+Ro}82awV61(~sQyK?~LkTS%)uW%DsfES5#z7PQSK z&-rQZJWTt2+P>T|=3Ft|9!g7oAk~`|*LmN>P!+-EBM~oPc3Gc@pjSi*L3E4zH{viL zXzX#R=Cbb%@F zm?1l+{h{n2?nQw!$h?=fnr||pg0LQMmvYHBRYdB3z@8!b--(WByZK+tKxlJXQlj)& zIO5XOVm-Z<^D5zIzr5V!kVU_}5m?mF{UwUjDR$wR1hiv(m#=C07#e41XP%|K8&frI zA9=eTM)j)Wdxh#CEUzE(mTKW|99+JNZ2(^mK4My!&yVn*oS#SLBbuxejT~toRtR&0 z{v~eCAAqL6xT_szEo4!)eGvxOvDHrlCGHNPo`x=p91L29cS9}?aksPDrI=oN^g!%! z`C|LTFkiw-X?}P`TLX46HR8gBiUL{`TrfU?bzRH6?sb0imsy(pjWj$Y;@Dqmi^*D6 z7d(hg6BnXe3>LTXtL_pLa<9>X+Gk}D*n(VQnVdOS$tVQn$EHx20bu&P>tW2-1_i2L zY+5(DXqt=LSC04TZ-`|m#{BQ8Sh$9k`Ggo!ibBxl&+fme{Yr6&1fVQl-E@UKyiBue zBYr>i1CN5BYsTd>Y9nDxefSl5m|W<56rA<1xY0?FC2)seA`rdkle z)EU!Ry!`^Ia|v)3wK7h5a?~s3mRpiXwJE1G<~Z-Mx+*I+b8OW-GI3z)U|Up_$7Y{S z`FOh3_qJvN-h*i0ge%j){M@{1DQ~vTSIPeC&43NC;(W%DgMw7Nu|}HVo0I%Dcj>RQ zxiXr%#J7bcnDEPV%6gR6u@*CHkcQtvDJ>VDN|w5)umj-l45z7Us?VWqg2%Nf8s<}~ zl~SvItF~CJ6FUG-$!?&u?DUP2z*JJTdX(a|KtS2nCaV|J& zyuAT*Tm|FM&a*Wt4Ha@ zk%ofw>~gPpOewudxSN4HFDSZqQ@3#z+8|%dz6LG7yv!V7eeGxQMFv*{MJt`?cX`bz??KRXo? z?Wwp|*})gMH^Eai%rI{J%A(js@Vc51*9{r3*qI>+D|$SqgC6Vi=2yFd3uqf>3Z8J@ za|~}I_lcJ(U9{DV7V!|>l+MAf-*L$)8jen;AJWe&d8_UYnPGM3wk~1FZ!Sg)_Do|_ zH4(@z*;>HoTanoKgfr}5YAVWSy3VL&e=y1Qq;OL{W*6frgWeL|Gbf^3j)J(UtbL!! z#((eg^(wk2l+?d6?p!l33zHs!?`y2ZR0Bh+Dpt{?h|Dyl=PlS^wB76xAEd^6EMO2uvmfM zb0XH*`1Vd)9ZTWdM%aSO&xR`zH%6P+ZSB;mkOf!FeM5`j@~DS|)oHP-)G-1FVYP9n zL>pw%g$*sYeNMaPgKMo{6SwwLn@wK!)ZdG+e8S$c~oW*#On7dp;Z<^_3Gvm1xq2j8-uPI5h_wv@=(4A~zl6oYYxs~cgC4bBoBsfT zM$cMLTK@pFfzvgGzU7#|lRi3?nz6CQFEOTkIkEBLV<9MQwqS%2LzWwv-xgI2pKMe! z_#T>XUr9@Lxq|g9-L$A|{{UP~Z+<3Lrfyt-#w}Xj;~@_t9$!Bbdz5#~Bpxp=g7VFUJZXBH_>{;)nk2yF zam-~*+CKjP;#O@DX!y!1CaUIS_J@TzQ}q+it4sd?z9rWia*mE87l^d9vfeC(8++O> z+XiKh?UT?3WbyAJ#l)~Hq`zsIq31T;#{~!u{m>}+^kY>^Vc@BlSNiw^$1`_BakjsS z;Rk~GsCAyc8msn=pE#zsSgf(J0(Q$((+yG$iSO6kvF@D`eH`rzWs5Z=Gvm zdFRkcZv=hm%sGnhe{2FvtscQ37y^n9VBM1`N?EmbuI7Rf$|7kKP(Jw#;BNOb*lH|p z_kQdK)M(EYZayH?*t$Mge_BS8W9%2T3vux(U2?WM%kLTsE54Bb0FAUmE5CL+s@7_{ zE>Hv(G&SkvE&J>5ULckbX926N`Z!??T~2bZMcxyifv;%vEHwi~9Q@Bcram}GSITK^ z#)P4k7p9!bI8CaDRzq-F;YHQpwA*DbANb?y_$1($=HTKDRFGK;)WlK|bxP(fm`*p4n7B>TWH14r9Dy5g!q?aAlKuB`B=5PLy*7^v{fS zsaX6gN)vzdze^Ox%C?!9pp{B>aP+mi=;{L7AU9VFHpPDBY6)1;#>g6o?7?>i)4|7= zh;CeppD~G}e?b@?a?9j!x-MQDY|9TC2LbT$2zdUW23`9@+wP5(n~xMGQ1g7mRXC2p zeKM%=+`Y_I*6bYkAo$f5G|l;$AuaA$WIqhRT@H|57pYI&94-MSbxo z(t2t=CnE(ukT*jIoUk64X8I@ir!z?bt3zI3th%X(>rn2siKBbtV zCXCTKaVt+N1CRSTl6)~LjO$tQO#c9ldl(#JU4e)J)yDu%d6cPyp+Zd^)j zbmqg)pdVdMOJ5mf0c@0f5VQ&exjIFj(TC=(vgrqj#9$~C_7<4m5nFnTvR=yn0OdsP zRGQY`8^guiD!>FB7g)MDi8L*!XTXppmY}+2{Zw(4OM?{u02xzn4UL1JXx0eVfWjdY z8IWOv4`UPE>cjRhtwv_rBKSCz-i>A5R;zZr#oFk?8PE>mcI9;&^Dy@fvbJRNxm!^T z2khUN=rG`PA*Ev{lWyDd*1f`Ie%m({TiXNgZ9_+`Ja<${9r}- z{SvH=4k4z|3qn?Z;-i@v32Ii`$ORKL@O}&mnf*e~KiTn}0o*t$B?nQS?xk|gQq~`| zxCC9g`+_y>CS}wpL@5?OMAy)XzHMP0H;Q(rLY)GOg$oZe0FAPqDbHzr;;8Md| zziGD*N|xKx=1`FusbdEmL&1b-#}xger4gERV~7vGp#rl~x)%$={#t+lU@%M9`VgFK zfMidFPF&PhZ4tu9mYMSQb>io|&UZ6wFPQv8L8&Rajx1>xw3s^_gD<>fqBAT5WzB@? zQSWe<>%$+3PqMTVTc>seN81q;C7xm3p3t|!b8TRFp#XM>JG+OTjy|gR9}r$l+~S->~}5+a4p6g!D6N-mC#z3g*)o&%(2wg^38fxU9w%zEf)$CqjHAXtf4^nR-o`(5kul({KPMH zM~L|=Rpg68=reMFgk^KQvPur_xoRTo;%h?ZVW*5lGI=WecP{3J?5X&Gur?#lJxw@M z7qhv7N`m+fsX9~yYmjSD0}p>jJd#2LuW5Gj!$D zq;46h&-xmhmOOR5WBQ#EyJctCK}VHeq_n1BUby@~YfVbUe5QzIuY7{!Yo^+7y2$VO zm5QHmG<}d)HA`279rY=FFuw=HbR95`FEx0%W{PLbxiC0`aF7f;_Lk;x`xllSN~`6I zHU27?;zQmHPs}-nMwOG^Qh7Q0R||KUbi4>wH;z3x2$<5Ct??>Kw${#m$uY^Ockd9k z>VF)447GljZ@I)yh6Wf=o#In~MtT~@9}y;IqSplc#jr&bTPXIbnSoIc32!{7T)g8q zUlD3wH?U~G6Bc4I;{5wV`)zl8@{qEz9baq)-JX>n!-uU>$c9Y;YY@9>SBb=dWT{x!G*nm3A&wL+nsox6^`&EGKV zOe8_6z1K5mkwRs|pO#R3?VSh;Sn(MsI5DYHAQyedL07n0r7WcZhH|||8H6!Oz31G- zxhYl4RhsSsQ=eAXA+O98?!u(6F^h3W)C{J{w0yXj^PNiXi^tSmV73-rY7YlU|e-OZCP_vmA6@zCYl?#`sMLmSHM28J2_3z<&xT#* zSL#s?7X7qGO}u4Y9Y@EQ2ePXkoq2}^;2`t-33)wrY*Q%PA29BJ<{PVPWC|;l?A5`+ zhXV~I%b3h(^b#AHT@0T`#HMK!5`~RdnBznoNxCr0KR{LfCFG{@wso&*eSUPVr|Ksd zYo~!L-%O&m)qExY0QPRReU9pBplVGG4iXRwyX*qGPcXNPY zS6p>GkZV2v0EtQ@whg|03$rZSgg#s2`wUZHyb0H}z_&2Tp;gFiKVO!{9%&aXMlu@3~b-;C;3 zrHawbra6(5@`n6PRdr26Mc%|~o>_*=y1x}M71oY!Z#(p?9oVhmx0I~WS2tGGB}#9R zxm^^M3ZDjUth)5C`^}ZOWO$jA;&8gP{$c2DV0?zX<_+kpux3^CJ(;MCMPCm_P?onV z1G6Fo16iSg^j=Ci(nG2PhBED6%&dUUuZi;jU4)0bZ#jxZzHLR%%q`~X6K(zmt+4V; z7Xx)KkMKl-VB9-t;Wys{ISpZ7iG~p-Zw6FKiwIgAtU>5f5q+PzJCG4hA@= z1)xmE{M~s~WWuS!@7QIpEz#ESyCGKMx{WUjzl)l$io;m{0MMD^*5@O3zi~;CWH-K7 z5T`xz?vnoiGT3M))5_fY&)bqp$N2E_&9B_QcXjdhSbIVF<9P>{4+*3P(J6RzOQPaM zTS{X`E6iTud!x#swQUZ={>z3|IO4_T=)}9BsWqyz#WxQ*d_voJJ?;T8iC%&xmDE=&OVilR{6`c!F;l$CdheB7 z%ASKj*6dyWV-C3sd^d&@o(qnS$bsF7zEHoI;gv+eYgvO*B1{^gW}>xkgxi@>fhDTg z%=X^L6+!JCbY$>WUAda8M(5q>oNSF+Q;#u~5vtB{)TBKZ9N9LE9YWR|TqUL+&q6~( zg#Q3+`Mz=NYT&xl^6%422r4Jk)5RCVSKP&<%T{#vE^~_Png@f>s7a=3m|D-G>!e-e z@KUdHX$>EmV{z;iwp2ybh_{ZR9$W4=a*Uv4#3+5jJR)9tYH$SP6QlL{k(*@SF zGYM?ESG=XNV~r!-o@czcu{*Qf_Z4Q1>B*?2@D}1sa(Wcf7^(^$A^0sPBpTFRZN4G- z%LSvTdTiWUw(&2X`JKA-qAn$B`>jer#yamb9xzi{zUEPXcJ4i1>-jdheFlfJ8v3T6 znA1oWO%-sPz-6yi&&*Z8-p?pT_`#7f+Gjj%VhC{uzIx>Rn48zfCO96GM+ z()AF50sN0d44tR3jp}rMm^=!;`udi-N-=%T2(Do>>UGdVxgs`D&2Dt9fki9;MG*f$j2e{k@X!v?c$(pQMcqk;V+WTG~c=R7nP3YeVRL(#w3 z{KwMM0mj*B0C=IQ^1MSE!gdw&-r&jwTo*Npv*5G_Es%M5o7c_wlQS?9#w;f1qIoKS z$zusllhm{MgS)7SLZqJGY(SU+3pX_Rfe_Y;0u&Ey)eiwJG))p5T7WCgqxYU}5w}Of zppB2>i4L|8=^Pgz8&>f@hkbOaTXNPfs8K{k}8#tCgY!h5*JQq z)=I8*XvZ?<3Sn3|A^i!VNNh7+XAW4UJ|Eg>!Bf$S=bX&S*DQ88HwZ#@Rg7YJ%2)xQ zh)(SB@x{nE70pU03g+B%KjvUnyRB;SuG@>o8eEfGzpcXJ(uOF~?bPEW=r&g7?NS(O2loJL%F%&kiS5qTNEA99@su0qvunR_#TydsSZ>`w6J|->hw0j(%a^G&c3$P4n zl*Ay+0tDWx?pbNu5}67w==#xxT}tF(y#`vw4}wIX;Yk)YdN@KgpgA4z1Oh+04y?#TtrrmN1K@75WgiieQ~&U zgX_cHP%H@yyyPx;#B3tEDl_tAjyn1kI@tsji?amY3pc83%-?}Wg)M`oFH=W1TQ>gn z99Cr4pDqSrsg;~e_#Eq+f$A{nW#n^@F#d>T7`J^LhNxCj}K>#p?L&Q?E z(kt298j9FYamq!|To1?!b>Z2QF@$^Kqe1@ww+kr>&C%Qtk1o+-&D`8t`9xsO_UyaCMipHAcd07}WngkNs>IemXg zfAHi-vi($R)>weY9eqL&=IZ`3*fyg~G|#xu{;Z*8^)67G7j_W;}KT3=axp^jlqK`pBq*-qkZO-{ntq;J#taGYyRkf0Gtt4AU+Fw8aQ&r zHy!7N4{BZc__?vey$c&TTv@=F&7FwLCoy@QT|7(v48lIH zYhpWW=4ccbaEK_rsgd1(>z8xfXkqt~uz(HtAp>SrrZbxHC0+cb&^apuea$v<>RXb$ z(g;yUNa}-Nq~Ep_Y=?i`n2g0b^PGEJZTh*vNC?IdNzRN_>}Fd0BDrw@J~&OoH#bD& z9Ddw{0bfWRO_i|IoIi=gV`NNwu5zssWM)J)dEL!tz-@zM^sW`O2q|{%EnM4OZaBu? zSTQ^YoTkz!@v*B~)^RzRu8lPo1Cw{NwsVzg+$!Au0!b59VBcG?)1epdtFo&*pcX87rcMthg=l zHL%cqgaz5VyqEa+Tx8oS{mX{wZ)ctUWpxVCr2wu$jATq=*hsaOP*~`VKQL3W2G^^H zPz(VThylvX4#0;;;G6#z5}3E&7=O|&;- zm!9JHRTW`}p691bf5L7?10QMoo|Leq{$q99@irj=i|z=Z+XdI(xsWi^H2e6h_W?d2 z0qTDTf7@|__7xUaTaFq0{CW}#e*25%g^*{&7z~wD)G(Wh zA8P70dDw&?C7@9eh`O@8)@vLj7gL}>z*kfFI{uoOIwjyeVi9%@>~U;SHbDj>1U}70 zmcXSpA#Om?G1LV`rj;x@I-a~^Z-r2efnRU)5CpbJzV|P3Ex0Z2RwgUneNMs6Uoya$ zX&>5itOP`EuPA<|DD_%3)HE@sfFc<~YRi!(z{&G}7yg_0zxIq7Dr)Z5bLI`~v#2PD zp?kAH3FzFi1lgmK{Awm7X-X`z;rN$kzH)+=`qZ$9(iP3_F;iwqZ5@0)@i(g~(y7*_ zT$_PxLtJw$UI53W<4J2xOA70CTvJHpEOKpN%4}|I@vMjA>;t= ztd)e!tV-WXifMj#7x(dm%!dhYhUYbB7^E%NRszHMXKe{oXQbx@Ycrq&VfP|;zkQ(Z z*7-!umUo@0Z$#(&RHz75OPF~1MSyjfywC;Db7zNQ+irgVzyQM)+)(J=X43xvxib1@ z$V@v0fWH=DvL2jym&HwSH{9CQ@jJ8|n62LR&Y_LWKx{QO)>xGspJJ(Y^VzI1mu0{V zuFE-M#)(@G1(%!8nCM{{TmQ z?h3GP@&;xc>QsN-^UHlX^DlVge-UkXWvtmboGm!gi2>Imm@1u%kJF8)H7niJmc z&L#mr?LowO#EHBdc|qe}Qs<@N#yvu_#C^lEjzZ_Vd`j(YD~Bh>y_1c2aw;C(nUnxj zxYkb_|83V6$ITWtO9Bh~mbh5RI#oJOc+y!F%Kz6c-tAk z6DO;z-*lnka>kwb1x9IGVXCb)w^J|OiYw!ZQyA^5h6DA_W7ydF7@anXuH(D&cLvgh zJIBf1<$nl5It5)@?pPvNfl)^mmblTV>4528*<({*AxSNo$5o?QIs6d}ZGGsFoGH6y zQ!LSDiI0VrbBAbzhLbBihjk#IbSQJ8Z*~!-!$)fO;T-v=JDl zrE2N|@Zoz1peQY84XO3mOpmFS9n(6E(e%qOwHu{2F^^%exEGS>L4cHf4WF=Y#4sjn z9l+iB+{m>tm#wdbx{M$UtSyT=xZt>s?KZPjaftmgebJzLgxg3%Q2GKTh?Z$mvm%3O zd#iAG)Xxlx_@*<`?q^KzD8Qz1V!@9e1P>y2i2Z)~jf>)j&JB}kgx)-}zOJD|CCsO) zN{q%nc3$o?S8tiKi0nR;B2Y zh1h!d=Mk>jyywkzF*h&V*sRf}>u4OGg<>LboL^&M7~6TYzzGQ`)P>>k8%<6lp) za(KnD`_lrSi?SQAM9~afIhZbOL*wsUIk+YR@vD>QcXStmqQ}u1REDENjiD zzu*45J~2=+D_VMBs20@HkuFA%-&>24&i5*W(be2j`r2})FP^F@z(D2~#s2`iR{EMa z4)GBdA5D?0prEOmO=4p4m+%vs{Rl&oOluzs*@uc3_>7C}>>+g*$uW1jYB{-FQ!@dP z8r-?_3LH{+wqBQWcl1pS%gTf5ukEJ4GUJU)cY4Sa*PJC)8hA^biqB1Dw43ZjmyOjL1VOQV$HohJlA`*eiDI0EL8z; zx!j;2RJEujb=|W4UJB)ASrL|HW3wW|vP_J0XXld2vlevQI zdb?9fb%fwkMZQC`Zf;zh&HfGH`mRzdMp%F{rckXuE!EqwXCB>*0Yl_>7(1&R3?ZEFs|5kj%1x~X8P zCv62IO`S?*E<@%M!9#xf%FbDnIvy_Q*qKjNvs9=zV zm2Cr;#H-kHMQet=+*jkE-cy9%xuZX4Hd){1;L~x>P62#T+?rQ-m2#BCql(vBREbL$ zwW^IQ(C09obPZOV8Tzk34%lupRbHy4Tn?(wK}DY3>MI5`>hk{pgaka|-VTmlBUTVp ze5eBcps)!@Th_BwCz<3Z>+KS+#Z=$r@9?*#Fn|$Gi6@F%6K$18(5ubRHhVL}+ zKyY2Am&01~99Px6I0`zV7o)80ZKZXLR}(wysys5im5cBoeNHeh{16LpWwxVNWm;(| zsZpy4#bKFf;-<=s;D8T)=KR>Z*s3s~xowQ|O1CU8^UROi`tz4-GgBrZXQh1a`omCc;$%lhmmgUA1-xtG^Q>L&gh6 zDb4BbFh37l-;lI19aj@#r-vg=znPHPRiwK}mS7bV$U1Q~ZvoZ+0C|jyVH*PEtlETr znm;m?x^i}BY!ekhsK5{JM{J_4yV2mg5yI|XWH>!zE9xZypGwwiJQ-( zuNI7PuqLJlYXYt~TJV6f?z7P_$TzRq=18v@6*T^+)7Lw7OSqnpz(4f{@b6$Hlb z8ilhXj+6z}d!NUP=k|vhBkWx(>6yrZR`VR?o}hBGTR)_2bVToX-`?>nfi|`MuQPk1 zH{(^C_c1l!PUyLvAk=T7`f@6S~Fxl_>XvV6;+3p zRqafVqou%gC^Da-_?lDUQSeSwzp$46PtUbwcyn_}54)-3%r{8mRx^4+R~Mo#q5JJ# z-4gjzKFLD|Ek}51k`0po00au>qs-e(aIcv5nXGxh(7Ichn2~$K%x6BP2WvT3<2#gA zE4!I6!?mmM_cyS!y*Uc_NPocoi~K-@HU_#nM`T)5{P0EQpc>TZqnezEAE%5zybQtZ zYO!>}wV-uweCQd=`(k=D!rOuY;_h&{t{g>p<;426T}F>!lh`<49xD|#QcT>ZCO#Cr zX)!KlaTPL511$ABi*YV&xgFV?h0|`-^iaRVG>%hpLd>di)||Q7@bsC56P}x!pv!gJ zPPExV$k#I~kH%cCWcVP9e$hCCL&-cPy97~iVs^**trxu~2KryE4KPOzHkj)57qi#ow`3*H#5+mN=HUjj*)g-g!%0KgU$L;luOb5hb<$m48;0=DEndQ| zDx*D-tfNnF#!sHr_fW6(6M!M$vaD(k=f~M1r-f=)^%#|mINVUo*_+~p{wEap9%TlS zy3|W8RY%GV99Yb%f$G3sd98ChY#;0vRp|+;*PbPuR3Q{n!2_wunv_JhoO-20K4)V0 z3)GGh&O`VSF=492yah8-n`Uvx{1=Q*=>n*G2irtlkh4ryt-rbVMem3* zgkA>X$c>bj_YeV^vcGAaaiAAgxPo0-d#-*gQ;BHst!!p%9A6RDWT5&S#nOuIlB;lc zT+PW9Pw2mZltXo&%3$pDp&ez{o@~8_3=`_;mL$L7SpPNb&+s_tikZ= z(!juPj6D5IN9Jh{cLV#ual-NDN8Ov?B<#`RYvTIyU5&H2@J{~#UlqbcmZ&e)zlNVF zNkZ5ihGcFVRS(~Pn40lg;%s7!0U5U#hvjIewr67QuxuhC!^n`#xCF@@h>{naYp1AYl83oS;q0E0=V=KlcF7J;`? z{cJTIhgkmrMe1AtH8Wb6h^V;*-6r(KJN{qN0)9W>zeWBgZ~p*TU@FG!Os=1`Lne%QI*L5V-3e4dyUp1vAbz2FRDKvs|keDAZ6=p7)wJd);6rRc~gHyFV zf}^mQtd}ToXQ@F*2ZXY78@R!GmKa~B+$n8i1Hxhk00F?G;0;j56*r(`O^Rv@)pczG zi}MZuIx!3d$}oq_Vo>w+9s7U`2cb7?TVCD#69@K^g%BfD$tjkLcGR!QuZkWA*)FV{ zEZ4)-69k*5%yxiq2JpD{uVUa=N2ykgl9T3K2%^TpmK4<=@MLmMR~VM1&cjAQ34+w%;1%fJBN~h}_FeG{A+Umr z9Ljr$AMND`$-HY&Rsv@@+BO$25K*%c?ViuOFk!p@ZwxH_yqLCM< zH3b`^pM=6n=C1(a;gT3l&@mJEE1%e8CL=}q7#G;sFbQoeSg2dftHBU$J0!gMm(Gz1 zWkXXyxg1-VLKTiSwPMv!TDWx!NEX$>p>=bsoQn_*R;pcCh-HE{EH{4B0wJ&$!|0CT z_tJ@2ah&Qlj-Fd_yU_5%Fm%c+*^lX~{T1_H@O)#J=`igz9~6ORU00It(gceHr%s=6 zkOef^OurcQ36Lll=W!{5o$lU{)}zmJd9(C@u!mt9FRz6em8!h9=|T^s6nr|*iAWYi zvD9r@NtxNrs_Vw#DM?bZr>w%%egGvs6!tnLa450*Qv~4YD8}$ci}ez;?4ilRSS+xu zw&H0*hxM6d%WJ{T zD^OCgJ96x0)eD=J@x@FBE+K3oaotqbTrI}`0Dl7mTKA7iRE%f!N4DjQDW4U!Hz+4S zwqJ{3hEvCUpS4QA8d%$=Z4^x**-H5_`!CHp|Va&E0?W$q5nFA>Dd#o8MGi7sZuSy<)A`qwU8G$-r!g7Wz&gB=D<>o-{t? z46dEL+tmDG!ZD>eOzmqxzC3D-&&KWgZCdVF+!i>^tI9{{W%^ zwlwqNYmFX#P_@-USI52SOQ(<-jccmTBGY_{pJzM)uwfAtg}=YsW=YCv`I8mQtVA@4 z$Q04PsErbY$D`OVSwIu>pZxNuvYDnspqP@)ikqyiCT|_AS z1l7|1?7TdT+SMEkeMYEDB<7Eo_J^xrC=NBhtEQiJexaieVlLf(bU=r>@}RvE(e$>a z%jq7>(Oh3m#DNY=MB4>#S(T7zwPRS~6Y{*KR|=2%fd2sFZ7g&IhKMh{x?%eg-4tbAurWj2H;a<|@zds2 zPVe~9jK1@`Vg#LW(D#EtC<0}o~28$cj_D&u`;+Y(8FVIZ*wj>&;sPm z&Kr!1Dyz$ywRZYsxFTc6pd8?;)j_<)k?5Jp7SljV3g6r z&QU{P4JEi0RmCEV!A;Y({pvngR$?NA&0b+rhqzBsD(te(SB57@?`8?N-0nA1#XW__ zQP^8hZO4!HYyJkMf3aWIuuP~PWBb4a0`o5yF*Ko<8iN|Q0IP?8&L>q|Rp`6NcL?Xj zizRNg=3sI6q?NpH%shE4tzxi@y(eEU!jvSA4}H;78_hh6OtVmBVWLuMKVBoQ2VkZJ zW}CTo5YG)Nr7+N`0aFNlC&;WR$wpo<}}y-z{Lp=Lp6sIrk&)>QuhI9ZSt(mGHr=1XTn ze$0i_h7M920JBxCLhanCEad~50)3Mm5}~V*TxSIAd^nm0YbH|>b?Af(O${C{{7u7D z)~GBV!a;IZrvCtOI&wj>!nI3Fsv7Mj@6t5)TzyxWNrw^S3*b%^Z!!#&+11Md6_TWu zlUFjKJ6=MDTqTbTHnl^@P#4itD2%PUNO=sXE!=t#2kjA{3ta+lnGARuzHKm&7QP~m zCY0l|aA0saxxCfIx8P{zLg6 z#AE~lCD&IMXdquSm59&6(@XGpm3WTao-0urvaoP&CK{8EV|M0P9vNHrxrQ^2H9KG& zUsIMmWxcVS@9Rmd8r0$}B+?8j8iv~w2sGrmi@yzK^}yn8eANE{!XTa!VHnHQTj?8d zTqlW`pB9P{vO*_3)%F|GKFQ(Dk^A*QDzW)GegEJtP14(x-et2F;YorZewikon zJi^$ePnbH!d7PrtdZ_!T*EZlXg=GFFesp4rZN|d{>)QA|*NDWuM-Q4-okMF^K}{Uf zEK-@DWbOh#A~c!lek@9K~`=m08#^{=oPN!SBo9ZW{Ui?I;xi2KMc3jEssGp)KgJo zIjuDE1R2#KeWs$No`EpkBA2DZ#(2PF_-_!T2prtEW`)(sPczQGeJ`M#oJO|fxZaq~ zFZ-pA7*j@+`M4hOk|lP>HCdHQ{{SjY&&(Nmm|<&l3dzJu2WQkZPMZG!YO#Gs6s+2|swHq%B7;q?U4^m3 z`AV`Z25XQT7ZbnvO7f0>6N-R;s^7S9qSehybmkh&y5MmuVmQT>PZR$DYHB+Ud)>2` zc~&J@(N!%fVJ+fe{=^ttQOp@4M~DL+vv71@zzaPnR|Fdb$+B9mpK{EL>ZOVn!;y`q z*fr(&oRh*OSdmfCyK#@y%mnEwa$i*xw3ipDJ^TghKBJmLJx1aj*FM6@T4$YNJf6v3 z6~0R&k3qu4d2@+f7ZuU!)t@{kyEMfV>6r?*#mK*q_ z_M`eUSNOBtSnBY#ZfE}Szx|TR;M-!s$j$+Wsa5z;)IB?Zq_)~t76RE#pwpODS7+=O zk>Ep(cQ-Q$ux^j3`gGp2#YEk%#xJ)DWLVfQhM3buJg zr!>GvGmzXI%hu}?5JX@XS>UzrFnG>PvWz!T?lR=egT_kOD9^WE{jK!aneHxYA6PQ7 zzc!Hmy1w%LRA%Ir1L(S_r*`{`*;#t4$54$Wl?z=R656h{Gz7ZtG#(UnUaHXnVsK{* z*&W*uG)u{eclRoUJ}IExm$n7h5gnx*nE1}RQSrL(xYeWZ)WQn)PUZK_hdYj2F4aX^ zt+265S(&3QnbfnuIVQYJYjaV$vt`6>E4Em%q}5N&a_6|c%9Ygb$Uqt|Ezjla5tsrr zXj@Kb&LIbAc0zgh%JqmIve-B=Kb|Z~PywKeAS#U)uY;n#aKo*Vm=*6ea5NL4g^UKD ztFpYy?563cFoe0n{w^1ORUL!cH6Wh|h+k%PGqRlAO0h;-!BN#4AhuHJzy#6Mbp$kh zqrCqB0WT8O?f_b?TXhoED)z>urm#a_*R%KX%xgHN!I<6xy-P8S-XN?+kxkYw9yph0 z{u>2@i`0**YTzK)UY@=Vpath3tqX#MRAN-!5*= z2iZie$lhL{Fzh4DDY%=^OI)CTZF3icnbp7R{*4#@K6UhN+_p4q0|I1-<%Q8zXsbptZ4w zSg6t_{3jb>RYq$@INSdlz`VT`$LOW#qog zm`BqKGMx-X7V}PvTVJ;__49RaG}A8zG?lKN8N5dRqMH3&AiCKA8u#-Raf#8cGu*BovWKBTaV#PD^-YZ2INgWjPim$FYlZisJ2f!5?-+KBq$j)N-iR;VHLt zznGX(#xJ>kn1X$;cK`unqx{ML0K$`AT8b^og;wcGG5CXdtwi;3?9CM&U%*B&Y=Y3f zvi@d=AP?5&(h-lP$Lsb9elJm@#baUCCk2|PT;GP`+e4!V#l>t@`~s*K{dYGZ?0}{P zu6`YXK2&T=@%>Ngzr+esmB48yFbrQOMy z?)VrXb~Q5shlFyKE@c;#O|#8hb}^+g@H5ZAf)h4Ee*XY>DJ&BPpHRfY_#bHK3HzyR z*+)@!g>=Bk^HYUHkyP~+15XUZQL-lhDRCBbj^?g%rC2f4MmM$<>iAb0$zG6@( zwz@vriL{-`UEzM=cE6GsuEV(Ep&jE{he8uLbXEh-sxuy><*7Kl(ia9m*p$4RJ|8yo zR7if}PIpstWOwCfrg8{*cm(^EP>s&ySE`L%>)4O+a`Q1YTd{r(a&tG>{_?|^I(@=8 z?|!~a)%PnDDpb-HFSa9c;kNfyfg?90{8YRsrBy4XyGMl2sz}LYZxaC){^+$&)}-d;9|>j z08=~k3v2sLz?bz`uJ9uu+|1!qeDbWfVXM@p*p_d&KE#R_f2JAB*fYDxC$P&^b9Z3Mgk3`rdm~GcpMp`VAU2@EONfW5HPE=?!(8=&(=$@8gef z1k%lZz8J#9o-==>b+1y!ij~@B0NclyBtp2DFk2--SzNx@q3&$&bX4~&!|X4xqh2Qn zzb7cM;W>p-MyK@0{uDP0)eCOS8bag6Q3foqN37BNU zUXsz)w3_FPZ+`F#srcDZ(aVw;6F3mvs2zU&VwQX;j_z5ed-z?Yact^k`M19Cyp<0cW z%REj!7Bm9U%@Es9S^Wx6xxgvGW$dr-0INlqDaW@Oi)D8I07U6#${(qq%4~QYOyQq& zjlR;36;Qwr+cM2Ha11l?0O2g=3!t+17>x=9l`D0P%kfcJpuxn<+z#8L*$G?K&RECi z7I#68X_`<$5NO`(TDe>U$KvrHf^hRYnj6+t%4ye#s-?F-XIWVy1FyNK{v~QpDC!4! zFhO6PQbOEqD!eh!s8W>9fNy+DQtdHQXd|TD$^GkT;31-JyvJ2lMsf9Suc@mldLh>e zrr9QOC8|iNuMENfkjRZ(VrrnNwzqn->e8d4=kP_Eh14$tfvtFBxqvFF`yrtc--$JrJk6eXOT)9s zsffoIWjkqu?S>?6K3ldHcp})=pw*#ZV+fdlN~pz!^ftMAW}noos8fDp;i#Xyrml5_ zQ&(Wx1%``wj;8apycgRKQ%vs_#dvKz)x-Vzl`H1Iq6wQo{7!VaUqOFyKWC@-~zkzq|qpUkXOI0?1Uaqg!sBRd$&v4(|vx20fegwYmu?$2_b23Go3tG1q zEwJx&xWgs#)LSde@$j2fNa0Yl^o+F4ae@q}7G`Q^kJ3?t2smO?84BKrBWsQTGEUfW z4PR|P^si5((4&-rwX6~cT$wR$uH9e^$A%F(HFY0B62)qO0wi!sD$orSg=R5WqM<+x z3=*qQuNPqx2N?@pDzqpGSE5vkRTh<&((^jZ$8;*%#6RJkzODU0KSTkg;~R_z7F1n6 zQInR4Qs&3&4(6hr4$(v7a-^VCGi6!45~HqGI!iCH#O-qBvCC*|gQ zOMc^hv3?d=j!DKR3Lo9@OU@G{IT}S1pN>=DR(XM2QQtjdEtXtkt!uJ_4aP|Bh>LmG zSgp&UTn1$%d}1wGS5lIo*JFsVGM5$1$6^4x(--?@R*X2#Xm0~=JJF-e9Yd6&Y(V8z z6*N?R?~J$GGJce93$Kvz7do^;rgF=jN96GF5BVR|e~4_Os)XO?>N|i?QLt>=h!I># zZ8Em8!6<_N01ENcumc4OTwAyzIjnPc%46tF+r;97I9i*R69;*!0BmgpfmeJk<&$i| z?k*8?F8tvj;1fd?u-3yZI5L?!nO(b`Ije7~s+i5nR@w;ZLwSasAUX|ACZ3rE6n)CE zO5|A*sZ_NwD@C{uZ^%XJNak)!Wlq>*q%K|M3EFxnPQx6KEK8vKA3iFTrh08~*@WE#j+Wo_I`4g`qK4;TE7RiI7+4;~~u%>vw#S zlN5JlQ=01ws2GI`7+NW<%QftFfq7w$OaB0q<$6DX9cW@#&jtO|pbaz!S8K-O-nXAi zv&=7cDTOzQtjG%XE*wdvBef2t+|i+ius|PIF(ief+Pd@bIe5Pn2EfwJKE4l_nV{uU z%~oe>>RhaizffUnA>;a4&k%EvgD#spS+9qyR2ckNm|;ly4TZN(rKMI#9z%l5>I;n% zY1Lk!LqQzxS+$>X&rL7cWmN<6Xe^JG$3^RLw9P$4upDE!B?_EQ)xis}9G$dM)!G%> zMwTjDH(|uU)+^7d8oZd^VJ1T*MF!Ba`nh~4!OrZzXk%E>)k-S{$c942{t2(R%t78h z9l6&K9zZEz0x=C(X$oR<@52{rE1>c#q<EdhNho^(Ow4-WIf2#$1k+TCwYJ}^C^PVm2rBM z#_VK{$B&JDfnaV-(M~=tDHEg{orVeAiMr8-C+}@QA5x%Wv;aJ9f<&@3a$DBh_-FU$nS4Oa!f%7?!8FZ*e zxA?zxQ;9u)(q#M;eM?j~b~-;+@gKzzjtvpUR6zi1>$M8lDwN_pC=y^{Mrb?S#1D1l zIs|$%H#!NX1I~{3htMd^EYFrUi}pUj;d#eaXXp5 zZeDyFjIsIqrqE>@_bMZ$)&~Wk)z!{eXA@&$5i>A2%q8Q-75Sc-t1vQBrJn`O&x#Uq zhYNNanXu++po$5Z;elNI?~t$K7?mbHfDrtyvm263Y-SXj3Dkw1f}caNUamt%Nk9RTjIP=YNPo- ze+qAKc~nJ8mAR-Fa)qwDp}GyuThU7vS;W0ElzELlNlPslVFW>Hmz)d{Pa{BWl%$2V zpCPzeaXn69J{PWRe!=-xC42?5AUJI(q4_iLVi#l`{8z%W@lw;2WlH?Zd;q)x!3!zp z5X%REt7^+88F0=Au5lLw?*MNc#DP#!n-)t#HT#H*YSHm>rAq#{^&jF^0sva>8n<@1 zg;p+JM&WZqvwXnD84&*f+syo#7dwr!A%}6Ec}mk(Xlc@6?wN!O{{Y;ohujkn+b==& zQJa0oT57NJwL6oAWiOy<{7P*URh&##{l{nyhY_F9T?b*uWU=l=>h<*Pn5VR7HqhJQ z8!q|hf)qo%%BR-<0D`OuS4I{0P;$X>I8hdzV5H%xkwq(6$$Jx=GyZeyk8#}UZ?gcE zEMMwixwA_neg6P6Cug_0vF2XfoQ-i;Y4wGn0_g@?DpUWz#JhQMh9ZwQ))Y9czWqtR)s5IGNsuj2qtgMj${3I@4i%wIAzD@S69k zbbu|m5AsWyBf3pp%*NM`o*`blVNxAEyk&=I9gT+qySMT$FZFi(nOE8}`C{#CVvi+z>Y^y}vPv8?W(8 z?9*NI&44yA=)OOJITw=IHKO>%%1O=W3v7@)BrBz$=34PUzx5}Os3OFtZJ?sm!X0bj z3GiK#a{TPJ?pC1Shdt3BcM=8Ljr{=Dnw;a2gT?Rq#t)+J)j>24iuHURrY2Cb*-pY- zxI4bh?y_s^;@YQZ%zt_0d?wTY->j}u^i;Ov%}ZAC1@>v}3x8i0%r7zB<_6bZ(22 zboCjWB=%WJ^vm-FRd|=@<7I@L=p;S?2y5cr<-M}r=YP3t-etMZ#(S2q#-*;?mwfzX z+j8FK`~4_+}bu$uNp8 z?PYg`?8KndrHsPJjJ-nbVy#jd%#_?zGs?OPeX@4SKFN#whbz~l;r=%PkBp5lNeP|i z9+>Mv+J`ooxM@jJg69rmr% zRn5a^SW3P6i4-=aCb5cl#J*IA@O3JCp#J~|I~I|CZVNW=!DW8tW~IZ`^xlM6sSpqA zjAo_UI!S7%-oBaO0uKvf*U4$7XQ_9etA;bo)t=c`*}TcAciV~GyOQyd9xefGe^Fo6 z*&V+~bukp@{{S153?jAzq7?>b1zF!5z|di!H+5G663~HD=CsNPHoBj~uc>&A9u_W& znYxV`uAwF4lK!6wVF#M~C5`^Dpn-@tW} zp{x{&A}EVu$23>cc{;6YY%1+r^Bs{^JFR3^CcX-|wEI^ue)S+7Be<<0w(#@_ZxZj- zQdp*e`#>2O1pOF>H7rszdv{8!d7q1Ae$Ko~ykEB01DdyVKrI>&UTbgX;N6)4pMw|8 zZdSFr3;GuXiPBG`vz>P_aWr8b4*bF%B0P-*HFa*flIzB%lt(Ma z)9d4Z=u@DNpY<33ulj7@YvW(-3Az@(QDDF71-j4szo7pB5Hc1PQO%sejR0t6a$l_7 zz&K|k52}pU-8#EqcvREhD79lU4P48+tgg`HzAHWhS)rZIYSaCwy!1iManoS2ys>I7TXD^M){{Yo-+q zcC1c6aSX)oO63W~IpPe9=NtK33b=&Cv9vd({X&lM^;}b~T5$`iA{CQ)9HcYt$ugO7 z{rt`rcRmG(ELC4|TUt801me62P3Zekls?aMmhQ2Ut9M;+k1=FZjISVDJoHQ)3M5sS zzc@rJsJ%MrPMJ)&Sh7;gM~4l(_}eD!{=d$U;{ zqv=@UV)yWxyq@-nfU*0h3 z860y;)o9Rs%Oy67xx)r95pm-JWGlZu z;w@@Pvd|Su*4eaoT6(JK7!;z)-@$v-WWi|C)#jY085@}#;r6Wb6~Ahz_PFeU#s;92 zzif93rdnih%}slld2MiU0YKfn`s3|@q+3xo9&}Y#m`m2AWw;z;KxORZPl$yStXiNf?}Rw^MdlHpfC=ZofH!M!Cypjz&QKT?-L}=pa`$L zx9(yxHbSge-91j-$z{U{6?wP*{Q=^^VcLfM2r4w@^A^?a1)+4o#6-0VAw0&91%nq< zE<6y;3r8mYh8lLF<$m*7Qpq2l$EX8z_Z~c zpxJ7`ZS-mges|Nj21$I2Bct+BE0a~fMd9@rwkNY=bM~2xthZX+RT!I1oy`mNHjC@W zMXlE5UZ12255!dHF2v!9sY!CjZm}IOF2&()qj^_wwl}MNO`siEsRGba;%XBwK`~}U z^2@{m8mzfne>rk0Le|%Ts(FFti*(d4iUE0KI$G$=CmAFnR=?1f?A>M54G)qIDu$3j zL(cY_4z_qzyt0Q3u-I(C*E^IUbfH~~n_$2iYFRJZ$BTNMmwC#bpa4Z*x+fzB5(BV9 zr4<=eLO<%hq5Ct|yen(-a_8H(F7GSnm=ps>BWV$~Gcm>LsV};Nc~k zzi4e+GVo{k=;13Kgh|I@B zEHEjDy#uj`3J4aOcM5S0{35)5GG*ccu>A+AnDHneL`S^%(;+sY-}E_QK7ms^x(W(z z?$C6A9&1%F51Azk8B@?#UAULfqaU#m!=B*6)**eEGZPU}RPJ24gk;9lG4$31$;75P zn*=9?Un>n$$_B)g^Sow4y$gV#DZx<;XxtmQK?1=A#yTL9TE;x>I*&wp5gp~?bu936 zKNGi6An*pch@Ax?Qmu~|Ym>anq@Ic*yN}vfGukkzahgbh?Gh#)?=|Zkxr88!Aoaz` zJEZ7J{DLttW%zm~$itsv;&9Uo#)Petb>f-T1M{TU{33%TWk^#DAlgI|AnL z0ew&lqbSGC;Rm3;E*;!3^b(K+G_U}2cvDBfG8O7V-{^PVU`$d1`Y!~*nF`U0BlJ)F zCFQb#a1{alRNVYk+JprK?NeR@jP|t!Z4?Rn%w{NtV8%F90%MQR6%-Roz-q|VqyThm zp8o&=K_&voBI;y}G(z)A`Z8P?T6h)dLR-!KH%g_{G8_ZyG4ZTPOol042ltQ1n=csa z27PPx?q4YY!6GW>+|*|T2dx^H781o|#V=*xxG*671$obV@e_268Qf>F@97_|Il} z<*Er>BLZ{ieV|t0AU6&s3=3IRA?u@s{V@HU`g@OW2~uI(c9aOB5do-~!KUV-L4R#A zu6kXkM1c$s-aejyRt|&m#)hFU0+f?$e42Y-oNW=JD0PD>W$K~qSX8)g@4ucvyq5jtHQL z#81DoF*0JZAKWn9{{YH$_roqf!O{Id=1L$oi2ne%j%cM4?_dBWSOyM)3RRDaOEO9k zMxy+VZaRVi81#RlO~7WNlRrj(pe?}yDuWzc8&}mqs#SnkC>KgG-AAIJG+NN8zX&C8 z!b8DPf2w0ZR}_G&0bqAR$;=XfLFvCrr33kt#`NwfAxNAM5ia!G3JSjpVc|FKk8`}| zXBiVtOj843$5RSPW;j{VPH1#OU7#6`=Hp&bPg7>E#-gNo7Y3gYL# zOjkU~Zn~G_UH<_3sEofSRr6T5@(lj~BBx)0;6`=)3>Wkmb{_Rnpgb{3SU83zidt20 zB`UixuQ>O>Z5!4R!YvqFDF;_YJ*wo7*UN+Zuhuex^KyIK8KDZr>V993REg587jWDf z&Cm2t6vWNFOB9(dC#~n?`1~y^Rx8|Q)nd;jEQ8Lbei{Xi1fug?mHur&mkMEFbp?C2 zh1Ru9X4_DRLX<{@8AvIb?Q$pQW=_aa(kKF#!lztZQHH52?uxeg6PWH9vuwf0aLqYJUT+zrf$%bv4&h zQ~v-KS}Xun+Tf`sK<%MGJ58Qt2hRbNDxvw zzb{iIS1uIP1%XQpLCndQ)9c}P95J;|r|SsoEaip%m?EMMu0<)&Uw z7!gtU0awB|9-fyv^A6ICLI5Icq$7vrD@gPpR4meyFhB38qXLhOj}aa3H3BsA z%vjP`8l(6DS3~!fScZrq!5k`Fs+O#!)fK|OR2-``DnfxO)1ZYqZvOzs``lspIH_<- zqqo#tDFq6Dg8X$zJX90l2t$9(z;^iF=qfN4E4zxA{kdT-G57qx^Zx({{{SDWjqA;E z-9 zGAj5}KWTs-W1&Z)evuvAQM@M98=i=M^dfk*Wylh{1K#C*w zo;<+Q$x?KH*<%7r)9nHXrbP&5xlg<76_4aHkO#g80MtM$zqb6)ZQVdq$*Zh)rE$-e z;;!8r@nT)0QZy6_s;QWi1t22;cq*W4cBH=>rTF8NsNd}CBu$xh=!9FEGzSN?R79Mx z9P1zVaX+1O>BR3TVi`{I{{V4gpUHH&0`!HDi$$G6-@RA0;ey_>#AgpMD5Y4H{{RBd zegz*)E2JR|nNXwqH(pHF6%>9fhjVJbiLF*sGeO_8JSjOl3iL$PNL#-2+)LzRHz(^V-F8!VuUOGy@C5YXS3}6 zyfvD;pv597blkgFhl&b3%+ak%`0=4!a!?~O``|o)UG)C|XYv04%C6~pH1%@!5nh|Oq;-e`}uD+H4ih+aF;tM@@@#@FZ+2PSi<2=9uAprZ; zQ7$2o%UZ~wmVoTLzLLz7MOuO+*IsiI+JqCNV=rt|m&eaTKi>5C#OZU8lwe=MAxC3a z$FUFiT&C9suH&_RYQ7<#dO7ed#~(Zwmj(#J95$O@cgOlzf?=!pe_^glb;{aANMrYSg358AgEQr6bT9eL(0yF z%FLQ}!Y`ny>FJ}xDUZ_9t&K40xnPpCDzeE`rw3m#<}uombSpr9`X4FkjI=U?M;O?g(6g0Qdw6@Hc3{ z%|oLR==5;lAQi!)Q=ure9+@#5s7t}62jQl%d<4HL&IVS^-dT21^a9|dKqQGoc!~D; zG$u_CvP2Ldz)u)m?W%OYje`EJ^a66W^aB2Rz3IBYfNBw-q7<}c9GXUoh_D!-4}-4@(RH&{U`(#e=HSv#93TUH;ejUC#oUIus}v zgJ;c78ARhqTjz3)fXk+^^AjW}& z)e!WxlO;S91ZbinpE&MiTysA_0g#o4I4cv}PitHSEdLWQQ#3l>!tRas(U@5C{#SzVVkp>V*5`|RC5TqCIP!#-*2rUp2y-+xyrW=Vdepxu@ zf*EGO1hDE<3V#Y+#59fyf}z1&IfF-_f%HHt1}lRG!Xloi6@IT>O?X*WAMd6yWstGB z1Q0>UT;PbGBPsZlmKn^h%PSy2MF9dXb={wVIz@+);m?A_azG`>*Cc|g&M2kvlA6YA z{)7Jj%y$mDCp)f<`sMn6_G_+*$D7aZsAKql^?q3&@G;yRmi3+KxT}=`U$u1)9MD(5kOvo^;{O1jxFgV5 z2mqB57(25YO(+18J9Dq6Nn6Si=oMVuyQ;)s04hL9fDd4Y;VB%24Y(A1Ku=h6@oI-8 z1b%EX%E_XbQ89De@b$^?vZNs@3|wsdK}JF-$WX*xB{6UmsvMf0vk<#%ssXE-@}30> zAKSa{7vMuCA#e@XE&Jra!7-|fRO zNfAj$iD=Q*`ZY3&fHC4CJu^0vXiMz}CyHrpc1lE3=|Ys$`o@^+FhL=fi@DqawPKpi zk;j>6biited4D)ezO$r7>3f?eQ$%zql1e=xHO3U?oeI05cG!Cvf7>~K$Wh<}%yj0t z^rXkeA&UV!V<(7SPW&zd?dz&8R!hU*;vp0WHijJuoWN`XuSCoKdaq=Dl+D;@V}Ruo zYwCNBjbjW}Kv^_)chRVB+THX&A^ z!~EEXITDXB;N|lp)S9d_4~MVf15l+B;@we=F8~8@{{S}5_LcTok^_jz$^q5;`Ik*Q zi+dIa(L*=Fxz{@>oYE+6}+`(t%QA)zUO5kxK z?`4I43KHHL+No9w2zd;Aqg7J_07xoM5y=C|tqqGX-DQo^I)XBkR6@iz&v*=F$)v`h~db}%nW9#gZq4#W0vZjeJ`S@iJhOLRR$`-6pmzD zqI$#bsw;9rly5YE5$GSR7ga(fM&5(EJtvbm>Z3SuAD$$|`$GC=P*|W<3aXg3wQ-K{ zI6TTf5z@cV2mvAZ4-Dhu9t%4ogpL<{cBA1%{4a&GC|-Kp94IFrdYX3--2!Vjaa2mb(mAXJb*Pnw;+ z7s#b%5yKW_TJZ+opk;M#N+iSrD9SA%^Bkenz!c(?a83gS$T<;Ls@21p>M&g53zd;~^I;yW#tD31JEkuN!=25&`YES?W;$MPerSj4U0D{kf zGjUb@D%vSpQyjO(v4K^90aeotO3<$Yk6L7voh2(8!mP>2>+$kQNS1?Na~bFeIyvm5 z7KP#fzbMKvd$cO|)R)0c2o0uv9NP?myuL+ZFVY;v(Jhk-L_`20Av7tq=+yv%t5tfq zOP|*8LJ5HxS0TH0P~x6PdnYmas-^`IS)_;ES7<*#8q@FO8EU8jRYMrE2m&HRPog~> zGLxao6oMZCjkU$tz^zS-rzUD4&lA9>C2K)H=L2n%K+v!o2n(jm#>#ZT;1H8o3$r|{ zfg{iiV)PnQqXXdinY=tF6MKmk&KAQ@ZZ0%HJs5y#=zwEZ^x zU$r>H7d;8a$|MJ@L+ywTvOexX*h!@N!C2IQJ&3f;CK=S zx%Koc<4c-H(!+y6<;(7I%u6LPM zE>?KRKWK7?qVTq{JPDemD~@i8$0ViG)0ECzeuWFF6lJC3Ug0?!@Yt+L3Y1fo zb4bz|d)z*PT)kkAxL`V58h9VstH%TK7Uh$+hY?xw2OkK5MPk#HmFZ(j0)rzFGgj}S z2tTI=gjM7)5ve#!Qvskb5P%zkBOBR{8zhXT1QF}Ad>gclEEerfy?F#3awoVOZTRCd zMK2it0Qqq46`CT1=p5F=wbMqZAfRH!Sovj!8Ft?RQj3K}ew}R(s0)r&MaZ`52dR^{+YLk-X&*E$vsAJKs67$`o#tR#RiYSR z!KR1t58`Nl6CcTs_7Cd8p81j7MgkywOjWTnLJf)mQFNzuK4AbpdQ3v<&`W2F#PY&2 z*TNExM?nG(c}99V6!`bOcYB5pQj!+qj<$b;wC63XrA96r+%mI$Mc^amn53;x>KhWK zqNsj?0s7Pw7?zcFs3P>f)hW(!oylI(jF+6pp9<(eB2(Ls7LmtLz<*bqUZ(ra7Cw+m z*HPgHW1&z46vNkUqg7MX)y2@Qh>u`|@-fi|6?JT=1b3^J9OFvh`b2uN6s-m>qR`X&XKwXJ! z;Mec|E|9W<1pQ#^cElwz5RolZvi|@#tgLcSNf|}G;zmh6rw^DfZ|`g#r=w{ zL)rSjwm#kG4B`Zp+&6urIJ9s2Kz(Bc4CMH5e#@1<97$ zX|#%N5b;cDI3h(zNT|po^KO7j0U`(hX|i)6Xx12re8H4|bn-w-%EmDG$D&_=1CYrO zoCV0y1Qiru>w-`HP>Bz?t}M>|szHnZ37 zSqF#Ih%Frwku3xLgFS5n({2382anb=pRs7ziwDmaKjq!k0%d`XTja-ia~~imq4C}{gqJL%y#nDig)=7ejVatgurYZ=e{{S;p zLmpJ5wssYSqIig+){s;mMk@@G0!W90dd|e!SwI2N;-El4RA4F}lsZ}cy??{|$J&E_ z?O_`e_XmS4FsjZZK}vW**dgAS;Kg0=1y;dq5w)@ScnC;Ah2h;%3L`osN4ST+DI}BB zFWA8Rj#uMF`z!O-x`aOIx*$GaHn@g4^_%igNvZASGY5G3loBELh8sdC!iW_V#@qz- ztB_>`a9`43txpO8Fc%s<#@oTCOeQxJD8jmC@FMkB>JLD~x$O8Tge2Nz%F0#=D`hCH zF10Y6lxAQ(W}icviWNzf*B?YSYZM=R zeC8Xh8W<*qnjgww(7`{Mf`0)t(7~pLhK7cQm_LLxG&IoA{voEAH2x?506PvSK=oDu z`&=iBJ}+Xo9NA2qVE(Tb=(}nbw}F4Aal^{-d8(B2hGP*qs19kSSjiZ7!={^e&>+eL z%@vjesFJEU{+QiU_jhDIGAw4zK=W(^bwnkb3gs9Dz3&%JpPniN41=sE;i&)zSOEo4 z&EC_@vDOC;ewiLp3G)NSrliu-{?K^9RhrJ~KBSb*ovA&|I#UeH*-eGwgr(CN#!kOt zQa{D6Y4#3jrl?9*NI!0TTpVjxp}YJi<`8N_n;RAcs_bFvBr(aJ2XJ zT`y(`=&AP;F4r(Chiz3x3`K{YxOtFN4{<9M{{VP1kif1biNyv;3Z^*G%oqi%P~-Sq zweR|te-VvW_%P9yu$BU2>f!|v<2=V+{XrQGD@w*<*P!S= z!TQJggEd7Qq7E}@TBQA-sj6S&#-a)0WM`nmNCNu40f4m@2k5aY!`zu*m54~!=?9sU zbMb1Y2mPeLfr4!)4~k$)_D)qC0P4EIhMe`x0P5qp{{Ui_`X9fVh4-MC{?+KFw&|RB z=8xE7_Pcg;Le5z%W-T7DUU(`g5eGYl^DRPngy>||jYLqq0t`cTA0^+tZtd9+B_@Sk!l5fo3^@8=q!(Ug!rdKvrX7;3#@$i zkjkZ?Ps1??{a>p_U5e&jMro2o22sbK%N2MHSM&~^nI{1CD5JTKvjtR?9y#amdIOK_ zbT=7DNRd_X9X{b7Cz|Iy0uDbs#Xwc;MkG9Fo~Ym-S6*1jOAHVKxEzMxr&a+*gct)o zPp3i$(ZBRjFa}%6QXfC4Y~%k~MF29haBTcMoRg^1tvl*krS$w3w$M=qqS zC}Af5Er2#p`F?%S8T5F6QOkM}glG-pvT_QCWx^Z2z0iqIMh7l}Co zKH@4cP1JeA%o04vHwf$ab;0LlUCg*Z9))dAk1Ni>`XuHMj{CiFB#s4 zl@$ShyHwVFnvY1`D$(A4b!b#_iYU4XJGLlZLA&FY+$eeSrWhBvspAcmA`WAu*4bWZ zKy=Yg&z0Vaytg`i14t~+Es>6++YO@nlCQWkvgyFG4?fvfdyMeXr+l}p=P#L3h;Vvc zV==3!(Yrx>%K1-wfN_Z%5pXJ|u|g)X?^tYKUUKA-MhjKdM%#si5nQwE{TXPhK=U4v z-wdiAQ|JdETcQI(huGQXd|HaV5l0+lAXZ!uGrNg4`}3DmX&Ieh-4MVhl3WUR+ad@_ zeuw^u9q7P%g(vqiZ==6x?N7wcF{Fqn@E|l&jSVFqPW0*16(1-R#xNSkj^K9-0)7Wh znXB*wv-BS_mubX;W2(7dB6K)ZsqCU$e()u^&S)S`spQ=)heQxx$`Ze~<}~^KbcBDf za%f5m2&7R0m3*bn6z~-KdKc8ajW=yDEI$-0`YWVSrU3pZkXR~!7lQG5ucQf052;X& zSHhLthe9cjh|q7yF_lRP?8WDpTW0Z?cP;l&CEg3I3*mgIr4rYln75$L8) z#DK;is{|^Hhh5?9S`H(Z;U3N-xF??3NEW(>9VO?b7qD19j7*h9FtE$;1$j^dPvB{j zO)_cH{{RyQ@h+J({wHz#B?$Ei9Yu)zN1{Z0{v$!B@i2ery?@`N%Jz3I=GBqF&ht7EE_$1EZMRVG9OD|VV6()PWoZmlEdfKRr}#d zonMCI?nIjsFHd5bZSOr7v=Yr3so~y?dBBs&u&=%lz^p_>M<@xW!Kx4i_G{H8%*6Jz}mF8)M&2NTeTxC3fAj)e%NB7YKa=T zxxlUt{{X2>5xp~VFFE|Nm(;AhdHA>h#hw9#-j%v+gEjDUb$6=Jui8Xc>olIHa8N)4 z@-MKj@IS>r&|mbA{{R6002fkSPxTc40ML@fP*UGR0)N9g^kD$%K*t`dNA-ftcKs*( zyVq)^2dX?z!;|(x6$f~T#LWC3@Y|2&qXK%3P{#57-b@Go09bwh04)Cc9f6E9^i=|i z41aKF51+-jzA&%`I-~XLqT~c*IX?dY%Hs3xWoC#d2xdWebPN}WDNGytL_egJUsO}S z%ShtCuO_8`gc*fsP^o|cQ>G98$NB#N>$?8{aOA)T;NP4&m|=$NsrlpM{088E0vi7S z16b%@f&ox9q1qsn1Q0|)2l%=TW+(7x?hpa_e8iRW5k!HtNF#M4se-fa4d>!4SXb!@ zs~a(`4vr9IX`LVyC9)%aC>($2d%&Up0MX1ll{0_}Pv$qqa#bHqr9A;0To-~#5>z7k z$y5H0h%H6+jY^Bs0sC+b7uSI=NBWttAMkAV!DsQ@&ZKwjhhb-;)qC!epqvU8K*_`-{F=i*vr+k%&bb6S(`0hzB95BYn1!g ziXc+O=vPYLEG%_OuE~vF>24ER1Kl9h0JBD$+!{TCf2vZ%E14g|p@*yh@G-h4rxZ#| zH&YxJq!=T9ozLxaY3*lYF`d^j#2+{{S1LknQHzT%p|Af}A=pyJwj|d{K}X zJ6r-wwSep@cJux1p!G7Y0?9SPEN-E?YxamT`#Qu4`0K3gGjNap{1BP#s(!Mt>-zFU!p1??wY0GR~lgc-i{TBW8m4__8BC1_w`d&kxU=@A9!~X!1Pz(6}WI5 zy0Xt*4M#F0;8FqkW&q$T&;q}gAW{L&3eT6^F!~TwoFJ?mACGuY{{SD={2Hg@i0@{s ze#|pF10e(A`h(E@u7;eP05*d3CE&=opo~Tz(vrWcoU~U=n*eT;+yS10KoqiYDD3^* z?NUm_JSsdf3fThT`+wzwZw(u3InPVp-O`( zbh7INf4~zd?4pMczTQ7iAGliyU4U82&alcp@&ZouL~~r#5R_+(jZY6#<_|L|ka`1x zcZo9d`w#J2p=h{+&O{H-01SWlh)MVQjr)Z;hz1K`4i~d1;VYu3_~icp&gO_A_`K@=n4f$qWF~N>aKCK zwyePffy9I7fET@pm%ie1Ktu>r#J8u%(+_|upaQ+SOT%D4#aFAF49y%IU<>H@f*v%c^o{4-C)P?lnMbE zG=7>R5G!2(!FWoP0a(`f2*j82u{BW4Th>x znjU)B28u!`gB2jclcLty7@WK3_#5DdV`d*rU zpnvGyCyj#33om89nNa2P{WdB7nQ@AP$Oqbm3VxRdUf~RQ!3c;F zluS_Ygf2KDMXn!Q+KJ_`Ph%E6h!9nZg;0NoudfP#y@(LvgN0iYc&~ExmGhM+&_dHI z^&7|~i@5+TD+!IGit$jWz^o*?00@8{00EZU94)FN3^np6pa6&gLrl!Pf*=?O1WZ}V zEW)fJTB($OA{vMu6;;t;L0FHQx{D*-7L=1g8^x>pc zFPO-yI!V1eN(v4bUIa+DXaEVs%RkYRTnSFDF#iA@gAh(X=#Qdk#q__xR4V>I>(At_ z1%C#Jzx+=6r|JGn;hq#g4Nv}rOIgon?_%Wr^G zs)SWnA74eDgh4Ti-7(sXSR%6Gf43tTg5^>?9CUJL&gx!oae8LCOTfBY)T9#0A4p$P@xf6(nXWFx=NvNBGQS^hLwMF zV^9q#1%@p5V{rngE6mw?DI#b=_c|lfZ;TMguvcFH0C0g&TlA}xh+eBBip6{@)*^I@ z3cfJk?g7(}{{TukDm)K7>9o#Z3_rN))9fxuN>F0#FYHQyZSD@=+m{AYMIIi#0^h zeWu>aRXgkP$qaoptTuc$O6T10M0H;{v6r>}a!*$@3@%K1<6o%&l`$#My1MlI7aW_w z3J?ua!$*c|h7Ln{@kSfD6@*j*9t1`ss6VCEl%jocN8|31sunWEY0gxN0%~x5V71f2 z%Rd`$$6wRy{LFg(f(zC#Mai=0c?1I2zA7~P{@`JvzZ49a*FB6hwB|jxUi1F|DF^ls z#bCHI40Gt5i?A+UO$sLlq6f_}>QTTY5aJmZmB*ly=n=$MrhHKCPP{)ZE{2;T$ZSZ( zGk#GTCanf}5{;m(;|q_2@wNLMF1z^sV${lSklMhX7e$v#xIAYGE;!w3)cL)?!(hd1 z3fHr?gwNk{!z}nsr&IMu8pMB#r#E9_2<<@g2tjpB1jS`kRmWYUnFSG4RfGQk&U2!p z$BZ%HJ*aYmlCDrP`w^uaV4IBlPq2k9QT1z^@_)Xgzliu_Lrm8y^aXvy0DJ&{M>St% z55S3kV(HJy!)5^y)teiQI^QvXh}Hi9!tDU&CiyJOa_KM|90P*FSfbFcOt_$e6(9!@ zS0_>`hkOk)-utY_mj3{_9GJNA-Wt?}LgtizUCTM`Q(MQ^f-H`A=TtzDh7IGJ@2G3S zIKJ|vKt)^vu$b^au;I9!>I}6BIq}Kg8Rabj5I{o?OTfuOx=#Y>A57^X(6SL^QF?20 zj`VKV?2p989iP$Ap<3M-)zeKRaubTO34B2x7$k9FNR_LJuoS9QIHIgYT3#Z*^6o9V zJ&^SR=gz-9B;$xbV6M5I;*K6D96|kK7`pjI;i zH0*l;dLGk|%}$nyn}#&6$exKRzhlimiKqII(9`*vAH+Y2r|}Qs7$@-$^fv~Z%o-oe z{{TreG&KGuFemE zhdm1S1ko+5ZJ{t#XI%G8T$|CP3at|}J{pXB->uSSTKd=}* zbsZ9G!{{qSYlSwaI#lcdKv(81`peo=$obGz$8stMV$NU!{vR@Cnl0$!`3ac67eB}f z6&k6_Cy&w{CR>{T0*(W=`cYDa^MmUeuknV zT>cfT7`zFJ8CvWhYJ#0Pih>&kp-3c?NcHdi*vy_R%(O@v%cP> zB}k8jUk2_>WB!T=1Q-7RVc+y{Uu}SRB>w=6eU7x4c!ed)I>MJhPzW#WlLF)c=$Sy? z+(!n~c3KrMm0l8pA8=R>KKK{=4RF~zv8~b2W;s^D_RnaEKWN6krcs^8> z!PI{E_Q{HW<2H=sQ5XtG7pDIJ$N&cX{W-iyi#;Pmib85k(oYM5!w_II4)=@6BGK69 zroCXz$f%Fdxg)PMWYic5)XvrBL|zC66>%PCK#+{l#FGC2!j?rOEWyJSfhmbJ*L}nh z9|8(Fr=lYKJm=Yl_|Cxo@|fy@I>tZ>u+~^SLZp~ZAHzF~{;B&8P-zWo_#lI@RoTk4 zNg#qx3PJJGZTPnBU6PKsb*6(AJp^I{r-k^-&r|vsxqLkpaGsOTs|(oVhRQ0}ar3p1 zfDi(FAeyRHuUG#7VaxuZFG*u8mO1BoD?LD;qe+{B9n|1}MHqxhNLKwP*d4Sd|%-Mj!~qDbPRS>T>Nyd z!3v;+(DsV9fr=DPp}#AA$(@mUhX*xUrTZa5wF3p`3!y@mg$!IqCIZnP5h*9_w&xr6 z{Y4zP^|m4chiZw{M>^0q8Uv@?!`p9Kl}E;IMvI zJ+zdeiZnhE9_x0*^Bmw&>GA#-{{UJk1bjtYkvz)0D)vw6x7Oy{U%BQ)_lxrV=5!YfOmAusHqiUoZs* zN)P~IxPw<6Jpqe75j-JGn00tn4PihBwGARD9y}B(% zfWN&>nnB;R(moSWtLKftu^@i7k=C7%L?&t|nm@kb_u*ARRp=G2r?1qR5~_h5e&n!* zIz*hf6cIj2HTxRk2GS!JFUaz-m5;iz=;rG(L@67`4_@ZB z*bcCbqw3H7O_qStOr||6Dq|#ybBLbn{8-C_#8T9HrdD2_$$HN;w)^z&Bp!uF@9;w2 zln{U*B!?}f@7MSB{_pWm=9quePxaIIzxoS+I>J#s6YVK8&uV{;}HJ< zAci@eyix97r|uqv{7<3NR+g&V6}?EKqXv88<8N#Eu1qQWTo{kRW7n6=ZRS4Mz$jcM zY$vtfzW)G7;VYCt!}`7&Ndlsu-V-ifBh>(opcV0V-tMc`g`ovpZt76_J-^?MM93^5 z{{UwSSM3VVY?m$8A^S`w_XPFM0TBjuY%f2>!gTI(ulV=Ot~25%`%X+NSJ$x27$R$% zk3d2;)0|hyBfDWGQnL!L0tk!yKdNq0zk-kVzV1JUx!M)_1M%++0n6$_s0*I(4V@$s zif^FYwcpXmztGZ}bc9RzRx*CUu%!rrA2N0Y;&c4Y=|p-WM5lc}xMHP2Y@os5RWcF$ z!Tks39tes8sr5Nu+nqinBItd;PvhQxa}dR1;6YAMhxMKsZaJyJcvTu=>!JbC9r?y! zulP5LG;{Gmz$?#qV5A5=Mrj^Ofl&SEs3fSN56WaVE<~=V`~)?BrdAFt1(p8*8BG*= zMaYbpprN1BCdT$k!&2x~9eNlcFVRKcVDCtUvMlB>^-u2_A;&{Hf9HUH>_PPAdpjf$ z<`=;g>f#=9XoNyNfvz)B=EUxzMy?d@HZDI$213TG=!fd!GEgbPzQ1SP8Tf8iB+En% z$NE)Q+T3*g{{TGUy}AaVfnnvt>rV>cbqlvO?~kax+`Hfy_ps)9tp~1JX%h5mvqfIn z6?);COIFG>K?5-1c+#W*{065PvdJ3)+f1*ey2zdCh3v}{nKO{D8g_lScbg;}3t%;L zJuSOu*?TLS-;@}$9>lSOto@ABit-(EQCm1+7`!C`dWa`MoS<^o^?WA{ta}9+(bpE2 zx#!rf55kD+atR4X;(8`;^pI0@{Fh3vza$@Xho)YnS0@1&!*kfFG(lGJ9RC0&PD5Un zp;FCRPzHh7@EJy->xk@JVsxIA^#W7-{{YfvJ-N>C?_RZiWI7WmAc#S#R>x1Z^#E6B z&XQ(7$##H7C$52{_}O4gbor;9&Oyqjem&h zK))i$->F&+&`rQf0tSY^^na$kIr4g-Vm0qg33>n$2ON#3tl%IpuQH*p=YpFN=@4}` zPB)(E77L|S$patc@mjWZSrUaM-zJF(SsK3-%(Os(EyfgDZbFh<3>*Idr~Yy-pr`m9 zINXGP6Gj3x4E%NJThIBQvlR?H6@S3~A*)u863m~nmcR;dRvBQLDi?;YDohA_XbFiR zNEHD*3j#uZ9R=gDVtFN>sgfk>bWRKvF^WsU34b8Y%>3^Vh51uhIPq4G+?gpP9fhs| z1VAizfCN|h8gK8_xAtjuq>_kT$UyF8^~L`H+;=jcJ*oBV&uOGZaCkqE-yMAle^cDG z9qP4T+v|-S&Uo(gegFBo-iq2^5i$V zq=@|K(;R{M9k2O4%RN>{SRc$TK6U~)L-SdQ(hn%rUKV}1((pDxL|7;a1#&9Ht$_~= z@AqGwf3^=8&zg=`cu*MQ{h(h<<(Q@1paSti^Vcu3^y0mSF$?UXzMaDk64)62qnY~O zQV8?G>68Q;Fs2nkctpXFd>B6aC5GqOVI}al{2kR=ul;$GN|95K?LP)|QV1zfJO}V+ zV3=-%j|FnPV2$cNi|l^k=WP*0{*E1dG489t99Gxc=F*l>Yw`wz+P2?Jk6^J={{S!3 zjy#9wh@rsuz~PkFkcvoo&pA*qDDIUJ5$QKfjEI&1YcJ%?7hOEqCBGLHFE74*hka#-Twg6<2Da4 z2t7aqKcYQD{w`w+?6|oGFxvngVKI{jQ~(?u)caQ7sQ9ry!r*g;_h6dj1O?Z^QGCGC zp(5e&$MWvk7qv1e?>zqiDdU<$WE7WaEzP#hih2wq^JD)2&h>f0BR1^$hWh^ixn@Tg zebRbfy98VW%;l#;g#k1ucmpOjjZY+InwNyJ*U4JK_u}rbBk67a{83U`RSP!CU@E{< z75k%f@d0!mE(5{4Bs?(Z{J7qB5HJE(fOF+}^J1Q?9vJVZ9)UL4$43vE%v>xFe#Z{V z^Xb4ek$ovVA8}g6C_eQ`moZkL!iIG+apSB>DiDdU5_M}s zt2js`juG2JGlMq1Q)GVZ>6u{}_9Ew{|4hIU(RSBh7;*}_NQ%}r^u*;2tkatz@ z=`(p6ITxjJThXvd6bXz$;{p|&W}rkR=l}aoORVEArL4^Vs?LHRt#)!xL0vz&wf^S&o zeSL|G4RY$(OhD*`2s3div$vS#ML$(@vAwigfRwlLclzF0SF1tF#R%Bs6oI?&MqDi#j~CQZ$6<60_i z;@`HvL78veo<5M^&Y;Bp421c;egzdkfj35EWYDfus^e4Ek*1NP2gxsY`RB2nn~`3y`HsHfzn9AC%qH_d{ zxi4C%i7RQd7i%>EfgXo}gRXL2VG>tDAZm4uh>z3Po?JS{02&^p>{~8b=T{L*K0(d{SL$5l(9*d#m>FUe1O(dQM<9bF;KTQJs_aN19DIU1-qnWAMBo8eW?8EX%cp?A@o|v!O)h=SV zS)S(1nNrqL5ZPs%!{OEwZQXBNy0;A$|Z-k}ggK(!TgzyaqE5 zeB_%CvO$2KtXu)=(Byx%*7;IRfL23D-&ylrv0M&6soMBUgO!Faf3&io?=+Xu1Vlu7 zH}*d@^WU_P{C6bVT{X-P_*`Hyz^Y=K;SXbun9N127WfkN4Eodl_M)rK%9 z7C0gG+^iJjtsxph6;s?i31wG z!}FpQcu#Zhe01SxHSj9#hu^|K_)cpWAKMu+@Vz%$z5|Jt^%(eyL;!{)<2)VdFrR>9 zec~vTuF&kyP}m=EAdZB^9@*U?MM74gUH~_HLy*+r}Z*__Q>>yq0znnF0Mlc)*b%qifM88iN33`u zW9jP+)Q^gu`1#R|q^rqHE5B1XmRLCPbjvcz|a(h-Xi6w8}sz@N0#muz(+A>4U}{ldRH_;LxrcnmiOf(-ugx1>!-$S{r!tRS zNR4?AYbIml0u#VoUE}@h_&`1l&H6v^CujGgkk70JF3JEZq0auYua$lO0K!Me(0oRT zm!~h4$dZ1V7ARnYF-8UXGejtN9rZrZpOS z&uOdW&DMnz8kop;S?*@<$zV(9GZ^UYNGDMAOvzH_sM;lk6WfCa*X|VrPjJJ*$WMFF zDvId_LtaNRiP8rWCD7k6Kz|I~$ZE(_dZCkw=v-A}#z%4HX|8u?0;|6Q}@i zHM0|SI(nbc!65+Usfmw{9DTnO7j9LepuzrUfOjde`RvC$?ck{uXCD-=>M_9uB*_or zNNHB!?1V(t4(!O_rT+lK2Y%D;5p0~j0Aifw3T4^Njlg2ZfYOgd%)_jvF+xXqEnQ*< zCE!udctH_6#04_VHt8@aO>#z4U;hA&LP#N9<#c@Z2nd6=fDI^cAAc0wO`LKejBKy0 zJ`%{ER0OucK0vzBHwgDvTGT=x!4KjZWYEyi(**uup`gIPG&INl5Mic2wSoQA#EDL! z`~J!Y^v%#L(uTz&A83@P0HcLc@E-`*WXMw72o8W~;71TxS`cNmIg_d&v=77vlQLu1 zH2|t9!u?^MyB?7A3ZuEk_?hYKxN3Zhse0`_J;$jekJZsz*fsa>zk954q3!9YaY=%WrRPmy!VaK&>(2E`L=DR&ACc~G_{GQT3$ zVXyxHa1~NND-F2`kt&D)1EkdaLVwt1`mvvu#>tHwh-WN@Mim*|UC{3Z$D^P?R51#~ zBSgWY3f$69^|S>_fGQ9YH!;pzx%V=a#DNqVKe>7;xGzO@GxcVT!rKwGp~MTBDn}S) z{Bc?-5>k1+a(XoL>0Gs$-L$!fRQ@c`T`fmrVuwWGW(vte_ZrO#bB383FySiOGGF16SoI-I42cjJ0b)_#-lPxFLi6gG#++-2ria^B z+qv^)$~wdG#(cHO4Yg-xg^V~1lj=8=K{$t*~2 zCEK*H)FoUg_R@AFr58E-9RC0VECC5;ff3FUfp7o|D*(cO>6|}6ekntS0$c;@)nh(@ ze_6nOoP{51Y=A%I$S8g5ks){5WEX}s5hoxQ1M8wBs-ip91_#2tJBP+)9DuJiDTgx` zqQ>Ye2rlq?@6bS()x(2$r3MZrG8ZbzmY4-8PjRx}rLn;IxEhOxqLmYq5rtj#t^{+a zU#?+t&=0Id1VPRbRF~+7=D7Gi8UUyySYgUKbD$vrhe$JH;yMI^2Ld69po74~QIwPB znZo{%Gd5BM=%7I-gIepX;6x%(PcxI%mWTifi~v_3xKC&e7zinb<{A^#fN*5CBAA+- zm^K&QGNX*IqlPhf5UL19ujCmlq@~~oRsMjt0JJy{ECF2klNs~dyb~eLW-0ljBK#(l zZKylm!Xa~8ecM-5gZTl%r2_E&z8Q8tj!4HKfunNLvdTp^elbtS4+6}g^mxtp=v5UO z0eeiJBEg(~q8=rQ{{XU5LGT0mv4b3v;93YhQIbnR1Qr4ih9z!a(COfP1<)TOj=AE& zxG_i$XaFEai-Nh%AdXZ%g1?~1Q{iQPTsBvku{ps{et)y?uH z#NdD|ASpr@0Lho_^i?B|pX5W#Ay1G2#&?4jm5?1spwKNRNTWbwm_z&LeNYY5GY1q( zG=U#skFXA90nY43rV%vC1_0jehOHC4D$ykKOB}i7fLs`O(`L+ zJ_z9A3$N4zJqWHmV1%T=Ak*}66{i%|5GY20E)FC|tOHtv0Hd6vMVr)sKPqFRO}sao zfmH?sN-RoL5LFbJiOJEhI%4OB!5mJ^JSo48T}0OkWfK$IXl z2q=im7bOb3{{VmS%wE(>wSrw95YU`sVvuMWO%xpWU=@Xs+A!hWgT3aE3%@b*tCo~1 z3J|H!#Rr@DRYDNAa%F$fWQBskTpXNkQve}K5zJ|v@#-3i5fMag{eSBZf9vZi-b3p6 zqSN@tEf6WQ83i8+h~cVu%shr>Lb&-~Ub>(F?-Xx)5OXi|FiW#7i{eN-V}pYthm?208ZDFM4p0cBVqu&)KYx(GH7GR6?| z!e13LSRG-wV_RJY3JNUZ@AAQS_D|?~l3P(BL1c9lJD>bI@W37l;Oa} zB@y&T(diyPbI$t9t42%CbnCo={+C%N4qaIvJjehyZrwfq0BlD2M)3$#&jnXh{R5y9 zF&XBE_kpI*8doY;tbLc2&@jIf`h0M70bVGE7Or{%4w;GByq%Fmy-v8&xJ-R_G@F0e zxAsR%Rc%_c5x-B1nn7M-i=E6t&eBY9v;RQhTo$wN+6GHDiy~EQwLnUVZbt z=lSEw-}gyQ?sMPQ_qwjn@|lf!J()TzN}*uz=6&JSS2afpm-6HYPHXs?qep4h@D$%C_&bKnnfi>L(7&yUa%~g!6!A6Rh{ci+}AY z2TUJ^1oW_XmnY!4dCtUn_{mt;&4S{|y@E47#p;DuU`R^fS1F3U zrwbatH3nxtbW5CKQoJ@`hzI+MU$Vku z|D(OSO=QluB|uRm&&f$0dZ6^ zJP&P3ak#isDY3TG1(_HY5j?|_VXh_z+G=RUb;_HmNJ#(W`t4%NYfTDv>719#% zS~i|6{J>i$=)|KmyCWpP=2}$?Ps@nZBV+EK{~CB=z zdLwz>I=+U@Qd3`_%aq-P`lj%8r%pQ`N#Z7>+Ki(W7{GLLsr*=zdewOYn0q3x$yG+Xwo1%dIK}oP_Zj|JCj1 zFiH&_!)Gx%3|%e?AMzUFIiVOSuX9D zD=$0AE%7kR<+y}DdnDuVq1T#(5Z{sc;K7H3D+3DEvMHpHaX;EDmD3Z$P$mdy6hBdYd>?F&5{usvo;5`jl`A2bH z_nDYCBF|zs8jQ~iCgM%o9t=J5b1fSY=Kt^ON7ugOr`)#wmgCxMwGdeoOXG@wI{Z+%$Z5>7Xr9+NL+Cho$}$WIu(`KC!`)jY+Mk2saQu)*c-c zO7(XsJQx`7rsR(*Bt9qSV!n-k%z^dPd{ZCC@C)G$p zI*i!Ozuhln3fw0eO#9{4a8Z=E_-1*2kloY`lO!CwqS=8>DtS-nU*J!<`W;G@QoiE< z@>)KBF|(N=UiqUoNm=QdLD0v&JE4~v@m|j^1OB9%7X4IHEaBTVzR~h^y(zWtPWs;` z>i;b_zo}NLp#$z+a9iJl>2J;hCAa~(f=gB(bo2M@l*vAN3-RTgK2rA5 zva*v7-oX*X9M&!TW4SjBfkzRwr?I!j%2E+I(p9rZ=3GpZi;ci~@E-mh^! z&QZ<=$I2sE6If~~Rn@Uxv7(Q;rw3J@io-1^9B+BC3LCJhbQzGTg*Pgm2_?`qQlxtB zWoGWg>>F_)^%}Mff&#qAn19bxqt7aBBdp~$ex9_NvZo=^O-DNk?hz%!(ISs-nU?V1 zSugKaFucD=!E+Z?n`@!3x_nbCNDD1Y#}HdehK92STBszChO19!Dw>?@FGpCfe9IJI>NacjM^!oX1X zhI;X1q6X7QkbDEaumJ_K}|ZOFFk`AIlAr}E;n62m8ROZQF(Y(b=yo=hb`!tJ^etU&fj;| z*KhsteG(J&Tt`D}0@hUv(orW)WB>&ut%!e^s>+zn5`fUEa$r`)|ab~R< z2;0mrRrPIT4eHd2QM~;8BHS^(pNTAnc?-DtOz+d8nhkwmi8{IECCE5`uqlS`S`0Oi zPojCLI{-mT*>RF$AZ;;1ZD+~7Dj})$U`QMMCO?k1d^$GP_qv?ZcKicLY`NO47lX%^ z6(SlA9)ol96qP?bPG1;`hAUhgcX0y-Z}nfCzAQs>E9SBFj(&F7HMm%`=hb>~(R9lE z%icl&QDd7TW%@cicyA=b5{ppnUFWCH$3Z|A;cnLF*N%0>)1E|?u<^5Zho2ecI2}H? zWLEq2WlN%rm0`d=U9K`F!zLl642jawL;jHb3A!p`y6sz5P|#B#JsIGX`0&B=-8&0> zKdu`CiXc%o9hoF&QVU-*`?we^F_&iiRp#HEa9h6U{EYsU!cSC>hOcG6ioI(QG<9c4 z^lusK3ca{{cO-|KO!)Yl#H*y$(YRmsgZfpRKi)OGGhnp|il*;>OW9v2{nsDjC25(# z8drkX&S3la1Hc+f{}{ode@pB?D?3KyxYBA)dpzGX^^5%L8$7uNY@f@RT4%-8emW>i zGm|)%Epi@CJ2C35(7E5(%zIe;oToom)8-z&!@d5$fP^PI@(SOKvv@O0C!!6;lHT7c z?1@O4;u*SCORS9)V7jT;92~TFmZ-dM z`svi^JGHL5vP5Dus>hHnFp3-yyo(Lto+hdV&>MoeotdGP!P4%+Y^hYzu%XY`BRN*V z5>!@mdEY&#!X`-Upu04qF)}Gbjp2tZyI|Ly^f;xQ>Is5kvj78jL*WMXECV}U0!V-HcQ5u(QHI-H|CSr zH$dKheYw>owcibBjJ#c=>)OE)?+P!@6J9C9N~rBmFJn+a57n2L-hq_k#Xl!ocWbQo z&fKARcyhZiTHXXtXedX-8=(b|~Gomagy|ZMb$F5PZi#F@Gsf_kQV!PY{7tJ>qMWhYOU3#`?%PQ0)AA6YaI=T2334 zP6|$3yPhpQBUi~2Uvx(e#mOu1E9L8hloY3N;@9Y(AVzy!-nXuVQpDKkWXE&WSRcGl zmn@APS_565v!E^zlcAt!`>(oMX1dhi?!C+}&wN9#FY9uYVuWx8m!${#?1M`s)J&{h z7DscA1KnD(t)IuH=&1wwA!q=aon1CgZIw^V;zEp-hRV3C4>5_3u)Qw#`_9Iye8*ZD zkI-zYKUq42`N}BFG5A9I#I=|@A8ccME0)yHIO*!CDrzX4D?)rBOMUG`j(7UD^-x(y z%%9MnOSRAR|4HBKj8SK8yREo!iz$2~7CA{N>gb}Ackh>*DDms7RVQuRv;2P)d*PM8 z$ezHmj1MRARaf`a1ObwN*~-V8IFe(}=qQD2h$hv1C`^f;b60Jc@tcP&H2LiSj#l+d zxgv;ZS%U+9&1T}x(%gB}fi#&luhmv1mTw$90Q$=UJ@c3#W_m1>%W|(IIic0N4FdUZ zr2I<;AK=KW<>bN4iF{9uqOFL<#feoJ z+xj(fkT;Zd4a1wy%iqL@VmL>8fh-H2B&8~k&9pK>r39g`>4-`i z%Ngm)iZQmR#`p7+>g|4C#G2$SZ{AZuaAmlOlx|DuU-328h?+Fv*Xs#8$XcYQP94B) z0RUcCJsv&T2Q%%<>M4CljL>=uK=2#pStN!>^u@@4{Cz~bLz3atYgZYUQpRP&2kB-1 zD3sU4-tJ$moY)aN(|HAo;iXh=a!O{Ivq=YqT$*yy@{d*Fzm@1>8C&@XDom#lhVR#O zd7HB5W36tk+WD$N)Y_H4CH%Pn|D)K|EE14zF0w3&sxcN_ZINj$?D8!DRH(AoTSzLw z7EST&Z5K5DSG2pZ2HNJVMa_xyGZ7#=$m|tmKq`6%*IG(5%9T|FHc9tp&%Ap=*0~nE zUWa&tJA~CL?k_Ix

mI@fAPT;-ApnllAvV(JSNrM?v|qvY0BDi%G*w$}{TODceJM zEMFF8|04eHF@OAXesOaqbD!qsARftUny}u)r>{*5jPtIG(oychXz2fBiVDPmFwn`C}n=u7p4aZjK3o{>NciR?-<3JtIgTP{?s7v zBt#}E@+>6wRC-FBgVj4Jg9ceOATS!*PD(?TrzS4p^}Wf7Mm33`lS}$B&5GZV#|qRI ztg9Di-bVG=qq2pgMa{hWk!ABX-OoUj|@kxQTq^vL8CX{Bq~^BB!flyq)popsycuN-~L%1i`qE3Hqb zU;m*o;NlRw;DC7fk=`CFF~1zw{W85i%Cr~DYvad@`H%SvWCE~bAhorCar`4tCJ{^% zkTd-nxR}|`qt7|?tL8H2K(LOo7Sn37JFZ@J=N=R!g~ zLf&lVDV_003=2kREF`jod_lIS1+hW@QGkc@`mZd{oJySfY^D4yTLQc_yxbS`_Q8qT zOh=1BGC4G!;4p@s{3vh^T(hx?PV7d*6|(npN_HO8HugH3`is9R28}0-f~&~~`yPM| zk76WlLnJG`K0Z9pO*s3#kO{k|K! ztK4j!uM)7#7eDvBK@ zwp3P1RkIRkY{T)pWH`QLvRd^XM~l2WL0@~0|J*UDGa0C{1dR#9S{7xicdUzFUy8TM zbE;BeR(2fict zvqxDl>e;+G-vDnj>MfJ0(rO#C*Vg#AJj?f%m}(R7B!b1VjH$^~049`}CUkP1bfg{~ zrjTq9uYgK@b$#dC!{49dS~(HaXR{p57_SP-xA}v|uuV z+9JcU5_?yR4#i3gnpD6F7rx3AOhX7^v0Bq7%>0Qg;;x)7mW6Ck+nCF%@0pyfQWF3X zrT6b0cNBI+oa}sN);-ui{fz%dVP4`UZ-|sf5gh`*lZ8TLqFwIctiQ%9fpqKQlG)cn zmcc1GYYdx9l_2(2LQ4HvY#`&rFA1BBpwZg506CQbxn!wNz3B~fE=c2IbU^&UMwyX} z6g)z-wk@|!Vwrkf*~Z_xm?z8MBmD2CJ<4agNeNRMQdE5acIY616N22ZhITu!>-!$W z;2bU5IT%1GgAHgEr#wlr>yTxJ!`Bba?} zl{%-^Va?!`dN_*$Rt_=s^%NU-X>k@DqIYX=jQ% z?(~j0&l-4#+{w{Q9$!p{7qr-yU)#_Y3HT<{CE@1osiN&@TrJ6jJ5fOYIKr6b(LJsd z2oPA5T_0UV1-OIf?7Tyd%J2TXRk9=Ah_lCPr{}RZCD=f6~s;FQ~(~ zS4AQxC&scP0L_s6WeW+jf_Pa2DMvCo+uN$B@_V(n#Zdx_a@HnNJU7em zOzq9EgilPIddij;$Y8qe90X7wBQ2*DvCPf%*O4_FuUj1rOjUt9;`L`bmnC_ZLMw!@ zEyVS6-Vkl^gThyf)z=R?mj0@g>>IsRCY*@$ZJ8a_o7=`t5LtAUGWBRi6~!E7!{j+; zwJ}|EiZ5sLi-?E7(i?k5j&HrqKYq>`PKL_ji^E=G3UIdZFh4%BKRD#!hkbY16zQKR zh1}i9sDZEhC7#tI?ILGV#)q3na>S}UH(Fc=$C=sac}6==Rl1Pbn_!v^k|U`3#W)dE zOjVY4)6tTFm`n3|u}_S#{A=w-jW0~Gm^**b*C8lxhhv9x=jFV~FOTK#*o1rL`J1-T zxPuw1T047x^idCrnt)AWz`6tin0N4Ka zl~6{J*3VB6%yQpi;<%HnN50kr=!{^4uEAE0i@A!Q1KcMQTuh&MO2m!EwSL*L zv@p5`mx56N%}gq?hR4LL&%OCk>&xBwOY z1V`k=X0T7jXcM2I|9L=m5ABx zscpI-TdW7iU12X1jp4tEO_JPP_4VU|T}nCHj%L77VtD~L8BS8&kox?kK#W=-g-+?y z#pOPb`kQ$Q{0q|1Tx!p@dtPwC{zWV<8`2uRgH_t(Z^cW1xBD_>{uuQxaPV&%Pv;0B zh`bjeMV-*#l`+A;I?IRhy~WbuB^w6~2S?qR?9cK(&s%flFp-%py8c#L2INKup(bzU zf!TNU=)X^t=^bR;<85X&>iD*=Ov&A-+VBV^7w_uGWTI4ncq9<1RnYq_oVkr{A{fR5 zDXbNWlkilSiV*)YUD-I~H6QT@;YWr1B%u0Gsx4QEcRuEqKvr#A?3G*Bu?%j&Gk*Yj zZ2Bja#w`HGmp)^+i(e{0$;Ucqt{~5Cm2K^U;uF(4`W|%F?85lfdkDOzmQra6Ri|~< zW#Ln&XH0ly)L<~pA>mW`>iA}HSz1aukCs-D+`)hvc>1gqq2Z`ZXV7Ee~n&^ojvNHcRBWB6CH~*XBUfqt7~p=yB!dVJ5h=iVG%r* z8TDp77I|Ef`a{E}5f&wyUjObBS&k!arSYsDW@cuobeMbz?Hx~}vubw~f%7Zz{k589 z?*X=CNS$qv@G8wxuR!A$t@VzZJLCSZ*lcCl-L@N_hS)jd6j-`bwwcURk>TDk+#7oFK~poZht1sG$rrH>}NzZHwRP zo7en)gg!Q1wQ>m8l4rBC4M^o#L%;P@9bVKu=4ugW9B{hq&c`IQNH-WGCMq3w5zGF% z-vUsFvl0>cyd|6R?cAGM{v!VIV2v_N6?IXoI$lgMKfZp{r<9Id6udnlNp}vM+=<6v@^62i0V%=8PIFW+-nX@wXh%GMDw+w zj_HuEn=02^qNGE=&m%>Pndek?uLxt@E;Ed)4YeEHMX|PmcDfitK8+PLjB7~qG(Tp} zPQje^lg}w>5Vz1kP%u5p`7sd+3p3q7SJocv89a$Hz7e;B6?u7ltb|s`4eM%^WW_WMB||t6WDq5v~!rR{bF@V;Ed28+*&_V7rhhbTsY#MKrO`T02U!m zoj4z|uRO2*x;YOB+<7&Q@|T|d(Agq4rx(cdMXq?8w}I|0!1q-+ZlS36J?j_#n^&IW zW@4nEctlK9nXrpL$pS9;!?OvWl!bBAt3BKwj~;;EI3strF-{spL5yr;WP@nkJQ{=n zz$9}$EwPH&R|I<>v=?;g3SO?ZP(NI-+qccM&E}F|DB%fxYwC^JyuKXS1@ZuHGeiI$ z2>qJ&`neK?Tayq*uTS{Z7DI;{5kA zk$v$u5W_m}g6M7FmUum)-qgpfevG3_P`6g33&16%dhW_$3JFTSf5Mk>5ZaiDWPU4N z%idfUp=8o*odeN{P=@gN*IsprDQA@WNwwTN*qbR)%KmkoC>01v)G9c42)PrL9*{tv zPQ{N|93^fVl1%SHvH2`e7t3#3KcL5oO-w^8LrZVxkaf_vbgN&O%G)#-YDLo2+eUAj z@wUVks~(|z-lwDOKrbNb%Y(8}D$*}=a=uQvH-iZ%cCDV+QtU#fTHFpy(snJ5+G4ti zefok|PejJL7&Ouzk7na!>MI{Ej%98_y?};9j~NfDr?6F<MfS3)vnw{w=@LYJc1a%QOm)I-`*z!e52ztrR|?!miI^(mtZYKtTV}W(3)JMMrM#L3$6}gkK33@ zpAU>uXf~T2#U6LuikD`$7a}GZhJwGx%fw|{WKcTes{{lV55UT1a_G| ze%?Y>C3S>puhhp)H`3Pl0mFn*3uOxS8cInF`4b=Dfc zVuPJO5C8uAnR%mzD|4qBbP>7`?^=fd<9_?VT6KF`(#EjQmZ6ihbLhXw-I5B%tBl3O zgU95NsziT!%*eq(QHw4ke8aBPHEsz_=DQ%`zP(KE2Hvxe%RhD1@QmkHgpDg|Fphdm zzTsFvgBd0Wd~*UroH-Y$Wx4y6`n;m>+qyg0qgBl{Tc3^CIkt}V2WUSoSI8NH>a9+y zAk%9Wac3pWIh4~twPBmr8|4akq-BWi&2|}oEB5Vfn&wVzEdUN&OczQs-sYBiO9II_ z9n9)Yp6VKS^WJ&SbgHnc97*b%AF5lbE-c-5av;JB^<#?^X&fCd@GOLd#qfF6es(EU z3E)B)_pU?um0jGfV9P8=S2t+x$>UT^z$-}FRMWzn{KQL{iS%zu$waOuv>z6$Ad^_U z?|Xq7!`=&s3>BM{6xgCe!aE(N5{=FwJ0O4KV%c*9oNT=%T~se$ohcZ4q0O*3v3am` zBma=w*z``B1UyB{WnOa)5*81Y@weSY;f*Q0#lQG*{G4e+y^_ z)X%X1^jXalg?sp^E~C_PesC_J6{kU5xfS#eYULX>OnF8{t&@9CrLy&rpjy3sL^vEz z)%ck0<%jl3WuHIu4Mp_?$JIpR2aXR&k51b8iC9auEHgb@QvF)Ij8d+2!*HlK!-Nyf z>KF-m7M)uXS@K27bB2Aj?CYmoVK3Q5%^Ii;di5*;IV24Dmg`{~52Z1G10HsxcSF>^ zwKGK9Pw#^4zQYIO-MnJTv^psRdK$UXRgQHJU4dO(*#6GBmMq74ERGoXGt&_qYtcnt zG&S+5Ap#s{tnH@DH+J}BPD)toDOn7;n_8s|sG@f4F7ZTDme9Q@G-5L-0kTzxi zwur3$1nYu+qE7NGQ&qe3G8Gi*qa1u2W@HH)p4S?yl>^+YK~fNL4)Xw@WNtLT9(xxemkEb>Rhr=aOQB7 z$DCIz$=AgxHF3^1I4(um6EuynUz3sb-;ezSg z1?^musd;27tU;Fta-N7IbIH`pf|GYmZSSVLEw#A`r)I?yc>lZqCdG2)VkemcZCB!5 zn<*_V<JRu5kVi->0yrjSUZS+48kFautJgZshHw8(*oCH6rsV-q3heIbnF<#K` zsGm?T%)eN zoCr870_nY)FJtbzM4$U9#HSho9-~+HeK~W}O%#ihag#UxZX~Fu>*cPpRbyumBrUqo?A)qG0C z@GOCsDq>e_6r@#-r!S8X^L`??_}lTc)Pc!JUtS>RYV4Ic@B_xH9T-!Gkbjx(!kTP| zltgotgV&%tE+&YH07OwI{BEZTG&nVJy!H~bolmSLX)}?YuqVTEv%^l`fbts^_t>IR z%}_iUO#P}G25jrhtG!P&OQN+8ym`ufg!(erUh~63Jz?`fuyLrqmo1#9ay}X#tQ`>b zJu0q18P;_`%P8WdzVC~C;JtIEG@`Zd8~ar z$na7-W2Kx!;EOj}Pg>p;T*gUW371&1#YPl#Brmu4b2urB=?nbH)AW<};-g~);l@)U zhn9q$_n|xdWrJWdFu|Tsw;;@mxxN4WJOQL)C?frE!%--n8T2827RJBuh-mMFZ56xq zfH=z+|3D(Zg)jh`{dl`_^$Av0)E7PMv|3FY##L`w4vJ-NB)Pl0=zagn;BOiDDF}$~ zxXI^lJg73-wL$2R}dHB?Z75GTiLoc)z^8e9Ua=fcD1;UAC>z9{` z`pz&yn3skV##@V)RZpsSuXF1w6GgbM-n2HhO;?s^SiQh1^#mu{@t;&<})_4+1LrrjA z8<_g^O;HX(x)=&Q5nZFiaoYlI)7a%cDT5u zI4LTG7G0n6H)S^qf9R?#AvjFA+By( zw^8Vk9TUEGhbL1;qAO7=_I4Hre^y>lW1f~TUmvAcrc3N>Cw7eBkI|J5&OTT>TESTI zQ~>5IIJwqR6d%B(f{OlvZx+f`(P@X(2XEPXxjKo(W~iE*`osdiPQ|nXzA>aeQS6;@ z7x!RL90DyU&7Y(rKXFyh)mP6zjrT4=VtV>In{E~iqCu*c)+bYN*=xksjO&Z%+Srf;yifjo+#W}7+`NiKR&V|X>;;s_e%m7df_;YF6Wdnt zih6O?L~fh+O(t|AR~R?dyvJT}T=#Qe+tdL;mpTKc$Fpzod5Q`HzPxaDd%}mbWwc#B zhn;Jg3KhjmTZ+!E5{nfd%ihuAwlvig`HLUvf>~+01Q?r)a>LGYc3to#8*s`rAya(8 zv1G6!p+%~$DLB-hSuyfQWTsk9%5)Rc9^>s%vR}D4p4xV?jcF8=L}St1F+|>>645M(c^IBe<275ee`FWy{36q{ zK+rN{7KJ>~AxHb|iMVOoia%`Edags-1dTe6HnO(_{xlr zEw2A)tCQ+HcnG8AYydNB_rWt0{rwzIp1h16I~afUpo;Ew@xk9nVnq&-;})-{)%Q-q znSEht-G`C#14&O%{@sJ~DL{QEzBB?==8%rz$ot-wXuzzfgH#j^jzO;_*`Lz*Vud4! zIgi#C{G&-g&ac0kP6Z=mbv*$!@K8@xCcJ1HD@3j4*=h#Eh25*d*TSjjeLz)D|JNAX zA+^9bfRRZyjR(ZKC#R4XxDTJke4YMOyDO=9B^oH<6O`M<>OyAM1H|#Xl5O7s0ufg| z`Hzt#`DwWR&PE%WbSSx%KHh>_8o0jMK;8o!S{l2jBu(4EpzI>!OS ze45Ih6)NfHfz4og^$SnF?C#a@B}QL}$w$`ZL1nlSzmaGyV51VZEZBTU=O;tEfd_}; zbGKx{sF+MxRwEAs1tdLo85W`Ms; zgUM@_M>I|VOxg1t8>4*m3kk76Te-3iA2u+1xT{6O%W}mY+v0*AY2Rs(gb_#e^xekz z#`@s;fz%r$rTiUGn4;2xbyIA=UHg%3^twxVfj=xo^y!*T+T@4S0)7x?7j~T;X^5q_ zb;j~>>HbiwF+z9-(wXuy9*y@;E1w+E#K$$TSW%-;vHbRs8@e~)4Lr#cvrzzB%a1CS z7?|);zzK)fHG((J@K*uBMRDH!2 zo0;?ev?`X^@@`pBtFYTgh|iG}l^e3VK?}hH4AGZv3059&qz}oX?fB(J(plr2mg!YB z^|KO~sI`%9Ndxk#fjiSc%eu6=wE2}ceP6B<0%LZZS3hL0dMpz92yH`;&*4xkc@0>d z3#(N*z{;Qb3cTxcssHq*htGFee%Z=)#2794yf2p%}=4prKeJ(aaKOg%29@d-*Yura|5 z#er34Gz~2k>IJ;@8=x}rb~^WFooYx2kaP3~8C7_*oY2ni81%$Y zqa`1nJCT*PrAu&t7f-zHmUScjBp@`cOFV5czp?7JURv1|*%hpfpjm!ME7@mSus0-B z0jt2NPKL1DdvtH}*~Fy;0GyU%S}>fA2%+#wqhTM}Ff6$uTIReC-32C0r1kID>TA?K z@vKeZOZAL>Jnnf#tI6wqS>zlfDQ}i3<;M#xy(??MTRi@0JLW9Np|UQfMYeGL3&u-70V6yx7fwi?U@8OPRSwdZLCfv~+gnNyT$=O@*9w-o?tKcsfRdW^ zfnrj-jOeQAx6ATOHs&kZ{8Cb>!0Uu)lVaIW?-t1fhlFZG;XQkE(4^xKuiNvqFK{}s zs`%h)HGlt_R+HQZ;K&(8^{b2uME~Z0cou{dEJ`)>OiO`20!&3zx_yN71>$-{CG35y z+uv5tK|{~SXdwy9H{wNY86%tMP6Z(lWFnLci5mg*h z3m*C5uuQ{bK&C~M5SCR=dBYSWE%E@qQv2kHVHF%M#(uefhDzdX%z|GqRaSOiQ`4&+ z?FKHjwnFi*pBC5ogK#0Z-jAb7K4dOb{nd8gzACZ0P+RhtT|1v`IcB3v6kw*Ky~345 zw4=G4N)wZ7T;B=KDS2|2ckl$Pyj zmQbebn8y&K@dPuD+84IlFXwePqkaz5!&_0rrnpdm_-%M^T-OXQ_wc?UQnN-}m)8)z z?(w7g1f2f*rxw~{e6w*N)a#5Xd=7{pF3A6S3<#pvz~*u#66M7av9EMTylSHIVh={z zT5fEOM`cN|lq}sT+FVgFN^>L+}APYGmzK(S@h2&OJPNf;G%WzS7Imh%zETd5z1j9L$D zmI~pa=ys(HeT1Oaq!V$l*T2~fuIokM`<8jcgSXDq zZMn*v>~wKhHgYiw>Rb6*H|&sPK1U8w#%zE6glosd3?Od1Ar4(rXYShvngZMO;PP__C41k=!d`yv_858qws0jrZ8B zHm!jB#Rze|f#_7haOQ*I8*xewg?c*BgJ5SJOk(A+LT;-PmQ6JpxdEwBulNaT!pU@2gwU=@6{Zn(AW3 zKMLPGkh z5B(-}l`D%mVnK(d2!H1ZNCO{FXzvpjf8vHqJ{b`G`T5f?WM`fA5NV(nM<5~V3(E%S zOLs8Oqpfq@J`xVkf*@Ip+zR+{656?qg}d4Z$^xhz@XoRpNp9fH(Ed5Q zy#g&WFBn&{Pl+Ecn_-I5@I~grb1Nl)xjRI?lkM5Osm~|WD@qH!w*$at=|F(EAkJx& zOVSiw9A#Ojz48PtTIA#(od)lvS%j9Wgcn$f{R7{-DA>XC88rz<(PM&Rm*c$!M{fD!C20@wC;CwR$8qchCuk0K zH>y<_zUhj(oz%&_+5M#^`_@(={Pn5{iq*kNux3&d5599h;`A!7Z;xB}{UJuMqF%m! zy^FV`>-#Fs`CwMlP_*{2MZRWE22n-~o}8{L*_3LqD@#AgeW3a|8F{Ej41yXy(hceh zePS@s=)o2UB?4r?qx+DDJx7n#2?-On_d>R@Q`iS{{4uTRry64-}FDSLEM z!elR{FEl65gPo`UQRoF}o+lqkwN#4vNquNANV|6@Kj^!7sCRnlQTyQJwxp$1M}d#B z`U!cFJ15LMyH!zSbt8b>eQIr55d#09bt&cLgPFGu`>bXi$Mg{D}1bD6*FHUBm9bW8S7;l)iw@IhQEASLL`U z$&e}A1BPwhQx1+@v4RbQ#Z9zN^ve^w!vcS8E#v1EA%XQ*$BCCzlfCeThy&-?JQ|)<*D+nhcgQ+ntxFUR=Q( zqz21}3E3(#jXE7u$D0LZJ8KdHNw5>T3oDr*r=$meDLGZQG%7GI zvDTSLlf{r4?EOHKT^m6ui5>GY=e$>3xYuuTTOJFqJa09>Q4@29K$);7*QF`sDP@Qx z|DFAVMH)U4Li9)MshaQ0Zb{o4zqCf}iOjE4R(^{}Hivoy1SQA;qlTEl{o-?9aL4R) zv}3KNT}HC7o6S3>KB7Ly!udY|R|BS2DH&2@W5mKIc!IZli);VO9P@I}-7jfg@-2TS z(ieP%&l=NjWas-QYVSI3w^SsgIM1g6t=5YSq#I^<`6}}Va@Tv=7jf=j$~N^DsB3)o z`klnM`&gVOPUJZv8{MKlY z?oQpKbybtPX{&x}=-9Xs4K2*)ALAQUdF1~-5ux0E%uDH&mNmb!0}@^^P?$R5Pa#jH6`AuJCz7z5 zCK0E60(Zxy%#PzrllM*IW3=EN;dnyNY zS~`(4zU4sfX6CQ0u9qpYE*bTFM~U0D)!XuNf)&+aW{i`hE5c&>n21(Jh(^kMW+0hq zsHVy5Ieg%#D0bu=#7A@-ay)ZLu`3<oswl{Bhj-;TTDl_y|Etj-4LqsJ%;0%d|F3l~JzL z)+hlQlX5)e(Z{Jw6NF37_|+h_OYS^h-el~tCdwa1WT(v5$~RHEZnAB{ywl##Ts6$^ z9@#I49*|`+g7x^-VQ!}(8bsSN?Z~tjRFe9mm17<>)=*o##a0%79FQq1aP7YHqIL%f?OTnu&q_|v`$-q3Ux1IF`vMHP_(84iIWgH!EJM^fqo=AHS72JSqyB~%-sj_< zo(^Lh(#-Ma9F_t_Y5B>E3!UZjY19sbhjizRKcdAS@QKz-0EQ4}N0@_crlz>>pp#K^ zzzkQeC}xP}kFF4?Ldt(0)FMm4vw%Gp*DFxR0r1;ujZ~FV! z@){aTR(o4DdJR9l_O!Vl)AdwsE(Ou*$zd9u2ELs-KSiqsq~pmxkEqg%l+_aHVETW( z2aB~#lZZn4p&eX6(1TPut|~PLdK03;E232aLUWM@6F)3pkv>jo55(mW%V?`t*%Ej?`#cxI6XgkFh0kU0 z`;dzwutnrZZoh!nUJ^@!2}`7uVacVTtqI%_rKdS+C`bPQv!c*idh$~ow{?y&G?n#D zNK%wr!8MXNL~zDF33?InCz&E1l00^bAF#d1@I#hEf;^Hm$Z;V@C|<+&AgP}sB#2_k zJhEJ}S+Yb4`VsP9gV=@}8XF?6(akg7$l-T?@>sDeBmV#^Wo3|fA&Nd{&%DL`V_hQDdi}HDgnDVG7^6d>(S&s+HU9vFw0e2Mp%e5SKil6qWgS+JO0@NM zt}2LJdM0Z@E}cthJbRkCNYu+h;Gd-U5m-M2eGM1>-4XOr4q7>jeUV;5^ozRSTp_*> zkrk2VB-Kqad6_)UL{-9vj15!TJgWWm`xp0%AJx;;`*TwK8ohezH5{YLKJ%xxStZqL zZKTO{Gm@*|U4EqeHxH&mVAj)zxO>P|Y&#T6c|6@I_$VhH)1YO>c{A)AN%|vIH2Fj|3LsKRP*feA_?lAFY3p^8WzAEDI>Q z!6mmN!FGt_$cwC#UN$^BBf$M4z_}D@c52le%k7KO$naGO?jo(<;Po^&PR6|G)$vv? zXln2|Q6-uisqG(#yoq?K65AUeh^1tgf$W6rQL(+LidNYDBH*k((a9K#x4zSdLv*a+ zoD+q`r^_^YM+Dy5 QDvR>7mb4*5mNm=;r3$?D+BW^4{$C^7HcC>i62?^Zot(^YZfbv4a5t0iDC=snLyZtw!YX z{{80H>FDV6^z`8E`Q-82|MSf4>+AOR_UHBg)q?=>GA&P z`}_Nml9KQ3?f>f8?*0E57Z&UE{O0rB>i6J(et-Al)pc}p|MTVD<@Mp=;tLB3C@CrN z@9*L7{LReF{OH=}?)!Utd$GR7f`o(vnv5%Ku{iHVA0X>R}c=^GmwRasn6Qd9r% z;PU_fEGsHYcgMcU)9v`&y}Z2A(9u3WK;i27IypGV$H(sc<1kc@vc=w`q@}2-sjk(N zx3{;guB=vn$sZpdxzFfJOiLvuCc?qOGc+{#{{JK+BA1z!Kx3L{Xl7`qKHAyY|M1v9 zZ^T4KMwGqe)#2~9!mdq&E8gAR@!!(l&A*F|kg~F}bFxTbVqwzb^s2PHE-x?F)zvs# zl3P+sq|Jw=$?cw~wASU#VTa3FUSe#J>syOKP9?@9I}xQ=gxoC`^oEiALS?|K`@r zgr3HAYGIJ4%h28G#oF-6*5%W9LAk-so12@Go1|`(Rm9T7^7Hh@(%aj^wZ^!t^s9OQ z_R&X$zTeTWc$LKe|MCC+;_0`PqnC?#p4OhP&-c!;?&{~o;Ise!*imn${kn>CdV!X; z;B1Vx;`!##+TVz)+O(yZ58E$j+{Vnq#KTz| zYjS29L4-9`B8Pi%>hAL96PbC>zt$>_tQfum}Bl&(Ty zFSyaC@OUV|&9}a=p}|-g{r2(MlVz^8&w4Tpij#$3eM?@XFt^RXrMQ!Twp+O3o0Ux^ zKy@3h*7}jDfYqUbu53leW-PjfWAxOSiDOA=OE$%wF9a7pB7H}i00Fi;Nklp`WQ{@l-*`2-0fZ+N0H1z=EAg2 zc$o*nf40@3)zbm^0pK$=594~hT-KQ*%&SdSL9LG~PH`p$GJKJy;4@l=$my~V1($dT z943P>gdrjlps5quo_)<|!cJrWAJwdwKs}A0YL4AA`uV&6_`JO2huGy-YN@sK(bdo; zZ)ml!T*HpZA{P8xchqoIV#Q=YUIl@Lt}|Jyw;&VEgYZs-)I>HP!P962=pG!vS8A>w z*L=%Zzox_C{Oq`#Uoh?uBOeG2`gLg?h{gI}8^4dV@bY871uV6-8rM3yEIg5=qSw*| z-nPYtzhHN>blGPH;s%6L>$P@VpIuyVbPe+yGZYQLLNg)abpUK|Ji&lPNo;7`A|wu7 z1gC}+Qb_`EBH+&0O4}HP9~@bmYEffUeo3QP;em-Xs2VXr-1`^V7Cp zl?R?y^jv2XfG1j=Q7~IcRJd!}5-lh)L18PSSQ7=j3>=zd&K457|H-jlq_90{js=Fjb8e)Jpzz2fJ z0~%E@Yfc-t(#~ZWQ_UHgp*3eU*9;no&n9T64QV{sSonVrhh^vBXz+K7_~55q)G%Y) zptu9N6dkpNOgA{`ruUX&Cx{?~?}^|9EgaXNu+XcJ<((C4PNZ!U35QvGGK%nRJg(-N zGwuE1;CFU&FktRG+A0=|=|pgc=@QFNNQvcgb9!%+5deWECq3T7?S>`IKB3*rj?Z+> zW{bFuykWWG$29{>isLE3gPH-{pX#N5<{QKE`dH}M&(Wq$AN}h4(8lJVJ7Nl5!;002 zpo4=x6JU%4nXV$1LU+N!hN0sGhql62A^_P7KxmS+d6}KrN90U7Oehr%z)&;SRI{|l z|2gpPA>A^VUP}`J)c>!eZqZ@e3~LSz3Ni?9#gfquiU4hZBeFE5Elh6eeCKWkkZ#sS z%{AjTdal`l{Z|v4TWD;|sb*I1&-KyI{5kRuJ7+f9BLqLlbD6(vifPYTPM$#RLg89O?s zCn7N$(^`PW>_{GzR&dBQJiV^dr0aWQ3oUQS>rj<{UA61mb10-fwSK~!Vya}f7F&_* ziC(MQeeVE(ti}u^WJkz#EaNE_aHPbHF1(hoefn?PiV71N51g*3>G{3f(4(hjTo1h? zJm2IpFcUic%eo;+i*Q?3^*T(Bc(O6fYaD*Jh+-qiz-^$1m~AEUd^?=GaAqqTYXeE@ zf4T(P3aV+kg=NF7LpZ4-dR-t8Dw$o-Y9n0Q4l=rP9PTiM9c>XqmsQ(Oh*m(%TG`J8 zd&RQZPzZwQf*C2I5F;^P!9nZRb@H6L%ID@hqA_SnFS18jO&9JuT`%Hpl4t9Lgb8$z zeN$yd#>IwSRrw!hu2GIq6rs{^<(P3BDEV~rarwI0wzal`G|u@+MwC1gn`u2aX&k!1 z4nL``LoZCghTbsW4-|0McTQM5r&@%dWt0!N4$Hg=KAY;j{Tpjj90EXM;!x_0CW z4gv>8j~WifP@f_p0KuOfV$vW%5{~k}jxhy6a8tv#ez1WlHwi9UusA*+4`;38!xWj^ zmTHO{T+J~97L(lYmJOY0^KF#y>emWt`DJfS(M&;HXZ-XD$or`~u3!c%Ac|H#2yl0w z0<~c700ADk1y4V+07^I|9E9QZ4yO7yM_sWT^2+j0Sw0%8?kSGla#eZKePTu3LZ!Z)a9X+uZi~@OsX=I}F%KA+5yB zN0HoojPJ+SOV{yzob0T&6}5VRK$p?|KzmCHYYR6yk6z;k`pG&TJxgDh#|Squc1%}QU6o|w` z&w4xt)Tv|`Khj`iK#_87CBJDpvm;V{0*AY^*5LvCvfE8Q60^D+#XNIkXgGnx-DT_b zR5{E>Dmd`lH9vQ|&t;uRxhL3jlGlA2uiZn=TK0J&xD#>7mHKZC^{eku$ZgPy$_CJa zks75!355<){rHoDAH+jgkC|2mjf>$YokyL{^o_`M=CEulJOEHdAD`+Qb%oMLhyE8p zPHBHPK!>=~KMjx$=@LaK(OL*gjx8T7MCD6Ky8s*pInp+Yq{-|e=GMi`H_Nse{}dc1 zT@@T6$)>e?{I<59UYJPkzd+CH>z|8->6HyR;qGHyBr3p!78xlik zLPSc?xDhl;i9!1oH|3DADdh74VCAZuc@(+A3O<(RP3SRaQL2*{R--QCa9Yk|AVQZa@WXZIFVuq{_XxH#I7f zaHO8Sz8^c7l`J>`u!`Y@(5S1kcBlsy1dt7cJULpk!G@(nm}_8JD4~)A7cx>9;*Z21 z_>un>A^8_;B$C28T9&NqS*`lZdo3f(yt|o^MFsNn%*%WAyi`XrX0YR;))>JaBZp4wGP-P-li2m1TKKUSVMQDfg5&jB3hRe>cIGNzBvZ#Ey) zXMdwVqu}7}sbkOUmi+Oom-auGyx7Qrpf4tJ2(Bc$)fLy#tu|Y#In;FMpBqfUfevS9 zBWL8fQi?Uq-FnZF2tB*9P%_fgoa?}hyD~_@F;fkU-U1DugO0_)%n6h?R)}2Q~6%68?B- zz#$O@2PPy>sE`D`=T@&*%vE+UxSExCy0)OfLGpo}bpr#n_4cZR3D8;VIkL^6va;Zs z1Y~1=wE`NTa($4(teI+L&HFamj~y~WZ)!N=a;az2N0rs~dQD!WRWvvRT`sab3C|W& zIFQUk8>s`FJc1OsV2k4H)FQUnPP7b+#26}Thwej}4hYF4eSUA2MrYjq!T)mna}8Yo zV&|wfilDH-UnsN+#n##4-A=t=t_#bHheeGC`r4XC^TmQ`x2r{o(&+*_u1JX3oc%vg zSS-m0SV_JG#fhh!h0qNN0S*a2t-5KA51xOV*i~Ks)?8`VQ+o^!8Pk;qf?&qTK!*|> z04M2H=1ENCFn9Shz`;Esa)@262rcyk9NZW7nXJr`1{}n%G&t0Z#+toTLkeA6>(%66 zch-P|Auwz$6iU#5133LuqhivS4SS=;Zo=R|O~B!m{UV382LcDA2jN|o(P^HZhmv|Z zNG&{1M^ca=yznVNb2zUrW?5690a|1_(l5+HME7Au%z<gG@ss=~Tmh=6 zNv8imPe=z5laM-dM^J){Kj=vsEu<9(3AH;>6HNuoB-5vF`KntQP1G(T2e!q8qu>)L z&M=9>!#)8VN+UJeV{jOr2o5o;6~d95M&Mw+XUlz665fx%fxW?igO=UDT~Q8C&IArT zFJ@mYxJKo=t&)xu3JN%wTN61L0(7-`N;#kkG&nF_V8G#%Au!x#Pq~st5I7*X$3_QM z;S(gm5>IN8&74qW0U(C$!mBaeSDu%9HRKbZNYMd(AP*8A;gN$iMbz=1kPH!$d}Q|f z_SW6oebs(cEG})@ovkgUBZqqX(#>YIsJ*ASv~}-M^L)m#UluIM1~9z?6L>Qc=n6^* zgp#1A(Vd@`unkBgc>OehL;-Nm#7w$gsE)_+`Mf8@x4!C{)etlHQ6?_HVHH8^D9`hfsEI2A{>PsS}m zf`4RfbkPl<0%T%!!64^-;|3xFjElb5ir4{|u{M-QKxf>N;{Q5_z^+W*ikWZK?Xqe= zREJB&v-N%Z=CY;JisfzlO?`XnG_7}Pv3aGC&s_RYgg4PVOMC#qK!{o(A;kuC!FdD8 zEX(wiP8OV&v?T{X5k)Yq$oc+QrPZGA`3DJaqK+?Xw;hfJ9KK-l1omA@(WLoIjq`p= z8HiEWMIe>}ra{oL1;RfP;F_Y&_`Xw16WA zM||sBT9I0^=eIhuNm@s2PJk20tY$>95lpK?alJ-t63%I zORT~>dwcn?qc-o>&n+))E$8zZ9G(pB6!V#UzNOvf$-w?txAKLDgQDpX1c-oBM>bCx zF_tn}4;#6-dVxs_(WcJGQ`mWtR*dH+lVQofpb5#Yn(@*@_1PaOH9_QnWCR;Ev2g=+ z92{Wa&1Oh(k%Q-bW$)-(+c<(Uy?!JT2Nf%ovyg2L`LG{^K<=HvRfCLeT!J0!2MLSd zs;!9y#vqVeYQLo9N%Tn=s}KGM?Smis+CS2l{)R%JKcI6vH#4_)mo!ar>$cfk_HJiq zc2^-YH($H=Lk@&P>G|_7e}L+FhC@;ho8^#XU0Qq&&Z^y;d(iP6-?Gpli4jI}NP^{1 zvNv;^TeAh%)8>R04#-v7tavBHTt3MiB1+gr!U0D@vtsVSR_@8++>Gl(gUkwzLYjkB z`f4TD*>GJyTnm~N%1{#ME)1Wt<_&fyVnUQb!~_wZ6-22-Zn&NLTC3eu$<38m$LY=d z1(8_kg_v?H?sNE$gd_iID3g2+_o_GN+r6*dIKLeShL!AsedVI1(12wq~vO_34=BcUMS4>Ocms{Q=S@$%|Va(^7LuT zip+{adnLDT@A~`C1`41KX$}_Ds5v;h_U`QR(d^phVQ3YwS-}oTjj~!>KALNVD55#Q z@T56_#cr?8ebFkp&CP*J!hzW29W5M`_qdhYJgN2fPWGCSS>Yy6lF2?A!BhvNEAg9C z+ojB%)DjTM0ydrcusP3Kg}DPQNsM=bn4$u^;}#CfqCYPj@^?mfYQE5%d$s-lI%_ILK^aQ*Z4<6`llz5Z}xu^LS5kBUr(oS;5(XnuFM02fVNW$wo%u70k`>}3^?1}u$!a)KN#`Z6%a4HGFibP6k4l=`+ z&*2X|K$W36WI80-5|&UVG)pI2xzR>$XVV^lBB6x?`_nL5;ZK*H3)^v?H0$Si)Eqok zqLmJ1mZo-XXkQeblon3rW>9mG_qG-ees9GYtPH%`rhVax&jBtrE8L5fe*W<|*V)|N zS2J;+!%XJRN_n-LZ)uSQruA=F_FNklNeUYgu;H)GD}(0^=B7HmTV4K`#Jb`oL_Plo zpTjMyA`-3OO_Sy7%EJ0hF#zU&Se@LQLPP?D!&2pD>$}R*4G4^<3peWvlk(E&i%OX? ztI-J%vrQt^fteXGpi1o1L?~^%L~Ic_F+6OdmQQHu^y%XlrB6#r21Z?FoqRflYyqW( z(DUo2PiyaMwnI{w3^fO(?8V#@dv8DYV4y4vWwQN@c@FtUpXGP#lU6Rjr}Y8!IUp}- zha@-)wm!?R+I#if;urg#MR^W-9Ml{_luz<~`=mW>JQ_MyBpjfSS>ZxGwVjf^x7FLL zUw9nK6b3gn0=x_iqDQ<85WrGfZN^ioW)MTKl-FzAmKn$VYI^Q z9(d|#I`g2taR?KWSz(Do5_VWj9lzIBZhz4pDhg$qS&-%lF^Qev=5|abNgui(Pv*Pa zv{;Pz0tpACz#bV*@bJd-O1DggNJ3ybN#AU|VzWZHw9>Jk7Q>foxq7pqcNX4T0^y)N zhUL@ByLdKy<>|)LDp1L*p3^G@iUZ=a2n@3_4v~}WNZk=N`FLvHvl%^rL-P2Q%5O!~ z0*t9h+F*qjXOVNB7H!#n|M<_9_qFDb9j!1dZAyYFs2R85xix#>Lo1f&fSN;y=>e8s zYHSThC7=+8B$*WvN@0F6*ZPcvgT(fAhNMRE3&Vr6jWK{pI7lcnoDXRIg4en5!N!Sv z4y+Og_ZlLCDjth?J-x+=5nemT30!o@O9`SjaYEX9zL$Y|MglL~soSwhiQZC3pB?Vl|_{TSeYnZm&gW%4^s4YoE8 z`wr$guy!3~NFv7khv%Oj)_{USnQ)QM0Y_BUqr*o#9;Hw7Yx13j5-WbT|9Dyu;UGQ- z$&PBBKWckc6s<_v5azE)9y%?N&onmmXQw?GZh_2f34g7r;={xb8yMqaLm-T2FJ_k03 z*&!(oI5nG}bph;f`ZQ_|p6Cd*T5Xo{9GG22ID~`Dd9xDA)L=3zB--cHd;qfMz`{Y^ z@C7KkOA7~q6BDM+XpoA5A`X~{d=IJf9L${Q)aFh@IEb>4)DRMSh@n%YPqspM@)X`T zhrJ{<2YF%2sZ6Ab8;39p(-04l8G#TYrbhhO1oJd$Mj-4I;}V>oa4N(|m*9TmexN(b zMz^>xcr$ros+hNurI#U@G(kBST~*9+VJc}Bui8aw(!$H;i$jUFO%6%Vab3s33n~|` z!YB%5qUO*mgnjrIimnUbJzx%iJcn5%MJ&Aea7-f_RD zF*J&fRwR-P>0++OZjl9<+pz=(353`ji;0D`St_wvVa2ZOlOYL|K6P4_Ll7R}#Ys@N z_wmeInCdUjEDlME-m!O%Qi`F;naj`=y@1b6F%FZaI-Z;W2&9*A0;FI|a)IzFVsRK$ z_?Bb@W0S-}>|%FeT;zdh*^(bk#3H)nH_KSEY#c^JMe$*y6=W_OTS2RFRb0+Fi>+>5fXmp@jv48#oP@-}{B5PCpB!t{sww z#=x9r*ag@WmgGVm0+Ln|CLnQ4NEiAX2u~2xRMN0AhE49pjDM%=?w#lG50*TK6yv*^ zMbS^6@kk$S!1M;8pje~iG#FpRihCx7+X+<^6s)D2v9V!#H$_;L?@!l`?w1kDiF=2vBO` zVxyuzzl&4j5O50@j5YQE1oR^z|<1zWZa zL*4Uy5B~a4>l^;{R80MK5d?L3=La_)h|>dvSXAr5oOmVxcWYF*;nbGs0S@-@NH}S< z7(mH51_7y~yJ1l%b#`{P-M2Fv_iZC2k;2iYg%py zb0e73ha0F1Axx=eLug?RD4rFKw-2;K%h9wNwvV(;WgP$vWM~KjJh;KU06U|9AyIfZ zL-ODL3I11Hh(NQu*(8Vm{!4eQ1ne}FO;Z$oWUV7w_ICaArQ0l@k0;wN>E51}BzPInB#LBrgOaR_sEpbHqh+v)T?b)oTGgt<`n0Ix%MWAH+TcuEZ$ z0CSMm3Te9uT1V4rSl@|gtscNEH5f;*PG_KymhFo_{L1!!A*Jaxd=6z)wL{tP_4-{a z(J{`VB}5lixbn)=2>Fxs+bw&WFYVskOfpVX8e9-*tCMU=&918PyT3!oxuYGA#~UY2OKF?Zx3*Wivoos|ct$g`v%8k&um}M7`o|8u ztaO3G!HR}3`TW9byT9GMftQs=C>)+Xg5Ta@oz(bgtG1g7l(1|z3}&?svh3=L0+BY^ zUad6=gxKD}ghWhpZH)sOWFXd3rD8BwG#F{Kt_vw31p*V6b9|-u_e&PB9aca!5ibJA zpO6q#LKTuc>gNd$TTtgnVkX&cQtnC6C|Bj3lC6j_+NUMkqB-D~ghLe^*D1b^G8ctE zOa$bjyHr&KP+XZqU6+VBcjxUcQF7v2aZZ^u!PXC7$p%rf6^v|wk0lfiWpH#5X9_#u zF)A9G3_~i4S_mNROl&AQ35tdk`^ZM;no7fI8MDwQ{WW!-; z<=i^U+_Bx8wzONu7I_h5zkGau|8bHLx`>yDl9a*J!0?r5NEQw&>p5Ra`1HS@yv6nj zBpe)ME z31L8SVT424tL#K(Tyz&K*uy5-L<7YnA_Oxm=GHU2gpE0OpyOYMk@Mi0U3uoHcTx|r ziY+7@%&;HbU{?O~!goo3rfr^J#04`2s;2DY8wh2YRl{<>WbNuqtyUP(a6h*K7nwCh zSS6Po8qAYt?sJnZV(3;`uZlj;B*B_BPU}{-t+CRx3|q{DAb5ZG?(E&!*~g4WE}h!X zZ0lb#5bK$X7}<%UG>sxnT!Tp2 z0$D;B9Cx7+=m?2j4b*{Yb+s1}GRE;4mLg*My2#vf;b6QW%LnyI6a{TCN5UbTj$A+@ z(sx+)&?t!K6{|QR05nQSW!f1ultqQAZynIZdyHzGuvs! zNx{ktvLDwrdHDA7%v@U~R&Vb<>bLjF{oTE?_I|JoH2awaWi|2}zge;>P_X=IRpwHJ zB*#3GaNt+2%WqZXMwiY0WxrX{%CH+k5Q!K$tPhU*p?M|e*h*|o&CG3%&il;?q@{~# z$i%LSHg?kjNoNO@k``?KuyBy0d#$i-32~Tf4(Ho9C!Qi=`qHf!K-|xghZFYl!g7p&}*%b4+hD5lB#`-nZ+xVMxo<|Izm2IVX9%{qFE24=vL_nKcT{Ayt=NABCT{7BExuBe#IYU zCI2!)a*Z8gC*Dy9LXEvc`5Vv+HWNlxXzwhrv=AD1*aA&OG zD7J=3MNlL^_*V1BB^Y7P5lq2U!)arqaSVo71POY$BMHn;DL{bbXy>s5zv0aQ0q_UL-CG+cptP zi2#Bi@nt(W@yqSh`l6O>|NF`N(x~j>kX-Nu78QvK6~cZ4yn5cegv((JdXo10;KF2> z>%+AKqH*7g+(~KdUIL<&dSMuPJ3GZ$4AUL$z)b~IW2XH{3X@*$xe)XM*y=!if|3HZ zzo)y3%7RyMNXnA!)-f8=w4L(7R=T!wmW`fP+uQ424USlYBaaDEB?u-F>;*kQFj34O5-Tgg>XpL zcTqTa-#12$9_$yWhsSll<2gEQ>;f{HIQ|uuu#A^NIPgX-VotDolB(p8)M2CET!%^L zxE!&skP_=Z?3|Q&3%V(lxLI-A623u!%_gP7JNV z!w|#*FAk>FRzce{u;jQ}!0JeQlO??>(k6KsZ*`tcv2s;1cm!+Lg93J}2zEK!HL%wO zZEk4CiRJRQZFrV=LDDW_xkz)!s4>7{{IyiA*4Flu`0VW6yLXrspNLu78VIw#y~&)t z_DY&)wt~tcwEt&u4&EPHvds=j_AD-5zshid5e^6IQ{4+;V-x(%^w@{d3B)jH!Qd3? zjk4o+9Nj&DHGdfT<4zaAKfm#Nnd^AF{$Zn$XMTL{PhjJX*J#6VdRSdw^E4Xk{v?tY zHk$&hIeZ~o+4IE?b|B#pjn}=aZs+`Jy9P>84{FZXMN-ruRn3pYkZ2Q>sF^sQh)J|4 zSRcpTeh_NcwXv~`2q+q%JgA4EJD%=z*q z$Z83qTQEb*Fcaa_d@Zm?@e(vEELgCDhnhoX!!R!~*I}|wp_6NX+ti_PL=M(wKm-9Q zGx1+YuO_WpX9q2tvwIcfZ!s&qeXDJkjU^F8YKW1+*w(g{q&aXiKhdXiD;@?xk?O#j z!;)P=rMYl8Z9$Zs9H0R3hcH2I@DHXisCuXiz~+z2kJ7L*hN}@krFYO64E)(m7c>%I zHOBi#1Mpw3Z3FbyCw`o!u!I*DP`?Y1_$G*od=Atmw_0F} z&tVZ-T%ZN9p|WfWsxNsZ^Q*FA_DT|t-NM;b^C03!2rOSiV4zaPI>D%xOgcL`_K>|l z%M)}+-sYEgAM=%itE`#ZWo<@Wp3*$0E{!L_GBM+Q5)Sr~ZTygM5Tjq-jF?C`)Bx5G znkT2I6Tqb(LjAx;;gB|l*mmNg$*(IzxEZA=98M2{{uE|iP{{viR-h^2`LAk+0K@gV z9|OQe&A-9h!Nw_|a1f@X0my6xZ!Jj~#d`C?>4oOtMBsz~-yiJXr0ghAi})?>UEGW^ zEx{Ek^9YzyF`j}jOd*7{xQ#3A{g#yGhvw&(v(OYX zs@sWRx~7U^W;aL?zZmC`3oxS(6**VKeZ|d!DkwadMKVGIt&^NN&5EzUjH~(HER*v# zNMSd^Az3+>j(L^)vcN{J%hvsZEn>==&Sb8qk{y(l1lCL_ z4~KmYFJLdof30hK4@ovF)O;3_lftSZ;V`Iy=O5Qcs5$JTaPZpou@BoVIH=Tj{a)bb zkzOCdaW{qa^IgAL+6`tEaEvpr-y3a#kNCUw?j}?Y;*;LR&AC5L9W+ilfu@NB4j`br za}ZSVS=w3N+D4u=2glIs@~mCg%8c3c#dE8e@iUn#M_f_Wkya8FNKY`X5{h<}roLEr zz}!@C4T5&G#%9tSo>;O?O82;&Q7m(h(|%{%dnS!r&!cqqd(Wk-<7ZLM+heKVVg?cp zc2ET7_R00xc_$87wJ7y^W&)CggSFz@4g%#4q*?rt%&ml*f4qD9;rjl=yR#1&VTz3~ zPfW~$!bXzVC`Qf_JDkj*uWAn7UpT+od!eUn|G{2lEBLRC=D-|8EJRhu8(yUF{mg@M zcWd(mzUjea1RIr62Y#GE>9l)v3~@IP^BuAL6H4kWCJ;w$|QW(0|~s1WkVQ##a+sYSPV(Oy{c!Z0%26?v%#MHoa8 z773w~rkNKZPO6c?($G_4zjLYJQ;v}26L;~)0s^5VF4(MKa#{Z&vkh!0DG3ZGbPn{S zGF2HYmV?kZq*sZ3Vqu$%9gS=Ffw#^gWEypB7vb1Q?`_4w$OY8B)kPdgHf+L@>BnR%2fkDE7rzR-Tu$lKsK3G*X4-)^4y<-h&whEu1HRtRE zLkI4r$doyda%+=pOuL9K)RZ#Jyv#y|7OW|i5rq{Iyz8DpNkp)$pcJwOMf(%=qkcs7 zV?jaf&!D2;QA9z{InSQ=*=cs8E7jS2-}9W?`|kVR=Xsy|dEBN3OqrOpw3(yzJ+^>) zZEPaC^pk;CHV@pFXb^{;u>Qw^`(VZeKy`q-Lmy3Y-y-8FwCl@Kcnb_8p@>4>h|}Q+ z6PSZ*!KGA%_$vT$u0h|_@;F(X55z$gJ0TRiUp2kAyJP3hD5OOQouCls`ugn*wKU%J z{-(mJR^t*(36!ZcG1ATLdZ8AHfRVjr*BoOAnleqbgdY3f4ZL!nVyz&KU2zWhaFLTC znE~nXY%?EE;@*1n6HoY7U`fO_GM^yM;99zsev5A9-<%Ma{A|cJAk6%|-KydS;()PV zN@GGY3emR&HZa-}l|>2#Di)ilF<(R6k_M4@C<4x;OH(_FbYq5zN2p13DIyLkqly`w zU^)@>#YWc>OgpsAv;o;-1;8!4+?w>DfwKgg6?CLHvsb%-@;qFQLB0ew8RW*iZOrFD zqxc^Fw^lH5ussJ{5!(ssl#;M{ZnK-fQws(>@^o`Wb8;(N&#@K6J}|%uYMFE3*#@34 z+%RNnkly!*CpfA#KD{jgoxtKS2{ag!H@+ zfJFD~7k$WmmJi!W$x64$&fz|xlrSI#ss`dqd_Kf`pBoOwmEmNAjitr;n5VX9Xn@}Q zt^411o-w|YX!71vP1?>%Vg*m7Q7i+Pv1n9`-K;oLYO%M!yJ@ts^CeB^UNhfBYfxzq z(=5B^f8t;e2Qm`^=P>@)NHpue&^^Y@&{{4mP`e%H4 zaf6KddRFCh$QZ3|-vE8N{p%LG!(FS3VWxQE&R)$Q`fgnaX$Huko(K{J)x<$zBjlJ$ z7}#YbfavSLYQV@;uQ>+|AaT%j!xzEaEz{6Oa2y$*f<&*$)C;-Jw^r-{p^zyTM^=wpwmldg`c0|D?+U+&gj3?MlE0Is4Yn`5y1 z<7Y+*XqlWEO1=Ez>!bS0<5DRk(WRj!WLq`BZ4>7}yg%a!$uacgn`{B1J+t75C-}t< zRcAcWAVp#jO3`Q|KhDo_4*0kOaR6ODiWO=>yj5Bms$^pg_wVk??9|mF9O??Y0hdF* zP`C?D?k+K&PV|Jr06~abxFg&Z4t0PR8fbQ)4RU0J!{GFul3Ro6L@0f=yDbFMJK*RO zD>n&WNCmvb1(|ZmV#{V5B6&z009!^AW+*s*F`{FCzq`}WBB&(LtdQ(992Sj~Pgpgw z=JEP1;qBd5_l};s&BJLj)(m??OE4+ZF6I*`Vx);i{JHy9xc`hvZWH=#!I$n|vR2T$ z7&5g|dEvEJCVG2q&@8AQ zvCFpMoTG^Y>lolet04He5-ai&{_@j{?&Nw#z>mnS>IVj4YjO?j1g~z7CLXXfEQR zrM;8A+eLF54O-RPfGdPh8t{QoK?8$odgL*MYt482Uosmypti4#TKUb}|> zIBjcUdbF?8v^9_^lf()fO+(DA`lbqzK{MEr($M%uA;_a?3XS0_n>k<%6Q{xM5OFXJ znF{!Cm@~NQi!YALE&+Zb<8n|+Unsq(R@sqf$axV9Ppn*Wu4vQ5p^K;<>r+hjffSN} zA!-duJaZr-&VjW!z5ttWc8YabNUSh&5UWjTh6#X`AWsMjl2wOq9d z^@3F_TeQ0sUewsg65XF{iiT<85*_4JVXGgp!moTABur<}_OM83JV_86L8rC`5wQ`; zOufsOL?ru}d=B{hg0(^k@;2wt!SW5QPs?iJ(3=(E>D{c{5WC}|NA_GTFPRn_SrP9S zV(9$&OTZsZ&WQ1AgfG1|m(K3ScT#ggQK{6_8M)nAk)FOcEqnv=YI(O0qH1NlD+Ha_ z0p~#O6C6hdU~?Tg2l&i1=fA}-W_}vVaWZS>225*(?s)#5-(Nl-f>nNhLHd?h&|zvn z-lF@z6texRw&1t3wZaLBgE#t6=6Kz8ytH(VGwN-y^{#+p?=Ej3dUzaFRN`h&gTHq| z+e3$$!{xEtHNIS8V7qJ)@*BS z5Ui4NL_n1MH@k*txvTKRYCYV`OT6fH`vhVEibF6>7%wA3gO4l%qdlk*Dv$-_X~2W{X4v z-pqw0~;6O&+;dNyI^E@{%9R{P`EB7+TH%9~dVGp=6hXJoM2*d$+h?C4i zQKweQdjiXMWlnC629Gyyz5oVpM)h(@-~ei6APU2=_DZ$jwer9^0KJHLla6AkJY21o zV!%xlbIx25=!a!hsuuLS@)i&QG*R%!Nd$uInCe8#RG<>qe2p+GD)mw;h?W)XnKI{1o*-9_BkAA za}HotHa zSGi_6K*p$4%Y|B{;3!oq?TRKGl&f0zP^vnUE@v`Vwi;XK+F4#)cqUMPp^alJ$O;H& z!?f6DGsx6Ip2(~_pf_hj$-ubDctX>e=wL649eNa2l+FSFHCx3Be4CCaLmw(6mGiVB zEqo{Js|D@8{9Hm@w;|jM=VwR9#m0iWbGE1VlBqS(4{Sr~{M~!S{H2giYwlKY)t#bi zWnPS*f6b*`_x8Hs7Q;KUb93XuH?by0=B~Y|OWZnp%}pY;Eq1cLw(#2QT=r6*5p-r} z)m4Q#2Vew%b1+ndP@pw2vZsO0mhd_(2WTVIHr7edIiQU~{h6t!Da$NYQM@2iCLkHU!}(EH zg+cg`?4Uq8`UW_*0+%2sZ2cp29ozM0 zId1rUdVw|pdG&-3YGG&q)PxeKDb4{&#@O-ZL!duUc8I0SVdmyBJz5b5{l6CtWKQnVHVuc%bMxj&(5tnK)={;b zujCw1iw#%HMN1V|rJgH;%27!Vm2$0Cb9te)Sg4nPCIIQk)oT`}Ps&#;{os(Q0lxSc z5q+@zqRfzOBa4d)^#B}!;^uU+Y#Qi-_6|}ll9*weE*?h~AuQaX4#23#Ija<+B8()g zYP~XrY@eRb0VQ&UaA>VCvr2L9%GNxUG^*iVVHQ*LG2Ok46>#l@{@rji*(7@b1X-+r z?9XHbpcDLvd#_87&jKCspaZXTBga-euZROFi{Oih%jDs5r$%f=8wkN045QbJhARcq ztKxF(M#7+HqTre2%s|u%Uz(;tMI1PXq#4Q0A=|$Jk^V7Kz#m@Rnmu>2EU)n}Ll_yo zee=qwsstwQ$|&i=V-p8QAnH12X>-Y|NP`nZj&q`l9OO&o!kaZ<8KA@}^#Yudv0CMz zQZBWt{sYiHRDoqsr3S>pYbmt1@`aj($PWe#;vA4d=V--gVfa~7E z!u|rt)us^x5JRkh1W%x*5iNMq;al9_pLe?#c7gXN{1OY-xXEgv&m4|M-=EFrfExT; zF<)!=d}Yeyo9M8$8qK*V%)nIz(m*VP&)mjcdbGWR2d#ydqF!nu7)=-mqKBmB5=_N{ z-u7-A93u%d)&70+^L8L0vXpE3`T-p z)r_MX{DMswP+R6|2SS1~=|B@S8>-Go98^^1_F6YPTo@M3okpb{%Uc00-3wd_6Ia|K}7 zm7B{!%Xv^Kf!SerGnn)yowZ87)E*lyRjc_*sa!(B;jEO-sDc=g1O}l1@&qD1kcnXn zT`fqgK!%d^4Z6Yfoo)`<=Frim&H8+Ob93?Z+=?5#jYC2?6^BA@$R6(#;ZTp9$%jHB z)NvNHg`h(~3JM?s=w=-92_Gb+dSaf%(;)9P_eEjtxhyXS41OlVPj1zp|O6vd#O zAQ#8a=p@q6vh{5r#XB0rfu1U&`3&NY>7H9ssiRx`nth;;!#}$nba`)pq}$sX^+~GW z$;3oDgOBY7X>nH*jlc#CxHNIV92C>JP9hHM!SR0%NiJ@CKM2A@m)1GhQhxO2%t_Sa zTDmJ5@J78#_2^nZJRZAX8P$}5z)9w2PxSiCL!G5czFNuy4RCDwt#)U%e0HoFXTLW&fEt>* z!eXUYoHcUmy}tbZ)or0%F1G;j2(kUe&~|)rK+NpVhePw73p25q{C;=%s%dwZ8yt+? zZEI&-*8{jA=f!AT5eIkv-sx~xsO_q&1$YdcwbOCd!a`RU$}UI}NJ+uO;dHSAL$(_^ zB;D64g(Pj_gVCLwdkWUX#tsd_nLyRv0O1zUpOGU<8l%-9Jqg$9;I#ly<5g7`ydE@S zP18CWEmqM(QYj>jjj9Ibhw^JRg`~1pAaSs{dwkU+13_3GnTyY6Q^8~~ z=!NGX=m%Qh2~O^&A};?ea%*nohNM5}0dnE-`~6-=64bu_BrqFEr`PWZdcFSj{%ayX zra$aOhq??!9L_^h0E~~hL*D;RP%XrPlsV0Q4h}EW5JLlj%;C)$rB8t32iwKw$zA6x zFKr$>q5+uVjh^$`%_kpzabvYwa9Ub@U~S%AsaFo_|5UZ8T zpmOly`!#7*^9~D@3Ti>{g&>D+`{u|!3)`_p-7lFke;hha8&R!nh-oR~SIIe<6{Ap- zi5@wZzBew@^C>wcH$+y-jq9NOlGV8E7rk;;?sUYZTnGbkSQ9tx0&x%v@_ae#zB`_l z6ITt2YgfCkof=HJP5)_D0*V>A*s+Rv?c%XCBCLLkt!;)IR|X_%(=O47ADwb zlE~j0kOU)S`yN}YXb^jR;;^N$b5UgONKE=YuJP>~JpLPiQ}6_zNIg22ia3Edh{=e@ zf5*{_?ZoUS z(E}nB)Yyp1ghHAF4zg6lVNFhp0U>(giE$Ygg9|%S2$4&xr~@%CbF;fbY)iKQ+yBsN zpUgqZ>_og*gdrV0&MU5GIz7^(NR_0p^z6xwm#3$=%HDtX1HZu4CYp>Q%&nw=Tn{j-EG?4I;C> zlOF$ra%DDs!wqs`Hto1fW-pQ3LB%<&dpyZ(dO{wp&&q2qx+Engq$?>OOJACm!6&3V z|FUZ?*_FEyCsLs1r&%@W8WRV-v*=ZJOc;Ou-ww}ENFo*Be?0|euod^;4jwOQeCN`t5~EPx+gjrR#$hKrz<=7$?f*agQPe(BRCMilurb1|YJNpmzXo z=>sciM@Lw(V$DLu63Q4#te|IxLNW1!`)d$opjVHbecDD<{%X$Qk}h9(V|7*bDTl$` zZ80Y|!t?3yq73^W3}Loki1Dm&#}~!I>byMv{7ZJb)AL|O0M6m+?tMA3FqMnvL*m+n z*$n{-NeRM6=q9;xw~wOu&qu!+DXfM=&gUBnlM_gaqCos0cNg@nH zQxq*N3pSn`Bo3@?&=uW{Q}nE6iv{W&*gcY+pE{5@3=W9?czz_^naT<=wJpSaJT;RJ zt;sgg86TM&2jZ|R!tot(@9K=awY$$Z5T6Dsg4Q_*xp39gnYyw#u_XlT*>kQ;2$8>b zMUbBr3H!*m5OHw+bl{ba%4Ex5Q&vQR=uyQX)JFZM#4&Lj)@etr9D>TS-tzdh<0`xmgxvNbf`^n4nsQ=GPZw>$WF~~$bqZm zRsA>J8yACH^XKMfS45k+MXzDXM4soA9k0;P2kBSMgvln}@|Ugfguy1dBVcY7+ma?4 zAp{7AO;sF?;spsHNob%c9Sd!H*OH5k6%hG646wxtO)nY>Nf(n(~LCyh%3*sEu35uj5UHzF%Q2)&>_zBm@#mK944hRRFdv`?@DudDD_)t4K zSdQR*SDgs~Q=Xe|U)p zclf5OM^*>#^v$d!+%t27D=~4?LO8r|lL&3CiNy(V>(<~k3ump2_+Y`%rMpw$kGylI z=$RPI&bY&=ORk-sUJ?1Jo!P#RrM`c1`lF+)XqYV%(Fw>b( z6-T$EXs5*)p&9E$lg4**Z#I+LJIx?kGy3au67q=N5rj<+Ri zMfM_~0~gBF9d}-)k54=%x3?q`he=oRVL9=bymWoHH|Pvb$j4k#Mjn%oT?(9oe8TUL zmmbO02x=ZtZoxkQ$RG!x<{a>@^F2+`;!X0}|2hY@R_G6gLDN=q8JAakvuU?SZIXyy zx%67b69HvHvAtUV!Z{3@#@LhPH;29%e=y<)<-?Qpqlbc)15^mb`XlctTLig1YPDsp zUiNx*-J!(_9kT7HmWPvMm_&p}QzlXia@*ubhy$)ng=7NVm?^rDvpqK4i@O9A9{^_w zxAr7fjUm;hJd(ll7#xK|iLlUZZtz0J$=XjjEZSfuG<7IYk8!J+sN}HSmk4A(2ggt4 zkW_nRnwvVEj-jUKA+@(|AIJOv9if2^Oh#Ilr4BxutOOMuctzvRTIfgTCv&fR4RLq{ z7R@0XjkwbJ>~};*DbTzi*AVA0mt^pcs8{S$pVv?tpp)g*%MpY zaliivl9GEKoo)9Y9R>YI*`wZvZ@B5fr2o;`OK<7*vuh0cT-t{3%TY8;fRBaGx_1ZaF)u7@h9ts{V#fr*62~JSO;cC6wQ+jpJ(xDaa zJ((~0Z{NJV=!DP-FE~MY)gxf#iv>$LRL3d@HL_12aZu}9!+8kcCdS1M{8CO(cq81! zUJXb#u*QK`oDqS%7b!M^kQNJNgQA)~VLzubpg900a9tX0rgEpLa3+`TZ_E z3~C?-t4=@pPP0P{#Z+kDYzO%{X#Rj%9@gOer?%k#Rh$EUQ9z@RZ3?@LMzz0FFi^bv zowvXC#&WgT7xika5c7Z<((g?Mdye1vX5VGS?*97iN8ZXifMvjtZ408Zft5(+i%Oo* z(gkKY$PdF2Ca54yr0q?PuoSWlg>yv``OxF&3+)uLP5#j>I!S#(asc?Nx|`F+(P)~C zqZ{3Jf+!wO0t9vQrD!yOBvw#VCX08Swy@CCoTU}(rg(uBKAT(slfy_B1uAA?(DW|Ct?)hg0K>hie3vM!r150CYCWWNK z3!=m^7%(9odk@T(cLz%4YESXyZ)@+BD?Rre`yH15A#j~k@e}3uJj-{zTCTs>lUq9U zCo!J`x;>(O!lgp1IG?;#DO$>OfwY1AnrtGK%vZ^@iLH3%Wb|~Ox^S`=)b9HgZSQ9Ews%-)GEJeSkvMS3Hk&fh`oy?QNbV$s z6E(8pS@j~+Y@f8B8-#Ems;IFKUnNKx@WG>DM&Ol6ZS~>5okOx!tWbsg$tYyo>w$!D z5lH>EcYB}+86I;L$naJzy?VRf3LXM+0QLrDwA`Yy(<)8l#GiClwg?HI*G7qA)!W95K zpS<~S^YEc3RbJR~JytpZZ1+RHmoXK1HFo=JU-`Y2GNga2$7&!oc#_AN%sJ~|_>2PQ z0B(;Ec#ZZ6h5Cfiq;s%J^}JR>7}iPnB5@!(#cBqAgE*OuFk&mzhhOiHQ{o0nMQ`#3 zlw_83JWxU>)N3MyBRN47={(I#skL}CDkQbG{3rLVnC>570ze5@TP%=2Mo~s;do;Bp z*gdq7>tx2DHRU6}-;FhlG-<}t>=>Qv)R~_~*C->Du~jxL!g9p`u4RO5i?OlwvENsI zdWFf;N3WZp`nws3gDQ#rkv>!3i`6FL;MRwKfLp@f<5BI_j5^{Xr)SJ~9GR1^_A<{v z4yOwEHsy6rh9vZOT$#JX;qm0jrt9qb+0}BbSb!d{!rlwUAo=t$xJ*SigR%TU0pcet zi?_YC5#BrOwQ{v~tI2c;9OjTzYgkk8M|yFdeRpFI@vMJlz|q(ro50$ z9cVIVl%_>EkOG%v<%9;2T>L`FHa>tlha~f`r}-N&R+?Kv+Fgu^tVHblfxTl1iLQ)( zMGx&x-=gLUGk;xmxM~LGI^R8W&ha$b`!SW@jQ4)8^WE=)=gj%eb7rkoCNg259-Rw9 zNj$l((?T(jiv;BiJBMaEiTEf3j5^@j`9`!AaIE;`>BcDU0dYX@SW>vk=4;i~e}f~0 z_ZL9xey@d2WMos|WDZw;bIEP5eD}^r@4xZUr|0)M!q9?)=)C>>b+5kp)~O(sgt!fE z2tiS;apkqwUUkWbpM7+nvv+C{F9<2XIehdMP#P-8t>Ba4=EB6OF%~6~3x*{5~;+Bh-2LO|AEnb)HDo{L14(I7P>EWaprapePy-=z_2uyJY9x z&z^kh>6F1^-`sKSn_wLI_E`j4H|ME0p4C0Nw7$XB z^C|m28E>+wl&kLJ9Ad1CbXcpgUK9)zt4WH4f>f%|7y|R`7{_43CMi3yDF+R)b7(%; zw*s?34a_E>Qm2bh7_ID-G%>(Is*0u{F}2zk7dHv!I5c$-4+2G+6s2JZ>U5lgi~JPd zPj9qUmTU)8@artwPU;{fg*Pb)sj0Ru{mX4Lchjhzq~YrxrMqymsqqz{4mtz)kt$QC z;OArfnL|rs=eaVre)-p5b&oABuAc`TSfIkyvDaaE9SXLkAn^JRn@dXoduaK34?qtu z#V=}=^|=c%0fchdCbth9X|B0!W3macM^HonH3>vIqA|%dA7k8C_6t;)8xde)CmgVQ zvRjxQz$y4y<{<1^oG>xfNf@b=hT2j|D>K#ULh6c@NOTN|L{cObN;N?XYH-;Nf%BuV zzxA=s&e`G4IFk$glPk&=(N;)O8(s>ZukYIW2&;#VL~y*;gQo~;bQuqk(0PpbUR zoPu974Wi0ySasvys$%unzaCQXSH~x@3_SYiz~td&;L=Mcf37^CTJ_)qKR-^x%OjL! zo82;-iIpJ11FJGcoPZy;a8fAl=CO`bcn4~5NOl@vC>>;xm*)JI678OYy)+--TR{fY z;sZe#sB{<9uR9qLHUBRN-PxZKq*pRRAuBwaM>XFfk5V+Izkg;Z6|dViLo0i3j6N*_ zNhq?kCWiW*v0neq2%g&L?dhGxg_*aJaSg0itTt&Y`qGpC67IRn5dchRK^^pyP z*jPq12Oz{N^I(^4C>TRh1cQPmn!6RHBvAvMoBpQ^L29zls>A_G9m?JKr<5QXucSks z>=%amQn)W3$D%kBxN!{0Xm;eXO&ig`+5kgYOVoRQA>#m9*oXS!Ov}`3>75HdKuzgF zNQ@!L?;PEaWDY+KFiU58M;s_dd4xrH>c&gNe4}5o3QER5zBn6z3FmN() z8!-|h1OYqZ5Gkm2LavjUHZ)7FOxb6QOd2>n6>Xcu2h3MA@%M4bf-j+3#rR3RGi{yg zSYkL?EF_x`Ar4U_C`Jv&KtYJwfO4`>=ps0&vN*QYpN1JO|g(G)|ZUfveBLO>p>w-rv(o4UO$W+<0ZJ3F4c8Ckh)W zeyCQqv^J1!qxYK}E4@F}D^p^HJK}(7!6JPXe4SEI!_80nl>LN)rPn6uqdQOw1$Hi>>2)G(!C_96jaS*C$QlsFHL=AcPYE5R zi62*!^NCnm9Incw|5j9Gs*^vIdbVDal-MCX3@pQL-GNEQSB*LHC>f@v{>@apziy0TNcZwmY-1NE(Bj}I8AHxNa%Ap{pfBO$l(vG64v1*!KQIz8yuVt&Gek_$NFn?!h$u?tYE+n z8tkN>`T#<%bo#E@PY8@Mhi0;^WP3_7u8!&`a}bjyT9vn_L$U}dV{Qh=>B(~3;cfGQ zL%`^9O}gnShiBR5|GwpcIG?TYi#Op66ebEf2!3fHngUkNo6;-OT#+&5prH_r){Liy zr+fRgK@~yq0-^xLNKl=8tVju2(C@5dqxzk>sTC>xy-wnw-&q|Zcpr*fC{BcdI@iWi z`VFvp6{Bm9m-9ivrIiLl?dv@Iqh(0EtZyx5s%&fQ_9LhQ>>} z0pA!vA;@vKe+Db$m06a<;VGIz5BrMgGOk`aoHW3X1FEN+tKFN2a2ahzu zIY=%~w24wa-pDzS1_EvfjBC^SBp!h(%MQgIlP=gsU>=3pxGKAUiyVdEa5k@riF8jq zNajFsl_VL=fxp1m)|w2uF$Tyyo+G36l7D@Cpe_cKOFaVv#!vkNaX~e`Do>a7Fu*M5 zt-u-aJO6ljd+pCy1tppkLsG@al0>?VV#)Y7FYT!YbL%2=zz?8%dDJAQ7OI_va@&d- zm?^Xsj5l5$>}<;njzfCaX)m0e?3wDDEtCg>-r;0mW~af7|B3)Uot`h3Siq>`Lx!d?B9_;mIpADjC)2RigfG>51rIV?z(DRvHkx9q45 zA6MwyBTdn7;%xlX1}Ux-J5-58LKGS}nF`uE7G=_iBU++(;K`Yq zbzFm1+AD@Y5h5(Km^!-bt8_kgPvV3W0?DJ)Y#mJrji4mUHoo(HPyPwz#y@3|JK6C{ zsf43y!=>Q=Cu2yq&`&aR4y>dw)sh*onc~`9Tg93VyHv8vE19;^bj4N*>q}KzH6=UN zJ7ec3YP{8;x1owib$n6zf32>gK=nm6!>q`?9a z8=-zR5C^6Vk27)L)VUN?48K%E&Vf$~D(|F9x=k(c$DgV5D=H7 zigMAw8U|uWA|({g;q7m~c&9PS_urrCHYA*bNP}HDvXDF|1wRspCiq~A)ff_oHRFe7 z#ol>!CSjI323#cOjqc=vO`2J2N=0L)ZD6u(=IXh)oees&o!w;v!bX$Hj@VH%z2uLq znlSs=YSx%|!B!@c0jylS-8jj^=tZuKE;!W%5QipTL;-Mh#6d#EsTBMe3QPo&tQi%9 z20CQ8ZfW8$@(|pl9%ylc>?p;sn*L~riHKNKiTU*5S++4uoN|=lBRO6U;vb+W7i{cQ z-QrY9HRVI0PXsFNy#DBDP@RP#I0m(@cFLiJ7T3X9)tM!ZT|^w9^7gl1dg(?exevcT z8+nD;+&yM?X~T}K&lgkOYo1f+*{N{SuB|QgWi|pcF;pEjF_(R6%D~bd+(4hX z+%x1#55#kgh(pXL9VDwVIpTw;F*z^z$zma?js-D9?j|A$~nJt_608jaXnX*3guR@#imE&L*amIdVx5Mm~ifDAPy^b#hnqdvmqdk0hYsLhi8@&8K`t$-8Kee zh0K&4HzxsEIRWGZa;$KMOAr}8NJ{8O&>f%=)XBs-P&7$$6;mnr#UO(@1&P8^Z};2; zrv)->lu??EklZZ+<7jsDjftZduLb3B&H>fx=#_~gv1Fr{aCvYAc<(r*!xKFN69yHM zQuq|~t$_P}mu*KJei?st^Bk#S{TDz4P>oQXiNl#SD=4(x6#nff+WdajPSNS?rmuOl5mEGOfNX4%@0JmW>CwWy5TTna-Mt%wxUfAT#Y0 z5yn<}$7}*}OP1|wF_T!a)#`Z#Yc8^_$)I2_qGz#v!Aixh9p{S~6j?|jo*_lxm{sCo z-pomDloK}MDjgAGQAK<Gk2CNJ*KZW753d;>hMg*Y!FNi zhC$DQ85j(bJ(ba!uy-H{z-Fjg9qcm$JxOo1Qtk`ALTPkhrEFs50!SpH2v{fbr6Q zsQI)QFaVi=;w_bW+ZwwJ+5!q2S{2WrT)|yM@JhTgqe1ko5aRI5rG4wU_MaKtIp_5J zb6Z`%=O7}-3b!}ZR*1z0?!ECcUj2i`Yhi1{h&YHQ9}pt+fb;i(g@BR_V9*PaQ1zj+ zDTu5my%6MzQPzi8(lq(d7y2f^XlV`z;A4+R70CZ!02(?1P+U6Lm~wUWF-ru8QXo4A za<;+?TToL7IW&TDvk_EBY0yY7Zq{xtqdURXx|N$H8+M7RQbJPFA~ALoXCtUXIS2e< z>WlEpn=hC~FJm(pn1Q*T=q_3o9p9R!B-Yjv(2x}TD04XH*GLP}z1_Lvk3YRV-+p>K z(15P4#b{on6q`1tVRj=kurkk~BgeD^QhL7qsRm%XYoC z0t%T=GdgL1D6A4HbI23!p78+*#|%=A+VlZ-&ChPb6yk}&T$UP@>oUGX9cVrn^JM8u=z9*2gT{@hWlV|dUq~& z{OMhv?{4Kkzp<-*>&f(eJD<0=Z{7FY=k3SmN}23^+l#yD{F6JM=b>|F>%OfUcW=l) z_gem!NW(co9HOnqI5a)K&k^TfG;!eEw3_DzKEt-vW z4J-}{`!tNG$`nb1Y!q57rWBcjY@3e?*u-wyHn-(b?EXc{3v9|uVoOaHMTLH{WZ7;$ zG~*R$D~LD*j#hERe36UBaCIiZ%<(`$^@<-P$(00+KqUqZuEo^Z4_ptb5DUS_*~)um=rQ~f6DxixJ?^r5^2P&gPZ?3+{Y_Y0U5T2nn) z9(4#%{l*wrzjIx|ckcfMmu*KJgmY+OttpZKr77_dE=DGrC=rbYgZRuw(si(tDAo|? zYl}Ky-|qt%5cD9hAD>J{i0^wah@h+@;wB!rh&Z6OLR&~65@|~;sX+`$Bo^$l{cB|o z#5tVfd7FDr+VocDhOyo48|{s^&Tj)e5Qi;$LviOmyS06veX?}k_V!NZf^FMtcI@^q zX);9SfE$g=w)aP&Zm<;m|D=T^2X6(L!>VblvhFws%7aqNkEil%V+#H$(^zH6**U0V zg-&V{ADB(Ky9b^IhDHv%Ifgyz%GPg%BqRv>v! zJ#lE?S{lplyax5OX}>93;9Nu>qoT@l(UYBkRK*#`BW!7si4R zZYMQ)o-qS92%!@sy&;SQp=ZLtwV)4>ANqi;Ft$3JnJ^u5#)p}3xBzf7oiU;3g8|xn zPuZ@)YB5b+Fr^1M+D60yO6%2~%3qWXv6V%qQe}$cl@JHPRbAD@VQD*^pWDoB7PmLY zx?f9=?4-N8azY%QwAmeCC32_RE$Gbe%%yWo9I*VdK$0RsDbF)k|G(Qgct&e0Nc?~P z!xuJMxxR%mB8pE1emI+ZBX0YY7VWaF1(MDO+Fc{!z=3AqEAHZ?DmA83HpdN4RZi$A zjgFhOsjDlqVK`l>Zj9RMcsNw*O_7_9Gb%DMdNE3Gf<6#8@+Fg9Wumh^f#%)9!Y_aI5=l%VhvO_PMz*b#gGKzaEL%s z69wHm-4O@7v-{jq=`~x--DflBq&Mwr&uz9BKZuA!@q=6%q!H<*vCkKm(u%y^{|n6vDpwed`u|B2!edk%l+pVnxlMe-)%m3F(|TCx$h9c}V33(5F$ z6k!py6=V=fKAuX#*<>cFreL#e(~DP+TGMM6%XVeKPF2^cwk;!wbapUeTe zkT|%6xFUhn;yWnPBiz8Wop-jsMZ3v z7GmF_pLK`Qgua0G{WQQS6$rsh#{ z=j0iZDhEI6oh88*?@PTiZkyi(hZ#JAlpdoB`69R&l4$4Pv=w-s;QTw%bYDjt7Hk55 z{c-CJ&Kr}NT5yKxlNqyNVjt}(KmPPI)WMJ;8g#d=Ts!>ba$QlY9hObE1Jo}t4O$@S1S2yWs7TR{WB>1C zg(@D?16MhAnkvT_{}Ex%`002H$_fePX_!k5S%`XQ_%n{R5E9}ngqPJ%b7P!3S2ltO zDB4W|U_e4oxYD-Ot`Ym0^5C!(gyN`E^>0mTRGdJ={9gDrhq`Q23`yi1IF7I8>zgn} zFA@hYSXmmG0OGLdyj09;Cfm1Q!-7pFjh!rZ7&EkJ{Op?PnBM@m*NL)c2i(HO7EH=6 zJT~8IgQRy}2ROp`KpZT_zc7RLgaF9^S&TzV!esKX;y4sYs))|`DpYiCEg}xk6wU#O z!*49@udSy~|3eN~GOCFK2D6L1TV3qD28-wL4;FFRhR>NV+=epEH`T z(c>?jb_NWw@xht*<1mk3ai%a04vQYv!*3otzqwgFs9Mf^MKAyGF`@}{mnQ8>b}eu( zoGHPg9PPON8olc?&fs@$cH4gDUcKvMZj_19v4<|#<8PkXLf)ZpZ#nZ#9Om-Su_z5R zfF42|5)tQs#|kzs*dLd>pQ2aAkT?XUZO-)D5Q#%*%v|rJ4L0mDa1Z|Cst^Yt1IHQT z1$Ht7XeV>zf^!2ATnbs{rni2MJsx=?~gHuqqSqB}#`UY=Tlz z@JEz^7@FQSkT|fN!*4EN1^K5IQDs6&q$GBR#HVXCRpvifynRQ5V=vtM@F}NUe+rcA zp`7yY<$z|0k#6|>o3450G8lpnd|i3vu466CaszF+G#yQ?hs@fqb@>w9W)dcPUwq z`W#G%9itU!;8UJ#v?%b~3g?|A@HrGAVb0QriBjbTSO=X)H^vhiA0=LjeKor@doT9k zo3YsRjo4^n!vrnw&3^j9IOr3oIlTAajq7lz7S6scz5P|}{lo`fJ=jfv2jYcn@or-F z-p5~LDSoQU0dZY028W7(1AFgutq6X=4oNA&kI{;^Uj57Ml~*sky^KBw%ps{0{j>5j zc4392o>Ly__(cR_QFA~kanD-oHRWj$=FVh8|^eX4IEmB5-JO_dVeoxL(;b`xJV@|7GiV-m+^Z zN4A?!?0D3Mo^1C<(;V~1D_`dA;An>=Mk`2jK*3bPiLnBFP~5#;nz^4%m1>o{o#Ne% zV(if#l&9E@?c9gloSjN7R;u017H`&ysm@O6{k2m3(fGn=#mTW03@+6sCu=2mT8z~y zonjWw#w|?jbV}LDy=-c%bAR&lwUlJrr0t&K{Gd>#Vf&+rP~i(*QbQfsmLuCbTJh?y zm!ka^$ONV8+%P))wQ_iuX3x@vxA-vSat7%afYhvxZwh+aN>>~fCE{A#nT$^PNWHBsQVT5Vs`7n|oUA^{o*$S1o2@fNtY)1u_# zvIi#xpjxtFTeAfszLKW`T8?vVdo{f5+2gW-OjH~;SGGoU^EbD1?6Amf12pNj)z~!i zM@1{hBA>F7Q@XKRqVrpvIYR1c*@NQ6;%Lfrf~39iR1vO9*;E`dZDmWNqi|6gg$Iyt zuLQ|`v+&nTn9Vni$vDjxh*a+>JTl3FQ`tCdj!cch7qk*CO>mHsqp0@597Zd!fS)>{ z1T7*fJQ6U6DWL1lOG>Uy*%|Y<7o$r!&ou%?LYqgTMOf_JLc-ZfJ4tC0a6BJ-R1!t z=r{D;?y(;x12}N4{OW2mETsSrXdD=6-Jdd~@4nyVID3|V3bD0g^^T;sG}`@44C zv&$YlDL8~D`98M2`ieWmdFB?I3=RTmWimRP{AOb@42b{^!~}rDrWc|w0yqp0I(S7n zBwwUZCaVdYB0qT0L?yOkMG8;6R0R90CWJ zACxK494K?0XbzKyo`8cTM}h-}GA%fi{u}umfM;{?O2;g9{e4Sf#2^V3hh(@`u-aLSRT}=JXhS;p?o~dA@R4l>hwAd|L*FBCAVXik z!QX!{BH%!0rvuf63LF1BhloKQD+` zQ6TSNz^ur*0keX46N*_;2$&UC+R^9GBC}#KXjV{}OtLAIUlQ)vtk|DjJnAhA))04Q zww>j=r+f~EE}DbaURIg|P9Wgme|P8!IG|;t%!=ip&tWKX`!oyFn7tUCe&^D+uQ9Vc zL>&UnWT%r;vK}^?RF?%+gSnHCC{3KGGWfa5 z1Ql^)8x{1tg@kerBAIMfkU}2~4h3=&4gv@Ki3fT#I5^-LM7x8`3Jnf0Gc%~OIpE<8 z)*L*rgsP%BIHX*pqoNggDU}l$t;li2)QyZ*6v!+Y9IY4{cw`%MNQxdVNk-^4vgzJ^ z4{b`8=?!)bL{A}CdM;^taB^pQCW%#Qq@1Mm5`ojV+N_`n@tq?Fix!~#o1yWYTu9tl)xKS1%VV!LJzU^D`){pk;YgVUf*>&^uqHrjH6`4o8Cn z|7=HtLxJ+J3<3u_*Ek3q_{2n{=HNOU$`t*9qN@&vGN}nTa9AS(9NNaLK>8qX;Dc3e zG&u0K@4?_O_{cUUm?gQyFZmVJZgsk35)J8BHX9g--| zPJXp(=%+$HZ8xqtv=thn74@v& ziIh(5j=xr^j3u>Ds2USIF$s}q6++TkW2ZJAO<}Y`M+RgH5iQXIW@dWKCg1GT)@tx) zKwD!{bkpR{r&NWLS&mmj$@-0edD)V@1p6CX2o{j@YOs!9#PskuYyCRuY_yOHfbCA_S1f@ zU3zDuIW!`;#-*%T@jXp1Y&QysP4Hg125~LpzlTt!p@qSLByi$pEh8#@(19K*J@NO^X~cu zw%Zjlo0JDrYgY1+cUF+w8$Pi&!^0UQFJFS$f3L;+9$=0YEB)WD_O)-*D2n!LHKG9> zsF-Cgf?QAcl1D=u=_lh>LW&FY)19=a2+W8>nPO1*g==Z;G?mbUuZU&Skqr#o`FVCKkBXAVQ|Fh%u7YEliu5ylKyZD?m^`uWG-b zavn%*QG^@?eZ7-OnjDf6Xdogr6||s=SG{Q`erNA$R-6ceu!29CBzPE-fj9(1(BNfQ z6t)-K96T?2@E~|o5b>fO;z4`@kBgXt`xv?UR35~$-Sy?mH|5OWzbIPU+f#qtU9(WV zGu1uSlb;9|(`$tW*Po1r7=Vkq<(_G{<&cE5ru`334hwNOveR_0BhFg<}c zrDezv$^?bJQpLvsNTF0571kH*prYBkH6YWe z{&vq{b6l93Jr8ZFsUa{M?J~Q;)WqL&W@b;TP3ZV!o2NP)jOgwJ5lN-NqC}*$#ov#U z1NlzVPY!KBsfHpN9~~aX$wB^02L^+r3!Y9=4oM$UZ8|Y4@YSnYKnPr`71X3Hnhl&e zK2x?v;-eKm*_H`veBF7v+&*=Q=*JXDmVx@Re1rq^)K}03-famMOwA_GF zyk3=gJA@8?x!M%yo&)GOT?|HBF-VPMeKyPj%mHwNC2Ul0M&h@8c$n{U=Y`|tH2*f3 zE{_Ws+__qD1rO;L2Och40oD*%eHl2#pIuKqKXXb-M4%25$d`sfV-S-RR6Lm9nm4jc z(>r|>0@ukzuN{=vf+`{npi4G|ZV1#iIcO2rZ=c}4Sq*I{Ew_xEf|R%2ztikpL6H<}e(;B;Yq0l@ozZiRyZoy;!qL_G8Zu?y;a zJ=<%A*;2TN8!4j5FBtPTciUk-TX#s_-BVf9{z)vt~M?z#IoyekXjbGUbE@Mib)FP4( z4JZcSOLy09zCXUR7#ZPw-al?oUNg_z$woFgsJXHJZ=d#u+AzYmT~^@Bbo~h)PKnT8 z2qFDO+$bx1CnfO^Qk5tpWA92oA39zh&d!RmjUPKuZdt`mfun$s7H)$~n?7I&EK4{o zEUJ?dXh;AJXce`YsMG3x)i2vcVSCSApKPadE6!}s%6vN>_r?ap@%U}Rdl89Pqz!tI zfP^Bdp=sCn)_zDG^ac6b9@P?umacO8?6(hon@v5K9ij zuPpLn%{r>s`Dp*~)A#S6URrP>s2SqcG#g)8Q7W-~CzfWPT=9zN!R4Y&xM?i~kUH7Ir}Iz|G)Oa$=QedOkka_)5EONe3^ykwA~OI%Td zZI)VLm@}eQ%T^DeqSXairll@qK-~~nofZ8X*9t~KIU0UwU&r?8bR6Dkp0dr;DTh5E zkw#5&$P~0>nl_Y73X&q}>7a2abATRsJlT5)>Gq^vH01f->D-F+BDA`@(3^oL%)mj;|EkZSt$w|BUK>nRY1Va*5f(`LdsD#v{MG7HUNC`0Lf^~V_h#Pjrzdh{hPm+!P{by7Gc87_GqRqhQ_4taFt%#yY^76hmN+$77bb)TV}|#YsG#0BoakSprO0SW zR@B(+4p~vvhDa=gZ5&88iAhE0j1G}&^fH6rZywpkeGVzOVf1O_Q{+%W(p8KOMT3P1wfkY_VuwSUSPCSQf z8RF)UX8|>burfggOX{uiPyy0;m>kF=?>>Hto_;zw{rF*%?2gpoyiYYu9LWeAwK{l3 zB>WE(civlpyyh6OC0B?wVk*sXNR1$9x|m@&=x$uFjR^6&Y#<+_YJoCNBVeqCoTx;F zK&B*z2|a4W_Sk#*pp*>5Sw^Og!K9=rI1V9KQ0Qu52$){c08c=$zbVVuCLc8zOmsq7 zUS$4^88B9njhR@PLP%s1T+TNk_!lVKlH(S_LH(K|PK8n$RG9i~_QHj1R#!cg1_0&s z{_b6-!Cx>bi;3W3sv;^IJ(j$R10x#Xt(A$hpZa4%y47F#`}*S^vLO_g#PX$%pKk~rn-k+ zCPd&@$W}2VNgN?zf1b_Zw?gpaOS%|U1t{3?x-Di7;L$noy9ZCsbS?-MPATxh82MA+ z6PKZ~?$oLL{B0^p9!wiSig1`z9JrR#clK^n`6(}bT z`2J2>kIN?=^aEWM<8c_YsEaRhCP!~eGz+MW!_?5@nuOqwgRn?soBUydR#sO8KH<>x zQk-2V-n+PzZ6RHOm{>Jnib7{HFns4QI?i{#a*<_^H>I3eKYfv22jQS^W}kk#tcILF zlNOQ(n8KSNNFRai(`pKW-T2qN6=9TZb@bRD8<&Jr*UU9O*UYPP%jp|)#|p+*W5hK0 zajsVO+PrUAme33n@_E}5!m=D;t{LvA+NF(C^p2r|E}G-r2A>PV1>7jpHDV@Ynqyh8 zx(QRPLBpjOJmQaUNb#3+s%uHMSptWKc`~=mpmALo^Q%UzFXo2X2mtjG2V;Vw)w651 zXIk!ZDXUtrVha+SnB}(T*4aQZDpTrCf!FM|E0z?-%mw|1WjdEkL(=YAF*z5#N`2Gj zAh9xEuG4D;>h=+Q>koyBzM`+E8^d(!qMlF3^T#Y6-PH3k7)$;Tko0dP*v3|r#k29_ z3mea_Y$e#%g>;0a@|V+kHj!VCT#=BO$py;xwRk=&qtYKAEFky~Hm^hy<0L@e$dxh5 zwIYNtG?to!Gyq1k#pg$esC#ZhQOMjtIO5p#hqq5}9Nv9-yqUTfefsYB?ePo9-oALD z6O>NqR0`dQ*F)9eU7acVE6z^vhiMMKF$iGz)mn+iu-&bUWkSQh$#( zH3XC=R<|ZS*P4o1tzB?^znZjad@?<0SEr+Z`GdAS0RWsZ6gG0*qS9=7Y+Q8xMpB4k z^QI^k8=loK_k3$MNY+kF4;r*dZ)D(LO+jACOznXrPzp0M6XqnhTb)H{t&3cGC1twMVO z6gC5+=r`^O)9X#9X5ck^tKO^iMX_MYcrbO0)-JDH3PBAz3Pgt>&v&V^4XRox~;+e;fN)`Ji>oxgsdkl!VPizndkL=ybLsseV?Q}$9Wfn_P zIBY@BBuh6-c5=sN!_0<$IdTJWv1B6SW+Xy2a1mDKt;1qAN=`y01qjq8C6sMV3WwOG zsnsrb`6W^D`h30@bSopjXQ;zZE8l~)=UbP=)GiLVTeJO&Kgjrfdz8F?!e@LIv^?DN zOq-aIRnWF)Hp;#|AGK!VVAQJnYrHz(qEq#3&*Q~*&*cSQbOOJ0;?Ryqypt9|J_;!u zoayzsEe17KY~D#$V1g;PY6FutMWJWWEZ1DHtmG!1J@WXV(T-Kh^Po|;3)ObeJIMrf z=loJi;}M0D*RC0SR35v0=8N4N5Ip0fTF$L>e7k;s;P+uJ?im{W&F2qZsX#-A@Slj@ ziZcijgtQ$|Nc>A6-P@_H2Q1IoEq(;RxQ}=`x_)He(D}{uK4;C+*5-ge$ny?scO)w2 zlMdsJZ0Z#Vhx<)FkisD|Oz-ggP5Z|2_EF`YotMb}aTE@?sDiUlgedq-MA??{91_nT zZHD8+P|$}jEtBZ2SRUCPyxiZ1?{^b+qepeSb@%qIohWDyHx9BdKHWGtJU!LVS}VVd zAClNK_+QtY{$*b%+>VjbU0yph#7D56VDDN+$vR1++3Xniv-$H2)C<#_-J(gsyAoy8=Js8#iNVmyY>E%V< z+F=hk)F;x6=NBm3O_nY_VEYnfyVv5^_!iS)Fu(g~$T#+Pqa0iod;9GtFwqg`581-B zz`*99qyTzS$!n?zQMeTs4^syiWh(UROb!g36@yn4-sgmHOl3W0?MZSrT z&}-2FAF)ZlaQoJS^^J=gr*Cw$IUGLyq@)zBN-~Q+i?3=Ws|el+@hTiGOXh!;x8ipi zB+((D8Nk!bZ?VacabcF`tP5&`3&Nq zz(@LHgxsl9zT$v#n5_H~ob3_~Ohnxit{ldb@G07l-0jX{ajWFua$|4NI2YF?+0Lq& zS)q^Zy@j0;cWKUNDbn?L$(2LnUW6iSdx5=}T1AQ6{C*042`DzW&w;15Ud=s*kY(9M_|Y*ahk}BxTCk6MRk+cA+m|%ghQs1i${^ep@;p^Kb_VKgxw{e_MU}!Y^!_ zuRt|0hvcWTFV*UsFGl6<*{f^$W%BZ?KdmR}TlE&q^FB;D$S%?}`d7?~-@M5b{4iUA zF-~yP>7gM`BJANSpd7}@9GfLg#3>YcK^@`Y(ivf0$H)&R`k;)LbKFUm3ruim<4#GF z_I?iD2s_;l(BN^1lOVd^34q|9j@_qzfEV{?Zs<(o$UNB}M*7^6#OH&piqd@n8`;L# zkFn?68sVnvnP%^bS=O9Vf_=Hb9jDO<-tTND+ruu{1$QT*gN8HvrWfhy-DTH7Gv8K) zOCx`0{`WX^jz{p2oq5Kg8Gr`E$$0EKsXN>1f!VQahD%$O#A^;oVeOvxIdC&VOd(2? z0nb(mDfr_o%l5Oratg{}#zqdechKIQU0kc;8-<$~&AOl*K;BJI4qhF1jpsXt$g+Lf z+QQEdJmd@u2e=jBqbE0Bn>8~=kQcMn!jCRgvs=NvgjZ5ml3cU)r0S>?o4j6N&t9}y*EPoiCz5RC<1BJ8 zO8zMv?5r3m#jci{V;5d zdcAogS1igEZ~M5zz;sI}4yd!a+wP-6=^%$zV$* z$$+Ueagpz)!xIb4!pl%PBy#{JDW&X)hB|uU)ayrkP2a3{e1z^kcpwoSH+w1pFk>NYd?#>?CF?Va1>b}?X z{Mvhwx>sVUD#8IFvutN;NM3$)%)yEBD-C`SWj*(havg%8BzQ5 zhqJSnFW-D>%P6T_Z(dpt;?6hbLD%b?i=;MERFt0(!(WiZ?@}tiNdf%WrIn(98EYG` zTbUF=#ezx^+BP(a5iSapQc0zh6je5nN7Rx^N|aix2}*=bPHK#T&pb+xf$$#yg+(+% zsme&^=0*b=paj<`T2x~t_@UZ4A(M_l$%qhN#5 zEHEo)D=J(XIl3XMOS?adl_*pz_1Z;%CqO&}RO*G6QHSFtrFOBdK;eC*dMWD#U!~q^ z8#Oq|#sh$_G|(kDX&4u33K$aCt}C@l)hM(xctjm8C^SPw+qt)~6RQ-fdpbMOMfzIX?t1i9$|` z$fK}MSY49+583|8-qq_w55z#xL>DR5r%NP`;zL)^A*yS53mRU528lPIWd1K10^UoV=2o59 zk}N}G9&tob=BBA(kvEGPfGgX~#QkA~f^W!)v^=qfwLTlPoEMk~o|OzS`@!{O=Vx&%F6 zfpVwEhHB4KoL_EHop8`LZEo*vlCXX;S2;5Tf9%=*g{u`mbJ=B^x0fI7O2G{E9q@~e zJu`$k!!)7XXtK5OcwR$#cRI+FC68gd#7$v-NS(|}>yjdM5@fo}tsYaf*?BD6)rP3r ztnQO7rrK`s3mL>KGAdJYva>QOZbNRPTsa=lz2w{-SJ0#48O2cQ&7IE7g5zi@+^LI2QI{Z2?JjK0-AGFGAukfHua;XD^!u|StYeY-OcSGuWSf&4 zgy4@SDdle4jN5JAb-e23xli`4U`2`;h|Z2M>x~ z6!fqcVL%VMU$8&m#Xp$OG|98-&BGq{APb6{?&&0zs_NuWnXV*NwXGUParW}$N%hK> zGxN)4*^>fB)tiA0tbC!0tHO}bWLSnpFEQ4RWLRal0&-aeD9zhwGN#8I0JUm- z@d5;x9G0BL39;V9CZwcvHH~LqhC-qfgSvK5M>5NS_h_=sc4P5h+g4ce-3qmornGzf z7*Rc_k(=SP39i^tyYF{u?GSz*AvB)yIno&nt)4oQXEoV2(LMQe-JPQkh5!OJMHgbe zesRnOc2(#2m~|AU+bkMh{qh7={W*Ml`tHNuo!(_PaaC^Kr<`S>Uqy zWjH*DpZrKI#NNne0OPA^9G_uPRCXK`+j-Rzt3W6A@b)6c)d~CGEnvTlX2PA;Lto>kWKiBiP_Dq~C)QqSW91frCFk zCvp3l$^+U|^G5Z!Z?+CqJb9U%sfhxrgzzuf1PgQy@)rUd7%rM7&2mPm+MJ(`KTgmQ zS)627y@@P%qe^1-P_0OQB&=VG|2{3^))>(bcnmx_-w$YG$nJUv^lZ zZ9o`vm?)E1dB`zIY-KV8zjPb7PB9?Yj-WR}yqS6*3W@Psu)V^A1z>Vv8_yeAkXj0={-gRQJ_V?=+%_#l8+Uo9m4&@Qr4N@Kf#9YmO)JH#1WKKC`{dL2 z@%$WgR2NX9N+IEKe9U^z`at2qiV2d9Sr!<4bCkStvuguy}E{+88aKXcD$~I;;sq&oJ^6+U8Ay2&Id0qhAY+R{^oz3Wy_5NFu3Y z6^+}?c}B50E5bEBfWJGNll2crA}hg0I6U-UNba^W57i2;@!PJyZOiEnA5{zpv4*qh z1e@;R001`EAt_^>(>Md}VbFyXs$^)<=nlI5`1}m$7_$nbJ_Trd1 zf}aCyAakn>)TLZZQE2>3s}2T81P8j9>UsY_&7~3+U_#lwPgep=4!=ZhzfVXt5BVdfvgpAh=MB zj5|mVR{4?R;~WGk;tJ?oy~y$Q&apX(R{a9v>ME{dW41Pi@~|-%&O%yBGUmz)<X@W!Uu)-(=X!wE3K= zVrG%?;sDr?Y3Dv}Psg)o17M)-!}^eRSdH^Egxxrf*D<~Tzk;{UuzCFW5U?0OR#-D- z*d`U&Pxh{+<)$bKQ>5d{;B?!?A*I};M7T#`B&AG@m{2lOQtB!)Q3f(2G4TVGDgVPS z@Mo;OpR=B)C&x&*;+*&0XMe4I_I`W!I_rDiX`g?bO?@(D{NfCZX~-CoPC4v4^J!bI zhac-gpvv57?pXgea#(y<0XChye~Kp=Y??BO_CzK5a?$WI3oxss+SlT``C+(8g(NVV zv&f0yg*g@{!aa?_OLwWDLA`~>jJKItvl@Kbk@}j+P2rwdGiv8uwy}5CaRq}A>4C79 zj7Jdrt5&xw(iQ5CX5CVCu`equhFI;IV0YGj;TU!7#D=Bkljuvp6~@;rM-%jxHPozyq>0T~A-TF42+HS>&3=-t z#Y0*jJAc%B<6S7?*3VoNNHv;cw)s!2-#uD6Vnac1ju?`UfE^M!RLbGQiLL|t)#2KW z2g(@}yt6Bd?7c_81Y^S<6bJ$MJ@r6AsBk2wo>Fjj zbnX!P1bZfWaHVqE=lmzDFHS`Zv znI7&?Pq7*5AUZ)b2A+CMFrpl0{R%u-nt+{v$#I2K?DeuczHxI+?W2GJ!5ES#(PjH1 zkOMLA9@3?0WM77{ZSPWRoG;Z@A9~=7(p{gBMlFG9B&%3j*N$oWfaIUA+M8Jo%oVjf4g;>-)a&TMU%o@Ytd^{2- z6UY=8fw0~*%E1wV*aMLesxp^IjpW8q1VQuBhm~~D`0fFbMo{q-asXSwXdnoRad?PA zsS$D-57uqo)RULTv7z{WsPThG^d`wTJiFM}ivDmrf+@gzy2xYfD8~pix&sc!T`Q7u zV1QXCn6~f4t3N5uSQL`P9(Ng3xKYlvhl>tHTgWVtZNM=AOkm?Q>_fZ9OewOcm!UvF z7L)rJ-(2WrAj!l+M3m!iUYFXtyna9GL(_iyYh}P}v%%KPX;TTq!VlLxFOby7+I@uL zV%FFI6xqXs4|ZmKLk>2UNDm_*2<1WpH4+iA*2sj_&6jH;0=`1vErz7Y9FlUFg8H;z zqMQ9SMEG#!?3=OS?Fajx9KQHqxVHaoA6Veh{ln$Zt!Kd~mVk+*5f@j_JwM*UAxCub z^7`#q9$Xwjh)y4%axu+mf;&e<6s!U&VD{L8Z_{QF~t4s|<;WQGW zaEd_!G7c_+Fux$8SyPx03}zWg5o~2qHrg!<;(}yt0%Bod5kyj?woc&_*jU*50@nHn zc1|YFo*Z1ffS{sre|9Ew*%`^4oa=Xf+l8?esE@?ZB_s`aTFBv^L|3A-1D1 zV2p%9odhoAYgiVFEj3l6hH=imyp=rDfJQ|dN$rcpyzyP#LBu;@^kC?R$+tlz(LEI5 zAj~&S@sHAwT?2^(v5P*$IZ)bw6_UA$vIkflZy-&fx$H;cfSst@HS!$O$q!JNJz*lW zV+!7;QR56?9B}}r%Qg;{dprlLbC|qlTu-KuaG$>b$OZF##*Zga36%;fUw0C-mGnkH--2K-;GpipvqBvpOHneJc`rs=+H0~@)CxGy=Ae#j*gr>%%G%BMyl zIqu}tNkZH7-yg_1{Lbsd1WVNj7-%k>9&!Xw!b4&2+XVhJxOocUfd0%R9rkb^Veg4@ zktjL$uF_aF^G-btcy*5tx4003`(`xqB3$T?e@v|ximTxg@k(gURfy2rw-4yj%r0b_ zh>yTndgGowvS$(*hN{a0#9JSy&_(sjcwXu*#Xm~Ruf)1EGtt6XOB{CX?yz_RZO z!TaS=Dqa+`WPRKqnxb0F>Kybc_j|UVzdQy1iSOOOh94U~vFuGmvjMw#eENNhA-yLK z!=1^i-IgYyM>M#?dL^KRk)TSG!DxrTz({mfAVc*y?Cm%4Ce#fq+HTv{fa#k+3zL|S z*R5)|9(xYQMV@63(^Ou&DUOL3y_RxyA70n8zUIn!Co#$!gUUxu?05F;(LL+pp^$BQ zgeX)I8Y_r^L8AmnZ~At(ItMxz5`^jR(FsvUflLC`auBFz!x&t>f08VO07+YzI1Jey zbq?k~$V#6wUH7Z92`YFMFNM6@^CapZe$l{lA%yIM$5{q-7{twu$D3JRK~qD$g_-m` z+3)1hIi4O>O@CBv*6U5WRJ))~fz{A<6_i`3BiKf8JBMEE#beYt7>Vcp86?xV=RYm* z>bGXuo^S;CWb^RSmomEleWCgOan4XKaHs$1M7B!ORTFz64y0){X|76L-D^TAA{_o2 ze|n1>bLKp&p%*DT*#hspnLI%HA|fw zrq+}p9iC~jA=`Uv1p@lYw0H$Fo`>-2+FZza%!0}ZBGtgAP18IM(uXNj6r=-U}=Zk#JSXgt`+3Y@!XS*AB5;LhTI6Krn0qi-F*OnzKFHL zY=|fx{&7?0i9_zwqlQl#hlZ>bqJmtSBsVI}#9#^tb+c)AeI{#Y7+O3{VQ;S{u1(eq zHC570%%nu)ZwGs}_g8M+6uuTCJ4~M2r`%M1r-w^iH?rb|n*ppsefoI?Lanp%PAuHe znL}HOYpZfLSI3wM#ts( zlHfmB{l*9wN9`&oS!fJN!7QdG3|7 zl{kh7)05aF8%0g^0~minV3=$bL1QVLt(}B&@rX8sV!y}l<>4sEws5Q&%$2@ip*r?T zhPtyz3nzqn;2e&L7S=M>I*0OR<_K*n4}DkMUMRhK!#U`0>SRd_si;w%z}S!=4LkTS z5){n^cmD0mcYpZx{?h}!YBY9W;p{cTl%t0=!Jj^B;Hf0o)BM2-TvHu8Xk6My&WSEZJa4er==Ch5y!y&+J{zYb#Y0e-?|(6GoVn z0Ud)*K_^)lqj`?Sl&BNmG!%W*MMViMZ9oJGn0cTr6vQg(B9vI3T_jxvvUMQ^Hvu=@ z2u0{tT(~R#8J;^Qr@tKA*lHE)$8Bfk-nkz$_x0W1oO?b&Ih^tV!o^#qy-j<5eJ}6i zzOuKg_PaL-9K7!8w*5pljV4>#tJ_Uyx3uVtaWIvJv?u^^Q5Foh8r6yl^hn!Pt5@r> z_>jBOc?2|2(8|t#9hw?qDVT3f$RiL>(lyPpw{B(Mz2oJ!@8pOgwTW-FK;ZClrTp&v zI)&7q4l8%F)Y@qhMTA*U(iZ1*#SE<9n6o8-cb4Y>p+pJ}KgDy`BLx%g7r#qB5q^TE zy&pEd?zGt!|FF2ikPDDj2tyJqWU_G}T@$Z+^U>)=%W0N7ole-j>AK~`aFPDCSrf(W z-lQfQiw}66jn3{(?`FBv?9lB+%eCB_i_Y#w)1xEX*(f)iZqv0E9|)TpyVhdpJX5|G zIyV>HaCbM9y$qAABZefUS(d1%5d8nHK)P4eum-b%z@b~(eBfn@Cw(WIec~&tWOs;i z$PwALPD%up72fOGcXs#OFsyv&;e)Ac9z+(ZwgsjhN>GqYAX+veX^**C(E1gU)n;J~ z6WNzosmXbKWqW1Ya;$yh1+WjZs(P5 zXc>gLmE-Z1rzvGD2Inqi_w|q zUz?%qxf+s|*hIJ_DGFV4Oh4)v9H)ZogwvsEFdZ7ScU;Oij!UPIZksc8q8@4Y=~SW8 z)OsdEZA}|5G@E1BN&A{{U~rMK?A;71D7gjtaiujQkW;p^xf}sTZfQF=(sLkbMVQNFo!#m^=YcRsEEJ2j zy=4z9MtMhgbf81cRWl?{Gnn&v&R!(l2XH79Cag@kpUKcKXXAkHs+6Hxq#(}dS}JqT z07FiGp1xtgK`B||{U3gP{yBjIs9nJmlrq_7JqH*~SmpqWh6-#?FY4#|)1&nDSBZlNZbb0^j+X`q&Ti#yP%dNeCQZAf&*`hP4H*LOV$7(HWQ_@&gSX z{1B(S?4^BJTm^*+Q=R>acnPK^^w#b)i63F4m$h^PZnNvL4KXR*=I;_;8xi#sn-QE4 zGqO^R(sP#Lq$u0t{uU=w4ux@O@MB7TVDiCm-&}P{t20sxdWWhuNDjGYFK%TN0u*qb zh#4_~gKQ==N1q+W$sMUQNqk5mR;Ec}l_~#J;ur*5P5!mms4}gfEJWxd$-wZYV=LWh;vqxQs9YfFKqwF=q%?CYo&X zQko17YP|+acBxdddp^+zetlRpWP(}c>4j3XWtZ%wfT@dMDjt*y0qBWLd{QD%2>i;h zkdA>mJPBEp0S=I3YF*3O*T`SJvNV^2y-$zv9M~%JMb~}wQTI`oE_9W{9c5k4a|=21 z$&|P%=mmEL%n=e=i3Ral_HrQ7D9<%D}qOy=B*+R2EuN(45Q0Wy*I-Yc?sv5FnL$5$!Q& zq9-V5OA9z$lrl+jSWS9<(29M3br=N&KW-aR!S`Ec$IxF}Y4;LBg{eXux8nGqO6Oq8 zr?XF+iMjnsi}F!pP&^@Mpu2OKVxbjeGLu0`^BiEVr9DTHC;V4OT0{^Aj`)$U5zhg7 z4wMKSKK|f~E2r)kS5FU#o&yEag)$nQ{*SVKmgfL7c*iF0u}2`W_VPLdt`TKAA?ZYV ztMlHp`)Fv^+QR0`*un$SC z0cuYFi!uQYP%a|M+TQhCV`<;1J|2dyOSVeeqy%NSzt+6xl7Puj;>6|;!|@nV7(vtq z&nhm-cNP!O0W)0Cq7p8MYiUd29DLvE7m=KW zIjiv?XE_HP6np%#(u#OkkM`|ByM7ea25Xys?YMZn8XemQ1KKD|_3z}jj`rhvskdeC z?A!Y%_D+)Dit;o4|vOa-n7ZMo;XGE_U>jw1ecawH~d1vRT35%xX4oA4>fF@7&V|lr&P8r%Yri% z8Ot<0UAxZb8y+6TwW2;HP@{b}$%aC99AI!tEh2!!Juc=fo(c4v;dAp`$^ixl3=VS_ z(Deci{hg(L@uV`hW;gn6`*>%|E*x+5Dn)M&-t zFHJ@J6~A`akLrWiZVhks`}JYZ_S<$ln7T-KA(VsC`Q{VvGT7Iwvf$*V55M{5SO|XY ziIST@-&&HOri`v~oMv8n;ptRsTN#HMj}9gjDQJMM#W$O;S-IlyDF}*OXHT5dNC` zAkJaK&Qq=n=lMFs1}GJHnA1E(kT3B4B;%m9o-gnNkc68SC>5RBH%NRpd}CS8lvYZGsX{kc zGs}{bEJon*uH`HlkO0~&UHBJozV*}_ZXjYo~xfOrvkj>Y%!D8=9$lMsZR z7hD(aDCsJ&p-ss#NlVXJ-jNS^8YmdRpnee^l1@;TG{uSj9I!`M;Gl3p7h_t}Q%9w0 zzE`XbqsG=jsXoI4R}xr6d}b8kOT{$F8o`r;<{>QAv3Y zwPI8oCic=2VZ{aX3cvx284FMx#w{-s>7bn002~nU3Uh|9N*xwoFJ?SUIq!e?k*8*# z#QKL{D-_WWTG7%FzzlHU<{(C{pC_oRC@@@oTLA)J2eAX^Ub|wnplG1g8L(imps+$& zg$11eM3#=L$ybp8z+@YW%6Pg3qQ-e+*%^NyN;(FWxt{kZbU~DD;FzAXa$ct+2`xA%KptBH(={oxpfgvSWNz|yMP3AR=GPXQLFTMqM9*2YzPYt9AuOIVr3nsZ z@qc)vJ{?S5CS^=BKQKy88B+*+pRQaqwLvhKW&?AFioR(da0fpV@Hs=(X}&nHYn%4| z<)szSi^j)Sich}v`E%!l&7T>yQgop zxpQ)IGLqO;S2yCGOzzyjx%W=#`Et%Z_ntez

UL!Fg`4xDSc8Aq0n({Ps~%Q#$uY zZ_w8YJp6FisTGfq4Azvjzwg;@Bieke2k6x>IMAz~r1suychLLKKHd2Uei6VQ890#S z@!-2MS%dfJ4xAjxl_ywW=W;%RUnB?2d-H|APtQH`!b6}h2|=_EN%2s_FT~(r{DT`E z)IYf6;-T~`;J~K+m-jNmiEg4Yyg{CsM8oGL1sp#8+)I`28gydVLC2H?Y3(N@k$7*# z$H~#ZMDwHn64iKDCi4v^#3~q=S|6ep;-cn+q<{lr!_imWLO*>ba}G3cR%5#H2>wUu zU{)-3FYhH(cTvR!2gV0Bjfz8f*UM8(bs43}*<2@2P|Q|Za`^S#Zl@nmauC^@W13h; z3gq?!R4BWbyt^{p(l38`_uCJWJb>cDJY+r$*t^0qtCD=nx6^eH2YpyZruS-ybaa8ddEV`F}| zyEel%WgtS{p-XihmG_<7y@ka134UjlA#mulLQ=e=ym|BM?xz9vh|+37lK!wPPRFs>xshjm4N(9{i5>u!vy5IC?!Q*4HmV6&kH z2h(1wScg0~e3&kJ4QiZ9Xwv@7Z z%~oka7?a!b+qQ*Ol-&RYx4vrh7J)dgj3p`VpSE0ZBY=J?}gF1OqiVJl(;c10w?k{`bDy?Q)uV`(Q}y zePS*2tVakWmWBlO+`OSate~msVAchP&g?CD4~x9`Akh~Q9B7<@KCkObeLdc??Kw!7 zxz3dw=-=No1sITm>)W}-+;%m{Lbm2QLIkUtg2^C8Fo{{BB#2^R`iFA470XiN+ct5j zzU*|_aDR!77%W*XXAu#vnfM{KLQde7T?}%ze*+vc!^N^;`iWt&ed1}s95h(eaR#RnVTfLTnoNNVc8@a+V zf@(IsK7c8|mM#pxg+hWEG!RAnIxlEp(r4U&iW44C@mCE$K7!nYjTp-bWm(vPy$At6 z0||!0sRRzBRETRq)-K#Q%{d%e&fJZHW$6xm`Wy7O=nE#p6eV?n;tWO9AH`Y+A_@@K zV>QYH8Pt~i!CY|YLfWtdj2u$@0@;c`UWKjnTPZ)P?AN5L$1R$F<8(M{JN@F zt#}~km4ip0w&8s#eDT8I_eYoh5-x~Z!9FC}M&UUN@bQGCNDlnNOlk!Q4c=Kpzr7Aa zMzwEcQZ41Hc5u>MTCq)IGXGSdvbJSdE4hGeP7dUJlh)#r8p{VPE4a95Sz}wTEc#8g zA67&0gmSJbdvbfg_Nn#!mRd-t>MMTarNp>0wV=Sz;G>Yh;aNHKzj;!I=iE6B4%PtJ zS2OC>820D4x2h{U>p_|2poY0^k{KpQG-0{HYytpgv+_X1PnjesaBY4Vo1P%};8skM z9P|*+*|L2;I7G79$b*AH7O7v75q?zC6jGZH4h~H<$TP3dDvSpvr+Lc*5t}l_!7PD; zOc8X3htXI#iMLBA9$Z2U!0!dU)|1wKCePp&ZV$OL@0Pon7cjVRVUPjk+->MD@`R*# zf?}aT);T<6tD%9{f5k(D(Gr!-2U>Dq+1g}wB6dJRO>_XYzk9!4ukS{hazgl2 z9>sKjN~g0MYsYVGV&nlSi{K#plH{iqiE_o)Y;y^Oc%K%@LFOFJrO`VD1c&*fTh~TE zxOr*t+O^ROA03Sj9$nG$Lb3?Xm0Ib}{OC@1Ep!0Qs!Z=ir-Mgj?}X# z7o=8@)sh7WzGEP!OjX=bg}ED-^N2=UQ!u7>maIcIu+Te@w&XS}Kl7MoAeg{7_v{Ws z*QSSQQh{r;EMj7@$+hu0>Pmk|t!OVf$mXKXYQ+p*#PkP{E<&~^8p`&gToA%rHll-2 zL?c3|zb6`vh9Z&O%5E0f_!Q6iD1Q zvB@m;DE-PXSaOh+TkQ`2cW*=0sgc&>@P=^Cfl3=&w%dTil|KT9`Ri~7ADyL>^YJ@J z7`_-=E2D zZf?d7qSH{%M|-k)P@z!t9%h~N*UQnK)amqnR7^H+pGNybo3YJ8`S_ro-`F`m-R&vw z*2_5GI9)Ge#ov>_1%ZQSvzP}P#92N0nw2?AbQ9-U1SNA0=W#HB!=;-wUmq%d23WjrPzhEOc z6gPb91M2mZ)>HJH10o56!{v%nd1nB&u;a=UOd}QNp0g6*wtEsgCkR9(oQ=Lib^H zJYL8^JzIc{cm~tAowttjkPgG?3{(nmy9TE{;G{9KT*^Q(RD0aWW;o>fI~ zNXL``iE65-Adlc+#H!n%GE-_F78m0Q4Gw-Mr^;ciKwO2zv2x$*jz$|}A6D|Il8Wb- zpj=Jusnv9KXR%VwLC*H6iwna`L0T?FUGnNFnviU6wqlR*a#W_QbD-K6bK4tF4aAC; zmK<14AP}G%8EG+aFYbyh74(&NZYU38vY?s-7t|OJTmE1yIdpI^>-qhaIR_6*N;-QV zpnM#zAD_T({GM{k5yaEC;yqae1Y~NsR}c4VR*#>AuyP*4@r~Mjh(_SHij8MeG2!v*-ggKlG2x#Cb z+n>;T5;(}5gYax;a9FL)&#zv;f^IYKX3+Iz_|}uJT^T(ia5%eL`v6k6P?Sgx!UveC zJ>eisBnceOA(0$todb_v-tF-U$=}$!n%5?RD83r;gM_`Yv~t;vs*8zuB4R zb98G|3M%-z&CZ)Qv&ojuFK^%1e4CXiwO|!Kt!!>twAK^5xBVeC0( zTqqhPH^0p|eAS31TsIM2mCD8^y>rIN89XvvGj_V}k%_xAM&PW>=ToiiM(q@)yN9#U zn^n_WGyBMB%uL0mPURbu6j!MQ6;b$f@ZmsQ#-6Ibk}ETq0&qAHjm86qY6+SH2Rms2 z+Pddbj|HcDCL3m!nrxfUKtxRhkxIg@%;i$HIY&X$973P`H(BYH*)>yeB zHQNF>M9i-BsA!q(ftad)0%ph{snu}c3b2d~AfY;Jomj)$;ho+tMG729bJ(S8`{Ce# zvf3Y0OV3=daK&^lXO`S&&)+7!r2rhfr9uWCi*m|)_AX!c1UM7{4li_Y;H&dH>TwSE z$)4B;{f2Kv{)TA||9l43Z&Fl788JBMQA}jciyQHnA$$vh)*Dmkm=w*mMik$bphjmb=PbE~U#d`9#kRBxZ7#<+e;-6wFj*^groE9{i{v;eb$&e3u{ya_uQc>;3vjq$+2+u5I+pMEJSzYPp*h%r z&A}Soup$9CREqr$#+yyw>OlMG*!k0?xsLBO0?pyERj+op`mLIvRE6}V3SD3B_;(pD6;_2NCq!W z8FBtR@&IuX1-+SIX;_S#^h5WIGv70a?^%-iThL#KYz`r1;@Fq|GjR_7Tx~RGS}k|9 ztsP0_YcsZxXZPFZmE()qp zC}fhz7Y7WO&&#Vfq6vsds*OR#EPo%Ss(u{x;U^!A6EIoqDBrF~s!(=6sqxH#v5&;*y;Ar+BV}gD*ZvF(rQ}u8!$+wyQSH=x$bGF5<%DL zp55!$N?kkM^JYeNrB@p{O@HL|Ms=$_jFfJ}i_tWP&05iqEJry_bKrbzcDZQ9m$sig zktN`eH6%mAr++PjB8jd2Hm+or1)9Fqsi3?5hJ3qVA5oL zG`U=9I5}77J}G5;`3Ol`R51k(ISnAVXj(ZB_5zxRTJIyD}$sZ{<6qy9nK7d;DIye}0$k71uwa1PnS2XuxQ=@pk{UxMQM=3oBil2mZk0$j}%=Duy4Fa*| zAVyQl_Af5zjocHS<2@!*n^_$kBsy>n8$D%Pw>d=rm8XCeziyPOMTZ-{t`bzZD|0-_ul|)JsZONFhffnE z@U{;GZ+aru#Due3=9cjm{HwkWpEe2^VNS@115G0Ed=^4taS)3$%>Ykwo~*7(ERW^U z^yZ)5t4dH%Z}^@l@#_DMSGqxxDyVF}o=`M@W^wWO!os2X`RREFyiU`H7Z(@bKsr62 zV4|thiG|}A7Z>h?3#Sh)yus`{)87jnNaybe@zH}c{^?#>PJSI1h~t8ioU*NIFKC`e zdX=6JHbdZ$OIwbi9*T411aHZcy^*gKuu`ODlB(wx`B~cLj^_8KeN2I$xV{WEAo4j`P+j7>GSyn(Zh}5XL@np(S97K ziQAYmK_ye>v)k)IFM4pmJyGCQPi(OGpU)xFIgqBW*gi{iU>Ag?1!KU4cQw2yX{SbET(zWMr9&Iqujd#8Sg;T?COFuV5|+WXqbb%dDF~_MI64&PnIT*g@jMjP;4Xp0+}T^hALslQPwqZ$To=#)TOv8j?w0|wvi2#mm$ev zl@rvb8h<1NhsRI%{!BW@t#6I_PX0s8=fI{3x>K;tFRv@H|K!zVx6CSD?MU8Gx8AG^ z-S10rutGBP1z)?0SHIBTVe86dFC?>5X}bcXHX(umST2=|x)pM}sQOWrj39g|tSi+L zl#N{>k&{@PN@YC+Y32cfgV;)eYvK_WQXnw&XFpmioNgh?YB=PTkg{mcM+-{>U|clk zYpo98M)K@WoSmM_Sdq1YMEfu7rg`4S;X^v;j2wBu^*DnU`y>X9BDq*JNy5d*9TA0` zpjPP|SmmDruWsB~ce*@=Y;&m!K1*Ydi#|-xT$*kit8DMdO7J6Hd;a5e@50Ym<=U`& z&OV)&+9h!KOU#i!o_(4kpsD5?Y?2zz7eT4$l>t!Jmt8AP2qC;J=04Pwa}({rd8^#{ z_5^Zd87II>wJP%##!F2*jZqYc*N2S6R=Qh<=%PrOo9MS6FtT9V)!71M@Kue)j zT~L}xmKCT*FBvtl&pHRBbhe3d1pyLa1xHzO?g*YtFu{|Ev5gbW<(3<4_FNiBnV=>s zuN9g7Z_m}c>rY4WQj&RaGrltqqUkAgI_xP6yh^PVRb{wMt*oCF;7)FsyU|ITdEX+Q zoKu3dqt}n`Vab857XLFrF*vZLaI9}VfI}ra2+gLTY^kGF$~$Xq)UESU);`E;=2dZ^ zb1Lf6f(>o!!FSEVXcP&I+$1}*Xto4!x;^he1stWy?Lv83Nvp+ni6GOJ-dbf9D6ch6 zZ|A|ZPbQ+595T}-5`jVb&Bf>vIM7W3Ulg8AMoK>96#2<&$_xb9mceUnV`k!V8o9*n z2jt)U)!2&M$ME^v*K1E7ZteK8cs+Kw@o7O^%r!e5&f|_91BWCBKBBzP=VdBsB|f5w zO|x;)>GeYgH!@W5*sNG*UV@(=hKjLsbPvr*3-LaW{GtEpf0o3;k^{>Q^b|0uCgQMi zwUpYLASR_%HkQFImo8}O%lg=47ED*yZ3NrN4wg%^sGDfEoY+(q2o8m8O>CM@q583$ ziD;~r^Uj0=ld5*zywwpeEIveBH?#+|f>6}8hfMhaq;m{6pV~dKkwzzQNZVd5Ij}7LFL2Gxz8clk4rj-EE0oHEvHc675J;z@qk97q;=@vp(~U) z0w2bf+}OZtjZjS6&fPWk@K^Ch)-Fxl@AQxyWpxVNPByq698giz~?7UUR zZ|ZvWfDN6HjYj=IHIrHaxUMQKm}Of_(Mt{$vReb7LR&-wCWu z4=8XmO(F=$qE?Z3G$|wp>1IVfhmBkpdH{z9Yfg_49C**{E1BMWeu+IjdwM$Nki^$m zw}SuXFY-VT>&r0DeDu(rUO##A;He|D7){{)&;(kDhyW(r2<{bgND@fpFt-XEw*C_# z+pA0P7p&~75FGG5C?DJHlR?o4rDzr=I6z!D;Jvp-MXgPkA!vXft`wDWQGs6+Pt1Ii zjsV^%(G)@1sO&=20lF@D00%*|3kNPj+03*9-zzS+wjG2j42T_sq+UeWh? zv!sw@y(5K9(L=dnNmG#6)E!4}&bq8i82ij7FyYZWA}vkHUV2Ph9kR`y0X;bETyOfw z-u1nvb;a=;NZMM%5rT&3j1Ea2srNKu@UVI6A??YemALMg z{0$AuqyrX0u30OGBqx|1rHl96c>_yJsydA?pceDRSql!NSYfgX z#6HO4u;qXveWc|+9W`_Wlo8j|4B&A4kOkZ0krb8~j5_e>MLNJPQ#NBKVTi})^UVbU zna4l}3eNJaeKYAXx3HCiJMzsGvOPTeG;`r7e+Uk>({8UZqhBGMo$COGTBkxAY2xQ@ zuh%;rQ1$r+ZAPZb*Vgcy_L)s@2h@TEBwV~GF#3!_v6hhjFA9~lUBT1PJ-Ib`SbB=tCR&8&mXY^P@(?O@5+GZC-trOGx9l1y+USkAsAl(A16YVTJ4b7j>^F=Re zG6cA9E#Ch7%HpSEq$aRk*=aLIG6GX$er%I;Zv}1Vzv7>FpRRuWZ!hhegOKfkiO}jo z=fJGI-;sp%@@BF*J3XP!=CXNgY_@g{Bq-&}t?5>BVpnNp;S-fV$%9z^OwGsR{#Ldu zxdlK(b9dK~ZOs# z>t~h5%+H;fis;r)&uV&I66+WB-kC1JA*O434Z}*k*6r57IW)Qeh0|W|XSwtYoCD~M zdHv$F);YzBntoAfG)`+HfdjikqT(TIDzE4PqhG=o!AqIy5xgA?E}#Uh4-Zjzf``-} zwmP&J*h!jiCV!(ObL6WQpMLu6u6X%O?1Yq; zOR*KF@2h1b&-VnDo+vM45*)_9k_qvgi?PV9!o2cw-n|@)tpzwQ4LeQBffZXZ)P34^ z0quz3uv)2$eoEBqqLMnh5HmX>)#+~P7oxu-P8(t=CC+L9hsasIt~bQyj(EBgIXZqS z`b$fCJ#y5ap8;1fFS=SPmJ-`qtWk?a`r=&cMJi%@ByeCMo{yv1xymxo5p^SBT=?Nb zaz1pwSH^9vZGuTWZ!ePohe60Ee-WEflI@qwVNh##o$`j5Y>yfV{`P@)j&3KQmYe?^ zIB*ITV%aTg1zim>ohY5S5|pCNKkIM8p)nVSX#pBbSS#9j#7ebc#KhT;!%f;`AzW3= z!C}hbDxF}Mo0?T607%OyfTJ2$C#*Mr<~k|EZGxPg49+1zTGGLXbRyK6R0`Q<6N1$@ z6$wh@OXSdBd6fP!`s}H4OFVce3JJxGUBBH@c6JUQi^H^d@^CTpv>=`oB4ZEtJYX4EGs#(}3~(?_ufv_7 z_;(bsqT3hs{uUtO{6f?;K!SD_t7+Z4+Uk*}Emb;VMtj=RrC2c^OKBZ(s^Q!>#PL!^ zn-TR&O0S8F^VIQ-=$>M(vZT#7YLVVCScuiGa&)ebqI2NyY>Umb8y(mXGx=q~V^}MA zEeh3qJQ+}^3?Fba9$j2;4}gFnioxqDP!P>7Y48_`;+Ab*ZL`A!N8dqR1D(Iatf{h1=~+zJV=vb>UxJa}-hjx^cG zGY4?ky0f|&2In9iDa!nh-+l$@q9DXeY`aFgay}2}83i1OHK*E^`hX^{utZ=TLMAw{u)+9X-%8Ik$94{dbphy8VaOwv6R4rW&jBt`WrKxiHOuO*E1TOWNf!whJRWAD zb0F`A32DNN-;{A!#;XY&=v&DnMo|?Q5PQRs7Zx~7l2bGLNuoBv0T*Y8<&`CQi-vM{ z5o&|j9>C$wwt|HF-t{{MIJhM^EdLn0eP{8BGxEfFLxMv=QAGOLicdVp72#iQEhDpT zCbqevC|~S|FXqTz!TB6aj>U4H9%{;DZ-G&D4#!%pw$#(Ke*gIRRCLb(4iRt;tNMIb zJ5HSwIOvj}(653+L(@e4=v>n^?cn?vm~bX^tzm-0gs$z_YB{is?)ehm*9n@n(^X~HRfBuI=_OtSl2(Ru<0cIW2*H~*+YlELBf z^0Mf1;dfXmnzOP{=R*|+hiOyMKsEq!pQg*_{Qh!Q_2b|76Bm4?p<9486X39`O0~jC zQ&n=ws=wIIqMzGu0u3Nv#PhrHeOF#e502JJUV_3#GN;5{Cm`jf704I}#*%)Kvx(M8 zRlROHM^faaVah~dGYY;RP1^L5>>Kbtl+b@(aV1O_LOHa zW&KkN9Ngd>?g@{ggaeU+69x$%z+p{1k+=WdQKI6Wy!9`W*<28}LW+`lp^&bD14508 z-KU3(6>K*f0UR1aN8J_Oil}$bJE8$_sEhNn?vmCM-E&cI2q|$GVnwA_ImLPn+Jy^< z6{6A?5Hot6KB(Okm;}-9i6uSKfS>{KMDL5$RA(g4few(}fss$GIcBE7z3};=c`k>E z5?t&N_gO=Og%jns48TMeA}TSW@@(ySpiNBhQ$KRLJ}TN zs;kt36ZO6+6qyxNyU0Y0z#dBICRvtrZwNlhlII32zJWA`?-bh-Hiq1Cl{`HOc5eKG>9+ zet5_>d-0CIIl!%rAirQ(e7&uIt^M&RMTk>q_Do#=Qo>;q-iKt6x8>Kmftg zBdyZu_fKnjr=O|-T~gid1s-s8QNeT#l1jZYv)MiEiF$2xodcibUk72?lfQTi4?W>% z7~05aI2?lJFC3179u9}SOv3_`26ri52illL!zM|?l(7!Y_~c?J8X7m&xj~Cs>u4|j z14ltLXhe4ygShsK$)NGe!eh_fj5i18fmj|L~78loYR&2&Z>=KMH@swzC2 zN~ROkeN|0J&nVaRS%c?D%FGbOeRXO-3ny!6Ak?_OgkDvh!pl?yWUA(Oppb10*qmtr zhl4^#E#bm3JY?HS!{P~sJvSCSujUpGePh0G(Bl~k2iDd+fyo7*XKn%a^?Sy=9*;ZZ z@$8{-`<%f?L7!(K>cKNSUbj2!^o87>x#-%MGU*E(KA*c5^f>pf?cIw8CW%GiH>1Mm z_xv3A-`P<;B+2^`1_{�RuEdM8|SyAiAWI6N79}(*rO-$LK>wsrOVQXFJ3#?lhcMtYv?uD$q zV{)(Cy>};*c@&cCc<=Kx(b9Y4UP%jCG_RyVgING=+`UIMV{ZXXn7JGND+nC^)_5l8 zFtH*cikHQ=LfrlS?RISEhvM7K?VV-ROK&9IIz->1EIk=-~>vA zKv4ysKUHlfAq%uihI4ew(QM_~h_YycSD3!pf`jx1yrBKntSbk{Cj?jIKS7u`I+~ ze252gpxgjiK7zt8>`7%NSs^(*p95>yKW=PAt#Pp3KO4mf>T0y)ki-leWy#1FTEl6_ z0bhHZPRj$HPCo`XWYTFI@I!+Z0xiAg9!G6}188o825^wHkU@JS_hudk$K~FF1P9Q( z)?PPg8Gr+xC&3||0nJN$GjcCzkCE2W*lVuyN?Q84xz|gQ2>nv^VZ@4c5qoiaj5S z-A$P_i^Q6Tu8Ire-YBv1UN}^P*SFC~G%&>l76Zc3D2XKF5Fre);&y2ACrdN{O}=BC zL

KllVa*1cVIt;CUFBG+M`hicMRJ`h$SO*U&laT<(a=;${0?Wc#gXy%Ru;H*dsO zML8W2KNSB3;6N!8@JrST>BG{;SBC`lxff0|{gqad?mt~hc>(zWau1YWH_Vm7V zhJ;2k+m1u82yno`NHk0PGBz`b!0uvF>Koz^9k4G^&Vd6-!u(l814bl)XTWwbyN?H6 zCsQV=7|ggU>pNOu03(De_6vJg_tMxA#qWaMs=16HiID7Wl#rE1SR|%V zY$aM$wq4XHR?%8Nc2%TWkSg1%5UE?mQe?MFcVYLbw$LJ!O=t*W9~b%*`c??^!H2%} zAMm+7b5Blwj$Zp+yM>N=Gc)JRN7_BVd*;lUGwSY?N>z2Lh?>y0h5n-nH8D0p>xvZ7 zemo-m!2ySV%^=CfGl>$Xv)T!Y6b9O+xc+XItc>qw$^Q67_70txO23t|8YoO$Lq&V) zn>TM(r|A4dWr|!G(mLozv7KAbQHKR(oPV`_q8uH z=@AV&H0%`uhbyE7=pb_0?E9ojXv*ajc{f#O-*}gn6`C*I{yJ4u5T%Ap5l%;UKCXN# z-9TaD_O$52Y}qf;IwIZW>>JNr*1HurA^N<4*YlipN?bILY_qa}^VPD* zfa?PhSbGpX06GQpGgZ_Mhh6&8C2qxBOg3-b<-RzcgL{BkyhqtX^84?e_4K_Q**1?g z8{S1!GGq}XI31kyW8@uIFo)0*rV@jzF zZFD$HP;Tr%QN_XC7=0Y1QdyIZtg`Xz!WeN>@lLE|U-=Oo@1RC#W>G*YPjc(j}S$jr(LC%zKUH|xvbQKAEMMDio znAUL~C=w-kRny9YG>1FyS6r^p-Bq`?YAePY4YhYe*><) zg3ZB+?uLkNrDVgG(UuzQFbH^XjTu(fj8w~Y2pfZ1d`cuJ1~PXKNu&?n`SjUOwGec? zeIb5QghpX9f`JRrFWmej&BkT~IruraH#XG<md;E;Jf2FdKfp!{;0`6>Qr_2$EpRL`Mv9kg@54S6-@j9H zSr4TXVMn!&LhamIL0h8~FN(l{s5~R%Os0H@n!}odVIg#qRN$gs-~?734pk0FFgLV- zVq<1U_<3n7MO=cz0co&eP_gT^gLHlRN5DW1XZk6l+Uf50d-yg8tXsr{%ErmFt{tUVn|!9TNRZ zH!X|qwBaOFtBR-u3?E$w4aA$IK2YPTOV%kG5kETm;(a1lxe=}FAAP*=1qGzc-m#>* zT4Ffh)+@$^VkAkqw}NBD+O~w1HLt6ZJFaYF0fCz{xoE?#Su8>wkR=O3#M!_bD#O&l z21%Z$Yp@q6w~&eWcH^8E>bbe=&ISME?4(}^*g-s=_s%swh%*^|nL!jO0_?R=u>sz( z%rDIGp~h95SA@8uKS2bH&SiKlqdUn;888EG2syqp8Z(iPlCX ztd#Mw$=a)PUR3yK3uC$^By9m3=LHUCO8vNTNZS+7M1tl}+$5}c;fa#;H21Sov6wm| z{roiXE7X|!kjQ5diSU#*l$SihLW?$m_>ZVzXfW&# z%*UU&Vw*$Mb0u?gEfyAj1a$<>O-<+hiu)iQvfQYRou z{e#BFUaz4l8$TLidg}Dm4?;8FyWFdnCUaF!Mkay2%xMEyjA^67 zeCLq!;cZfpPIWcz*Zz|JxSxljQoRNa{iJYG^}51e#+#XOzc;RWdOIiM&Ms)Z?Jx5+ z4hI)+ZO6;l5tDXAp{$%YIEW(gj9>%6!CBk$MY=BqLw+aimF?C0{U7+<(+7KdL?+g= z$^Nrb{n=zaWW67B&#m{WVSEs6;W;@IR=N%kdk%;KjjQTp3z9hmEgnhHVdbJMDrAom z`G=Lx=2vNN{yi7yAetqFYwyA#<|*Jr5Vr>BDA>F z)8xhb7s>;qv}t%~QSveo<7Lj1!N>;4K$fLzED%sFpC+%_%nbGjAiz+VG!Fq*4Gtq# zoeB*sZm$Mfwp(n6Fj+P_885SYhkESLYRkZAXtH*=#WOPM4b_i4Xo=gZC*BiU9U19q z(q(&fY-7MX+2pSu8JfEonk*Yxj+a7p{eNV;1P;9Trz`lcS%I^Y4;iywDp2EdC!^^5 zwPDc4dhxyLB6d|Ek{u3Z;%o)u1CDG12gDo7JR^nJ^nw}=vvA9uIIvjqDnqEo4ZRp! zC&Y7b+pT~DfFTE=-X$N*Z|34%`0U2G$Op7#Gcde;wIu|LfT?pATh{CIlCUQNI8Y9) zcG%JiW;)Y~XlfLgwmeMLhilLXY|^XoolOqRR_4RtFs!^25j8!X=#M>!d&~SyYNAt3 zTOF|%V;fc*sTPZ|iO$$YM2&|xA}eY*ta|1mz0;jzYNV@kCANG`CHhxy4g2~N6KW*f z`D&+{jk{mL|MzPSe_2)dLp`MaV85p3z%Sw#2pm{iAY0hwHx4{X!LZyzBNxFGUffj6 zbm5nXA_#N$lgAZ1G%G9Vetpa|3yjHbYvnyf3he@^Ap?d6IHSJkR*vo2hGGL9*TLcb?fw9FEhU7~eQ6`+0W2hbyK_q0$g8jUOD~T}*YFOeN;< zQItj}3lAgG^s#HmeQ9t&*qI_ih$?|gZDe!=U$HUm6274*bEQMw?7PHk1E8J3ZQ~#p zzO!xgLnYwOuicf2_bcc8d$kwzPgzSxJb)oMe%arMhqEjNxY zwRo9l&ws&O*Kd_EA+$W~cetq)Z1hgXzsCe`Z)q}dNTun?ga^zG02=_@vE>V>TsRT` z6`~KwkqidnT5nTAbOyA8oL!y!!b2)tGV!lu@*&c*Rkb}6;FLjjKqG!>P@Vc zjZejj;9#$+<>2D{O03J8x)vzw>>2M4$48Wv7z*~P5Fe^Pr>)FFE|<&iS!mYA!T7LN9i!d)nGZ^L18VGI?U66RA8Pz?&m-v=YWIQ=yYgy{w5^c zEW>agBvOb4MbJX5bUH7-gxuIi5?E|Z9r-R&nTo#J;hum;) zaP!8&E15`fN{?88W!k)Yz-?8SCsjJ!f_rEz-QlcesZ)0uQ8L*?B9J=p=`AhVmz>=YJu3WaPcy<4@DNf$I6 zWcZoMXmCi)p%@%0WS4@HGf(m6#Jv^J^F>r`8*b372gtXeyr@D97{pICV0C_t zi?8g`{$;!zR3}Rv#hnr+AX}NIH3z(Y`E%>Whd;Ep8fkz3tAlzkUaTSS;Ld)lnHqKm zFYG}Z_tqb5wlWV`UjOiutz#a4KdlemXBLd~_8OssEafp7gQrla_Zr4Ucp0H{m`F>X zr{d89RG@{ROX(g7k)ePuSb-2YEZ5U2DDl9yBjYV29ra}AZc@GB>4+7ZAGoc;FHQLT zEsDV*Hm6>iOlWYJi}g;c(d8D^F&XQ&=2u3VBFoF_>WG@PVgQHp(;Tu3RoPr-CzsFX zW{wKEwcLz;*{5@ZRL)Q>Rps;POfFl{ESc0=evR;8lL~o)goXUz(L$Q~%I61jyIIY$ zrhB314Cy2Vs$I)y*27#?MpLu9`nj3<{osJZ+M+qIWcuPS)!piBYz%s=VA!Hi?7?u@ z3YR+p-fLqQ!ak2hkrt`W6YvlrKx)E8s-rL|R1A_HpvD_xQ`Yd9&ljZT92;QcZ9zPZOM(gWa`XiBR^;(-4TMZ2Ng$F3Q_(1S# zw;k$TS@hL427CJ#hY~Mt%vvi0i~VzLV`GsALy1?r>cW$WSLfzOf^Ct=?YP}PYOi)m z@Y?gz9P;@Y2OJ6ugZWf$v+6Lnkj;_^FYM$95(?>bIz>9dt_FupI=@N0RfGA3+)PzL zYYy4m;n83|f3Ta|P3H%*xvyxMLA?;MYBQJ7utBIWlTXvKg6LK0+~E$bK?d`GXtzS# zj2z8@CBmjBN&0NJ<#{mK)iN?W+iMMsF5h1EgePa~ytA>!7Hj)(kbw48LycsF-E@lEb!7T~`V}&huRpDBGrb+ZU;pUE* z00l#+&QSJmAK8XY@2AbX`w&D3TR;t*xoXV1m(3uaoZQL%X}ef77vHR^>*CniFtRo(dTyqX`VQ?ro?YhJzDIHD%{RAgLjP)x>- zSe=znQ$t}LqHVETYYv_P9lxTdT+Pq-4JfO#pYD$e_9(9oy<%|+YSqEhOW>f##*YSv zqQ4WLWI}j2ovDC9lIAN;k2tbjx{e=)6sIWMJeY8^9^hn{$VJ0hQu@Y{1!ud2o0k<8 zXI#M#XFLLjA0V5*q!M+s-1;yXb>Z26y623- z;P7SAjb-!39&8SRS)75BgIRNOe**}@jOL;NMXmyPg2Qz96p~DY%_49PK?7-zPC>$) zOY;EJKYGhHz+oYka=>9XHA4zPLF@b3Y%Z? z8=?j{a7UrXGnbj=N^|A@CFv#l!{$%5@-Ket!zfo?x)e=bNtQ->Do%j}4h05>=>C;x zvH>hJ0uDSHERUvg0QmzqL=TNcFF+MK#YQID8z(@GRv620r*gkpS32by{Gdw%iO$U+ zSyd(AkUJua1Ie@_=2{WZB7c}WWX(ad=GUt7xx=-Y)L<^3a}1K%d@fH_Rk{4(;9-`k zHjBMbcLWZ3YE1ax*c=XXgWBfsN5P?{bjDGGgN>6$tT|jvjCcb5YD|?)zv{92u36Vq zG=HM;kBhI@JM*261Z;|-!Y5I7V&alk>NL$q?EFZoI2<)jG?nv20fv%IoW z{qn?nNq_^i3%uzKI|H-|CY-_yBF7uW;T_f7yiM_OvV9?{O&HH!(ld6RhY>Xvf0fA&05H$(uEm1$hmNMxKqe# z&V_@+&4Y9*dpL8LO&{xpHY%#9c5`>Lin_?|7HEfqSg0yXq(8Qm$>G?<=1?xd(H=24 zjOzPeJgRp3#US~OP?M?dxrDb?Es{FXGEQ2;P$z-IH0cP7YL&>r#Aw(qcWe$;Y-wc7 z>hH3w?yi@{9L-@wOI^kmH$@V=JUoP!YFmYw;AP zX%SRJbHdqb>|6@B@tvh8XaeUAm^)4H59nIPb$G64E8zPF%^~`@Zt9i3Zbu%WIWX&1 z5ge+YJc%t$JgJCA1svicX{0%D%U9~g>iU(KhzV{ip#B^aIU- zSXA4}#^Z|0D8NBP8o%cKQlK*xLhQzmC5WL7pby{&T9>+n7^7q62lU_*J)tp9g<&Gh z03GLCz@d2hl7F2){d9IH3$%1_^5Ve&7BGUYlhF(Pf{1G7`C|vK-~?svPg-5`R$L8R zL+aYRs$GhNX^&xfeksvGo5|LEqHSWu>Yued+X*X7cS#N}*gCXmY;R&^V+8?83F`g@dhoAd);Ty@XN<=Dlel)jca^-_zk^x~%k%|VV-z?WL*5GKy> zyTTn7R7XV?dtt#cfol!uU|7PDZT&!FAxnb47E_sqFmZ0DGGWlivf|1^)zO}eRvX|z z3E)tfZ0+n!UZRbQW-Ke5Ua+iaPO9!p)y>h$R)7Ob1C|xnyVQH_(aL1S32B%a^)ldx9Ok0WBlbFEw^7TD?mF&&ciB0gq?$_H4`I zOP1X@5WMJ_e(6F05n8n*2BsU^8mE1fg{6;e4!AWF(dy|JN45)EV8b3KEDC<{1Og`EZ7 zz`&A%iclhWbW@3d@MCkc#LZq)=vuhZn9o+68P5S1EXlH>`hG&)h~C@ieAp_QgSt0m-M^>Wp*c87z#&=vYJ`wuDf&qw8f9>hw|U9taOGhn(QrR`uQPI!!C_Bc z&?TEg!}?e(d9QkY%1bk<6$5O~30XPktq5BDL;EV7X+6#sP zk{L!!%nL04@R9BR;w{_7JF5j8C_yMnXP*&N4W7izcG}EO#YZARqLp)F=?gIx5KsxH zDG?$;mKAWI=DZXrA^J48dxU5UoRi&1*{GC|_4Fie>|oeX8DNay#=^}Tp!iSVa0^NL zVw;1jLDJND1ssx7s24BJ0(nImo|$28Za|YOFFbR@dIW-g#3r2hqk^bBNxHB(A8=y7m`0ow{G@voOq|gx-9)K7q2&YsBzylYhwFy|}$Y#*E6GzR35!`>_RHoS!eSfAEbb&^L7B+`k31``*K)NAYGnm za%5W$Qaq0^*?h7|z(D|pN2VfbAoM6=oxs4xWd6=l;CbRCc+tm6zB66)&&vS3bOvtp4P7 zUo?7RSrIRS!_|A$pFDhF>BT3iD=Etg%_O@O1P%}HSHHX-P2PVzRd=bltbpgD5e@c~ zO3pqI3f z9oeR{QO>VT50(y_KOWgWyR_+NYcijta`ME*+>aa75wjwz?!w3I z=t+4^YFKr9u+lDz+kGaxefX93oixX-Gb}jX3f$F6c~kInyCjz>tAy`2DB-92%ahIgu{c{ND6@%|DaO zexGX>KL;^LUf*c5JkM0JUL|l~q+cZz|R3Z(fx-j_oF4v0e%kPxYs}4buTIg zNz*;iCA=FBJ{Z;j86HL038hAIXy|f3jttd*fkTD3{qiM` zPN}%Mm!Asp0j7*pF@wYFwB`!>qq3Y<);D(?JgZcX4~g>K$1%L=D& z4Gy89Me$}3a4H!+#32dX z+*p(J2X|Mb5XpfL$%_~fk06PR%)?Ph7nva7fOm28y&KP5iXL~14TE==I?dPD2!)Gjs$dc@Z=?QSeZ&EW!rgBwc&&Q@q}*l%^~=INe;S8R~{!F8lf z?IC&E3-aU7V>fp9&M;JsscQ+&@eYPlt}(j&Jri+BfAu?? z6*W9hmxEV^q3)-H58V^or?dNu-%eeFv`udDs3IK=tHFuW5JuE8@o}Mw8J-myt zs(^<8>;otQPN1n9mI-cbU<>%r*w|{2nW8WJp_xpSDVPZ02MZPxWYEQVEGx|09l6M> z4XJ@5z*>XBEguMd#1H*}#udWD8i$abZj#cM0}4Er;0(aQ=}ShgF%wrKq5>$K7>HPK zT^O{0;0kYt+*||m3%+iVe6oM4k|h7Zgqw>@aB0O{yti?Zu4jtx>^5oM4=MdmJJI~C z?~LIcx|{p78HByI>3Bj(-jYoP(mPB3=NRDxV31W#5D9~kk z5VvL%d3GMM`ru9lF)D=WFzD9c#R7&Bb5r7>G}*K{$lY=^zp!`xEUp_t`~nfjI&g(Z z)T~58AaL(GB@ctS5OE~IA;nX`>f#r0a1~TNNbXQ^5V*pHu_GR@NOzT+Be*e#+iUd! z3@%jon`BqxpT={RNkX_l;BEYL+MU@y6Fk42nf+roZT-^v#U}dWdIH;gm4aCPQ@$-F z2(ayV?UcwZ`r}qTvE$;lQlhqOH>u~7`R#1u9=<1=Z@Gp{Edv3$Et1g|xVAORPj#zW zCIJKdV`%&-^II2Kj zKIn2qgr|WP9cl}Cm?1GSZ7UN??XNIeISdXQ#0YLjz1=jr4WL7IbUN?JG)!F>N4C1} zFGr&KB&d4E46fI*`d#RgdJ~s_eD|pPMANz@nt%mGed6fV47Jd!CiSI}`)VkAy>YC1 zvyF8sg&wUTGvSgumDN|Shnn``>G?=9^flGrw;b5;WwqdmOjy&%f@l+0=Bzxzz`7(S zY!34aO=__HnGXc+Rt5x7S5ldoHT{sxQX?TV6KkB=ml;yXs3yY_cv)g02vG%2$)2C~ zK8G`_ZMF44Gb`yBMG4rJsE!xLh!hwqUA|(u;dD^OLHGgnQCYzlU@$o5%F5QK%D^L& zgyn!=pVYTmN5}V3Cyvj|p-+ta;UbXxc7v&_!vI7>S_ z)$@l0GTV=zKFoHT_m?-PMrOv-`A`me{gBh;(}#bz0*ccxDT{kZkRyUnWazPY?8NhA zDK}ha(2DfD>pD!5t~*8t3ahE;QVg3x0tHPngG07OIYvkxCo&+*O`Jkz?tlRbS<11@ zPk#F}zyScT>+{E|zUqwB7)B_}fBMd`GlGyJg(giUa3dZ`UsG<&0SN(6xFa~xCQ~3PuK9u*6jS|dQNejGhL(Ceyu{-?v2S^hbHy!T5O9g8rkdzaC9HXHgi3w!<*)x^dc@X$1qr z&BpgMUZiM#({|9NawokwmRBo2RD&iA@_sRZD6eQ+-UWy^Seb7+p&WGbkovTyd~kBQAX7en2D&(fElnymFRbF?a~Y9Fkl9;th(V zDzhIUF25Es*MP*G9+a5{6*8mQrw0e}iDB@IZ3nQDX*Q2>GLlriA`&-H<=C?vgdCO| z8bu}q^`S^ffhN=h00EwfLf;DD00YEP!F(M3anb$#biYDS)O9IwdX$!_HIz?Deo9zR=ryLAONz)fkH=q-g{I(>^xX=2r>^L zbDg4_#HDy-MMVlLqz>9buZovJPz3X2+Zuxd0TaOCZ2H|7`uh2I-Q0r1?QLM#0S++~ zgfAKk_$bur-8z=^&T`^nidP04I+7gb1{~_Z;h>j?kDIsP0B1s;vA~uteS6zc&o9%H z7di~hg$5TFx-*`iDnxYJ4?rYNnW7a90*6@xqA9g)zny21B!}US%*O^0Lr_rPf&SKa(MfrA~IZIQttY^fR?^cI75 zz#%lL1_xco3;+(Mdd%Pe)dx6GebzId1f^a`3Rqw^5jb4s(;uIGd-nO4Uq`Pl?kjNE zavN}nww<*@2@bS3H2^q}b3vz?0(513p&r#|x&;Tdt!*p^fP=d;XcxxTen4_JN+_RD zD1iiGf|d#@b|eY=9h_j1rU%>^CW|3SZc0Rw112(RFbsj0hs`phjC%%1O88(Yh}aTqHzHQq5LxAcFbmj$ ze$2&0Av4I_)qmkukRQQDdp+TKL3a@NrUaV1jUxJ04J^=|MxJ^4-2m`nC}>U{2VLLl zYjo-cN=?*x60lOl_ZPa8r#0DI$UVs1k>hdQ8 z4lY>^+e-e!R=HoFyn*3jEeC`#;8y78&s-5GAt3ev4*H8LqKDxq;C)!I9K!U35D%7v zYF@37CkK%cdL=gjOV=2MDB$U|*-hu!nft|Of3`+_@KaXwA^-+baBIlP56dKF=5~b4 zQVK=@E?}XWZmSfy$})q%0|B>?`5*quHi%sCH#b2!@FNs7+$7&w7Dbu5a4+ibd|z3* zM}C1fES0C+es4~}G3K@n>v{2;fft>X_N*mu&vs08KV9^F>dG|3Td1)+8(Dw@DYhjM zI0WX-ve)fjPr~WeasYY9tvFL0bEuaQRD_s=>+Z%JFbT6;p?BPh`jiN}6|oj>1(ETA zUc#0lx8nEw2k*tn`LzXy`_5z@ft6zpDnzf9x4q)n?!Bt z@ovoFxN(Q%u43RRt~0?4GIIG2MdD-v-lZqc`YN|iNMD7)0u*7GppIBU$(&|7+XI?IxzEJEK6+S)(% zwfRq(dDSVBz;U$~w3S^pEEeV~8m$)PY2f?XbKJo2wT-H|ZKDz8VyPWHTlAEB=4sdQ zwRW}RRZPxWwX2uoj`j_WV8TH}0`3fh1N?6ShbW!4;xJ9a6yQM032;Dy$L3 z`>lTZuXaEksw_7_Rr57=Hd5AIFVaA}i`g&BakfhJv>Yy5cKTly!)26|*{bC^795tD z_NTp7IoJ%djZRmFTW`kN9dBGm50^#D*S&eq&&#ar2b)QmO@e&6G1Z+_nwz$lsqUA< za;ZEFq~46lN4DSFZl6|I%KZGrr3$NwG=KFyRx3z>4}k*|^)NV)MzPf2f@Ks zC6Fa`NBEZhRxyY8hd1#w42?6Are9Yvhe+Xow|`)@f@Fr}fK&%0v6#W&AXY0_=3(P% z1bo+~b;m`#c`1<3aUkZvKWanZ(8({)!>!_8pSM~T97u@{f&^j z!LF_Y-~d~J9*~7TSgrV^y716^;%_g`V;gh8vA4aYOjr)^RvMQw18lrfe&bT+HpZh= z<5K3(E@d7QIJ`}7Q`0^Ng@p);32av#gJr25)h##ELJc-$zL>3z6)+ypyrMrCXF*_8 zemq`#MOrS8io}3Jcba6A)N&7_MUpSHXTf2dbyv&0IF8)@Ce!6+(9OpAA{fkyU@`3% z-e9xxi~OiqYX?gsI8{M0Nq_@ld)BR(45BDHqg1^Os!dR&nEo^atxf}24wWf#mw*9e zitUkJx0AbpBu8m>iUVghAlZr@H@{u2a9}>L6M|t|aA>vNT6NvHRap+(70`qG97yH^ zmV;Kf{C>}JfLlRF+YVU{6efNP4sTNzrl6Tg_Q7wf{546sp9T~ndatt_3ykb^Nlfs=X%F=>E(9Je^ z5$H96gNim~Ff4jD_D0BLQmw~wwE`d(%ttpvlWvALH}+v)9|#=Ca(FQ1L*f9JnMJYM zqPjOdTZ^bBS@OiK&ri3L2O?VP_A~0@jDs?61@&v6Rrj{Obt~ALpamCkO})MyA4M`9 z&E~vAvi$%ur^>BBKy#H09#ywHB+*4X&-UuG%yx%l{+N7Z!NE}9S`L4*b3Lt913~-= z#;_6IVGna~H{!uc6-2zOP>XtzLQzl}MKK_?ikE=!1pmB);He%S#-NvefnG((!K>GN z0O1|pq2I*WolK_M_mrv?l}_@K?9R+?_L2Q%XR|Z=G!q;sOjB^F+Gb<85@#CIQT?Q9 zl8d8BQJ1|XWltHSnlZliiyneUYRr?m(oH>D-9m}^B|maVPK}1ES<{Sl?7?9b=PEIY z&PqLWaQHmDSTy6R&Q?DT5gax=IDA(1=4O)8RD+@uLscZ;5Yhp+Y?w9BVEec`8Vo5O_U%1(Z!=`Q~pa5&uCr{4+=Y~iyH6bFV(M-9?At(qt4 z*8TadX5v3r#*v=XlXTJyvG+$YBz2W;)#xO2f4&(Go7Y917z779Y&3Xq7%h$K&5K#1 zM>UQQ%*SS>ndrP;S!puxBZnk?KQSxANwZ~S5w=8lP_ckR_v(+V-boHw>?@|H5cPZ7 zrsx9}&A0?+Ctt~!AfBRbASidBAPXIMgCfr}+`rmNiz1`|%;SUK0*BfRZ-Txk<24}~ zRi%W@5|xuS+p5c!0u;1Z413TE_i74#Z_1jju=j1=<3InGy)xN}DrUn1cTI zp{%UQH?zcZZ8FX^WaAQ9sFW#4GK~_Z7*}Nxd+FS!laHzLPkd=rjuJWOY+U4MR^(Y^ zGPFf1jjCt!$W|p?MdsqN9jXSat>5|JPaxkkBBQNP zdmSlp<>Jg%8571~Eo{Glal|4nsc&;$aiYux8OLo#aM*u^|2K5_C)l&WAqpEUn5KmC zJ7^7fVPge0yl&}92W|b64m66l5hgGbd_F*gl3*U6$F)<+(Ix6aEDBocag9v@hl3RJlJn|9*q zl`K1w<47C$bBan;x0kR<5_l`dmt$`tyf_qT^FxM6qZPqSBf8~e65h>*@3~8Q=W8KC>9(<5bc7@ zyYsr6D1Z+GlbYSp;DhTd2nqfqAtVJdIG6Sg<^UcTr4)gN3OG%tc8@|b3R_EN{i*J- z2dag1cWsUgBAP-7eA4zbLwwOIw7>!0kdk6!D~U47YK``(S*=s){9;}5p^Z<(z*OzRxThL;Z4 z+Uoq=@f@LQ-`f%$j;Ak0+Gp~5OP-XIh?7*v_Mnxl%;-b4v^A@*UA762B)*hj6^BD& zyTaW#y*PtT&&pO1vkRCvkrZQ}C$15SV+BYQtZ);|2SQ7+eA*=49-Az*V?v}u0!h!U zzqmP-&Og05olbvnQWd=0b)yHa4Nn{W^&^MN(p#VRuh7t;-|t`VJvZqDN760|9r|_E^3AXni>U6 zXC!RY<5W8n8gG=$cn-Ip1y~x?jQ64g1MA(T@p$tdQL_%%jXB%(Ds1lxHfl;-QOQ?4 z$&ggtFfPLFgvf0i&>VBFj>${d%i=wy`=r-0r>Z8aJb2?$Y(wrIyY%}6sFx*qnG4*n%!V9G9A9kTK8Z=x(Skm zJOz-Om98Lrkf!0YH^~_}PLk)Om0mqRoRw*~|2jE!hBHDgN6EAOFm;a8^!3T;KaCde z1C57zEpM{QaVm?k{+8**(JbxKX6j|tZXP&J!E7WXUqvJ;eXmsCJ{(H*w|VS@{ak2m z`7MD)RrN=j7Yi3uL5E~&vo|@1oz-a{p>|-jK`N&4=yw)fYwyfiHu&SEkffFx1gIgw z1Wd-j&9g*#l|3Q3ie)?kIk)MAgp}*9QmRMVbynX7M^tBVJCrc-#KDFef>K5mfeksL z$zj~SXIbio=;q?x_iXDk0WJF@aC84m>;NVKMnDc>pO7KEki6rU*8rpM5gevKz(IC* z{|JDja7D)C8H@;oEF2Ldw;{V{}-c(+oZ`k)Yi=WEcBq~ZnrK+*PJ)0>)HqwF1LBa znoSmQ^r=M-NeT{cNU|y1NZ6dZC~4BjrB+QeENt5io4ZsRhP#C%Xt&+Ei^V8+1!}vf zO&ND_aNamCx92=8s8nenn?)3D{|9Ma1{`F&p^COpEIWp=NUUn~(LulQHY^%j249r2 zF;YF0>a0kutuce@&RU4NW@1U`=*!X!C98)sV;M+O2&2JQ0%ct_cL0ebeY6n@M0LD| z{JK`rj)qEe&w4^jVMdI4=CZYl*8H&d9R8(nIssg{Z<6Z?JSXHa{4BMPFUc_g<|3{k zIrD%#k|~G`o{wOB_J|xnS3=1>Upfk(Lw1>|k={sfAP5frUE;(3qa(zeV+T%%@+b!I zqI*V85fa+|&nY~;>%sRPgsr!NkWV*-@WC@b)ZVSl{DP1Y=z6`bm{|6sIN0v-qSx<7 z{b51f#c)Ou4stIhec$E{Ni?7@BgT2LuvZ`z`)+-ie%#`c$yEi1UqD-G|%NmyX zgMBvBcA@b>LY`op>a%H(7TQ$tSA&dM*Xw?=p}v@l(5Ey|rr3Hjwqh$(-C>p%Nj*KT zfe|M3v}!6nmWhWnjMOd)68iWGv3g>wi!xg^bXzgiM8Aa>vex%3rK0(ko&lua?)ejV zT5$;1SUaPRbQ&H@6sW<0bOABo2riCJe#ADiThM-;!T4L^K%dOvIf4VK-UScfjL7l= z;5#`KbO^%9w3K*od2;gX8}PnC{tVKTAULGNgV!e~Uy=~UM;;{Pe_0+{I+xCT#DpB; zau~xh3MMf)Q;(D|4$Ap9Ey{1CQZ!#1oU zFLj_`mK)3?3r|LLx88OgN}K z{JRx&ks?Ofls4FciiVJcpwzYsW-CuYnR>0w_Nr}6RlZZ5svsIV9zZzJwrMs=?Vv%n z_^dt7FbtuhDQ9U2Sv5sX*0bX5_4GOcsr>>IjF{oYVkH#KXu}kniCs8mx}u7^$7u~V zwU;26NoC?47-Z82biRGLYy(DwLp*!dI#EAg)w0 z@$?Omysor&Oj9Ceye2X_`0^4mY2}Z|1fEK8klc9-#%iceowvM@%HM;mGdLyp?>qqS zfrJrHvVMk?v@w#o^NgH;Pp06=o*(tU&_7ZhS~?|&;J|GLMKB~#xr0R#S}Rxw1A5*G zc_d(FE#Mb{;NXL2(e{B|zy@aLZthp42djBNqQjEGngpFaSf57~Rh0E>Ob>SuZ-C-}QTCU60dmAkN51I;p>d0SPrt%&pBsI#wtv>^PCB!Y-Uq#O(J1T*jS2cSW%GLNg6eK zOGcNyTrBgoEm)4Eg-t=5y^Y|TwGe7H?E>5a1^KJ&{bIo^h9wFr*OpgcBSCqx;k&3p zsn)u^!FUo=ioj9B9xW`_cS|Lo@|e*pi4u1w#*H zCqS@J=<#W6yqd{%A0YAIN&>=D$?Ia1Jn$THFN36egF4~t0Z9(YKC}qg_c8)0xdTvV zA$^j-aQCpodqpl4k$Cs(K;vNa(`Nu*N!AtWqj*jn@Gmn=PrzfuiWmP)eATvT3z2fg-!@nFiJ z;r0h4fGpoaHopiWi;UGAYBEPpV?2jAglsK#yCHW!mRnAvD5t$J?L}1;22mf< z!Y-%9vUJzmEezv&k=5@t>WrjnM!^j!!$vmp*Q-e~d^qS_WYKz@wTpZ(AB5-GD52H3 zlM3zhn>Ye%4L7@$nH??|UdObE{mv}vCw;owZqv;Il7;dlR6i^hT|z5w74E8d!t&*6 z%~#$c=bN6eVrp-eu+D80J*j^6m)gzVPiUG&46&2u#geWXSh2lg#jMp{PYAm1A;1+m zLuGx>pTN_K`*479M0k3ZqPx(ZO4i9RJeIbOf@e3Kn|tJe6JFz@j|;ye!Q(ni9Lzrs zhv+D@hQqVVGF6F#@0^i*>LJTnU)HTrb|zIWYwUQ8j)rp+z9Ri>^sU68SLE&u`J~IQ z>|H_Yn=lmjQc4`;kb{>jo0o!~#<0t!B9kf(5AJ{2N!pnmi!cy|JqdmSkyzvjT`J38EH4xNg>8kH z@F#o_S;mnBhl6z~NYn~1qn1d|C~B=dn9I--RnPEq1r8Rek|~n?y)9xBz6opcGn`8a zS-Df-kern8rep$#*0Pu2pe%<&P-itBmSP15&1o=k4yy)eaBz^-9WBhL-%^~r4_2Hw zrKibN8XVTPUO9P6*_s{B(xCIF9Pfr}*vYHil}(Gg!7)%`7I)K`x8NXPOCj8cuFSj| z=U$Xw8*7p5*%14Vb!jZxw($VX~bcF{S<^VzgvZS zgIM@ZaF#Ve71dpE#!ZON5=O6Q2hT4H7gsHXIBf_@0h>p4POZ2Srz%?P=#plYU*S)A z@PND8Fa08mU+weUFM@#^_s8F`p2Nq$L?}s*;{Wu^6Krbpdi{3N=%GJK2uM$!SK=<4 zup3s7lY)``%RVA8ez}}Bjx*rayWw%I=KJY-c^u(qK_MEvobYU8+gtp@!{oaezBK8M zZZ|B!sT=uLg2Sb-{KlKHbHhnf%}vBaP%Zc@LCLLy#yk39P-N~yMuUl=d&;)t(R$CQU~T7!RN>Pj&0bk65`q>leHvPP!rrIxJCFPY%KT}hk?pK zsXB?K`or1#a!2oucDl;fyMwfF{Prs49px z7zZTBP8)PM6WPA8k`JfQ>fY7SY+>{&w5InS4w6Zgh;s=8cUZXc&~*%J2V^kHVG z-_kau@hRkV>`(E?)e`h=e_B2f)}%?7g$G*4h~1LY!Ve4KW72monH?wga-;UMQ%GsI zqhwK`!FS=)Iy4i}>H2tEV=qU0nY+$9OGax zZ?MqpT2;3dtwbny4(LgO!^?Ugc`5QUJ z;6iwqas=*4v#dXmGg+9js6F5$jp<7qXV*ja^7PPppN?Z`3tW#?uq})9G|{j37eUUi zCR_h}dip}Yzkl+*5r0w0_oE+_1+D)8kL6GQtBgZZ<;?RKM;l7Fh%&Vp17mDU=~|GQ zDr;048IEehh@+}1BSUag@e3Q(N}kyC6In*o*lZni>VSR0k);bC7DRv*0Sj@22t&qR}Wg?~^My5>}CGn?kYDK6E_ zNMKzW;igt}$}dF|=XkDqiu;@Zkid`V`dGzyOYDuux!9LeSMFmHo&6-cmFN99?9t=n zaJ*h<7gg860h+m{j;{j2{Qy7D^jyR*uZfFEeC(!Z7a#IS+K@r`O_Z#^Nztk#P;U5`;9LVH7sx@tu>`yT6_y1U z`L($a)fI*UaDXX79SF=F2?cyDKvXc7X8{~Q%ZV~pWX3m?AVLvcZ5~S4mQaT>zD1@r zcnP5D$XrV1T)lW*+0vNNqsXDxk&()@W&%@8BaP0rXkDeuqv~QD&Zw*~?$C{V5=L}8 z&~zn*F~kU^h;dF-Bn7iMr8VX1;V{JY$9a!JFoQ}|56&E$QU@{bd~g^i%w|L$-4HIt z5W;FgzI;~y*K_!vz6TEA4pUFf#L@OJSggRzG^}W|oT7fyF@yjXX38Iw903a)I>;w9 zfq6W~XcX8gU~ri5X^;$J%pU);YBu3Bqr=K6`hbp4hJ#3#fmCL~9KqYC$Aj2wo;mUy zcpHssAs7$v04iXTfD6KG4lK)tj-p!w14Ew$qB(?JKh`j{0@jxTT!^uifN7%JIqQq1 z2deROyLDEvN~o+tAN_&-@l%=q)9+8ftx%789pFv_8w$tLXb(w%5i0SrF8$^##tLd&UH4sqUbKe351TFr&)| zLc{F$$tpv?N`mFInnxViL0mK13G2mtR5YQ5e-%?HCn&7M!Vp_oij>2%C**k;W zxM3KIZrTNrt^+iSE{dKYK;csa#+;(du6z3q^ZoEZ7}G3@493AIfh~+%k@EuBM>DTe9rM9o;R3t_ox$* z5-z@Fn@Xn7sYGZi5O*%BMc&o7!g?{sJ|+lt`OzACneDt($mw#(5W{F!fy8ri*u|d) zDb)G&h0Fyy@f%_xK>Qp*aHh-*z-?&+$@BikSWX2NtY5`_UrzD@(tL5-t!i{X+g z-k)u=FsJMeWSo1cn(wV}t&*F1Fhj!IDKQntK`Zc0)r3`ja~h|XOUZ0NYK;MFfCPwP zo)BU?@XNs%Tt1O=*sx>CAv4Vv*3iFXuE!e+`gBvL7vJjYLxkvlF#GfC1P8y7(F#?9 zMSt|k$ng}y$V}+kpV=!%hl+P=V+){gAOXeDQ9Ki_Ri)*Ca;dJJE=E}Kx5yT1G4x(V zPV<;PMbnUgc{BnQrI5}omt@lwS^9tKZ>N{%C&8gm0@f120K&as;;tJatgl+6X9BG$G`oxInpFX+!y+Mb zX|0&f>7$DYGY_@y$nlnK(M#)r33GoSx;sqHFwexD#IRPy%=8sSXP|wD01};zP`mK) zs0GaFumm{FCz(GFW-3#3F>%fzv#_TXQOdWI8Qh;TA7cCTJzXh1FU#rW(~o(!BJ;nF zw!ZnC5htR{-K~Hfffm^o+>I-!dhW!QW*Q4M*u<(Dm~fv%IS1-Vz|WYgcW2$<)JF@| ztqtt_ffQyguXJ=pFVqSh-Qs@HAT?dixcbPfg`k6D41^33!8V^SHZ?Ld4I-+P)7MbTj%9cVl{GLOO7;=z# zGGs4xMpBn#&Oh`ZN4D?j8tKcgWqDaTeHR>jc(`5hEdUdJtuFF#L5m?XQ!-7msZz37 z3&f=6Ln_jEb+#G!7|LV-2Mlfucy!R8eUW`g<%*kW+NpgqTddA#EWuOqMezCL)V3^2 z&?C1_Uxh!Q$*|Or0-47+MX#OANP)}(DF)*0y=D6^duOwwAPhxeOdKbs>meC4abbuL zU@{@|5H5HMmacjG!Cp>|T^MYz(9-+GVTAVnNqbwRXoLf57VO%(N74h|RybD!(a$lb zmyIARd3rdY@e+g;c79k&lu(!zZ8<-S5Mf)NOEn?K&$qaXLuE{$BxaJ1Ev6O%t(~B1fOV#(9@Xl z?g*Zxam%(6nBPt^gUt;suU70`uz1ip;=*Fqc8`we1pE?eSMLX_a)l%cWtssn>*&>B zBYg%YxUu~RHmD27Y%z7g2?$F?EN-MQ0-zQAARV-dFm|vDqr>JCZqrW)-(k$qMN@F( zSl+HRzw6ksQh&7wi+abRzzy${{&;=?d*;IjOdX@H{d9 z6-u^Lo9pAO;n3BEqNF;Yu774QpU=mOBnIJjD2h`O-=-7sSzUkgRB`~Rp?^qf=&6o{ zqD)+m7Yq|mYW5HgYr(kx-lFvD_Lv$jRS_Bf=Q+Fu|COk*#jGf!>^i1~rrv5hQ&x>O za1XW()-^QO?k;yEOtvLx$WtQ zf6UYGr8aMld0f|~zSHg~Y%yz&`6bI{!I+omCyY7S$)XA8vx#)bGeK)tc3Fd`?s+3L z%b$a#F{!=y*`KjQ62J(THutNR@eF-R1ZZ9`sTW9x1X2 zb0{kof+(|smmXFRmIp(>8HO&+VPVS@Tqw)~Zv`p6W?*^}dhy(YFd{4wwui#d zgJ2Fhgp`?6Bb%`7d4Df4`!$l_c{rSCTbdpbhzh9CXU8@g&rIj9)p5hVs zB+h$HQerF)etdCWgFgJ;;3#V`d)D%P+$tLjC}>T~&}6o7cmu+deX&>eN7;r@Mc5EF zIsBfzQD)B+-fY@%UvlxaTz)eBWJ9}*-wq6Op?!o4%yaF-o#AGz3ogFREt3*kNe~oNUUh5h%O2Y0*?B)u@teK@Go3uypTwqzU)>c{Ze7c( zu+`XDQ2rpZW{U?GW&`1ua-YUO{yOY+!|!=3xSd=;srX9wr4)rP;r}JVN>*c?#D8+d zo`OwQhKIwiM8e%9II+E`KEmyb#THr5XE}NPTESsHVsKb+k@ix3bx9%_TWPEn9F|uU z4#TB^!;)wP2c{JqmRAAK0b_oWFdtEi4TsQg{S_RRNh>(aPYe#2*LD1LKwuKt!B}iK z+@cj6mPg51z@g+7%c?nTISVVWQADWbSPJ%C2v&mdCE9lQXnOK+h^_RGR-VI@D$SAr z`CLAza7W1CHaKh}-7xuLna7U)7jdA}385v$gR_Is?g|c%-g8Jtl92YSlM{_tCAucf zNj?oY%u=cA>D3#5q=B4678#DSixAQep|J2PD7SGHp)A2XHgvSJh;UfMZx&x-IQ$+_ zThv%M{(YW$J}Imy&DAq6&C|gz-C?u;+@Dr|{xJ;0!Lbn9(T!qRA(}yoy9P2W_n(1G zpaLd~1BY4OIh_c$?nsvJ z4vU}AOvU%trxb%j;5rA#X;;x-00)AKR(ab%#iHd@k!4v1cHyww=A&zsb67%c0UXw& zh{0ifZdy1vEWU1m!$^igUYFe`IVv3VqGlmPvdA*f50GH6(50q08q#zM2drqQY!N=1 zR&ZEOC4+E~W%psjfy~1y?y)!Z#qK6fJ#yuyEJ0#k1yB@PM5r%`^CbDc_uaj#4 z1Qe6uz`}|eLS=(y9`s98+^s68`U=&siV#vT@6=Gah;%%I1N{lZAt@#WG40g_@s?8s zu?JEZ4l;%}(x}V!5T0%K^B?^i4za1=AQNs#Nz>!-C;_|V=o^l}-f43i(|8U-wCfyl znTvNhx6=D5@4@1yp7hv`|I+_~gRFC$4Mn}ZHd>e48>6Ujz(mIwN`f)rv9pVG9aSCZ z7cTdwU8xGEeFU*9~)52t*BQ$!CaJb&A6YhHSY)!Y@ zdM{<$wjA&rriBBigAc;NiW}I4LkY?r_iP$CMDrX{((9KE7Y@=zw&TI!X6GUkFnYx9 z!bCXOT%s}3H|p0zfCCUBQdkkZ=!yPQ5JKg@+{T0Jad-~tgKR&2fcm1_I4`Hagv0yY zlS1r)6o$h$BYCrJbNWb@M~%`5g4Mft4r@bLqCH$2!(pS80N8MiZ)WH@NK#75o^ zFFQF}m!ylG4v-;}$!9*HUM~LRbc^S(fjN6hI0VJ6roKNgkS7EOQA~;jAt|7R*Gk0?12;xE3Q+UHqTKejCzSuikI&Khau+bGKX_xIBepz_6V=> z8?*BqGB@9KzW=o(y+d#n;LUffo4q)NL+0Jy(T};#(aTPJ;Z(YZ=>b^UUrxuG2i z4wG-$#tYyxFgQd%-*~^@O`4b*4ux*N(m#1#>6cqE@f3N{gh=5~`FyqaUZ5>7IlDS8 zoENtqDw!FELxby$u01$3)_APnDXdtla~NXX;TR4Hg1OP=HQB6Bk&sMIfN#XCil1tGlJpCbmhZ$yY4)B2SGT$L+C(_y+~9=2Zm+=5gjy6KcM)N?5R)M=Db`5wwI9~|zk zFdXbrgL_8dAdlKm$dPjnplJ`=Fj#aW>S}bHkjD;(q;QP-+}lgBd@jZcwJU&w=Q$LP z(Q)C;b1;VZ0(C2Igjl86eW0vE7!DgFd80SthLDgXZbI@-aM+;PNL!|GxVCM16T?1Q z8j-~8at_i(G2Z(5MXUCr10e?Lq*CBHw0YrbHT=lo030Z8wGOXD^ckG-?75g0 z(**%Mhf23tNx}{@{R}Av3Wu9=CIh+1{JZ4}D@ylvokM+H=C9ZF zcDs#faV+SwRn) zkraG5fa*3BHe9x3Ls7Rucex0Pu2u(QyS`4>s@t?qyJwh?tZg@HwTyI=?WA(WkCF$6 zU(k0)hZ0m#CjR6UhJ%0YnLL6xkrtD}3YK%g)ecv-Bg>HXsSjq z4{Dm4W;C0YrfaIHJ4zEPO~tf$tZPuhYzi1?4NUAfgwW*_$Izf($HanFcnuacOjU!H zs$omPn0hRp4Z&Z5!$SR~rzt&$%B!6>)ysl-cv488qPN{Hy4rK$aE7|IlfnTy7R2m{ z=pLNtbU|_sOtN2w!iqG5!*pVc&F%`xRKEDz(G75+ff`cjV=w2zp(9<;&Qzzi_lV(8 zx~F?9SXiN{ZZt43t%1p_iZZaA!N62hU;z(DuS$=0r#XP*TGyysU>U0Fm?}U;)dmU~ zXfOkq0AqwMV_?HDczs|g1I3^w8a7o}g{ZKo*;HUaH7yL@gEnYt1H-j3oyQke2*<_# z)mbG8p+hT~>~|~a?DGl@E5t(g?dL-JJ$fT-_1!(c(%lwe?$nz9y%j91n3w7Rhe=^Y z{+D7ZBS9teF`LgzIX4Tj-HGREXXmCEhn_`q`@f<^y76iZ2UGR@2RAUlx&enR9MHU~ z8W;|7V1i*;hGk;SbZGPe7McnTm_c~eG5`UpsSI>F$YZDwRUPO}7093gD%z&017{%8 zP)!HB%Tq95)#8Cdgf3tJYMfczm6@1qJ?FbJ)1S*iL8RACA6``iQ54TF#qo6xVgVkP zG?EbA6Pm{l_!h2*P@H?~)MC&izlrvDGR zH6e-J3;~qj9;iCF{S*#OjCB(ZZWW>h3(BUV0xTRr0)`O59K!*-$F`@-3X35c~ZC zg~M*zHA^1&SqA7jhx)?_;IQ9}Bqi-(??ZEb`f!|_1cy}PQ!nS$GQCF1t(3++xlj3g z{0S-*|M1VJ=hs-%?{S}*;ot0Czi-+=6polWTDl=C2vr`i$yy<1#;Q}VtYt=W8)bVE zTcU_Hq7qxi&K=nAZ@3}D4JV{@W9eVe@9BIvG~`IEh8mYo$-DO(Gx+1q*oWM5K@z9( z9hvKuHiYw7+sKgv@!4+G>Q}_n0a~z@Wh6D$0}sdn6*3}l0x5-lkd`ogn88hvCkJ#P zLSLp1CWj?SOdzm|fy0qQEtVB0e=IBJQ{lA198$zBmf07P$5}X<$o?reY~S~7I%sV{ z{)Sp%OvB-<5eU9`o4C`?T8Eaqaz{PC#e4?ZIc|ra7VW38IPEWdJbAo_Bygv-#u*H0 zBdwGMIjkgUC~^gC@Lgn!^9ig$bl~e7=EO}l9Xf( z8EFuKCOKp|i}2xZwIu`HWf;i@hP)cWth=F6#BMT#%*b_t;@Wd^?nXIsxJ?bo;gHd~ zX_mfcb8yxC5vUc@W}1oP?$i4~XPoxOPm{yx zAUm@pcV+$W&sH>Q0}-dOC|%ocglbz?WxkscM{+1d4(BDv;iV5=z4n$bHM|MxkRe3+ zPL!|hmLP|cGlx#QY6zW;`#$Q$`TZi8klq%Dy*YEJJPvH;P$P0^q@sNFPI8F<8aZ_C zFD@&F;cj+FzFqDKAKr)KpD(;N-gspD&G*V{%weoJ zsfd<(}wUoq4=nH`%UA zC4Jpy=lz{G-Fd&6-85}74;(h{EgID*$g4T??mu+Zuhf3t|Mt0d(x=^4JUUwMuPhSv z?|iTLg>CsH>3_<)+dW0kX}|b!Ex&WHjmAw+a^}W7wh<0$azkF_Jg+In4USjV%IQR% zC(41t=GXU&MlY|QiVgXsH~zXK@%?v_8<4y6z~NXe7PSX;z+us{R)gQbVe#LwJ-6U+ z@8`~>wMaBx#kUu$i9REmCEe}6YPF93wx^?UweO_<4`y=z%VrI5h@%|Exh8!<9E*^e z1#2J?nRzu@O9*Keql-bDj~imCrCQ*yql)nXvTP+Wj>(DSdKOC&u$gGlStqVMW>%xL zdAq5DPQCdJ94;(6_5L$BB>(81|G0KvEf&2!9UL}4wH49PbF2QI9=1)D+O}TTk9AxF z9I`SC2moA0g+hy&1!~9vBo4(xY_Nmi0)hp=f%woyqgUS9qs57D)#bK#Cxg4``(v^*PN10N5srsYeHIt>C1Ddu6E0ZMi14&zE zm`?)_DBPOdiS-%)WY3~8Z0H`0H&S3vr#6K z3+5x#2M11@6X72;l7Ispq6{sbO+`fE!)HUINOre}RudlrF{VHa=xq0PTThoYNuUG{ z#f^h-mH&HOaG+$3Ba}Q_@6cduYf&H`@c0^U^8ZNA5UsLg{fFaeHrVfG1x?_aXK1W>omct)2O$xK`xpSI%|~0k?N;+ z(i_^YkCP~KkBl?w>4q|N>0BqJae?nS0H1fC7dFK(`< zMPVa~{}z@INO&iQvU8Y~WfY`{uxVHj8w9oNL4zI?MbzLTLN3CJphxM!FW{lytY4@3 zv#(RBh)XYRce6>}yqSqf=Kbcq_Zt^+h@kh~9$8-;++@C+zCW`hv`5isw?1d z-E56%-jQ0kt^MJyrUnvN_&2f@&iBFLaze{RXFFWc@calzLu&|E!*P_QokW<`AfX+{ z6%MpY^hlMKfQE8aQYKt^oQvCm__K( zLWc#BcVujZ?Zdtk9P+L?T+1ys`BMovpxN@GUFDqyn7y~CgJ!VrOo*n>{Ys@$@LO{l zP4^Y51>Rn$`Mecl;00b!p|OOdvC|r&>MgxboV_VMwvT$+f$gGSL~fL9nY_P0=inPi z&7O=23S27_4FVfbcJrgl1$@sYHIayn9Vmr}OE8T&Rms>}SqOs2gsWl2&$s`_^@Z}f z@bew;aU~mZdCKbl50zPk%`*X8Mk-nwTMuZK!ulwT8d|T0v$%(D9BohDe1*?`JVBw| zhz7+|RNS=tT}L(hvjeu(|oKtr!@w%sO~$5S=7Y)CsrkG+h{eAr>EHS?ImliKnMdPG7E>DL(;wI?r zIhJTI{SHCLFT!wlSxsXnH`xjfD>!;L?88?YL}uVX2RYCPcL-a~fHu=63h5oLrUE~; z9qK^S5i9Dwc-{Itl(#)lN!d1d@ZBy5eZ<@jJTR8`| zdDmn-+zZy#a3;LvUMB1L+IiHL=`JG`%&PF-o*kQObLf^sg?uJ|K-DL0**B=npKn<_ zo9<1BxE^)z;da;8f#m;?kTiy5Xe!Ich=3Wb&a?B0wjgRob5LV+ew)UVCY_43DA2OG zpw+G4S@;t>> zy;!0%tJOlNxF8sIY}+?cc3Oz=E^x4a7;uhg~+b_7DPq^)PT#CQ{H@( z{hov6XiEXn0%I((owx|3&8dZaGa?gtCV-M~l|p@p9q?mjS+XXW6~hp4n)JMAnI@CJCAc`7N~*MaI@2Mg`c7^f+8 zWfC8s)(_5re39lem%zyrtx1=F2YRCi0G-Z!hAfi@X%fop+$Wp$O|~N4ayb8*&U2jP zI4LdXB%@jOkm88BXAVLx2U_NIK>Y@&2az^rtw9w_IKK&~@{@4?$s#BQ1|}qKGaIJ- zrrPxKzg!_%&HYBE-kEJ37kf?676-yQq$a|u@&+31qeZSh(62Zip{Q5scRIUr zsg?>V9W#rmlYtx=6orq5R|2M1a-HzN{BVS~mCI!iAHhjRr^Fgnp-bAC<1!A{AJ zx3M=vWvHpX?zBLi-mcd65Py^nzm>lAh-^K;bh52P*@ID0b-i>}k~I>4f$s-cRvd zN()iZFT2<5#J%5h-~p%01ubM*>8D>Eo+8Zxnq1@~6Pr9xZPX`^XCH&JNWp>EIq*Pt zMY;c*6iNyqWxpU(efF_=3gy~&g%vTfrY0A{9It$BZBXjKDS6XQ5$y*2pL*>eIH!eb zKq zLrHxT92jf)$HbxL$r>SlNE|oi_k2lV@~NRWjmWPjYsp|@s@S)|;qOBF#j4w2!L|Ng znwR~+s!QFEH`A6z`1!xRLel29efgd-B;?cBeot9g(W7-`R%s)uVMI~o-Fl_6K2RYI zDB+|KJA0YWfNZ8u@F^emf!(*!EW6T4lWNlA#7I& zk^GtA$u^juu{U8{RvzJ~*2)mBf_!Zc^_Tsa+O}2ukits4Kh=DZ>5X}tzV6I9aEjj8 zRE}m6($WdCK?9C(v?ZV*;kYWLfE+*wNLxVJ3aJmClqGrzk@lkjBw8rffrd)tM5moZ z5|*n0+h*^{%1jR8#?qIrRm}DA5H}UkswbP@yvH^p9$NS5i1+NgznVv590VzM+Xt4mS9%R_uZhK1}`WfG;M0zH_nVK#5HHzys}+s_xyguzq;S z&Q)jYr&g#7cGWkp+ojf_aoUz8J7(2a?wB=i!LFI=s;%n09ZRR3U9x0V{qRn;3LgF0 zUAuNJsLkB9puTeZfy1?h>h>jvYnu;GJG*OPRXw|XfiQMaMXKnumQQ7(&Sd}Sv4wsL zsi)W#V)8fvV41OZ#yr>uzkll`%*dl8eCE~Eu{6{qjg^|l$xIw4g>D6JrWJ6`hq|IDny>;yMfAb`rGl7I)1 z4^qNd5K`JO2z8~xx(+K z4$tO#RUMT*0tF8>DAI!l_{;*SD)3`UKs@;HGV z3&EUBl$VydKVmu1HRDJp+X~3hS~lQdOVZR8 zI}r)oOqwbsq%1TF(W$CAmP0|Pa4L`vX3E3J*5KJ6@u;gu&<6X88ONTY z+kugU&|aBxq9`;J0Ef_t8SBMyV(^eVUOHmiN)$&ydb8^=9!G#17Vx;OiFq0SL$+eN z1PA4l;v68l4pGlqu)$JmibW&DzWJ^YPMqZcIlT@8Z zajH__u;=%b!0y{u%wLN;jUOqW78gq;>2l>eW{v45ricVeV>`NLR3L?0n)`>1NFpz< zLb9kT$&4Ka>nKUNYXq*#{hM*u)HL1azUhRD5%m~QVQN+sfRH5UYk_5P!;7Fr??-)Z zFm3@j3_X`CNibBx9^*Yd37F$}+|kph7N#DYr6A;K{M7-;TA}5zuEaThJ$u?s+&NP= zl_{$yX&61eB*nxzUIGmSOOBY64wHIP)f6czkR+YSm2xZAG#80w-L*!e0b6Nl;Ht6u z8f=jNl^k?zo@=2rH50q_;Xz(!LAP41hj$(dqPf2$IN@2at3?xD_25Q?RHMx8|mBy-)0qH;z(jhJp4f!&ME5jaHdaA^CMW!o7zjC=+IC@1tZHVn)( z)I9))&<809JN6?p1uTK4VUSn?9C{%GaL~r!U|RfU97Rq=$tj>P{`vPgEPi_hmsg$6 z<#zk@)2B}bEiF$sZ0mHMv=0$e;$CTAKGJDl$Yny8qwQ^-%j+j`*SFgTJDZkc%ZX)W z+zO#AlEP&ghHW&B9L2tj*rB$PZbPPECV9`m6_TXt@~OnJP`ZR$K`(jtcE3Kd`}}Km zi{&i0j_tm9@#6iPc|7md)ZOz(E^fM)%VhJ}jltmf;Enz*ZzE$bX5aWY7|fpivNHMA z^vWb~pl|}dxC3F8@D|N74B`?{V9g|EAh3f}C@rE>U#HY5u7q){3E*Jl&fC`PpliEP zI4uJQ%MUE(y1K4Id!ltC+YJ&1q#vYqpF1}NI6!#vGk4by>NF6>;bjrjDM&^KhYVMW zi*(B3AY4NSJKpNx-5_-Bkj zRjVkdN7~EXcXxR&-2M1{-*?}&kKa57b6BF?i=hAN2pDq&lf;Xo>&p-Q<@@0I^2rSg z4sUK89R=)5L}-?O5VHNZoDWCfK+ImMb%oMjBpVs+C$0V5|je(0g*Je~((Qw!A5qL!@RL*UlBr3ZdhF{4J)$MLH z)!JcSiNbEgV;+b6OS_lHin5%{HtFfrYEn#27bG!mwz5oDoAqY1Drbw8Q*0U|?;m(& zW2o!@o{Y%W>{5eeT_jVYsU#Gjcy9^-vu*3KtdGNXwzh3s+!x%W#f^<&|`qxi_GP%8Nb2(O3 ztN;)5xybU~z?Z)39?N8|+_-XeR*AXdQ9g7bs=G;dEJVyJsbs~gh$jR2X;gF{RgvII zC0hGgXTXzBBG;AD9d(n4V>GU!M2$yW(%-V5D-$z`!YIjghc8<|iA-m9hTUp%2-@Ot z!8EQr(^R+4QmbH-OAZ&U6RTFZ`+)<$9zGTc%RwJl*pz!=t{zVhqiGn6>!GtHaSnwz zye*Ot-MxmuDV(yL&DQu#ip8uf7K_3{hF7Mj-%M7U6>c};M5N8(TveXjV)Tk+|M%ytnM%n*_0h>o(CZ#1c-SMhKEt;gy30(4lKN$2;B{D z^E~tzzuu$_4pzOhUa!h^;FYHXR!k;^5wLQ-2v)O2x|yYAx|pPCQLfj8Q_OyaIn<70 zay=ub7fd1dbmpz42z4d8)4pgTa=7d*Fng;r()L*P0ec_zLhNa)U`rl}zSD}Ti~-|) z%frnQJ0@rdI6hy&x3PQw_CM*${02C<{*dRgUFoRmGDbU@X#%IR+fiXgu$RR-bqbKzMOW~7rbXrreQeRVpo`B0QNoq9z|9RUOFaBSW3UEdF2i&Kto>TVm{BRo^P0g zD-%AFa+L6FbkKqN0Wgr28u$kW9H2>zR1UG8&*xR!&AIOQa>NtUG3rPVhPe`1l1Cb* z3%{p%J^<)&)BWd}gGn@7cTPAv-%zvynsB>Y+uXFH)+8n>+*~JH2FZQE!O{d428j~g z5ZXoP@EBPQ4#=+050QId5<=xp$fk*UYy`}7FTe~|n@PF`MObVWg-OsEJXB9Mn?<@v zo%L!KOv<8I&t|0u$$De#7{6e1uue>(YK=4UAf$IsdO=-#%gAl;DlTelPbzJaFw@S` zhPp@=C0pD(Wc#-x2C)nFWi|75^9DL#_HPTW`ZaJk@O`bNizS_@Y&3N{nTUAY?!|(u zXna-5DQJRp0Utt$s5bz=5c9!ZX_y1(fj=E6A$T$bd>k)*Uv>coVGn3D`i9&eC;I zfJUSXs0{#&)nw%riPfMa${j^#^hTdj0*O_`Mo%#G; zh*+IDiBMVl>Tp@|Mi`6yI0E&=88oD>D4n9%5tG-!Y4qgp7$m)Zi?ty8c4F_Yw9pRD z%YNVanZ(Yu*blm}7VsnRzPWw+!Rfy}-8uLnH~3S6knSOE34@^EC3q-!^U%{?nnT5VOaDNBZ>H0+^bcK2q4Y6v z-g`6Su=3t-f8Y1}zW3h7&<*q6Q}`7D72(2}ZzP$lg=^jd?1`i+A1>Al;c&TkI+rXI z^WJg~3dq~xvqHEAM<<5_01aL_6mu}3VIZs-Wl}F_m_c<02f#?uho=E7rqD4sJki0X z1{Rb;HZ{LAxEPX28+6!tK5$5ZIq0>|QZ-Gk;5I|J@YrFR6`DR#NTqUs1oY*?FgY;Y z-PLY&^&_|yKpNnRoZzR?YNJ{O3c%I?f4I5qK7(6QBNaF3<6qBF1Qo4VpAF-4Ff223 zh4IM;igI#3yFG4%KA35gu{x5ITtyb+%=#z^34kL*`#<*|rW~+Jn%|(afFd@b`u>65L)2bbg zX2zjL)4@Mr8g3mNKAQNCm%jlHD9I8igDyrE;5YPfRbpX3?+s&eb4W;e>);E$Q|^Gw z=X(o~ENgi#fu8y1Kp%?UdMOF9qIXu{nuiMymq0qou7zLZjZ}UPUI_l46y7r|IGFU` z9Dhhx{=jClada>}lZK{Xe|^Sf8ys+Eh~uvwwn-E=WjpsxnF+*Go3GoA_h>!l%2LLV zeQt-OQFZ^~XE+Pz9MCG^Td->20AEDS3#GFLbim~gQ?($7K%;xxeKzU^QLtJsyRa(& z4BeaCM)jr(V7RSzFB^^9#>6!);{=Cfldvs*#o3nNC1(=$C?cM*nK4 zfJGEIWq?BxTtrto^W~So8^BO}c+1e>UP7ce32_d&a;g9o14vQ-5dHNxUaQ{RLVkb^0kr}wpaH4?t>W?-JPt2cyVsZ9iDzBZ zuX&sUn8MevCv+45Z}|H1dej7m>*oaroooFT8OOL;5qQYh){DxkW7{V-`wPAoPZqJW z-}dX4ki0$3eCIx9!`2#`i;Mf)Hg@7-V{96IQ(KeUKCA}jdBU~`%VRRj2ipgBEMT*< zZO7t4Z1bMww}QF2Q_396 zvZCtXu(ZOObft@`~CF$Z4MFtsT3mi1r< zcpT3V5?1~YKe!c)Lbp8L{OY}LeyDaKR%-5pYj3Xc^^F$yg!#`IL$`|HM=nfgPKnFQ zO9-5}*Fb~DGh8rjpweu&@6>pDbVjnoNNJJKrn?KeQ|LIPbaE{zxF?3@ zqtQxQzUUB+jzXFF|0Q!kD~aGF$dXze+C{NmEc8lVuckbd;GqnsPR~+B2ot3e;+zmt zS@ND1ae^XIszJy|ogJm}xQ`nm$s;uLk9*#5>6gGtXcec0{G5Re!#X4lzG)DAR3{l< z%6WS&me^V%ag*&mib=C8P6?mIwnnng8D%qZET=6V*|d>rYH)@terRw&xjL&CuZ#|87qwI>_0^RQ z4UiyjtKWTkGr?!R6;21PO1uy6;xq1=MElx(+4vew)bs~nPe9=04MS(^c5l%d0Eg!{ zD@+557sG_pz+sVAHDo2@(srdn8-#(+ZmjSfmrSNy|%q zX5ssjkVt6`4%t1%B9C^+&W^B+LFC>v-QzH9tfQ}^zux={<`C?7go}*`1+)HspE6E= zMU~oZ>i6RTHzA+I5mKl*%fUg&dSY~;W&`~dY9C7>RmgTKf=3biY9;L-rCg;Gt3jZEJq%B|)qDqJT84?nmN{}4U2wu}UmnuOa*LG5+pHM67>sBeFs)>FlYf;61 z@Q;|o)2B}-{v&-BnMwN6gZ3Q(ckG1G z-u@x6L)kZb;#_+|(;?~p&i91hI<+wON? z6biv>EA&bCW%mZr3lDP+PyrwVf}g6F2xl0rNb1G_Q9wu4Q0|8JISdFDgR+DM z4%lwgvbq!?*N!;F*7&wN{s+_Y{(*$G(EC#Cd<0 zwpZg!+O#liRu*UD0Cu2=Wu4Yvuo=-LW`@PGhEji~R?KH8hj~i#F+Xxz-L145l-mk* zTCFxZ=P%@Vi<*kg96FYGT8#yqE$2nBMQI`17lcS(Y6sQhz$i7S}*} zG}w|V^A)fTI&QT(YN8{|lf$3DS6}o!0*sF6I;s6cP+ecoNf+}%ywi!Rf~PP1o;f^i z{FM0e+oz4jA7jJ!UvB)QU;1$)@ZGTJ@b5E+Uid70G^aP_UUC(nJd_a$t@#vP&@1ck zK_ZsvXyNUn9*>1CV2mT}qF6e8`?SZ6iyD%x$|XDihsEPU-V2?hw*cax6NT`Zw}-|* zVGgFeIJhvZjf2CJ2>z9Cy}l<}qKE8Navjj>Bzuxzfi=cf63cRkG5F}z4^!;{`fjr$lvl=UxmHUxSsdAwXpuniQAj*W%b5-ZEyrl zpKwex1`@AsZw#4Yc%Q@a;5OR)LoWt69I+kRT$Wrr@;C$gt7FD62Z8{diu1**uH0tnX0oXM*S6Yum;JW6Yfax zM=h06K?jFBa##dwlN>@vVr~r_aN|;^5~5txZ>d-vO?F4_KtQ;lOg5@owE+%t5X>PcIx9H}b+W1;_Ju!S z4jxL-k54^;A9QYjPZYh52B9Q)?~f!AeUIAkQ~lop2fzV>9kj4$juRujaJa{FD~=F2 zeN-x=Rr2<(UIG&E9E?;ReudLy3Zewm$}qHyRIOZ?yXu8YB?L=g++MTz1IK#bT7W3!Q0<((3w@#B9tNkrc)@v$ShBJ@@10JQ*4s zfCoP_cfFx)RZ%=^L<&uwAKDSEkWO0-UXt)(l3cQZ=6% zpP)iZ{1asp3yrI0FoJe!nS~S-Dccwf6e{eKUs|Q9--bA|miasP+avxb57N^Zy99$KEj`1xKf>5xb&OY(>=n$#5a0mn5xgFGA#vc~{ecL|XFT86MKw`LBx}My!OmTd1B3l9?{u`k)FA3#?uS$t z+_MY3#xr}g{!VS@Q-?-0$G-5(4;^`JzeDDhQaCibd=Am&Fb-ZA^r5Re;C!L)oL@WK zdQ^KYa5<#u;47YM*G^cb%)xJ4{ef>o+TgD6e7HN@*V+dO=TPf0Z7?wCKcUUHUhZ;Y z-sRkUx?U9~6>4(p|(&$R^}v2?wZHMpB1b-KSNrB$0nb5(mewQIvoIaR6ra zk2oBGr|{eKIx1Ur?{AaDK_t}=YTKcZjD%a7>n-B&ly;>1sSyYlsM3XBg*$=|?HkZW zwxJBisH4p1nv)e&?LHmyNLz&&({<_Z6gOd*fGsS&w->U3-}5_Y>CTJS$u7}*S8P7F zMDY{4XXcx!X@p-pXE0aYSMlq#s3{|HQcek zv$^e&q0!Mq|2~vof9Z{bg9mpv?K=V`&aT&YJ0956(DuPY4G+E^aXkG(gZG8!y7xFn zUv8sA{>ae-2cK}fwF8{luD)j+j)6nnA3S*APVfG`M+f)a(=0>5apyYSwpF&H(fNVp zs3+<s&rR@23|=PaN}{@Zigx z^Yb1oInkMfY3nOmce}P;)P{yMK^<~MtsPHvu*i;cn6w>M zNmQyK4s>iST|uLfvV0kRK6pr)agr-2?rJUx&0{LkSTGBSpQuZ~ATeP;4+&oect)j? z1<(odo0@~cv(Q*qAQMJm7_T9}O9XiPhJ_{~V>J3|7As_wufQ1&6mCvvsCyZ9G#u8# zTA<<3KF1we|ELz|dT>v`u??E?@cs4LK$`dg=XX-Ba%N=&}5rXbw$=Y*P=D)cL1k1^t0l zH%auU4{Hz#YFM_QW-os18ycFaOOsye2>-D*Sc|CZaEXQFVOCF@(b_Ou?-HGQVKfrT zP1>oJ7?Gasb_r%Nq)=+%0jZawukrE27KuJOAgq|Bu1+7ZPQatjT|Ot*;*K|99xyp0 zIy}ZweE4ge=;#FVSP!~c-(ErIP=vT}Ef*NNo&%)_46SM472KD!Vo_Ts;^1yCk%Zhe zt^2>xIXG;I!!SipHI`B}#K~p}odbNNh1cVFIz{HoSa29E0X1WqI>(`agcnqHT+Gfv zjPRk;k)p6{Gu6|9ePrNFchIyYg$RKqio+jb<;;N9K=M@j+MzWx3t$j1FGm?KAB`296C0h70(5^z&ww*&e1 zgDsAGnwlF%yd%Lo+vL}{)C9WKhHQsQR!RsH9_jDo&l>Zm$w2A9VxA7=XJLNP22;pFZg-S4PTY zPh3$$F?A|MG3ivxp`Z#hl&d-Iv^@Qsl0y!a48=C?(=|68+=cVsYb;g>j$ln|cY`>9 zB0vmU(KM|qH>95Jj2A5c~9b*f1t#cxMs9m*MD@xfJde2T6=0|@2RPV+2SID?9 zHU0kL!qkEjo>3tP8O}#Py|DKzD8bZ)3sVcB8JW+)BitT!0A__e!8*tV4?RMQ6u_-E zYhR7A6Q zdg>ru#A0N-5J8(YIYHg2G{zq_>RQ&qbI~OMoJE;;DosJp`rW11@bOSl+HCrS8yaeSF@8jeXYhd_BkAu%iM6P&hoOo zHX+-gTlhnNYqzNi$)bj~23A4C@$b?GuR;3|E@HMD!Ui-F)c)w!R@SLAXwN#_cH%&s zK1hNuO$i-*J;E`XqO4;RX%k8Vk)46=6R&dak&>RxDo9{wPAZumY1s9?&&vd2kLz>RkJ8h(p~b#DN@}eIgzrQdcnH$(w>{ zGLxk!Ycge${Q}(}C`U&S(G%HhJd+p}DH(`3kmpnqZcmtn3KBV>B1!VQ?wL!#HbL&w zXRtjBNXCTkWcE085+*1SX+NviIfzrtI3sXSK~Hp4^puh^0vX=Q@fDIR<6;XA7b=uz+X{8#b8hw2WACDi zt6I>kQoGc8eFY~d`$NqD?uHmFpFC7ISHo0<_^Qk~u;(KlxP(dK(jje4bR{%*6Zrqp zIn;qT_=hNmwL$$o(0qVTm|cNuMAPWO2gHG(m>b6v~~79TzzyJB-ws zAdt8N%mvsEWPeCg+(kzmTTP^PNr8zVrKV-cm;>99B^QW^n6C!0g3i1wgI9*0vgA9X zrvNd+;Z}~G5^cp$&SOSmtB^>nS-#{-$@f;Na|r3C`K%c`ys|Vo4d15FRERvIE!$>N zN8r^JMN@EL6eUJw!(`LxES>SR`eDXe6@{fW%9Lav}w1O87Y> zV%aoQ33UR}=);LGAYCL_k8~^x;y?)~ePR#V@kjw-+;ecUG-Bu-AFrA?kOG|=7nx;s z0;8woU{&ZAC<|R#Q&pJBo2+Z z&^lrXBS|3<@ z`X>N1;wiJ=M4+|>5g;ra4+7iC+=-B^*f6~gyQ&P?X7p5|w;7`tJ!u8A1vI5%tCHe@ zQ_e9du{O(dB8Q+oRTZ^4mE3!Ac6xI1dMhG~H&(TI()gyhl)HYqm**IG?!C{9*y8Nu z-Pl>Voy*Nm@~8rQiHq87(a*uf>P5Qb5I41oH>NfulV_5^`bS=Z#skH^A-_$n_Fsu} z*!-x5)JYZ}DrS#W_ozD=rt$^0MCEqobC|4oHAb5fhsOIJmd!v*ewIDE?O`rx!_{dJ zDnS)1Mk)dy{>Y@w*=?sbz#bE2ZLrtKKDH6~*r~@mKl&2jc=9IHS4G{5df}tU#rXLQ zW2jom`Kj`)YOQ{@yfbcB*sMag!a4W{y4p+ayVcb|cyJKoO0Db0)zv`zrqIE_!J&Wx zRaVpmgf@r+N)dcJxkK*d8w1oADi6Dilj;lUvLz0c7>8RHQ!)9^IERP7ZO>p+R8-c;0%0S?9;O!gXO`u&vLxJ#jH#SB0p}|1l>$h)$Ki*OyT}~!#p^-#S zJY;xp3B)=AHo=!AomS8jo0XEgF4kT!ZlftaK3goo-?zK7)Xu)sY1iJd1-EqGnznni znnJqbD$l$^+1 zp8t{9>TKr(Wv_~~5h(07+G{6Oxnj#YhpVe-L;>&-W{In-xX}n7A`Dkoax1GBF((r{ z+%`Wb<{X$f>~tu|V~>ZG`(B%+jJym{H>)>C_g7oz92jjdkm7*sG5#s+f%L;=yyHc3 zCy@&kJu9F(dsJ|q zm3x11xjXzHbKPv}$pbG5FKuS?rZw#n4|Y?NltI3MBd_?IZHwB3Dr*_UWdmYDzza9r zLwr+Y76r@?)wHP+hrhVHg4I@TD0)*$F^rxz%}_E6#Vv9mP}e1{aWSbe#4cUf#$LE2 z7{{Sz2zX;iFvQD8q5gp^ym_rKGHZt8l9SHA& zJIGvShB{4>V6^g^G8H+r!TS<^;=8uO`nU;2p|z4PS_~9|`OcLRBd*e#@Nh*vp;nlY zpgVTsgP~wL&1^&t%rY77knzsN^cd+}VoP!xAjr!*m-;QA*SK;xaf(aK`W^8n3duA} z(KnCyN!s8~QcRylE_m|}5ASXs9+xOXEYWSwy+KZRKy#13$6J&f9!(%dLhnoHA=66^ zUQK4WV%J`?c2fZc9JlEec)86RlnP=I^sTBr(h9vC+kM5D&@BX$L)z)nj2WL3xD;ru z!PAp^xy&n+bDx-Fua#LO$}Cduc=b6-4&G9M(1nKnw`z$tbm^mz2!N*oBqeO#^?|7s zefA>Z?#VEs^Yo+x?W&k6#*jlgg=C2zCGT{_OIcW-<2`9lR#{i%dS%YMZ>JH?l4#Ph zLVx*jeGmzhe&Q!w$5V{Ti~5eK>VK(>^*bhKSXBBEZFO&r0z+KMuQ#luay6|CqLD-}*jWW2{h zN?hVi7tx&~uY2~~6JxQD?-L!)Ys)JfY0$MW_4@Sb|Bm^iTY((~WQIHJUOYZNJTf`l zEYWYs!EnU#;pS%faQpCxp8K2B62ZHFo*cNI!%3X26yjXSBhPqnRi)eV3cP`BLWQe# z7e5$fv%LTsW8jXmK9+!S?6=_yz+$HjgGzrdr1omtLt32!0nR~v9iFTE63$z~*shGF z#VU3}CE;lSe0UP<(Z$m)U1E36^^`WzIXL`>rD==|+eUQKHdK(vBq04R?dkZu)sAhP?PX~d^C zkX8n*A%GSF6extRY{#+#^#Lq27@T4PXi7{MqIwv*I42U^yJyc57+3UE=LmXD{| zbA52#ihD|TrB)hNNGzb{hff?y2jb*JLQd-YXgN1= zGk1JQqm@kj-zSHM$H(Q(9ri7d9MCf4ute2@ktLSPhdbnhNJ|dh z!;ir1*|2wg{Px57G{24TJdTe+pG<2iE*4tq0uda1Kdj7~-CXuyHk~WASDQA2a8#~$ zl>m7OClcxn2%2&JKZm#9vt-cof?f0gpxyQor7S&F+458 z*w31L4{%TMnee&jf<<6|(?`mSM?+HS9GDz<+8>PH!a~YU;fHMJNOfhkA?MpN-)UxL z=lf|fu6Cd%`}LeWZOi?!jG=y1DHh;Lo)!Yku(6%Dl0#Dt#GHU ze9#N9CQzoaLD!H2?{d8b5bJokz9=>i3$6ZMv*b|P z(vUR4bL6nJLId_Fa9{#y;t@~W+#1K-8+*jg#S(`zZQ#2bm0m6cS`I$iM0>pC3rGI zdDMcY^Yh?<2X)GUQqvWwo9uEVl|JB2l^TsYn!y}T z8NzUqgK%Rl8<7LiD=V6Kxdo3$x_$x?HSd2z%)(c2Q7EyANKOt-3VAeQ^m@r*)@|+T zzm^8D*@b`Z_{N>4Xz5$`i;#mqnhcg4jv)lNlrSom;B-{0h7g?Z2VEH$!C=q< zs9(WMUP5Eql)KPqLp25i%VKq3)(tuAUjS6EFZKa;n#d4J2^58=n3XCw>OUc78%<>t zYla+1j>M`@uXEs~+5{=Ld^#aV@ESjU>C?~vk4aKuAqb+0Z#qzV+FMHy%8l1J(oLU6 za0v4WN&3(3&gOMK9Ky0O2l`LQ;e{4ayu9Q< zilLR{P~FkHUtv3k+;;)A^SUh?cybSSeo*IfXHWsNL&0Q!0@L~kF?%!(s>j*@4kd|} zJf1c_E6x-qM3(pnno{;4BfYjtc;dI-FO}ksK~1poyv{+9Z~j0L4+< zP{r?!TJ=HZ8pxEdJ+_WHVxZ%1B5I?PUbrD+>k?d+#AnjPRTA@9o70Bv(*FS`+<_j za9xD6{zY~QeJeDfr!tn^YYEpfJC!pbJ9&8^Cv#a=;g9ldteyN$d>FQ}66u&fSg}dB zf^M5%-?Svc{pj`1qeK|YqthJQ@5?<|vQ5GnW#!UyVj8UY8}l3jcuB*8Lu*A3;(tXB z!c+T*pjv~jJb-%j;z&YmU%}O3A+E`CtGFA=_Mwc8)Cz+E%;XH}W3}HI)pHcS9i|kbb5w#QkRoR4Q0Is21Y#Xpj2+l-Xt>l(xijb_~ zg4dq_j%RK$JOw8tjTwWNVhexCcKWpyiTT`c-P2)qHwtTm>%*-{7UwQp@5Y0RC{vl} z?MkB(2F(ktk83Kb%z|!Si%R3|G1M*PCU1aw{ zDw`{QRLkNBwYHt7O-CzA&n|1_CEEgh3C5+E>}6$clAk$l@|gIhZKmW@60do*;-gmC z7C;^I>Tswh`JHGdTc78HVJ*_RX1U&&!$E#WPvXW%1*1!)<~{f@ zzXji4!z4S1Te?!Vy@XmE=g*-x4^*y!19U;N{KaG0Sf{fsWO25F$v*(w8Z!ph6(7lu(zySr?^&x_= zq#?4Kfa6K)_(;P?c4G%Tp+J~(x&#_i)2N(jHhG+!JG;JbOh*A zDB3#Aa^F(mDFm1aVDkC!KwBZc=Q)16aVH}eevt_lEZ4%6Y)E_eeh%R3trqs6NBv8 z&^gS%jb`uOX&7B%`z%f7_Po6*x)v>DG||aRV}d-~f+jsV*u?fW(U@vm<;Sy8iw_G?nP6tc30w zt8SI8?&!vY;JS9DLXnC;K&m}NgbD$5mm)z|FRRF2r1TaFqPTeU;;G_bFE~kB zXWq0sZT(rlTWgXxZ|2$Leex!0Tv9AwJIH9?_Du)3cL4{DynZ_8wV%e07!Cmv>boj9 zl!9ygbU*uBx#KUWQ0oU`V4-GaKd)%4uAt8BF~U5Q+|?u7E9yL{XO}+&Wkh`y+*zmj zAD}M)32059_oj7(-!}L_ez1kkRyx0x*<0Gv5gd9g>Lr&KfkWtb-m?e^EaJYl@1w*tYd)sEQ#w}~X5(EeIAU|BAZltw`!{H^prQ4lNEBtRGJtYrq1?S+^{@LfLzu-Rv z`jot$t_~3318BA1dUep+Uk#|Yn!E^P{Im1Zy7OxCG&BK$-`ZPU88`>uE+j^8S&E@yy`Ov>7mo z*`m=m0UY-C)-iiK{I#RXA=FFi;F0a<-K)K(x3RWxhz?!tIo+jUi&HxEuf6~gT6L;^ zZIWPOW0!Ey1U9*=Sw^>$lqNVRH=g&JuFLH!hwy=kUebZE2o8awyL(L{HuP@PyQ4HX zCeM3JYY83#E$s>purO$Cm#H0=0MJDf526XD6@ip3p2&O061V>KzU20oRj%LYHEFU4 z4uN7RC_WEUoVetr2Ix<_-wn)B&Mt5lD%D-sG2XqMKTr!iEe{M#gWkO$$!Nu$IvvDt z(7L`w0o}zl12l*b1mQp>12v~2CxKn;56xn`ShAuJsst~^BMenznP(6uj!p_MuVdPv zv5kuF-H%%ghtw03mHlNb3X`m6g9e2mkVj*2Xtu2}jtv=LE=FORj#)Oa=c&6;W8b*2 zUq<1u3!vZ3-QAE4O<`|?VJ{zIMb^2>We6(uzrbP1l2!hEt~L2<&|T*9T;L7`d+jc0nI9DDHwG>9aG-3MlX8qQ zU%-JrR;%S$p@#)JyC&}6?6a)9R5Gm;NU}@?*ee-OJESbL#G22Vd1389!?r)O{Z1EP z*s+Bg81`XdCP3HpI43_8mlvhT8O;ShUkf7THrECD*Cs;@Bua%VF`3|j2*AMTR( z2T$8T4IVap?~Wu_(g^a~!x! zxMas;)ctN@3QJUY$|EAxShbjw7@_HcmHJOAht=0s(-nSNp@DZX3dC6Gj3)A8o>qvq!3iN(L%BiTFS)ZzoF}D?L1DcUB&!pM5+MEvU>TJL!u~JrZ*bD5b z-$Y#4llBmW!`PQuK5TcYnxb$pr(7iYT_EoVxl@HGGrE((0d#^U2;pfnp87xF0JZlw zeYNp7cO$-AanjCVf7eLCKEk>AlL~Y1Br~ge+fndCUT&4il&n?A>4uJQa8hv%q@T4< zG5&-Ns?p!D>ju_eSF=o-Rg1*rjgEXt?wt8RZab%VXP4GD4k%AJ2k4+=Dz4vKqi0F2 zEGX2Rx25%l`y;5G1GqHLq_=$CoKg*yn4}=+^?I$E1#H0nXVr*Y|7<@{+Q4CVzQ$!I z;^2TJPC5gJF{|QoX3W8Xw?jBor!d#M7!H(aT)+790yAvXln@6U4i3#>Gvf}K>v?m5 zb#Fo%*^^Fpa0vaqZ$E^DaQKr_kg4B;uWHjz_!`QkF^Xf+{tg zk*Gbn@L4U5I(Wt}7^T5cgw%?~xD;4I7FP=2P|t)N7JV#ZVr&0 z2jGAuPC9_YoEtws5D#xw@#Anp4d;+vW#Mq9=#~zle0b2sZlVz5DsU#p!M3hj^Th%pRjTuc2VVk`mhiV)UM9QqRnvVL%7?@0fg0D zIRxrMs5Kn)x*emIaA<5rNB{>qI)Q`pKf+bT-ZF~;6>ojH?RmC`s3-z{P{x>pptUMjSL1Jb*IL)Mv9<6#oCbHz}PC1xBQgK|fiS0(Oz#lj)!!6AKo z zc*#Ofr0Y?mR&YSl#9)M04n6q)IkeSY{`Mhv1w8!WQTWkbC7Dc;njo1}W*uX~l zm>Rj4&TWud!vW1W2>pPA%)r6Rv;37S4n9!6*9?bu1P(`!JHzRGZX!{hJiSQefJ&tM zZs0IEyqN^;Trku`dfJc{sT>eOi&YM3Vum=ad84fsq<+PQXC?_U|xym6+ zDhGPD6v83YVp?%*hc2ib(8!e;B$!r25`#iGxWhXJ1k(yMogVoaIG~x)5!8S~su>RG z@#qX{z#-N=t>`~?6T$&bEAY*Gjk>Be9EKg-b>$H1aP<#exqxQzw1U9lJ2z+W+B6)+ z@zGmT_#K*qhbb%viB{ZxA){C}Tbt_7F~Jn|7As9r5b@9ovSC3fco@VeWdt#~7zn$_ z@DM0M@Gx@Br6>P{Ui}w*vAXq_U)Hs#pgy)P^yR(&3d^V8_uemUFRawuwi$VtIE_YU zIUGgiB|!`$a~bNWHvp9%-?qy8Zr1Qrg@x2@`vR5%R!+I8!y z^=N&6e+A%BN`*u53F^-d2La&Fw2EB*Y@@dPbVsb0{1wJFTo36u7Bn#hK#2T~_2UTToE*G340ihpyc9QP4&~=(Uj? z%5`_6?6u0u3Ti$;9S8YC+eh9|Zu2}r9*%1Z)D2Lm+(TwdQ$jR~G$laIaD)OAwc*G_ zP8$u)ZCq>katINuO;--h_3Qa_`#rrlSlE>NrS!5wdXwc+l|vzhL%mX#-(8{Lz#l3H zfJ37w%hJ*ks~ljC`Xjx{&7~X~RXq=iBwlnMy(s}4*s?-hyUIz&D+f()$-9suy{u5T znmW9x_1+KVK*2#Bf0|xHni4nKrFh zrr=OU_go(k(QN@76vaTcYa<7B6pZY!=H_J2K_l1Q=-L1Y!VB=QoWbU{>jq`41vwiT z=U0Fo9j~mo-MVcf&#Sqv=7m}gD-N>l;dwlz2{DEP*u+0N^c=R({(5hH)v}&iOZTKy zIDkp%JqLh8T?aTUtzKV0rmWC(t8wmXU9T?1J%@BSaMFAY$--{DXsc#{VA(Ghk0~qE zymVa8p|0<0`o0F0NqSka3LVddUh~|=o__VXhNP5FD=X%J!>vC#t%w_ve|D4!#*8u< zIFwT0FlC#>$a^lvJgwO6 z^>#PoMPKc0suUbDpd3z|R@C09@y9;=yt$c}R)_|`LEg&7i_XHLH|CmF@U!l7Q0Lm` z@HfGM8D-9~&w(u~D3fM|!}MFW|C;ZtXW)=JHHhKRap9szmt2KpFnschAQOB9+Q9{G zjNw2OR}1UI1s99Zb7<7sir|Ufu%YE>n+=fEvJQODb;)aSjv`Wr@>0>FV& z7k?}o{xS{8li*P5FImMg8`QPxu1b$GjoTIJ`=X_I(OcC`b$&Ro%HdOGP#TnutZ93^DF=P*I&$yPaL^O+3p5<|Dv9t{v%%q$9w$9YUARM+6{jcOReX@#A*siX ziTDZyhi8MycLk{i2{=@!U`YROmKC`Z8Y~UsHN{1N z*D{emLXF`)0|ppI8Aau$fi}9x$K?RqagSlxZHhq)69+}ZRwvLD(h4)E*=*tM^02-_ zSn0wl2db<{T>bMZ2UQvzo>GJ%T~axeq@0}YC0U)vO<;e>S63+mSY-{ z++cFi91Vw;3>;?J1aUb*f&kV;Te;D$GY(ke4oSAeGaVg%RyV@gSOzAaNwC9HssNY zr-;NeeM}r*AYNxVFh_*gp*i9qMBu&96NnLpHWs~((Ds(Gvl2SQ_Js^QZTh0^6VbOZ zoMH$BG8^I%31(9cuk{LLxjh;V;HGVXQVw_Yr28YK96l^0{M<9D=O87+Kg^;WUL~^X zdo&!toiIu{>@kaePnQ);Jgpr1$?#9>XV2}$)FsJuA9%vYzdXU@ zQv4802}OYC;TZlS;fRQE41Wc9{1Jj7NF+prC&f5=>=R?RI6jI!<5RvT&9F9IIqcj| z*bW7UgZqq8%3+6fQ=6ce_-8v59KLaP#jI@uQ5fX0kmUxgIZU`g=d*`GgHS>3a1=_R zq!7puD40T*fUX`2>ENZ{L1YUU{tTTj!_u z-gn@8JU{uK&i@d9{_2xCn54DfU?UBKgN^k1No#+CI)7t`Ke~Tbm`zc5^P=}fsmBIv zonpA7uZkvb-Wmba=Rg~(yAcAHZnw}i*WGfHwy6$H=;|h-9od!HX>SF|E;16JvDpT* zEWb986QFTz;)$kiA|qgq7Ig-e`75WoiS?v`RbnF@c#{r92h#2~aZzuBFfHQEMQs}zcd5JG6OT?%c&iAfqzh@{ zYA_43`0g+*&Lgb;m+5*&4^KPn#aMMdJK;MR9O{~bs}5K{wgMnH_<;z84PZ>=Fmpib z6V*MRVrUHlRfJ;bNoQnz<`CT+@ZN^d*yNK-VDagYBUdrMoU}9IXa2PP{PU9;u$c7v zM=11XXHiRV*vd-8=r4Oc5h^7p7D#H-pwh4)w1JmPba2RHM)dxdHeD#A;{^kYCZz<3 z+d*ieSkMh*!8%2>JeJ;k0|H2|>jX7FIN}>c|M-OW;4;JNk8fI%jfQrg>v-9f_Jo4~ zwLi2Y89EUE&UbceH9^F(EK9|Hg#fy$sIhZ9kmXALi_9TQ1B!zj7qHn&O0h_D z(lt0dlf;SuylS7xSw+F56N-dPbJlX-ie3`?VK9h20n7f;_hD>D;&cQq8X9;w+QVrG z>mK)r>O(My=lt#<)DYc1|DS@SXblv!*n9i>MJ4Hoj>?NGdHPLKCMBuIwQwYftWK|< zUwj$`^lnEcrIHzzgzI)%@Bp7V1Z;-?&Iw?ie2B672c2UBi?cvj5ePfj_?iI&V>Ics z{c448y>60nOdj8Af1o7@h#Dr2_VGCKJ=N(cynhRMJdK)R*vPCrw&EhC`@~0xs};gN zrfTnSB$+H$i|ohaN-UJD#Bw=OD@9qcT2k?DugXYprx8wV7{hgrs}WT+3YxP-e&S#n9sg>@9k(^MzD0oS@+yN zt1jXTUQdOuFCS%nmGeJ+-pgEP^*cdfT;6}Vo|>0go-M@@5bI}wdeQ#oo~ic{$n*`j z=EFR31(vgGgM&Bsz)=DN(meoZ&L{YH>)`3;Yohj%g38`}`b+P^ox~4J4z2GRb`WnQ9>ohe&=O=dlz`1oo&%R)a-cOg8S7#s zKg2RZRjGlErnyVm#lQ%5uQ-e z8h-v>E=Pv3TOtGNW!v;VUHwl$CxlO0OY@v7-!^jCIH@A($#upd18VP^OAN zT)KxfZv}^Yls#HoFt_)oDg}zdW(&;ni8-5X@w~;%cgz)2?QCVsSyq0qP8JFQh3)5g z+)z1nAb4wiAO!vp{EBcs*1v53o>wp8ujUm?Z$K7jVK=V9ECX?SSZHdZ#WJngamCGs z;GqU_SeczttyiGEGNGhr@afY>t?&GZL0Wu$`*@ z6=qvIzq3=*{a~=f7WVl0z*nb4&*zS*@4E(b9QZqHVCqxh<9ZHK!yoU4r`@t)X&81L z+inKKk_%U{w1}k0VPTbcrrlk5>_UyQmDFC?!$`|k5H4!^t+^WwcsU8`YQRK}!j%U% z8=4Kp%}h$TO&dlV7>3XbGqE1HF|KOB>t`rKIfAV&&BL zb}m@Y_A7k`F)k5j|7;|ht4LYnsKkPN*Db6FUPv=@dOOli@K zg|modZzBg@0R)fN2Y8wc8F_dFhvjg(vk$}VNZ+Z-@YD2)g=}?!tt(T&jJOepgd7|g zko%9Vtr>uIE)(1?WsFD5ezb0O9KN(uf?}5bRNm)cydC83JfAyLc{-UdKPkQC6=qvc zU;qw*dhZRt}xXQpm0BSKZC z@<7ms{UI*Az7fo050eAWa;AJ=pE?*5Ok6C0Pi_-jFIP~RAHkkkaW=~{&nm@O() ztwKYa0WAVzY4lKHqDIf5G~Kebuvdl=gACOsf0Kij`YJLCWm>D6R8ir?|eF<>&T(C{B!`Vju4vEJEXm-(&{>0r-gXxW)?%dnkJg7Ae+eIR&p>hjYL@= zX~Sd@qoSG(&Zkx`*bPWfY-){BK{G)htST>TBhN1nLqr4iL=KDE)G#@q3`Afy^H{Qd zPmr{qNo|~ynrKQUm9ia-;#}QNAP!BhI4wNua_vU`?9_8H{qyZ2df4c3Bwrg&*HPX4e zQs%ufm5rZ!0PjX)xgsa+7g^jSh7bsk6{SE4f7V0I?1C4Eq@x7O{b7FMJ5WOpPTF z=`kL|bhhEVPy!pS);#j&ROW8nT{FA+k>HPkb60nFnH-o2+^G5|YJ0^Ivlt+Gzd&l`c-&@LF3Fg;u z8gVj?u#`2$6FGqw#`#%)yDZ#D!(g&1${3^7I~8Nr~>350Q^% z{><=ADY1vTN8VZlWigC!YAC|OFP>to^U$fs^H_L4p39_p{!|yh=RJ!jp6eX&_0q!) zZf^$9V2ddR+s9i3g*gb{1RfF?if5=R&3?A}Y+N3V1EBXSVm%>v?` z0v6&O>l|1pdXR;lC*gOKSSCX(g|x`BtVg#)wkZ;NeuN(hC5fgb2_`<0@Bo(Z=t+Uc zP9mQ9&Y;7h8^N$BLaB~mRAYV;PiJpNCtA|t!XukJCX!@D7KR>6d!ZESp$AFW4k1g5 zBq`EjNV8acxFTEeSA%DWkBE0yQ2q1D85*+{VnZ?W-{eGy_bQ(z*;7NFzb`L5ME)eO zSIiUEe_44X3niL7pRh^e*D2S-VhPgpS%74tozE*L_NKlJJ^?K^qlxRc`@hy?C)S@7hgm{&q2B{e~nN7I|)Ps(bvLaU{%9`I$=&E|q$wA(aH>p%fTbjk; ziu)FaM@#ghb(%*__vh<6ICLXn5emh6f?$?Q2yLh#E=oA{S<|Oqv^#FZu8| zhZB5s3Hahm&w7+ryyD_kFKtAWzKRGIw;(DwL^HbwI?G5aHWFY)fLtmcgr{g&hX}Ye z8cF3>)$NB7QDx>FDV`tKyLM`L%Wq$Mpg62K$5n`#jWup&=6=?Wv{{`@T*Zu=#VWft zIbw`t#9)r(ixEW~k~Oli?#k|#;*dh(>;7aut#RM`1 zVnaYmC^XqL=+Kf5#?lZ37bhW({SNuSg~E|UZDR>nF5&r2zMjvc$9Z=SDuO|{q@rcS z$JClV|I^V*jM@ct|71fst6gW*$(ayBt`fCQw(&FSsmsfmp~W*~)N&I-2qA$}W@zW$dARXhPm*)qf9IEDYh{?R);!|_4coXR@1Uzc061OoHR;hla`*nLA7WhN z1<9M~NANx;(Hy9&%bWUszm957@E`9VPVI~TI#F~+%6GAt8M2V@XRVLG$iQDTD(>b* z%MO17hwJEl^drD?zxvoVcNPGVE%}?<8laIrW z4NyH4v{uiyblES;^J5(Hg7@k($KPnhe=O!$86Oyylep5U6WmHs*jr5?vC7zRGj$ieh|87wwp!}N=rM};3=mHPp!+-lLi3$UVYD1$1M44uBn?khc8g)! zqd72VHizc{C1dCT567ypy*b9YI8w*r0guDsJ-h{qBd~^p0uHH(#WkF_;90{S4zpF5 zYD-CT=c)sX!ExC8RN1tjfFNCta2Q%1;N|eVyb}QWqO8xc-D&`?o%u!DDJA~O#avY_ znLWM(8bd?Q<<1`i!{wg2fz$+ z%o~T;d$tC7F>=6phtw7*VTG;0WfTF}I$di89+j4jWJnJHtRhPp70>=;Q+O!NTihNHSfz^X3 zIsxDoxd|+r3smN1+=W2RXvPH`DpQ!(Fig>t!fIvQ2AJLfz0p$zXe?^O<87tPg21lN6mE7{a}pK=w$E)F8J+#eR>=kS>dX<2q|Y5(ZhXvq zcuaf_rG^qF5S?bgKEgd?jqZMj-3n-KU~oKPN#Ky3&`c(bryM7|n=6$Z0|GlnZC0?R zvtf!)$Q}Y`sw8E!XDl`}seFb=d1#5%8$eC=Y}Wydor1OsOEO{vaEK$_gf(m#*=~pJ zQ*C5}Pk`!R-QIDCkWDAX50NWAlhIlru@JyYL^inT%*-QEcQ_-phzS`)9Am*D`xrB- zkaw8y5ReZ(yBLYToUQ4;hr__$?N?~1ZpMqG^2F$lTr0+i=)#vCAHUk_NRFQ2fG0Ze zs8;mr%*L$1q($OU?e=}lx!|@zV?V>sQpBS){dPtRCV$E36mEtbpNOqe$-?Kv#S_mf z1&|wuuxL+eK@p{hHjuELFo$4*DLl3i&EVQQ8_U~%vZeenw*m))Fir_OniSSFZ}T*g za=CIp;S}@Rg7}2C3X-UPkG{(%#H`$t2{$M)hRuGP57^r-3o~0h000+Q@wDOr{vHRp zF-AbY-(Jc(0@KU}hQyZH#wn*f`$K$1hd%ulk_{=zwOfiF#f3+jCIxHV6JRB%1E48zXwD?TK6C&ogf#X+jMxNagdLQvKf__)jt&4B zP7e_qDfgHt?1&v}QyzBQ-7V%!9t`{})%RR^SWx0}`&e<2iiHM%Q|w%z$_Btiaj0|A z_&Xp_rGWRSafw+1(13_TWC?JNU^f9A?Wznc6+89?4i`!L(0+gp_!*8iAmySufbZkT z6rJ{`;9vs=y<(o65|3bxT?9bRO2Z563{X2nW4J!B-wP(tgS7`V z<|#d_EZ7%`#e!ci&m$A8?Dbl91ZSAk1KVCz^=7gX(sqDOQhWejw-pw4Swg4VVF3qj zA=o{zg4)e|jp3BSl>k1^trs|yN6Gp-92V>W+oaDqntlO?qbe1iPKLm`^I14#uHb1l zp(QhQeaLJD9D2A@9yp45dEHCmNnx&l1HB^K?RB5sx}*#6n*|(Vwz?bfedE~r!B*Rk zqWg6?Xl>O3s2)I7gU9`CzhIY8X#NZbm!VkAX7Spsa#cnh*Jn6Pn3mf_heM>C!DU=x z`z;&>=yN|&LIhBh$Is@)JL|U*n;1MJSRli zTFwg`E?h5O;SfhB&IAtEHqhe{IQl1MmaW!vx1y9~rO|`atn;x#XT4@vp0Hd0_vQI< z=bVol>n=b1CU6K?*F4XIy9oj|WVIG>u$)?+-eaNa5;{#xj3THJ5K~Lp!y{`D?W1u7 z)EEO(XASEH2TLsu-{HU!aOEPk!{TjBOy%w*!o&hLoe#|}|2sIuydIM7pW%?Aq4GNY zi3O3xf&eqj3FJ%*I2eumpo`&hs(Z_v{j@P)Ifpd81N1GQe#{r8SE>e3l|Z`=k1ikt zWh!my4IHKg7CJXLXq1ZiGaN<*9DISp;5{6=y*>khL~=3#-lI|NQ{0SEUnKa^mPzQe(U)!w%8 zsOE+5SaST8@(h|a?AsXIKz=|ZaHvAm{vBK1A#s6`8emEf+Vlku<@T2g~Luk!_G#)5SCpn$3%rY1YF+W@cErX1Q^6UZ*WMS z;n2ImA@w-iE+8r~stmf!G3O3rr3)PP8z87)lVCc;%)e!>Nw%bXfGOq=XlM8#^4SAO zeZVsi(|YcV0EA;Kxa*OS^$u}x_`H8u(XDp%IbEF`V8R31#Ec{Kik(BCy@SIY(c}rk zsl{=Vd+4ul*mC&%v7!SK*oTAHRml1?9CRS#!Q>e;`U9)l@@`RjJk@N5h2i;Z>cAGINxI2+?ht7@unoT-gL^n9>W;n1nV2+;3HAb#d5ao(gQy^6R0%q+e)FvcN zq$CYY$)4c=9%m@L}YFxi4*K;%h59bu9o zT-rD&l9d-YEGSQ@yW+MGZ{Yx|h|)NMP099{%m~cLiH}iR;1Csatw<+>-sTwg6H=QT z;|2%f6EYNJ_#!ENSs{-VR&Q{K3pl6{wm){LLF5)Sh`O^c+Q_|iZNK$MM}ss#Q%0+x zFpjkg$pBxHR)wfs`b!#a{bE~jNegeJ>u=kS)p|1wO8Bz|C{)EwzkGR<{!at#)M$WvmD_w#Ch4CKIsMsT`-NN^)b z!xTw@d@i@XLHh&rf16GSq@00YU=ESFSO zhEXu^Kr*u32b;J`m=D7!7zLwX6c7mpM{{RTf+6J}VjMe^Eae^oCJx)c9|G260i#|V zPGIy{`%yoSf>AIEMgd8Hd$1h&N#HO}YF;+WP1`No^g0)sv=Ac81@!haH-({1!^og0 z@P|&*ZLl6MOx>e)N76CeFfd4uXC5kk9+vrJu#`F@2@J0CeuRP1z$Y320C)+Cpe7^= QTL1t607*qoM6N<$f(BSKX#fBK diff --git a/slm/examples/dialogue/plato-2/interaction.py b/slm/examples/dialogue/plato-2/interaction.py deleted file mode 100644 index 6c4227b3d0..0000000000 --- a/slm/examples/dialogue/plato-2/interaction.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -from collections import namedtuple - -import paddle -from model import Plato2InferModel -from readers.nsp_reader import NSPReader -from readers.plato_reader import PlatoReader -from termcolor import colored, cprint -from utils import gen_inputs -from utils.args import parse_args - -from paddlenlp.trainer.argparser import strtobool - - -def setup_args(): - """Setup arguments.""" - parser = argparse.ArgumentParser() - group = parser.add_argument_group("Model") - group.add_argument("--init_from_ckpt", type=str, default="") - group.add_argument("--vocab_size", type=int, default=8001) - group.add_argument("--latent_type_size", type=int, default=20) - group.add_argument("--num_layers", type=int, default=24) - - group = parser.add_argument_group("Task") - group.add_argument("--is_cn", type=strtobool, default=False) - - args, _ = parser.parse_known_args() - NSPReader.add_cmdline_args(parser) - - args = parse_args(parser) - args.batch_size *= args.latent_type_size - - # print(json.dumps(args, indent=2)) - return args - - -def load_params(model, init_from_ckpt): - state_dict = paddle.load(init_from_ckpt) - model.set_state_dict(state_dict) - - -def interact(args): - """Inference main function.""" - plato_reader = PlatoReader(args) - nsp_reader = NSPReader(args) - - if args.num_layers == 24: - n_head = 16 - hidden_size = 1024 - elif args.num_layers == 32: - n_head = 32 - hidden_size = 2048 - else: - raise ValueError( - "The pre-trained model only support 24 or 32 layers, " "but received num_layers=%d." % args.num_layers - ) - - model = Plato2InferModel(nsp_reader, args.num_layers, n_head, hidden_size) - load_params(model, args.init_from_ckpt) - model.eval() - - Example = namedtuple("Example", ["src", "data_id"]) - context = [] - start_info = "Enter [EXIT] to quit the interaction, [NEXT] to start a new conversation." - cprint(start_info, "yellow", attrs=["bold"]) - while True: - user_utt = input(colored("[Human]: ", "red", attrs=["bold"])).strip() - if user_utt == "[EXIT]": - break - elif user_utt == "[NEXT]": - context = [] - cprint(start_info, "yellow", attrs=["bold"]) - else: - context.append(user_utt) - example = Example(src=" [SEP] ".join(context), data_id=0) - record = plato_reader._convert_example_to_record(example, is_infer=True) - data = plato_reader._pad_batch_records([record], is_infer=True) - inputs = gen_inputs(data, args.latent_type_size) - inputs["tgt_ids"] = inputs["tgt_ids"].astype("int64") - pred = model(inputs)[0] - bot_response = pred["response"] - print(colored("[Bot]:", "blue", attrs=["bold"]), colored(bot_response, attrs=["bold"])) - context.append(bot_response) - return - - -if __name__ == "__main__": - args = setup_args() - interact(args) diff --git a/slm/examples/dialogue/plato-2/model.py b/slm/examples/dialogue/plato-2/model.py deleted file mode 100644 index 437a708876..0000000000 --- a/slm/examples/dialogue/plato-2/model.py +++ /dev/null @@ -1,458 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import namedtuple - -import paddle -import paddle.nn as nn -import paddle.nn.functional as F - - -def post_process_context(token_ids, reader, merge=True): - """Post-process the context sequence.""" - context = [] - utt = [] - for tok_id in token_ids[1:]: - if tok_id == reader.eos_id: - utt = reader.tokenizer.convert_ids_to_tokens(utt) - if merge: - utt = reader.tokenizer.merge_subword(utt) - context.append(utt) - utt = [] - else: - utt.append(tok_id) - return context - - -def post_process_response(token_ids, reader, merge=True): - """ - Post-process the decoded sequence. Truncate from the first - and remove the and tokens currently. - """ - eos_pos = len(token_ids) - for i, tok_id in enumerate(token_ids): - if tok_id == reader.eos_id: - eos_pos = i - break - token_ids = token_ids[1:eos_pos] - response = reader.tokenizer.convert_ids_to_tokens(token_ids) - if merge: - response = reader.tokenizer.merge_subword(response) - return token_ids, response - - -def get_cross_turn_repetition(context, pred_tokens, eos_idx, is_cn=False): - """Get cross-turn repetition.""" - if len(pred_tokens) == 0: - return 1.0 - if is_cn: - context = ["".join(utt) for utt in context] - pred_tokens = "".join(pred_tokens) - - pred_tri_grams = set() - for i in range(len(pred_tokens) - 2): - tri_gram = tuple(pred_tokens[i : i + 3]) - pred_tri_grams.add(tri_gram) - for utt in context: - for i in range(len(utt) - 2): - tri_gram = tuple(utt[i : i + 3]) - if tri_gram in pred_tri_grams: - return 1.0 - return 0.0 - - -def get_in_turn_repetition(pred, is_cn=False): - """Get in-turn repetition.""" - if len(pred) == 0: - return 1.0 - if isinstance(pred[0], str): - pred = [tok.lower() for tok in pred] - if is_cn: - pred = "".join(pred) - tri_grams = set() - for i in range(len(pred) - 2): - tri_gram = tuple(pred[i : i + 3]) - if tri_gram in tri_grams: - return 1.0 - tri_grams.add(tri_gram) - return 0.0 - - -class Plato2EncoderLayer(nn.Layer): - def __init__(self, n_head, hidden_size, attn_dropout, act_dropout): - super(Plato2EncoderLayer, self).__init__() - - self.self_attn = nn.MultiHeadAttention(hidden_size, n_head, attn_dropout) - self.pre_norm_layer = nn.LayerNorm(hidden_size) - self.post_norm_layer = nn.LayerNorm(hidden_size) - self.fc1 = nn.Linear(hidden_size, hidden_size * 4) - self.fc2 = nn.Linear(hidden_size * 4, hidden_size) - - self.dropout_layer = nn.Dropout(act_dropout) - self.gelu_layer = nn.GELU() - - def forward(self, x, attn_mask, cache): - query = self.pre_norm_layer(x) - attn_output, new_cache = self.self_attn(query, None, None, attn_mask, cache) - attn_output = self.dropout_layer(attn_output) - attn_output = attn_output + x - ffd_input = self.post_norm_layer(attn_output) - - ffd_output = self.fc1(ffd_input) - ffd_output = self.gelu_layer(ffd_output) - ffd_output = self.dropout_layer(ffd_output) - - ffd_output = self.fc2(ffd_output) - ffd_output = self.dropout_layer(ffd_output) - out = ffd_output + attn_output - - return out, new_cache - - def gen_cache(self, key): - return self.self_attn.gen_cache(key) - - -class Plato2Encoder(nn.Layer): - def __init__( - self, vocab_size, type_size, max_position_seq_len, num_layers, n_head, hidden_size, attn_dropout, act_dropout - ): - super(Plato2Encoder, self).__init__() - - self.n_head = n_head - - self.word_embedding_layer = nn.Embedding(vocab_size, hidden_size) - self.sent_embedding_layer = nn.Embedding(type_size, hidden_size) - self.pos_embedding_layer = nn.Embedding(max_position_seq_len, hidden_size) - - self.encoder_layers = [] - for i in range(num_layers): - encoder_layer = Plato2EncoderLayer(n_head, hidden_size, attn_dropout, act_dropout) - self.encoder_layers.append(encoder_layer) - self.add_sublayer("layers." + str(i), encoder_layer) - self.post_encoder_layer_norm = nn.LayerNorm(hidden_size) - - self.dropout_layer = nn.Dropout(act_dropout) - - def forward(self, caches, token_ids, type_ids, pos_ids, generation_mask, aux_emb=None): - out, self_attn_mask = self.gen_input(token_ids, type_ids, pos_ids, generation_mask, aux_emb) - - new_caches = [] - for i, encoder_layer in enumerate(self.encoder_layers): - out, new_cache = encoder_layer(out, self_attn_mask, caches[i]) - new_caches.append(new_cache) - - enc_output = self.post_encoder_layer_norm(out) - return enc_output, new_caches - - def gen_input(self, token_ids, type_ids, pos_ids, input_mask, aux_emb=None): - token_emb_out = self.word_embedding_layer(token_ids) - type_emb_out = self.sent_embedding_layer(type_ids) - pos_emb_out = self.pos_embedding_layer(pos_ids) - emb_out = token_emb_out + type_emb_out + pos_emb_out - - # auxiliary memory embeddings - if aux_emb is not None: - emb_out = paddle.concat([aux_emb, emb_out], axis=1) - - emb_out = self.dropout_layer(emb_out) - - # generate n-head self-attention mask - self_attn_mask = input_mask - self_attn_mask = paddle.scale(x=self_attn_mask, scale=1e4, bias=-1.0, bias_after_scale=False) - n_head_self_attn_mask = paddle.stack(x=[self_attn_mask] * self.n_head, axis=1) - n_head_self_attn_mask.stop_gradient = True - - return emb_out, n_head_self_attn_mask - - def gen_caches(self, key): - caches = [encoder_layer.gen_cache(key) for encoder_layer in self.encoder_layers] - return caches - - -class NSP(nn.Layer): - def __init__( - self, vocab_size, type_size, max_position_seq_len, num_layers, n_head, hidden_size, attn_dropout, act_dropout - ): - super(NSP, self).__init__() - - self.n_head = n_head - self.hidden_size = hidden_size - - self.word_embedding_layer = nn.Embedding(vocab_size, hidden_size) - self.sent_embedding_layer = nn.Embedding(type_size, hidden_size) - self.pos_embedding_layer = nn.Embedding(max_position_seq_len, hidden_size) - - encoder_layer = nn.TransformerEncoderLayer( - hidden_size, n_head, hidden_size * 4, act_dropout, "gelu", attn_dropout, act_dropout, "True" - ) - encoder_norm = nn.LayerNorm(hidden_size) - self.encoder = nn.TransformerEncoder(encoder_layer, num_layers, encoder_norm) - self.fc1 = nn.Linear(hidden_size, hidden_size) - self.fc2 = nn.Linear(hidden_size, 2) - - self.dropout_layer = nn.Dropout(act_dropout) - self.tanh_layer = nn.Tanh() - self.softmax = nn.Softmax() - - def forward(self, inputs): - token_ids = inputs["token_ids"] - type_ids = inputs["type_ids"] - pos_ids = inputs["pos_ids"] - attention_mask = inputs["attention_mask"] - label_pos = inputs["label_pos"] - - out, self_attn_mask = self.gen_input(token_ids, type_ids, pos_ids, attention_mask) - # [-1, seq_len, hidden_size] - enc_out = self.encoder(out, self_attn_mask) - - enc_out = paddle.reshape(enc_out, [-1, self.hidden_size]) - label_pos = paddle.cast(label_pos, "int64") - out = paddle.gather(enc_out, label_pos) - pooled_out = self.fc1(out) - pooled_out = self.tanh_layer(pooled_out) - - # [-1, 2] - logits = self.fc2(pooled_out) - probs = self.softmax(logits) - - return probs - - def gen_input(self, token_ids, type_ids, pos_ids, input_mask, aux_emb=None): - token_emb_out = self.word_embedding_layer(token_ids) - type_emb_out = self.sent_embedding_layer(type_ids) - pos_emb_out = self.pos_embedding_layer(pos_ids) - emb_out = token_emb_out + type_emb_out + pos_emb_out - - # auxiliary memory embeddings - if aux_emb is not None: - emb_out = paddle.concat([aux_emb, emb_out], axis=1) - - emb_out = self.dropout_layer(emb_out) - - # generate n-head self-attention mask - self_attn_mask = input_mask - self_attn_mask = paddle.scale(x=self_attn_mask, scale=1e4, bias=-1.0, bias_after_scale=False) - n_head_self_attn_mask = paddle.stack(x=[self_attn_mask] * self.n_head, axis=1) - n_head_self_attn_mask.stop_gradient = True - - return emb_out, n_head_self_attn_mask - - -class Plato2InferModel(nn.Layer): - def __init__( - self, - nsp_reader, - num_layers, - n_head, - hidden_size, - vocab_size=8001, - type_size=2, - latent_type_size=20, - max_position_seq_len=256, - act_dropout=0.1, - attn_dropout=0.1, - max_dec_len=64, - min_dec_len=1, - topk=10, - ): - super(Plato2InferModel, self).__init__() - - self.nsp_reader = nsp_reader - self.num_layers = num_layers - self.latent_type_size = latent_type_size - self.max_dec_len = max_dec_len - self.min_dec_len = min_dec_len - self.topk = topk - self.unk_id = 0 - self.bos_id = 1 - self.eos_id = 2 - self.mask_id = 8000 - self.after_eos = paddle.ones([vocab_size]) * -1e9 - self.after_eos[self.eos_id] = 0 - self.is_cn = False - self.batch_size = 1 - - self.latent_weight = paddle.create_parameter([hidden_size, latent_type_size], "float32") - - self.plato2_encoder = Plato2Encoder( - vocab_size, type_size, max_position_seq_len, num_layers, n_head, hidden_size, attn_dropout, act_dropout - ) - - self.logits_fc_layer = nn.Linear(hidden_size, hidden_size) - self.logits_layer_norm = nn.LayerNorm(hidden_size) - self.logits_bias = paddle.create_parameter([vocab_size], "float32", is_bias=True) - - self.nsp_predictor = NSP( - vocab_size, type_size, max_position_seq_len, num_layers, n_head, hidden_size, attn_dropout, act_dropout - ) - - self.gelu_layer = nn.GELU() - self.softmax = nn.Softmax() - - @paddle.no_grad() - def forward(self, inputs): - token_ids = inputs["token_ids"] - type_ids = inputs["type_ids"] - pos_ids = inputs["pos_ids"] - generation_mask = inputs["generation_mask"] - latent_id = inputs["latent_id"] - data_id = inputs["data_id"] - - # [-1, 1, latent_type_size] - latent_id = F.one_hot(latent_id, self.latent_type_size) - # [-1, 1, hidden_size] - latent_emb = paddle.matmul(latent_id, self.latent_weight, transpose_y=True) - - caches = self.plato2_encoder.gen_caches(token_ids) - - # [-1, seq_len + 1, hidden_size] - enc_out, new_caches = self.plato2_encoder(caches, token_ids, type_ids, pos_ids, generation_mask, latent_emb) - - pred_ids = self.decode(inputs, new_caches) - - nsp_inputs = self.gen_nsp_input(token_ids, pred_ids) - # [-1, 2] - probs = self.nsp_predictor(nsp_inputs) - - return self.get_results(data_id, token_ids, pred_ids, probs) - - def decode(self, inputs, caches): - tgt_ids = inputs["tgt_ids"] - tgt_pos = inputs["tgt_pos"] - tgt_generation_mask = inputs["tgt_generation_mask"] - predictions = tgt_ids - - # TODO - step = 0 - while step < self.max_dec_len: - # [-1, 1] - append_mask = paddle.cast(tgt_ids != self.eos_id, dtype=tgt_generation_mask.dtype) - tgt_generation_mask = paddle.concat([tgt_generation_mask, paddle.unsqueeze(append_mask, 1)], axis=-1) - tgt_sent = paddle.ones([tgt_generation_mask.shape[0], 1], dtype=tgt_ids.dtype) - - # [-1, 1, hidden_size] - out, caches = self.plato2_encoder(caches, tgt_ids, tgt_sent, tgt_pos, tgt_generation_mask) - out = paddle.squeeze(out, axis=1) - - # [-1, hidden_size] - trans = self.logits_fc_layer(out) - trans = self.gelu_layer(trans) - trans = self.logits_layer_norm(trans) - - # [-1, vocab_size] - logits = ( - paddle.matmul(trans, self.plato2_encoder.word_embedding_layer.weight, transpose_y=True) - + self.logits_bias - ) - logits[:, self.unk_id] = -1e9 - logits[:, self.bos_id] = -1e9 - logits[:, self.mask_id] = -1e9 - if step < self.min_dec_len: - logits[:, self.eos_id] = -1e9 - logits = logits * append_mask + (1 - append_mask) * self.after_eos - probs = self.softmax(logits) - - # [-1, topk] - topk_probs, _ = paddle.topk(probs, k=self.topk) - mask = paddle.cast(probs >= topk_probs[:, -1:], "float32") - sums = paddle.sum(topk_probs, axis=-1, keepdim=True) - new_probs = probs * mask / sums - # [-1, 1] - sampling_ids = paddle.multinomial(new_probs) - - step = step + 1 - tgt_ids = sampling_ids - tgt_pos = tgt_pos + 1 - predictions = paddle.concat([predictions, tgt_ids], axis=1) - return predictions - - def gen_nsp_input(self, token_ids, pred_ids): - token_ids = token_ids.numpy() - pred_ids = pred_ids.numpy() - - def __reader__(): - headers = ["src", "tgt", "data_id"] - - Example = namedtuple("Example", headers) - - for i, (raw, pred) in enumerate(zip(token_ids, pred_ids)): - context = post_process_context(raw, self.nsp_reader, merge=False) - _, response = post_process_response(pred, self.nsp_reader, merge=False) - context_tokenized_input = " [SEP] ".join(" ".join(utt) for utt in context) - response_tokenized_input = " ".join(response) - example = Example(src=context_tokenized_input, tgt=response_tokenized_input, data_id=i) - data = self.nsp_reader._convert_example_to_record(example, is_infer=True) - yield data - return - - generator = self.nsp_reader.data_generator( - reader=__reader__, - is_infer=True, - phase="test", - ) - inputs = next(generator()) - - # print('\nnsp_inputs:') - for key in inputs: - inputs[key] = paddle.to_tensor(inputs[key]) - if key in ["token_ids", "type_ids", "pos_ids"]: - inputs[key] = paddle.squeeze(inputs[key], axis=-1) - # print(key, inputs[key].shape) - # print(inputs[key]) - return inputs - - def get_results(self, data_id, token_ids, pred_ids, probs): - data_id = data_id.numpy() - token_ids = token_ids.numpy() - pred_ids = pred_ids.numpy() - probs = probs.numpy() - - infos = [] - for raw, pred, prob in zip(token_ids, pred_ids, probs): - tokens = post_process_context(raw, self.nsp_reader) - pred_token_ids, pred_tokens = post_process_response(pred, self.nsp_reader) - info = {} - info["response"] = " ".join(pred_tokens) - cross_turn_repetition = get_cross_turn_repetition(tokens, pred_tokens, self.nsp_reader.eos_id, self.is_cn) - in_turn_repetition = max( - get_in_turn_repetition(pred_tokens, self.is_cn), get_in_turn_repetition(pred_token_ids) - ) - - info["score"] = float(prob[1]) - if len(pred_token_ids) >= self.max_dec_len: - info["score"] -= 1e3 - elif cross_turn_repetition > 0: - info["score"] -= 1e3 - elif in_turn_repetition > 0: - info["score"] -= 1e3 - infos.append(info) - - results = [] - pre_idx = 0 - sample = [] - for idx, info in zip(data_id, infos): - if idx != pre_idx: - sample = sorted(sample, key=lambda info: -info["score"]) - result = sample[0] - result["data_id"] = pre_idx - results.apeend(result) - sample = [] - pre_idx = idx - sample.append(info) - if sample: - sample = sorted(sample, key=lambda info: -info["score"]) - result = sample[0] - result["data_id"] = pre_idx - results.append(result) - return results diff --git a/slm/examples/dialogue/plato-2/readers/dialog_reader.py b/slm/examples/dialogue/plato-2/readers/dialog_reader.py deleted file mode 100644 index 00339c1f0a..0000000000 --- a/slm/examples/dialogue/plato-2/readers/dialog_reader.py +++ /dev/null @@ -1,436 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Dialogue Reader.""" - -import csv -import gzip -from collections import namedtuple -from contextlib import contextmanager - -import numpy as np -import utils.tokenization as tokenization -from utils import pad_batch_data -from utils.masking import mask - -from paddlenlp.trainer.argparser import strtobool - - -class DialogReader(object): - """The implement of DialogReader.""" - - @classmethod - def add_cmdline_args(cls, parser): - """Add cmdline argurments.""" - group = parser.add_argument_group("Reader") - group.add_argument("--max_src_len", type=int, default=128) - group.add_argument("--max_tgt_len", type=int, default=128) - group.add_argument("--truncate_first_turn", type=strtobool, default=False) - group.add_argument("--file_format", type=str, default="file", choices=["file", "filelist"]) - group.add_argument("--data_format", type=str, default="raw", choices=["raw", "tokenized", "numerical"]) - group.add_argument("--in_tokens", type=strtobool, default=False) - group.add_argument("--batch_size", type=int, default=16) - group.add_argument("--continuous_position", type=strtobool, default=True) - group.add_argument("--random_seed", type=int, default=11) - group.add_argument("--sort_pool_size", type=int, default=2**16) - - group = parser.add_argument_group("Tokenizer") - group.add_argument("--tokenizer", type=str, default="SentencePieceTokenizer") - args, _ = parser.parse_known_args() - tokenizer_cls = getattr(tokenization, args.tokenizer) - tokenizer_cls.add_cmdline_args(parser) - return group - - def __init__(self, args): - tokenizer_cls = getattr(tokenization, args.tokenizer) - self.tokenizer = tokenizer_cls(args) - self.vocab = self.tokenizer.vocab - self.pad_id = args.pad_id = self.vocab["[PAD]"] - self.bos_id = args.bos_id = self.vocab["[CLS]"] - self.eos_id = args.eos_id = self.vocab["[SEP]"] - self.unk_id = args.unk_id = self.vocab["[UNK]"] - self.mask_id = args.mask_id = self.vocab["[MASK]"] - self.vocab_size = args.get("vocab_size", 0) - self.max_src_len = args.max_src_len - self.max_tgt_len = args.max_tgt_len - self.truncate_first_turn = args.truncate_first_turn - self.file_format = args.file_format - self.data_format = args.data_format - self.in_tokens = args.in_tokens - self.batch_size = args.batch_size - self.continuous_position = args.continuous_position - self.sort_pool_size = args.sort_pool_size - - # random_seed must be set for data slicing when using multi-gpu - self.global_rng = np.random.RandomState(args.random_seed) - - # training progress - self.current_example = 0 - self.current_epoch = 0 - self.num_examples = 0 - - # model related - - self.fields = ["token_ids", "type_ids", "pos_ids"] - self.num_numerical_fields = len(self.fields) - self.fields += ["tgt_start_idx", "data_id"] - self.sort_key = lambda record: [len(record.token_ids)] - - self.Record = namedtuple("Record", self.fields, defaults=(None,) * len(self.fields)) - - self.features = {} - return - - def get_train_progress(self): - """Gets progress for training phase.""" - return self.current_epoch, self.current_file_index, self.total_file - - def _convert_example_to_record(self, example, is_infer): - # process src - src_token_ids = [] - src_pos_ids = [] - - # tokenize src - s_token_ids_list = [] - for s in example.src.split("[SEP]"): - s = tokenization.convert_to_unicode(s).strip() - - if self.data_format == "tokenized": - s_tokens = s.split(" ") - else: - s_tokens = self.tokenizer.tokenize(s) - - s_token_ids = self.tokenizer.convert_tokens_to_ids(s_tokens) + [self.eos_id] - s_token_ids_list.append(s_token_ids) - - # trim src - idx = len(s_token_ids_list) - 1 - total_token_num = 1 - while idx >= 0: - total_token_num += len(s_token_ids_list[idx]) - if total_token_num > self.max_src_len: - if self.truncate_first_turn and idx == 0: - truncated_ids = s_token_ids_list[idx][: self.max_src_len - total_token_num] - if len(truncated_ids) > 1: - s_token_ids_list[idx] = truncated_ids[:-1] + [self.eos_id] - idx -= 1 - break - idx -= 1 - - for i, s_token_ids in enumerate(s_token_ids_list[idx + 1 :], idx + 1): - src_token_ids += s_token_ids - src_pos_ids += list(range(1, len(s_token_ids) + 1)) - - src_token_ids = [self.bos_id] + src_token_ids - src_type_ids = [0] * len(src_token_ids) - src_pos_ids = [0] + src_pos_ids - assert ( - len(src_token_ids) == len(src_type_ids) == len(src_pos_ids) - ), "not len(src_token_ids) == len(src_type_ids) == len(src_pos_ids)" - - token_ids = src_token_ids - type_ids = src_type_ids - pos_ids = src_pos_ids - tgt_start_idx = len(token_ids) - - if not is_infer: - # process tgt - # tokenize tgt - tgt = tokenization.convert_to_unicode(example.tgt).strip() - if self.data_format == "tokenized": - tgt_tokens = tgt.split(" ") - else: - tgt_tokens = self.tokenizer.tokenize(tgt) - - tgt_token_ids = self.tokenizer.convert_tokens_to_ids(tgt_tokens) - tgt_token_ids.append(self.eos_id) - - # trim tgt - if len(tgt_token_ids) > self.max_tgt_len - 1: - tgt_token_ids = tgt_token_ids[: self.max_tgt_len - 1] - - tgt_token_ids = [self.bos_id] + tgt_token_ids - tgt_type_ids = [1] * len(tgt_token_ids) - tgt_pos_ids = list(range(1, len(tgt_token_ids) + 1)) - assert ( - len(tgt_token_ids) == len(tgt_type_ids) == len(tgt_pos_ids) - ), "not len(tgt_token_ids) == len(tgt_type_ids) == len(tgt_pos_ids)" - - token_ids += tgt_token_ids - type_ids += tgt_type_ids - pos_ids += tgt_pos_ids - - assert len(token_ids) == len(type_ids) == len(pos_ids), "not len(token_ids) == len(type_ids) == len(pos_ids)" - - if self.continuous_position: - src_pos_ids = list(range(len(src_token_ids))) - if not is_infer: - tgt_pos_ids = list(range(len(tgt_token_ids))) - pos_ids = list(range(len(token_ids))) - - field_values = {"token_ids": src_token_ids, "type_ids": src_type_ids, "pos_ids": src_pos_ids} - field_values["tgt_start_idx"] = tgt_start_idx - field_values["data_id"] = example.data_id - - record = self.Record(**field_values) - return record - - def _read_tsv(self, fp, phase, is_infer, delimiter="\t", quotechar=None): - """Reads a tab separated value file.""" - csv.field_size_limit(2**20) - reader = csv.reader(fp, delimiter=delimiter, quotechar=quotechar) - headers = next(reader) - headers.append("data_id") - Example = namedtuple("Example", headers) - - for i, line in enumerate(reader): - example = Example(*line, data_id=i) - if is_infer or phase.endswith("test"): - self.features[phase][i] = example - record = self._convert_example_to_record(example, is_infer) - yield record - - def _read_numerical_file(self, fp, delimiter=";"): - for i, line in enumerate(fp): - cols = tokenization.convert_to_unicode(line).strip().split(delimiter) - cols = list(map(lambda x: list(map(int, x.split(" "))), cols)) - if len(cols) > self.num_numerical_fields: - cols = cols[: self.num_numerical_fields] - tgt_start_idx = cols[0].index(self.bos_id, 1) - record = self.Record(*cols, tgt_start_idx=tgt_start_idx, data_id=i) - yield record - - def _read_file(self, input_file, phase, is_infer): - def __wrapper__(): - with open_file(input_file) as fp: - if self.data_format == "numerical": - records = self._read_numerical_file(fp) - else: - records = self._read_tsv(fp, phase, is_infer) - for record in records: - yield record - - return __wrapper__ - - def _read_files(self, filelist, phase, is_infer, shuffle_files): - input_files = open(filelist).readlines() - - def __wrapper__(): - if shuffle_files: - self.global_rng.shuffle(input_files) - - if phase == "train": - self.total_file = len(input_files) - for file_index, input_file in enumerate(input_files, 1): - if phase == "train": - self.current_file_index = file_index - self.current_file = input_file - file_reader = self._read_file(input_file.strip(), phase, is_infer) - for record in file_reader(): - yield record - - return __wrapper__ - - def _batch_reader(self, reader, phase=None, is_infer=False, sort_pool_size=2**16): - """Construct a batch reader.""" - - def update_max_lens(max_lens, record): - """Update max_lens.""" - if max_lens is None: - return self.sort_key(record) - else: - return [max(max_len, l) for max_len, l in zip(max_lens, self.sort_key(record))] - - def get_batch(reader): - """Generate batches from reader.""" - batch, max_lens = [], None - for record in reader(): - if record is None: - yield batch - batch, max_lens = [], None - continue - - self.current_example += 1 - max_lens = update_max_lens(max_lens, record) - if self.in_tokens: - to_append = (len(batch) + 1) * sum(max_lens) <= self.batch_size - else: - to_append = len(batch) < self.batch_size - if to_append: - batch.append(record) - else: - yield batch - batch, max_lens = [record], self.sort_key(record) - - if len(batch) > 0: - yield batch - - def get_sorted_batch(pool): - """Generate sorted batches from pool.""" - pool = sorted(pool, key=self.sort_key) - batches = [] - batch, max_lens = [], None - for record in pool: - self.current_example += 1 - max_lens = update_max_lens(max_lens, record) - if self.in_tokens: - to_append = (len(batch) + 1) * sum(max_lens) <= self.batch_size - else: - to_append = len(batch) < self.batch_size - if to_append: - batch.append(record) - else: - batches.append(batch) - batch, max_lens = [record], self.sort_key(record) - - if len(batch) > 0: - batches.append(batch) - self.global_rng.shuffle(batches) - - for batch in batches: - yield batch - - def __wrapper__(): - if sort_pool_size > 0: - pool = [] - for record in reader(): - pool.append(record) - if len(pool) == sort_pool_size: - for batch in get_sorted_batch(pool): - yield batch - pool = [] - if len(pool) > 0: - for batch in get_sorted_batch(pool): - yield batch - else: - for batch in get_batch(reader): - yield batch - - return __wrapper__ - - def _distributed_batch_reader(self, batch_reader, num_part, part_id, is_test=False): - def __wrapper__(): - batches = [] - for batch in batch_reader(): - batches.append(batch) - if len(batches) == num_part: - yield batches[part_id] - batches = [] - if is_test and 0 <= part_id < len(batches): - yield batches[part_id] - return - - return __wrapper__ - - def data_generator( - self, input_file=None, reader=None, num_epochs=1, num_part=1, part_id=0, phase=None, is_infer=False - ): - """Data generator.""" - - def __wrapper__(): - if is_infer or phase.endswith("test"): - self.features[phase] = {} - - nonlocal reader - if reader is None: - if self.file_format == "filelist": - reader = self._read_files(input_file, phase, is_infer, not phase.endswith("test")) - else: - if phase == "train": - self.total_file = 1 - self.current_file_index = 1 - self.current_file = input_file - reader = self._read_file(input_file, phase, is_infer) - - batch_reader = self._batch_reader( - reader, phase, is_infer, sort_pool_size=self.sort_pool_size if not is_infer else 0 - ) - if phase == "train": - batch_reader = self._distributed_batch_reader(batch_reader, num_part, part_id) - elif phase.startswith("distributed"): - batch_reader = self._distributed_batch_reader(batch_reader, num_part, part_id, is_test=True) - - for epoch_index in range(num_epochs): - if phase == "train": - self.current_example = 0 - self.current_epoch = epoch_index + 1 - for batch in batch_reader(): - yield self._pad_batch_records(batch, is_infer) - - return __wrapper__ - - def _gen_self_attn_mask(self, batch_token_ids, batch_tgt_start_idx=None, is_unidirectional=True, shift_len=0): - max_len = max(map(len, batch_token_ids)) - input_mask_data = np.zeros((len(batch_token_ids), max_len + shift_len, max_len + shift_len)) - if is_unidirectional: - for index, mask_data in enumerate(input_mask_data): - start = 0 if batch_tgt_start_idx is None else batch_tgt_start_idx[index] - end = len(batch_token_ids[index]) - mask_data[: end + shift_len, : start + shift_len] = 1.0 - # Generate the lower triangular matrix using the slice of matrix - b = np.tril(np.ones([end - start, end - start]), 0) - mask_data[start + shift_len : end + shift_len, start + shift_len : end + shift_len] = b - else: - for index, token_ids in enumerate(batch_token_ids): - input_mask_data[index, : len(token_ids) + shift_len, : len(token_ids) + shift_len] = 1.0 - return input_mask_data.astype("float32") - - def _pad_batch_records(self, batch_records, is_infer): - """ - Padding batch records and construct model's inputs. - """ - batch = {} - batch_token_ids = [record.token_ids for record in batch_records] - batch_type_ids = [record.type_ids for record in batch_records] - batch_pos_ids = [record.pos_ids for record in batch_records] - batch["token_ids"] = pad_batch_data(batch_token_ids, pad_id=self.pad_id) - batch["type_ids"] = pad_batch_data(batch_type_ids, pad_id=self.pad_id) - batch["pos_ids"] = pad_batch_data(batch_pos_ids, pad_id=self.pad_id) - - batch_tgt_start_idx = [record.tgt_start_idx for record in batch_records] - batch["generation_mask"] = self._gen_self_attn_mask(batch_token_ids, batch_tgt_start_idx=batch_tgt_start_idx) - - if is_infer: - tgt_ids = np.array([[[self.bos_id]]] * len(batch_token_ids), dtype="int64") - if self.continuous_position: - tgt_pos = np.array(batch_tgt_start_idx, dtype="int64") - else: - tgt_pos = np.zeros_like(batch_tgt_start_idx, dtype="int64") - tgt_pos = tgt_pos.reshape(-1, 1, 1) - batch["init_score"] = np.zeros_like(tgt_ids, dtype="float32").reshape(-1, 1).tolist() - batch["tgt_ids"] = tgt_ids.tolist() - batch["tgt_pos"] = tgt_pos.tolist() - - batch["tgt_generation_mask"] = batch["generation_mask"][:, 0:1, :].astype("float32") - else: - batch["tgt_label"], batch["tgt_pos"] = mask( - batch_tokens=batch_token_ids, - vocab_size=self.vocab_size, - sent_b_starts=batch_tgt_start_idx, - is_unidirectional=True, - ) - - batch_data_id = [record.data_id for record in batch_records] - batch["data_id"] = np.array(batch_data_id).astype("int64").reshape([-1, 1]) - return batch - - -@contextmanager -def open_file(filename): - """Open file.""" - if filename.endswith(".gz"): - fp = gzip.open(filename, "rt") - else: - fp = open(filename) - yield fp - fp.close() diff --git a/slm/examples/dialogue/plato-2/readers/nsp_reader.py b/slm/examples/dialogue/plato-2/readers/nsp_reader.py deleted file mode 100644 index 968da9ff1b..0000000000 --- a/slm/examples/dialogue/plato-2/readers/nsp_reader.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""NSP Reader.""" - -from collections import namedtuple - -import numpy as np -from readers.dialog_reader import DialogReader -from utils import pad_batch_data -from utils.masking import mask - -from paddlenlp.trainer.argparser import strtobool - - -class NSPReader(DialogReader): - """NSP Reader.""" - - @classmethod - def add_cmdline_args(cls, parser): - """Add cmdline argurments.""" - group = DialogReader.add_cmdline_args(parser) - group.add_argument( - "--attention_style", type=str, default="bidirectional", choices=["bidirectional", "unidirectional"] - ) - group.add_argument("--mix_negative_sample", type=strtobool, default=False) - return group - - def __init__(self, args): - super(NSPReader, self).__init__(args) - self.fields.append("label") - self.Record = namedtuple("Record", self.fields, defaults=(None,) * len(self.fields)) - - self.attention_style = args.attention_style - self.mix_negative_sample = args.mix_negative_sample - return - - def _convert_example_to_record(self, example, is_infer): - record = super(NSPReader, self)._convert_example_to_record(example, False) - if "label" in example._fields: - record = record._replace(label=int(example.label)) - return record - - def _mix_negative_sample(self, reader, neg_pool_size=2**16): - def gen_from_pool(pool): - num_samples = len(pool) - if num_samples == 1: - # only one sample: it is impossible to generate negative sample - yield pool[0]._replace(label=1) - return - self.global_rng.shuffle(pool) - for i in range(num_samples): - pool[i] = pool[i]._replace(label=1) - j = (i + 1) % num_samples - idx_i = pool[i].tgt_start_idx - idx_j = pool[j].tgt_start_idx - field_values = {} - field_values["token_ids"] = pool[i].token_ids[:idx_i] + pool[j].token_ids[idx_j:] - field_values["type_ids"] = pool[i].type_ids[:idx_i] + pool[j].type_ids[idx_j:] - field_values["pos_ids"] = list(range(len(field_values["token_ids"]))) - neg_record = self.Record(**field_values, tgt_start_idx=idx_i, data_id=-1, label=0) - pool.append(neg_record) - assert len(neg_record.token_ids) <= self.max_seq_len - self.global_rng.shuffle(pool) - for record in pool: - yield record - - def __wrapper__(): - pool = [] - for record in reader(): - pool.append(record) - if len(pool) == neg_pool_size: - for record in gen_from_pool(pool): - yield record - pool = [] - if len(pool) > 0: - for record in gen_from_pool(pool): - yield record - - return __wrapper__ - - def _batch_reader(self, reader, phase=None, is_infer=False, sort_pool_size=2**16): - if self.mix_negative_sample: - reader = self._mix_negative_sample(reader) - return super(NSPReader, self)._batch_reader( - reader, phase=phase, is_infer=is_infer, sort_pool_size=sort_pool_size - ) - - def _pad_batch_records(self, batch_records, is_infer): - """ - Padding batch records and construct model's inputs. - """ - batch = {} - batch_token_ids = [record.token_ids for record in batch_records] - batch_type_ids = [record.type_ids for record in batch_records] - batch_pos_ids = [record.pos_ids for record in batch_records] - batch_tgt_start_idx = [record.tgt_start_idx for record in batch_records] - batch_label = [record.label for record in batch_records] - - if self.attention_style == "unidirectional": - batch["token_ids"] = pad_batch_data(batch_token_ids, pad_id=self.pad_id) - batch["type_ids"] = pad_batch_data(batch_type_ids, pad_id=self.pad_id) - batch["pos_ids"] = pad_batch_data(batch_pos_ids, pad_id=self.pad_id) - tgt_label, tgt_pos, label_pos = mask( - batch_tokens=batch_token_ids, - vocab_size=self.vocab_size, - bos_id=self.bos_id, - sent_b_starts=batch_tgt_start_idx, - labels=batch_label, - is_unidirectional=True, - ) - attention_mask = self._gen_self_attn_mask(batch_token_ids, batch_tgt_start_idx) - else: - batch_mask_token_ids, tgt_label, tgt_pos, label_pos = mask( - batch_tokens=batch_token_ids, - vocab_size=self.vocab_size, - bos_id=self.bos_id, - eos_id=self.eos_id, - mask_id=self.mask_id, - sent_b_starts=batch_tgt_start_idx, - labels=batch_label, - is_unidirectional=False, - ) - if not is_infer: - batch_token_ids = batch_mask_token_ids - batch["token_ids"] = pad_batch_data(batch_token_ids, pad_id=self.pad_id) - batch["type_ids"] = pad_batch_data(batch_type_ids, pad_id=self.pad_id) - batch["pos_ids"] = pad_batch_data(batch_pos_ids, pad_id=self.pad_id) - attention_mask = self._gen_self_attn_mask(batch_token_ids, is_unidirectional=False) - - batch["attention_mask"] = attention_mask - batch["label_pos"] = label_pos - - if not is_infer: - batch_label = np.array(batch_label).astype("int64").reshape([-1, 1]) - batch["label"] = batch_label - batch["tgt_label"] = tgt_label - batch["tgt_pos"] = tgt_pos - - batch_data_id = [record.data_id for record in batch_records] - batch["data_id"] = np.array(batch_data_id).astype("int64").reshape([-1, 1]) - return batch diff --git a/slm/examples/dialogue/plato-2/readers/plato_reader.py b/slm/examples/dialogue/plato-2/readers/plato_reader.py deleted file mode 100644 index 4459e808fc..0000000000 --- a/slm/examples/dialogue/plato-2/readers/plato_reader.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Plato Reader.""" - -import numpy as np -from readers.dialog_reader import DialogReader -from utils import pad_batch_data -from utils.masking import mask - - -class PlatoReader(DialogReader): - """The implement of PlatoReader""" - - def __init__(self, args): - super(PlatoReader, self).__init__(args) - self.latent_type_size = args.latent_type_size - self.use_bow = args.use_bow - - def _pad_batch_records(self, batch_records, is_infer): - """ - Padding batch records and construct model's inputs. - """ - batch = {} - batch_token_ids = [record.token_ids for record in batch_records] - batch_type_ids = [record.type_ids for record in batch_records] - batch_pos_ids = [record.pos_ids for record in batch_records] - - batch_tgt_start_idx = [record.tgt_start_idx for record in batch_records] - - batch_size = len(batch_token_ids) - - # padding - batch["token_ids"] = pad_batch_data(batch_token_ids, pad_id=self.pad_id) - batch["type_ids"] = pad_batch_data(batch_type_ids, pad_id=self.pad_id) - batch["pos_ids"] = pad_batch_data(batch_pos_ids, pad_id=self.pad_id) - - batch["generation_mask"] = self._gen_self_attn_mask( - batch_token_ids, batch_tgt_start_idx=batch_tgt_start_idx, is_unidirectional=True, shift_len=1 - ) - if not is_infer: - batch["recognition_mask"] = self._gen_self_attn_mask(batch_token_ids, is_unidirectional=False, shift_len=1) - - if is_infer: - tgt_ids = np.array([[[self.bos_id]]] * batch_size, dtype="int64") - if self.continuous_position: - tgt_pos = np.array(batch_tgt_start_idx, dtype="int64") - else: - tgt_pos = np.zeros_like(batch_tgt_start_idx, dtype="int64") - tgt_pos = tgt_pos.reshape(-1, 1, 1) - batch["init_score"] = np.zeros_like(tgt_ids, dtype="float32").reshape(-1, 1).tolist() - batch["tgt_ids"] = tgt_ids.tolist() - batch["tgt_pos"] = tgt_pos.tolist() - batch["parent_idx"] = np.array(range(batch_size), dtype="int32") - - batch["tgt_generation_mask"] = batch["generation_mask"][:, 0:1, :].astype("float32") - else: - mask_return_list = mask( - batch_tokens=batch_token_ids, - vocab_size=self.vocab_size, - sent_b_starts=batch_tgt_start_idx, - is_unidirectional=True, - use_latent=True, - use_bow=self.use_bow, - ) - batch["tgt_label"] = mask_return_list[0] - batch["tgt_pos"] = mask_return_list[1] - if self.use_bow: - batch["bow_label"] = mask_return_list[2] - batch["bow_pos"] = mask_return_list[3] - - batch_data_id = [record.data_id for record in batch_records] - batch["data_id"] = np.array(batch_data_id).astype("int64").reshape([-1, 1]) - return batch diff --git a/slm/examples/dialogue/plato-2/utils/__init__.py b/slm/examples/dialogue/plato-2/utils/__init__.py deleted file mode 100644 index 1a9ff1098d..0000000000 --- a/slm/examples/dialogue/plato-2/utils/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utils.""" - -from itertools import chain - -import numpy as np -import paddle - - -def repeat_array(array, times): - """Repeate numpy array.""" - if isinstance(array, list): - return list(chain(*([array] * times))) - else: - return np.concatenate([array] * times, axis=0) - - -def gen_inputs(inputs, latent_type_size): - batch_size = len(inputs["data_id"]) - inputs = {name: repeat_array(array, latent_type_size) for name, array in inputs.items()} - # Add latent_id - inputs["latent_id"] = np.array( - [i for i in range(latent_type_size) for _ in range(batch_size)], dtype="int64" - ).reshape([-1, 1]) - - # print('\nplato_inputs:') - for key in inputs: - inputs[key] = paddle.to_tensor(inputs[key]) - if key in ["token_ids", "type_ids", "pos_ids", "tgt_ids", "tgt_pos", "data_id"]: - inputs[key] = paddle.squeeze(inputs[key], axis=-1) - # print(key, inputs[key].shape, inputs[key].dtype) - return inputs - - -def pad_batch_data(insts, pad_id=0): - """Pad the instances to the max sequence length in batch.""" - max_len = max(map(len, insts)) - inst_data = np.array([list(inst) + [pad_id] * (max_len - len(inst)) for inst in insts]) - return inst_data.astype("int64").reshape([-1, max_len, 1]) diff --git a/slm/examples/dialogue/plato-2/utils/args.py b/slm/examples/dialogue/plato-2/utils/args.py deleted file mode 100644 index b112acf6ba..0000000000 --- a/slm/examples/dialogue/plato-2/utils/args.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Parse argument.""" - -import argparse -import json - - -class Args(dict): - """Arguments class - - Store arguments in training / infer / ... scripts. - """ - - def __getattr__(self, name): - if name in self.keys(): - return self[name] - for v in self.values(): - if isinstance(v, Args): - if name in v: - return v[name] - return None - - def get(self, key, default_value=None): - """Get the value of corresponding key.""" - if key in self.keys(): - return self[key] - for v in self.values(): - if isinstance(v, Args): - if key in v: - return v[key] - return default_value - - def __setattr__(self, name, value): - self[name] = value - - def save(self, filename): - with open(filename, "w") as fp: - json.dump(self, fp, ensure_ascii=False, indent=4, sort_keys=False) - - def load(self, filename, group_name=None): - if group_name is not None: - if group_name not in self: - self[group_name] = Args() - self[group_name].load(filename) - return - with open(filename, "r") as fp: - params_dict = json.load(fp) - for k, v in params_dict.items(): - if isinstance(v, dict): - self[k].update(Args(v)) - else: - self[k] = v - - -def parse_args(parser: argparse.ArgumentParser, allow_unknown=False) -> Args: - """Parse hyper-parameters from cmdline.""" - if allow_unknown: - parsed, _ = parser.parse_known_args() - else: - parsed = parser.parse_args() - args = Args() - optional_args = parser._action_groups[1] - for action in optional_args._group_actions[1:]: - arg_name = action.dest - args[arg_name] = getattr(parsed, arg_name) - for group in parser._action_groups[2:]: - group_args = Args() - for action in group._group_actions: - arg_name = action.dest - group_args[arg_name] = getattr(parsed, arg_name) - if len(group_args) > 0: - if group.title in args: - args[group.title].update(group_args) - else: - args[group.title] = group_args - return args diff --git a/slm/examples/dialogue/plato-2/utils/masking.py b/slm/examples/dialogue/plato-2/utils/masking.py deleted file mode 100644 index fb6be80844..0000000000 --- a/slm/examples/dialogue/plato-2/utils/masking.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Reader utils.""" - -import numpy as np - - -def mask( - batch_tokens, - vocab_size, - bos_id=1, - eos_id=2, - mask_id=3, - sent_b_starts=None, - labels=None, - is_unidirectional=False, - use_latent=False, - use_bow=False, -): - """ - Add mask for batch_tokens, return out, mask_label, mask_pos; - Note: mask_pos responding the batch_tokens after padded; - """ - batch_tokens = np.copy(batch_tokens) - max_len = max(map(len, batch_tokens)) - mask_label = [] - mask_pos = [] - if labels is not None: - label_pos = [] - - if is_unidirectional: - # unidirectional language model - if use_latent: - max_len += 1 - shift_len = 1 - else: - shift_len = 0 - for sent_index, sent in enumerate(batch_tokens): - sent_b_index = sent_b_starts[sent_index] if sent_b_starts is not None else 0 - if labels is not None: - label_pos.append(sent_index * max_len + len(sent) - 1 + shift_len) - mask_label.extend(sent[sent_b_index + 1 :]) - mask_pos.extend([sent_index * max_len + i + shift_len for i in range(sent_b_index, len(sent) - 1)]) - mask_label = np.array(mask_label).astype("int64").reshape([-1, 1]) - mask_pos = np.array(mask_pos).astype("int64").reshape([-1, 1]) - return_list = [mask_label, mask_pos] - - # latent related (bow label and pos) - if use_latent and use_bow: - bow_label = [] - bow_pos = [] - for sent_index, sent in enumerate(batch_tokens): - sent_b_index = sent_b_starts[sent_index] if sent_b_starts is not None else 0 - - def __filter__(tok_id): - # TODO: exclude [EOS] from bow loss - return True - - bow_pos.extend([sent_index for i in range(sent_b_index + 1, len(sent)) if __filter__(sent[i])]) - bow_label.extend([sent[i] for i in range(sent_b_index + 1, len(sent)) if __filter__(sent[i])]) - bow_label = np.array(bow_label).astype("int64").reshape([-1, 1]) - bow_pos = np.array(bow_pos).astype("int64").reshape([-1, 1]) - return_list += [bow_label, bow_pos] - else: - # bidirectional mask language model - total_token_num = sum(map(len, batch_tokens)) - prob_mask = np.random.rand(total_token_num) - # TODO: fix replace_ids, include [UNK] - replace_ids = np.random.randint(3, high=vocab_size, size=total_token_num) - prob_index = 0 - for sent_index, sent in enumerate(batch_tokens): - # add pair label position - if labels is not None: - label_pos.append(sent_index * max_len) - - # add mask label and position - for token_index, token in enumerate(sent): - if token == eos_id or token == bos_id: - continue - prob = prob_mask[prob_index + token_index] - if prob > 0.15: - continue - elif 0.03 < prob <= 0.15: - # mask - mask_label.append(sent[token_index]) - sent[token_index] = mask_id - mask_pos.append(sent_index * max_len + token_index) - elif 0.015 < prob <= 0.03: - # random replace - mask_label.append(sent[token_index]) - sent[token_index] = replace_ids[prob_index + token_index] - mask_pos.append(sent_index * max_len + token_index) - else: - # keep the original token - mask_label.append(sent[token_index]) - mask_pos.append(sent_index * max_len + token_index) - - prob_index += len(sent) - - mask_label = np.array(mask_label).astype("int64").reshape([-1, 1]) - mask_pos = np.array(mask_pos).astype("int64").reshape([-1, 1]) - return_list = [batch_tokens, mask_label, mask_pos] - - if labels is not None: - label_pos = np.array(label_pos).astype("int64").reshape([-1, 1]) - assert len(labels) == len(label_pos) - return_list.append(label_pos) - return return_list diff --git a/slm/examples/dialogue/plato-2/utils/tokenization.py b/slm/examples/dialogue/plato-2/utils/tokenization.py deleted file mode 100644 index 605dfe7e61..0000000000 --- a/slm/examples/dialogue/plato-2/utils/tokenization.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tokenization classes.""" - -import collections -import unicodedata - -import sentencepiece as spm -from utils.args import str2bool - - -def clean_text(text): - """Performs invalid character removal and whitespace cleanup on text.""" - text = text.replace("“", '"').replace("”", '"').replace("‘", "'").replace("’", "'").replace("—", "-") - - output = [] - for char in text: - if _is_control(char): - continue - if _is_whitespace(char): - output.append(" ") - else: - output.append(char) - return "".join(output) - - -def preprocess_text(inputs, remove_space=True, lower=False): - """preprocess data by removing extra space and normalize data.""" - outputs = inputs - if remove_space: - outputs = " ".join(inputs.strip().split()) - - outputs = unicodedata.normalize("NFKD", outputs) - outputs = "".join([c for c in outputs if not unicodedata.combining(c)]) - if lower: - outputs = outputs.lower() - - return outputs - - -def encode_pieces(spm_model, text, return_unicode=True, sample=False): - """turn sentences into word pieces.""" - # liujiaxiang: add for ernie-albert, mainly consider for “/”/‘/’/— causing too many unk - text = clean_text(text) - - if not sample: - pieces = spm_model.EncodeAsPieces(text) - else: - pieces = spm_model.SampleEncodeAsPieces(text, 64, 0.1) - - return pieces - - -def encode_ids(spm_model, text, sample=False): - """turn sentences into word pieces.""" - pieces = encode_pieces(spm_model, text, return_unicode=False, sample=sample) - ids = [spm_model.PieceToId(piece) for piece in pieces] - return ids - - -def convert_to_unicode(text): - """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" - if isinstance(text, str): - return text - elif isinstance(text, bytes): - return text.decode("utf-8", "ignore") - else: - raise ValueError("Unsupported string type: %s" % (type(text))) - - -def load_vocab(vocab_file): - """Loads a vocabulary file into a dictionary.""" - vocab = collections.OrderedDict() - fin = open(vocab_file, "r", encoding="UTF-8") - for num, line in enumerate(fin): - items = convert_to_unicode(line.rstrip()).split("\t") - if len(items) > 2: - break - token = items[0] - index = items[1] if len(items) == 2 else num - token = token.strip() - vocab[token] = int(index) - return vocab - - -def convert_by_vocab(vocab, items): - """Converts a sequence of [tokens|ids] using the vocab.""" - output = [] - for item in items: - output.append(vocab[item]) - return output - - -class SentencePieceTokenizer(object): - """Runs end-to-end tokenziation.""" - - @classmethod - def add_cmdline_args(cls, parser): - """Add cmdline argurments.""" - group = parser.add_argument_group("Tokenizer") - group.add_argument("--vocab_path", type=str, required=True) - group.add_argument("--do_lower_case", type=str2bool, default=False) - group.add_argument("--spm_model_file", type=str, required=True) - return group - - def __init__(self, args): - self.spm_model = spm.SentencePieceProcessor() - self.spm_model.Load(args.spm_model_file) - self.vocab = load_vocab(args.vocab_path) - self.do_lower_case = args.do_lower_case - self.inv_vocab = {v: k for k, v in self.vocab.items()} - - def tokenize(self, text): - """Tokenizes a piece of text.""" - text = preprocess_text(text, lower=self.do_lower_case) - return encode_pieces(self.spm_model, text, return_unicode=True) - - def convert_tokens_to_ids(self, tokens): - """Convert tokens to ids.""" - ret = [] - unk_id = self.vocab[""] - for token in tokens: - if token in self.vocab: - ret.append(self.vocab[token]) - else: - ret.append(unk_id) - return ret - - def convert_ids_to_tokens(self, ids): - """Convert ids to tokens.""" - return convert_by_vocab(self.inv_vocab, ids) - - def merge_subword(self, tokens): - """Merge subword.""" - ret = [] - for token in tokens: - if token.startswith("▁"): - ret.append(token[1:]) - else: - if len(ret): - ret[-1] += token - else: - ret.append(token) - - ret = [token for token in ret if token] - return ret - - def convert_ids_to_str(self, ids): - """Convert ids to string.""" - tokens = self.convert_ids_to_tokens(ids) - tokens = self.merge_subword(tokens) - res = " ".join(tokens).replace("", "") - res = res.replace("", "\n").replace("\n ", "\n").strip() - return res - - -def _is_whitespace(char): - """Checks whether `chars` is a whitespace character.""" - # \t, \n, and \r are technically contorl characters but we treat them - # as whitespace since they are generally considered as such. - if char == " " or char == "\t" or char == "\n" or char == "\r": - return True - cat = unicodedata.category(char) - if cat == "Zs": - return True - return False - - -def _is_control(char): - """Checks whether `chars` is a control character.""" - # These are technically control characters but we count them as whitespace - # characters. - if char == "\t" or char == "\n" or char == "\r": - return False - cat = unicodedata.category(char) - if cat.startswith("C"): - return True - return False diff --git a/slm/examples/dialogue/plato-xl b/slm/examples/dialogue/plato-xl deleted file mode 120000 index 3225c24e63..0000000000 --- a/slm/examples/dialogue/plato-xl +++ /dev/null @@ -1 +0,0 @@ -../../model_zoo/plato-xl \ No newline at end of file diff --git a/slm/examples/dialogue/unified_transformer/README.md b/slm/examples/dialogue/unified_transformer/README.md deleted file mode 100644 index e8b61507ec..0000000000 --- a/slm/examples/dialogue/unified_transformer/README.md +++ /dev/null @@ -1,217 +0,0 @@ -# UnifiedTransformer - -## 模型简介 - -近年来，人机对话系统受到了学术界和产业界的广泛关注并取得了不错的发展。开放域对话系统旨在建立一个开放域的多轮对话系统，使得机器可以流畅自然地与人进行语言交互，既可以进行日常问候类的闲聊，又可以完成特定功能，以使得开放域对话系统具有实际应用价值。具体的说，开放域对话可以继续拆分为支持不同功能的对话形式，例如对话式推荐，知识对话技术等，如何解决并有效融合以上多个技能面临诸多挑战。 - -[UnifiedTransformer](https://arxiv.org/abs/2006.16779)以[Transformer](https://arxiv.org/abs/1706.03762) 编码器为网络基本组件，采用灵活的注意力机制，十分适合对话生成任务。 - -本项目是 UnifiedTransformer 在 Paddle 2.0上的开源实现，介绍了如何使用 UnifiedTransformer 在 DuConv 任务型对话数据集上进行微调，并给出了一个搭建简单中文聊天机器人的例子。 - -## 快速开始 - -### 环境依赖 - -- sentencepiece -- termcolor - -安装方式：`pip install sentencepiece termcolor` - -### 代码结构说明 - -以下是本项目主要代码结构及说明： - -```text -. -├── finetune.py # 模型finetune主程序入口 -├── infer.py # 模型预测主程序入口 -├── utils.py # 定义参数及一些工具函数 -└── README.md # 文档说明 -``` - -### 数据准备 - -**DuConv**是百度发布的基于知识图谱的主动聊天任务数据集，让机器根据构建的知识图谱进行主动聊天，使机器具备模拟人类用语言进行信息传递的能力。数据集的创新性是：强调了 bot 的主动性，并且在闲聊对话中引入了明确的对话目标，即将对话引导到特定实体上。数据中的知识信息来源于电影和娱乐人物领域有聊天价值的知识信息，如票房、导演、评价等，以三元组 SPO 的形式组织，对话目标中的话题为电影或娱乐人物实体。数据集中共有3万 session，约12万轮对话，划分为训练集、开发集、测试集1和测试集2，其中测试集1中包含对话的 response，而测试集2中只有对话历史。 - -为了方便用户快速测试，PaddleNLP Dataset API 内置了 DuConv 数据集，一键即可完成数据集加载，示例代码如下： - -```python -from paddlenlp.datasets import load_dataset -train_ds, dev_ds, test1_ds, test2_ds = load_dataset('duconv', splits=('train', 'dev', 'test_1', 'test_2')) -``` - -### 预训练模型 - -以下是 PaddleNLP 支持的对话类预训练模型： - -|模型名称| 模型参数 | 模型特点 | -|:-----:|:------:|:-------:| -|unified_transformer-12L-cn| 12-layers, 12-heads, 768-hidden| 在千万级别的中文会话数据上进行预训练。| -|unified_transformer-12L-cn-luge| 12-layers, 12-heads, 768-hidden|由 unified_transformer-12L-cn 预训练模型在千言对话数据集上进行微调。并且模型输入中加入了标识不同对话技能的 special token，使得模型能同时支持闲聊对话、推荐对话和知识对话。| -|plato-mini| 6-layers, 12-heads, 768-hidden|在十亿级别的中文对话数据上进行预训练。参数量更小，但效果更好。只支持闲聊型对话。| - -### 模型训练 - -运行如下命令即可在训练集上进行 finetune，并在验证集上进行验证 - -```shell -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -# 例如使用1号和2号卡，则：`--gpu '1,2'` -unset CUDA_VISIBLE_DEVICES -python -m paddle.distributed.launch --gpus '0' --log_dir ./log finetune.py \ - --model_name_or_path=unified_transformer-12L-cn-luge \ - --save_dir=./checkpoints \ - --logging_steps=100 \ - --save_steps=1000 \ - --seed=2021 \ - --epochs=3 \ - --batch_size=16 \ - --lr=5e-5 \ - --weight_decay=0.01 \ - --warmup_steps=2500 \ - --max_grad_norm=0.1 \ - --max_seq_len=512 \ - --max_response_len=128 \ - --max_knowledge_len=256 \ - --device=gpu -``` - -其中参数释义如下： -- `gpus` 指示了训练所用的 GPU -- `log_dir` 指示了日志保存目录 -- `model_name_or_path` 指示了 finetune 使用的预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的模型。如果使用本地的模型，则配置为本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 模型参数 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - -- `save_dir` 表示模型的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `save_steps` 表示模型保存及评估间隔。 -- `seed` 表示随机数生成器的种子。 -- `epochs` 表示训练轮数。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `lr` 表示基础学习率大小，将于 learning rate scheduler 产生的值相乘作为当前学习率。 -- `weight_decay` 表示 AdamW 优化器中使用的 weight_decay 的系数。 -- `warmup_steps` 表示学习率逐渐升高到基础学习率（即上面配置的 lr）所需要的迭代数，最早的使用可以参考[这篇论文](https://arxiv.org/pdf/1706.02677.pdf)。 -- `max_grad_norm` 表示梯度裁剪允许的最大梯度值。 -- `max_seq_len` 表示输入序列的最大长度。 -- `max_response_len` 表示输入 response 的最大长度。 -- `max_knowledge_len` 表示输入 knowledge 序列的最大长度。 -- `device` 表示使用的设备。 - -程序运行时将会自动进行训练和验证，训练过程中会自动保存模型在指定的`save_dir`中，其中 loss 最小的模型会被保存在`save_dir/model_best`中。如： - -```text -./checkpoints/ -├── model_1000 -│ ├── model_config.json -│ ├── model_state.pdparams -│ ├── spm.model -│ ├── tokenizer_config.json -│ └── vocab.txt -└── ... -``` - -**NOTE:** 如需恢复模型训练，只需指定`model_name_or_path`为本地微调模型的路径即可。 - -### 模型预测 - -运行如下命令即可在测试集上进行测试。 - -```shell -# GPU启动，预测仅支持单卡 -export CUDA_VISIBLE_DEVICES=0 -python infer.py \ - --model_name_or_path=./checkpoints/model_best \ - --output_path=./predict.txt \ - --logging_steps=10 \ - --seed=2021 \ - --max_seq_len=512 \ - --max_knowledge_len=256 \ - --batch_size=4 \ - --min_dec_len=1 \ - --max_dec_len=64 \ - --num_return_sequences=20 \ - --decode_strategy=sampling \ - --top_k=5 \ - --device=gpu -``` - -其中参数释义如下： -- `model_name_or_path` 指示了预测使用的模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的模型。如果使用本地的模型，则配置为本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 模型参数 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - -- `output_path` 表示预测结果的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `seed` 表示随机数生成器的种子。 -- `max_seq_len` 表示输入序列的最大长度。 -- `max_knowledge_len` 表示输入 knowledge 序列的最大长度。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `min_dec_len` 表示预测生成的句子的最小长度。 -- `max_dec_len` 表示预测生成的句子的最大长度。 -- `num_return_sequences` 表示每条样本生成的句子的数量。对于每条样本，模型会生成`num_return_sequences`个句子，根据每个句子的概率得分进行排序，得分最高的句子作为最终的生成结果。 -- `decode_strategy` 表示预测解码时采取的策略，可选"sampling"、"greedy_search"和"beam_search"之一。 -- `top_k` 表示采用"sampling"解码策略时，token 的概率按从大到小排序，生成的 token 只从前`top_k`个中进行采样。 -- `device` 表示使用的设备。 - -同时，我们提供了基于 FastGeneration 的高性能预测的选项，可以选择性开启是否需要采用高性能预测，PaddleNLP 提供了 JIT 的方式，可以自动完成对所需的自定义 op 的动态库编译： -- `faster` 表示是否开启高性能预测。设置 `--faster` 即表示开启。 -- `use_fp16_decoding` 表示在开启高性能预测的时候，是否使用 fp16 来完成预测过程。设置 `--use_fp16_decoding` 即表示使用 fp16 进行预测，否则使用 fp32。 - -程序运行结束后会将预测生成的 response 保存在`output_path`中。同时终端中会输出评估结果。 - -采用预训练模型及微调模型在测试集上有如下结果： - -| model_name_or_path | BLEU1 / BLEU2 | DISTINCT1 / DISTINCT2 | -| :-----------------------------: | :-------------: | :-------------------: | -| unified_transformer-12L-cn-luge | 0.2606 / 0.1576 | 0.1168 / 0.2977 | -| ./checkpoints/model_best | 0.2808 / 0.1744 | 0.1124 / 0.2899 | - -**NOTE:** `./checkpoints/model_best`是按本项目中的超参在单卡上 finetune 得到的结果。 - -### 人机交互 - -运行如下命令即可开始与聊天机器人用中文进行简单的对话。 - -```shell -# GPU启动，仅支持单卡 -export CUDA_VISIBLE_DEVICES=0 -python interaction.py \ - --model_name_or_path=plato-mini \ - --min_dec_len=0 \ - --max_dec_len=64 \ - --num_return_sequences=20 \ - --decode_strategy=sampling \ - --top_k=5 \ - --device=gpu -``` - -其中参数释义如下： -- `model_name_or_path` 指示了预测使用的模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的模型。如果使用本地的模型，则配置为本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 模型参数 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unified_transformer-12L-cn | - | unified_transformer-12L-cn-luge | - | plato-mini | - -- `min_dec_len` 表示预测生成的句子的最小长度。 -- `max_dec_len` 表示预测生成的句子的最大长度。 -- `num_return_sequences` 表示每条样本生成的句子的数量。对于每条样本，模型会生成`num_return_sequences`个句子，根据每个句子的概率得分进行排序，得分最高的句子作为最终的生成结果。 -- `decode_strategy` 表示预测解码时采取的策略，可选"sampling"、"greedy_search"和"beam_search"之一。 -- `top_k` 表示采用"sampling"解码策略时，token 的概率按从大到小排序，生成的 token 只从前`top_k`个中进行采样。 -- `device` 表示使用的设备。 - -**NOTE:** 输入"[EXIT]"退出交互程序，输入"[NEXT]"开启下一轮新的对话。需要注意使用退格会导致错误。 - -## Reference - -- [UnifiedTransformer](https://arxiv.org/abs/2006.16779) -- [Knover/luge-dialogue](https://github.com/PaddlePaddle/Knover/tree/luge-dialogue/luge-dialogue) -- [DuConv](https://www.aclweb.org/anthology/P19-1369/) diff --git a/slm/examples/dialogue/unified_transformer/finetune.py b/slm/examples/dialogue/unified_transformer/finetune.py deleted file mode 100644 index daeb700d36..0000000000 --- a/slm/examples/dialogue/unified_transformer/finetune.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import math -import os -import time - -import paddle -import paddle.distributed as dist -import paddle.nn as nn -import paddle.nn.functional as F -from datasets import load_dataset -from paddle.optimizer import AdamW -from paddle.optimizer.lr import NoamDecay -from utils import create_data_loader, print_args, set_seed - -from paddlenlp.transformers import ( - UnifiedTransformerLMHeadModel, - UnifiedTransformerTokenizer, -) - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser(__doc__) - parser.add_argument('--model_name_or_path', type=str, default='unified_transformer-12L-cn-luge', help='The path or shortcut name of the pre-trained model.') - parser.add_argument('--save_dir', type=str, default='./checkpoints', help='The directory where the checkpoints will be saved.') - parser.add_argument('--logging_steps', type=int, default=100, help='Log every X updates steps.') - parser.add_argument('--save_steps', type=int, default=1000, help='Save checkpoint every X updates steps.') - parser.add_argument('--seed', type=int, default=2021, help='Random seed for initialization.') - parser.add_argument('--batch_size', type=int, default=16, help='Batch size per GPU/CPU for training.') - parser.add_argument('--lr', type=float, default=5e-5, help='The initial learning rate.') - parser.add_argument('--weight_decay', type=float, default=0.01, help='The weight decay for optimizer.') - parser.add_argument('--epochs', type=int, default=3, help='Total number of training epochs to perform.') - parser.add_argument('--warmup_steps', type=int, default=2500, help='The number of warmup steps.') - parser.add_argument('--max_grad_norm', type=float, default=0.1, help='The max value of grad norm.') - parser.add_argument('--max_seq_len', type=int, default=512, help='The maximum sequence length of training.') - parser.add_argument('--max_response_len', type=int, default=128, help='The maximum response sequence length of training.') - parser.add_argument('--max_knowledge_len', type=int, default=256, help='The maximum knowledge sequence length of training.') - parser.add_argument('--device', type=str, default='gpu', help='The device to select for training the model.') - - args = parser.parse_args() - return args -# yapf: enable - - -def save_ckpt(model, tokenizer, save_dir, name): - output_dir = os.path.join(save_dir, "model_{}".format(name)) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Need better way to get inner model of DataParallel - model_to_save = model._layers if isinstance(model, paddle.DataParallel) else model - model_to_save.save_pretrained(output_dir) - tokenizer.save_pretrained(output_dir) - - -def train(args): - paddle.set_device(args.device) - world_size = dist.get_world_size() - if world_size > 1: - dist.init_parallel_env() - - set_seed(args.seed) - - model = UnifiedTransformerLMHeadModel.from_pretrained(args.model_name_or_path) - tokenizer = UnifiedTransformerTokenizer.from_pretrained(args.model_name_or_path) - - if world_size > 1: - model = paddle.DataParallel(model) - - train_ds, dev_ds = load_dataset("duconv", split=("train", "dev")) - train_ds, train_data_loader = create_data_loader(train_ds, tokenizer, args, "train") - dev_ds, dev_data_loader = create_data_loader(dev_ds, tokenizer, args, "dev") - - lr_scheduler = NoamDecay(1 / (args.warmup_steps * (args.lr**2)), args.warmup_steps) - # Generate parameter names needed to perform weight decay. - # All bias and LayerNorm parameters are excluded. - decay_params = [p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])] - optimizer = AdamW( - learning_rate=lr_scheduler, - parameters=model.parameters(), - weight_decay=args.weight_decay, - apply_decay_param_fun=lambda x: x in decay_params, - grad_clip=nn.ClipGradByGlobalNorm(args.max_grad_norm), - ) - - step = 0 - total_time = 0.0 - best_ppl = 1e9 - for epoch in range(args.epochs): - print("\nEpoch %d/%d" % (epoch + 1, args.epochs)) - batch_start_time = time.time() - for inputs in train_data_loader: - step += 1 - labels = inputs[-1] - - logits = model(*inputs[:-1]) - loss = F.cross_entropy(logits, labels) - loss.backward() - optimizer.step() - lr_scheduler.step() - optimizer.clear_grad() - - total_time += time.time() - batch_start_time - if step % args.logging_steps == 0: - ppl = paddle.exp(loss) - print( - "step %d - loss: %.4f - ppl: %.4f - lr: %.7f - %.3fs/step" - % (step, loss, ppl, optimizer.get_lr(), total_time / args.logging_steps) - ) - total_time = 0.0 - if step % args.save_steps == 0: - ppl = evaluation(model, dev_data_loader) - if dist.get_rank() == 0: - save_ckpt(model, tokenizer, args.save_dir, step) - if ppl < best_ppl: - best_ppl = ppl - save_ckpt(model, tokenizer, args.save_dir, "best") - print("Saved step {} as best model.\n".format(step)) - batch_start_time = time.time() - print("\nTraining completed.") - - -@paddle.no_grad() -def evaluation(model, data_loader): - print("\nEval begin...") - model.eval() - total_tokens = 0 - total_loss = 0.0 - start_time = time.time() - step = 0 - for inputs in data_loader: - step += 1 - labels = inputs[-1] - - logits = model(*inputs[:-1]) - loss = F.cross_entropy(logits, labels, reduction="sum") - - total_loss += loss.item() - total_tokens += labels.shape[0] - - avg_loss = total_loss / total_tokens - ppl = math.exp(avg_loss) - avg_speed = (time.time() - start_time) / step - print("loss: %.4f - ppl: %.4f - %.3fs/step" % (avg_loss, ppl, avg_speed)) - model.train() - return ppl - - -if __name__ == "__main__": - args = parse_args() - print_args(args) - train(args) diff --git a/slm/examples/dialogue/unified_transformer/infer.py b/slm/examples/dialogue/unified_transformer/infer.py deleted file mode 100644 index 3781f73e16..0000000000 --- a/slm/examples/dialogue/unified_transformer/infer.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import time - -import paddle -from datasets import load_dataset -from utils import create_data_loader, print_args, select_response, set_seed - -from paddlenlp.metrics import BLEU, Distinct -from paddlenlp.transformers import ( - UnifiedTransformerLMHeadModel, - UnifiedTransformerTokenizer, -) - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser(__doc__) - parser.add_argument('--model_name_or_path', type=str, default='unified_transformer-12L-cn-luge', help='The path or shortcut name of the pre-trained model.') - parser.add_argument('--output_path', type=str, default='./predict.txt', help='The file path where the infer result will be saved.') - parser.add_argument('--logging_steps', type=int, default=100, help='Log every X updates steps.') - parser.add_argument('--seed', type=int, default=2021, help='Random seed for initialization.') - parser.add_argument('--batch_size', type=int, default=16, help='Batch size per GPU/CPU for training.') - parser.add_argument('--max_seq_len', type=int, default=512, help='The maximum sequence length of training.') - parser.add_argument('--max_response_len', type=int, default=128, help='The maximum response sequence length of training.') - parser.add_argument('--max_knowledge_len', type=int, default=256, help='The maximum knowledge sequence length of training.') - parser.add_argument('--min_dec_len', type=int, default=1, help='The minimum sequence length of generation.') - parser.add_argument('--max_dec_len', type=int, default=64, help='The maximum sequence length of generation.') - parser.add_argument('--num_return_sequences', type=int, default=20, help='The numbers of returned sequences for one input in generation.') - parser.add_argument('--decode_strategy', type=str, default='sampling', help='The decode strategy in generation.') - parser.add_argument('--top_k', type=int, default=0, help='The number of highest probability vocabulary tokens to keep for top-k sampling.') - parser.add_argument('--temperature', type=float, default=1.0, help='The value used to module the next token probabilities.') - parser.add_argument('--top_p', type=float, default=1.0, help='The cumulative probability for top-p sampling.') - parser.add_argument('--num_beams', type=int, default=0, help='The number of beams for beam search.') - parser.add_argument('--length_penalty', type=float, default=1.0, help='The exponential penalty to the sequence length for beam search.') - parser.add_argument('--early_stopping', type=eval, default=False, help='Whether to stop the beam search when at least `num_beams` sentences are finished per batch or not.') - parser.add_argument('--device', type=str, default='gpu', help='The device to select for training the model.') - parser.add_argument('--faster', action='store_true', help='Whether to process inference using faster transformer. ') - parser.add_argument('--use_fp16_decoding', action='store_true', help='Whether to use fp16 when using faster transformer. Only works when using faster transformer. ') - - args = parser.parse_args() - return args -# yapf: enable - - -def calc_bleu_and_distinct(preds, targets): - assert len(preds) == len(targets), ( - "The length of pred_responses should be equal to the length of " - "target_responses. But received {} and {}.".format(len(preds), len(targets)) - ) - bleu1 = BLEU(n_size=1) - bleu2 = BLEU(n_size=2) - distinct1 = Distinct(n_size=1) - distinct2 = Distinct(n_size=2) - for pred, target in zip(preds, targets): - pred_tokens = pred.split() - target_token = target.split() - - bleu1.add_inst(pred_tokens, [target_token]) - bleu2.add_inst(pred_tokens, [target_token]) - - distinct1.add_inst(pred_tokens) - distinct2.add_inst(pred_tokens) - - print("\n" + "*" * 15) - print("The auto evaluation result is:") - print("BLEU-1:", bleu1.score()) - print("BLEU-2:", bleu2.score()) - print("DISTINCT-1:", distinct1.score()) - print("DISTINCT-2:", distinct2.score()) - - -@paddle.no_grad() -def infer(args): - paddle.set_device(args.device) - set_seed(args.seed) - - model = UnifiedTransformerLMHeadModel.from_pretrained(args.model_name_or_path) - tokenizer = UnifiedTransformerTokenizer.from_pretrained(args.model_name_or_path) - - test_ds = load_dataset("duconv", split="test_1") - test_ds, test_data_loader = create_data_loader(test_ds, tokenizer, args, "test") - - model.eval() - total_time = 0.0 - start_time = time.time() - pred_responses = [] - for step, inputs in enumerate(test_data_loader, 1): - input_ids, token_type_ids, position_ids, attention_mask, seq_len = inputs - output = model.generate( - input_ids=input_ids, - token_type_ids=token_type_ids, - position_ids=position_ids, - attention_mask=attention_mask, - seq_len=seq_len, - max_length=args.max_dec_len, - min_length=args.min_dec_len, - decode_strategy=args.decode_strategy, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - num_return_sequences=args.num_return_sequences, - use_fp16_decoding=args.use_fp16_decoding, - use_fast=args.faster, - ) - - total_time += time.time() - start_time - if step % args.logging_steps == 0: - print("step %d - %.3fs/step" % (step, total_time / args.logging_steps)) - total_time = 0.0 - - ids, scores = output - results = select_response(ids, scores, tokenizer, args.max_dec_len, args.num_return_sequences) - pred_responses.extend(results) - - start_time = time.time() - - with open(args.output_path, "w", encoding="utf-8") as fout: - for response in pred_responses: - fout.write(response + "\n") - print("\nSave inference result into: %s" % args.output_path) - - target_responses = [example["response"] for example in test_ds] - calc_bleu_and_distinct(pred_responses, target_responses) - - -if __name__ == "__main__": - args = parse_args() - print_args(args) - infer(args) diff --git a/slm/examples/dialogue/unified_transformer/interaction.py b/slm/examples/dialogue/unified_transformer/interaction.py deleted file mode 100644 index cde62e5057..0000000000 --- a/slm/examples/dialogue/unified_transformer/interaction.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse - -import paddle -from termcolor import colored, cprint -from utils import print_args, select_response, set_seed - -from paddlenlp.transformers import ( - UnifiedTransformerLMHeadModel, - UnifiedTransformerTokenizer, -) - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser(__doc__) - parser.add_argument('--model_name_or_path', type=str, default='plato-mini', help='The path or shortcut name of the pre-trained model.') - parser.add_argument('--seed', type=int, default=None, help='Random seed for initialization.') - parser.add_argument('--min_dec_len', type=int, default=1, help='The minimum sequence length of generation.') - parser.add_argument('--max_dec_len', type=int, default=64, help='The maximum sequence length of generation.') - parser.add_argument('--num_return_sequences', type=int, default=20, help='The numbers of returned sequences for one input in generation.') - parser.add_argument('--decode_strategy', type=str, default='sampling', help='The decode strategy in generation.') - parser.add_argument('--top_k', type=int, default=5, help='The number of highest probability vocabulary tokens to keep for top-k sampling.') - parser.add_argument('--temperature', type=float, default=1.0, help='The value used to module the next token probabilities.') - parser.add_argument('--top_p', type=float, default=1.0, help='The cumulative probability for top-p sampling.') - parser.add_argument('--num_beams', type=int, default=0, help='The number of beams for beam search.') - parser.add_argument('--length_penalty', type=float, default=1.0, help='The exponential penalty to the sequence length for beam search.') - parser.add_argument('--early_stopping', type=eval, default=False, help='Whether to stop the beam search when at least `num_beams` sentences are finished per batch or not.') - parser.add_argument('--device', type=str, default='gpu', help='The device to select for training the model.') - - args = parser.parse_args() - return args -# yapf: enable - - -def interaction(args, model, tokenizer): - history = [] - start_info = "Enter [EXIT] to quit the interaction, [NEXT] to start a new conversation." - cprint(start_info, "yellow", attrs=["bold"]) - while True: - user_utt = input(colored("[Human]: ", "red", attrs=["bold"])).strip() - if user_utt == "[EXIT]": - break - elif user_utt == "[NEXT]": - history = [] - cprint(start_info, "yellow", attrs=["bold"]) - else: - history.append(user_utt) - inputs = tokenizer.dialogue_encode( - history, add_start_token_as_response=True, return_tensors=True, is_split_into_words=False - ) - inputs["input_ids"] = inputs["input_ids"].astype("int64") - ids, scores = model.generate( - input_ids=inputs["input_ids"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - attention_mask=inputs["attention_mask"], - max_length=args.max_dec_len, - min_length=args.min_dec_len, - decode_strategy=args.decode_strategy, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - num_return_sequences=args.num_return_sequences, - use_fast=True, - ) - bot_response = select_response( - ids, scores, tokenizer, args.max_dec_len, args.num_return_sequences, keep_space=False - )[0] - print(colored("[Bot]:", "blue", attrs=["bold"]), colored(bot_response, attrs=["bold"])) - history.append(bot_response) - return - - -def main(args): - paddle.set_device(args.device) - if args.seed is not None: - set_seed(args.seed) - - # Initialize the model and tokenizer - model = UnifiedTransformerLMHeadModel.from_pretrained(args.model_name_or_path) - tokenizer = UnifiedTransformerTokenizer.from_pretrained(args.model_name_or_path) - - model.eval() - interaction(args, model, tokenizer) - - -if __name__ == "__main__": - args = parse_args() - print_args(args) - main(args) diff --git a/slm/examples/dialogue/unified_transformer/utils.py b/slm/examples/dialogue/unified_transformer/utils.py deleted file mode 100644 index 1049aa593f..0000000000 --- a/slm/examples/dialogue/unified_transformer/utils.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import random -from functools import partial - -import numpy as np -import paddle -import paddle.distributed as dist -from paddle.io import BatchSampler, DataLoader, DistributedBatchSampler - -from paddlenlp.data import Pad - - -def print_args(args): - print("----------- Configuration Arguments -----------") - for arg, value in sorted(vars(args).items()): - print("%s: %s" % (arg, value)) - print("------------------------------------------------") - - -def set_seed(seed): - # Use the same data seed(for data shuffle) for all procs to guarantee data - # consistency after sharding. - random.seed(seed) - np.random.seed(seed) - # Maybe different op seeds(for dropout) for different procs is better. - paddle.seed(seed + dist.get_rank()) - - -def preprocess_examples(examples, mode="train"): - """ - For training set and dev set, treat each utterance of the first speaker as - the response, and concatenate the goal, knowledge and the dialog’s previous - utterances as the history. In this way, multiple history-response pairs - are constructed. - """ - if mode == "test": - return examples - new_examples = {} - goal = [] - knowledge = [] - history = [] - response = [] - - conv = examples["conversation"] - for index, conversation in enumerate(conv): - for i in range(0, len(conversation), 2): - goal.append(examples["goal"][index]) - knowledge.append(examples["knowledge"][index]) - history.append(conversation[:i]) - response.append(conversation[i]) - new_examples["goal"] = goal - new_examples["knowledge"] = knowledge - new_examples["history"] = history - new_examples["response"] = response - - return new_examples - - -def convert_example(example, tokenizer, max_seq_len=512, max_response_len=128, max_knowledge_len=256, mode="train"): - """Convert all examples into necessary features.""" - goal = example["goal"] - knowledge = example["knowledge"] - goal_knowledge = " ".join([" ".join(lst) for lst in goal + knowledge]) - - if mode != "test": - tokenized_example = tokenizer.dialogue_encode( - example["history"], - response=example["response"], - knowledge=goal_knowledge, - task_type="knowledge", - max_seq_len=max_seq_len, - max_response_len=max_response_len, - max_knowledge_len=max_knowledge_len, - return_length=True, - ) - response_start = tokenized_example["input_ids"].index(tokenizer.cls_token_id, 1) - response_end = tokenized_example["seq_len"] - # Use to gather the logits corresponding to the labels during training - tokenized_example["masked_positions"] = list(range(response_start, response_end - 1)) - tokenized_example["labels"] = tokenized_example["input_ids"][response_start + 1 : response_end] - return tokenized_example - else: - tokenized_example = tokenizer.dialogue_encode( - example["history"], - knowledge=goal_knowledge, - task_type="knowledge", - max_seq_len=max_seq_len, - max_knowledge_len=max_knowledge_len, - add_start_token_as_response=True, - return_length=True, - ) - - if "response" in example: - tokenized_example["response"] = example["response"] - return tokenized_example - - -def batchify_fn(batch_examples, pad_val, mode): - def pad_mask(batch_attention_mask): - batch_size = len(batch_attention_mask) - max_len = max(map(len, batch_attention_mask)) - attention_mask = np.ones((batch_size, max_len, max_len), dtype="float32") * -1e9 - for i, mask_data in enumerate(attention_mask): - seq_len = len(batch_attention_mask[i]) - mask_data[-seq_len:, -seq_len:] = np.array(batch_attention_mask[i], dtype="float32") - # In order to ensure the correct broadcasting mechanism, expand one - # dimension to the second dimension (n_head of Transformer). - attention_mask = np.expand_dims(attention_mask, axis=1) - return attention_mask - - pad_func = Pad(pad_val=pad_val, pad_right=False, dtype="int64") - - input_ids = pad_func([example["input_ids"] for example in batch_examples]) - token_type_ids = pad_func([example["token_type_ids"] for example in batch_examples]) - position_ids = pad_func([example["position_ids"] for example in batch_examples]) - - attention_mask = pad_mask([example["attention_mask"] for example in batch_examples]) - - if mode != "test": - max_len = max([example["seq_len"] for example in batch_examples]) - masked_positions = np.concatenate( - [ - np.array(example["masked_positions"]) + (max_len - example["seq_len"]) + i * max_len - for i, example in enumerate(batch_examples) - ] - ) - labels = np.concatenate([np.array(example["labels"], dtype="int64") for example in batch_examples]) - return input_ids, token_type_ids, position_ids, attention_mask, masked_positions, labels - else: - seq_len = np.asarray([example["seq_len"] for example in batch_examples]).astype("int32") - return input_ids, token_type_ids, position_ids, attention_mask, seq_len - - -def create_data_loader(dataset, tokenizer, args, mode): - trans_func1 = partial(preprocess_examples, mode=mode) - trans_func2 = partial( - convert_example, - tokenizer=tokenizer, - max_seq_len=args.max_seq_len, - max_response_len=args.max_response_len, - max_knowledge_len=args.max_knowledge_len, - mode=mode, - ) - remove_columns = None - if mode in ["train", "dev"]: - remove_columns = ["id", "conversation"] - - dataset = dataset.map(trans_func1, batched=True, batch_size=None, remove_columns=remove_columns).map(trans_func2) - if mode == "train": - batch_sampler = DistributedBatchSampler(dataset, batch_size=args.batch_size, shuffle=True) - else: - batch_sampler = BatchSampler(dataset, batch_size=args.batch_size, shuffle=False) - collate_fn = partial(batchify_fn, pad_val=tokenizer.pad_token_id, mode=mode) - data_loader = DataLoader(dataset, batch_sampler=batch_sampler, collate_fn=collate_fn, return_list=True) - return dataset, data_loader - - -def post_process_response(token_ids, tokenizer): - """Post-process the decoded sequence. Truncate from the first .""" - eos_pos = len(token_ids) - for i, tok_id in enumerate(token_ids): - if tok_id == tokenizer.sep_token_id: - eos_pos = i - break - token_ids = token_ids[:eos_pos] - tokens = tokenizer.convert_ids_to_tokens(token_ids) - tokens = tokenizer.merge_subword(tokens) - return token_ids, tokens - - -def get_in_turn_repetition(pred, is_cn=False): - """Get in-turn repetition.""" - if len(pred) == 0: - return 1.0 - if isinstance(pred[0], str): - pred = [tok.lower() for tok in pred] - if is_cn: - pred = "".join(pred) - tri_grams = set() - for i in range(len(pred) - 2): - tri_gram = tuple(pred[i : i + 3]) - if tri_gram in tri_grams: - return True - tri_grams.add(tri_gram) - return False - - -def select_response(ids, scores, tokenizer, max_dec_len=None, num_return_sequences=1, keep_space=True): - results = [] - group = [] - tmp = [] - if scores is not None: - ids = ids.numpy() - scores = scores.numpy() - - if len(ids) != len(scores) or (len(ids) % num_return_sequences) != 0: - raise ValueError( - "the length of `ids` is {}, but the `num_return_sequences` is {}".format( - len(ids), num_return_sequences - ) - ) - - for pred, score in zip(ids, scores): - pred_token_ids, pred_tokens = post_process_response(pred, tokenizer) - num_token = len(pred_token_ids) - if keep_space: - response = " ".join(pred_tokens) - else: - response = "".join(pred_tokens) - - in_turn_repetition = get_in_turn_repetition(pred_tokens, True) or get_in_turn_repetition(pred_token_ids) - # not ending - if max_dec_len is not None and num_token >= max_dec_len: - score -= 1e3 - elif in_turn_repetition: - score -= 1e3 - - tmp.append([response, score]) - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - preds = sorted(preds, key=lambda x: -x[1]) - results.append(preds[0][0]) - else: - ids = ids.numpy() - - for pred in ids: - pred_token_ids, pred_tokens = post_process_response(pred, tokenizer) - num_token = len(pred_token_ids) - if keep_space: - response = " ".join(pred_tokens) - else: - response = "".join(pred_tokens) - - in_turn_repetition = get_in_turn_repetition(pred_tokens, True) or get_in_turn_repetition(pred_token_ids) - - last_pos = 0 - if (max_dec_len is not None and num_token >= max_dec_len) or in_turn_repetition: - tmp.append([response]) - else: - tmp.insert(last_pos, [response]) - last_pos += 1 - - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - results.append(preds[0][0]) - return results diff --git a/slm/examples/information_extraction/waybill_ie/model.py b/slm/examples/information_extraction/waybill_ie/model.py index d6d1e8dfb3..4903183328 100644 --- a/slm/examples/information_extraction/waybill_ie/model.py +++ b/slm/examples/information_extraction/waybill_ie/model.py @@ -15,7 +15,6 @@ import paddle import paddle.nn as nn -from paddlenlp.embeddings import TokenEmbedding from paddlenlp.layers.crf import LinearChainCrf, LinearChainCrfLoss from paddlenlp.utils.tools import compare_version @@ -30,7 +29,7 @@ class BiGRUWithCRF(nn.Layer): def __init__(self, emb_size, hidden_size, word_num, label_num, use_w2v_emb=False): super(BiGRUWithCRF, self).__init__() if use_w2v_emb: - self.word_emb = TokenEmbedding(extended_vocab_path="./data/word.dic", unknown_token="OOV") + raise ValueError("Please set use_w2v_emb=False") else: self.word_emb = nn.Embedding(word_num, emb_size) self.gru = nn.GRU(emb_size, hidden_size, num_layers=2, direction="bidirect") diff --git a/slm/examples/model_compression/distill_lstm/README.md b/slm/examples/model_compression/distill_lstm/README.md index 667145c792..8386edbf59 100644 --- a/slm/examples/model_compression/distill_lstm/README.md +++ b/slm/examples/model_compression/distill_lstm/README.md @@ -69,7 +69,7 @@ python -u ./run_glue.py \ ``` -如果需要训练基于 ChnSentiCorp 数据集的 BERT finetuning 模型，可以进入[文本分类目录](../../text_classification/pretrained_models)下，将预训练模型改成 BERT，并基于 bert-base-chinese 和 bert-wwm-ext-chinese 模型进行 fine-tuning 训练。 +如果需要训练基于 ChnSentiCorp 数据集的 BERT finetuning 模型，可以进入[文本分类目录](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/slm/applications/text_classification/multi_class)下，将预训练模型改成 BERT，并基于 bert-base-chinese 和 bert-wwm-ext-chinese 模型进行 fine-tuning 训练。 训练完成之后，可将训练效果最好的模型保存在本项目下的`pretrained_models/$TASK_NAME/`下。模型目录下有`model_config.json`, `model_state.pdparams`, `tokenizer_config.json`及`vocab.txt`这几个文件。 diff --git a/slm/examples/model_compression/distill_lstm/small.py b/slm/examples/model_compression/distill_lstm/small.py index 92681bd039..4bf38ea225 100644 --- a/slm/examples/model_compression/distill_lstm/small.py +++ b/slm/examples/model_compression/distill_lstm/small.py @@ -22,7 +22,6 @@ from args import parse_args from data import create_data_loader_for_small_model, create_pair_loader_for_small_model from paddle.metric import Accuracy -from paddlenlp.embeddings import TokenEmbedding from paddlenlp.metrics import AccuracyAndF1 METRIC_CLASSES = {"sst-2": Accuracy, "qqp": AccuracyAndF1, "chnsenticorp": Accuracy} @@ -44,10 +43,7 @@ class BiLSTM(nn.Layer): ): super(BiLSTM, self).__init__() if embedding_name is not None: - self.embedder = TokenEmbedding( - embedding_name, extended_vocab_path=vocab_path, keep_extended_vocab_only=True - ) - embed_dim = self.embedder.embedding_dim + raise ValueError("TokenEmbedding is depercated in PaddleNLP since 3.0, please set embedding_name to None ") else: self.embedder = nn.Embedding(vocab_size, embed_dim, padding_idx) diff --git a/slm/examples/question_generation/README.md b/slm/examples/question_generation/README.md deleted file mode 100644 index 41d62b2a8e..0000000000 --- a/slm/examples/question_generation/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# 问题生成 - -Question Generation（QG），即问题生成，指的是给定一段上下文和答案，自动生成一个流畅且符合上下文主题的问句。问题生成技术在教育、咨询、搜索、问答等多个领域均有着巨大的应用价值。 - -PaddleNLP 提供英文和中文问题生成任务示例，分别基于英文预训练语言模型[t5](./t5)和中文预训练语言模型[unimo-text](./unimo-text)。 - - -## 英文 - -[t5](./t5) 展示了如何使用英文预训练模型 T5完成问题生成任务，支持模型微调预测评估，并提供相关预训练模型。 - -## 中文 - -[unimo-text](./unimo-text) 展示了如何使用中文预训练模型 UNIMO-Text 完成问题生成任务，提供数据准备、训练、预测、推理部署全流程定制化训练，并提供相关预训练模型。 - -# 参考文献 - -1. Raffel, C., Shazeer, N., Roberts, A., Lee, K., Narang, S., Matena, M., Zhou, Y., Li, W. and Liu, P.J., 2020. Exploring the limits of transfer learning with a unified text-to-text transformer. J. Mach. Learn. Res., 21(140), pp.1-67. - -2. Li, Wei, et al. "Unimo: Towards unified-modal understanding and generation via cross-modal contrastive learning." arXiv preprint arXiv:2012.15409 (2020). diff --git a/slm/examples/question_generation/t5/README.md b/slm/examples/question_generation/t5/README.md deleted file mode 100644 index 16044bb015..0000000000 --- a/slm/examples/question_generation/t5/README.md +++ /dev/null @@ -1,208 +0,0 @@ -# 问题生成(Question Generation) - -## 简介 - -Question Generation（QG），即问题生成，指的是给定一段上下文（passage 或 sentence），自动生成一个流畅且符合上下文主题的问句。问题生成通常可以分为两个分支，即无答案问题生成（answer-agnostic question generation）和有答案问题生成（answer-aware question generation）。 - -本项目是 T5在 PaddlePaddle 上开源实现的有答案问题生成的例子，包含了在 SQuAD 数据集上微调和生成的代码。 - -## 快速开始 - -### 环境依赖 - -- nltk -- evaluate - - -安装方式：`pip install -r requirements.txt` - -### 代码结构说明 - -以下是本项目主要代码结构及说明： - -```text -. -├── finetune.py # 模型微调主程序入口 -├── generate.py # 模型生成主程序入口 -├── utils.py # 定义参数及一些工具函数 -├── requirements.txt # 环境依赖文件 -└── README.md # 文档说明 -``` - -### 数据准备 - -#### 数据加载 -**SQuAD**（Stanford Question Answering Dataset）数据集是一个英文问答数据集，现有的问题生成研究主要在该数据集上进行评价。**SQuAD**中的数据由段落、问题、答案3个主要部分组成，其中段落从维基百科中获取，问题和答案通过众包的方式由人工标注。 - -为了方便用户快速测试，PaddleNLP Dataset API 内置了 Squad 数据集，一键即可完成数据集加载，示例代码如下： - -```python -from paddlenlp.datasets import load_dataset -train_set, dev_set, test_set = load_dataset("squad", splits=["train_v1", "dev_v1"]) -``` - -#### 数据处理 -针对**SQuAD**数据集，我们需要将 QA 任务格式的数据进行转换从而得到 text2text 形式的数据，默认构造方式如下，其他形式输入数据用户可以在 convert_example 函数中自行定义 -```text -answer: {answer_text} context: {context_text} -question: {question_text} -``` -具体案例如下， -```text -answer: the Miller–Rabin primality test context: The property of being prime (or not) is called primality. A simple but slow method of verifying the primality of a given number n is known as trial division. It consists of testing whether n is a multiple of any integer between 2 and . Algorithms much more efficient than trial division have been devised to test the primality of large numbers. These include the Miller–Rabin primality test, which is fast but has a small probability of error, and the AKS primality test, which always produces the correct answer in polynomial time but is too slow to be practical. Particularly fast methods are available for numbers of special forms, such as Mersenne numbers. As of January 2016[update], the largest known prime number has 22,338,618 decimal digits. - -question: What is the name of the process which confirms the primality of a number n? -``` - -### 模型训练 - -运行如下命令即可在训练集上进行 finetune，并在验证集上进行验证 - -```shell -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -# 例如使用1号和2号卡，则：`--gpu 1,2` -unset CUDA_VISIBLE_DEVICES -python -m paddle.distributed.launch --gpus 1,2 train.py \ - --model_name_or_path=t5-base \ - --dataset_name=squad \ - --output_dir=output \ - --max_source_length=1024 \ - --max_target_length=142 \ - --learning_rate=1e-4 \ - --num_train_epochs=6 \ - --logging_steps=100 \ - --save_steps=1000 \ - --seed=42 \ - --train_batch_size=4 \ - --eval_batch_size=64 \ - --warmup_proportion=0.1 \ - --ignore_pad_token_for_loss \ - --device=gpu -``` - -其中参数释义如下： -- `gpus` 指示了训练所用的 GPU - -- `model_name_or_path` 指示了 finetune 使用的预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的模型。如果使用本地的模型，则配置为本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 模型参数 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | t5-base | - | t5-large | - -- `dataset_name` 表示训练的数据集。 - -- `output_dir` 表示模型的保存路径。 - -- `max_source_length` 表示输入序列的长度，超过该长度将被截断。 - -- `max_target_length` 表示输出的最大长度。 - -- `learning_rate` 表示基础学习率大小，将与 learning rate scheduler 产生的值相乘作为当前学习率。 - -- `num_train_epochs` 表示训练轮数。 - -- `epochs` 表示训练轮数。 - -- `logging_steps` 表示日志打印间隔。 - -- `save_steps` 表示模型保存及评估间隔。 - -- `seed` 表示随机数生成器的种子。 - -- `train_batch_size` 表示训练每张卡上的样本数目。 - -- `eval_batch_size` 表示预测单卡上的样本数目。 - -- `warmup_proportion` 表示 warmup_steps 所占总步数的比例。学习率逐渐升高到基础学习率（即上面配置的 learning_rate）所需要的迭代数。 - -- `device` 表示使用的设备。 - -程序运行时将会自动进行训练和验证，训练过程中会自动保存模型在指定的`output_dir`中。如： - -```text -./output/ -├── t5_model_1000 -│ ├── model_config.json -│ ├── model_state.pdparams -│ ├── special_tokens_map.json -│ ├── spiece.model -│ └── tokenizer_config.json -└── ... -``` - -**NOTE:** 如需恢复模型训练，只需指定`model_name_or_path`为本地微调模型的路径即可。 - -### 模型预测 - -运行如下命令即可在验证集上进行测试 - -```shell -# GPU启动，预测仅支持单卡 -export CUDA_VISIBLE_DEVICES=0 -python predict.py \ - --model_name_or_path=./checkpoints/model_xx/ \ - --dataset_name=squad \ - --output_path=generate.txt \ - --max_source_length=1024 \ - --max_target_length=142 \ - --decode_strategy=greedy_search \ - --top_k=2 \ - --top_p=1.0 \ - --num_beams=1 \ - --length_penalty=0.0 \ - --batch_size=64 \ - --seed=42 \ - --ignore_pad_token_for_loss \ - --logging_steps=20 \ - --device=gpu -``` - -其中参数释义如下： -- `model_name_or_path` 指示了预测使用的模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的模型。如果使用本地的模型，则配置为本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 模型参数 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | t5-base | - | t5-large | - | mrm8488/t5-base-finetuned-question-generation-ap | - -- `dataset_name` 表示预测的数据集。 - -- `output_path` 表示预测结果的保存路径。 - -- `max_source_length` 表示输入序列的长度，超过该长度将被截断。 - -- `max_target_length` 表示输出的最大长度。 - -- `decode_strategy` 表示预测解码时采取的策略，可选"sampling"、"greedy_search"和"beam_search"之一。 - -- `top_k` 表示采用"sampling"解码策略时，token 的概率按从大到小排序，生成的 token 只从前`top_k`个中进行采样。 - -- `top_p` 表示采用"sampling"解码策略时，从词表中采样并选择概率之和大于给定阈值`top_p`的 token。 - -- `num_beams` 表示 besm search 的 beam size。 - -- `length_penalty` 表示 besm search 生成长度的指数惩罚。 - -- `batch_size` 表示每次迭代**单卡**上的样本数目。 - -- `seed` 表示随机数生成器的种子。 - -- `logging_steps` 表示日志打印间隔。 - -- `device` 表示使用的设备。 - -程序运行结束后会将预测生成的问题保存在`output_path`中。同时终端中会输出评估结果。 - -采用社区微调模型 mrm8488/t5-base-finetuned-question-generation-ap 在验证集上有如下结果： - -| model_name_or_path | BLEU-1 | BLEU-2 | BLEU-3 | BLEU-4 | -| :----------------------: | :-------------: | :-------------: |:-------------: |:-------------: | -| [mrm8488/t5-base-finetuned-question-generation-ap](https://huggingface.co/mrm8488/t5-base-finetuned-question-generation-ap ) | 50.11 | 35.83 | 27.68 | 22.03 | - - - - -## 参考文献 -1. Raffel, C., Shazeer, N., Roberts, A., Lee, K., Narang, S., Matena, M., Zhou, Y., Li, W. and Liu, P.J., 2020. Exploring the limits of transfer learning with a unified text-to-text transformer. J. Mach. Learn. Res., 21(140), pp.1-67. diff --git a/slm/examples/question_generation/t5/predict.py b/slm/examples/question_generation/t5/predict.py deleted file mode 100644 index 7931059b88..0000000000 --- a/slm/examples/question_generation/t5/predict.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import random -import time -from functools import partial -from pprint import pprint - -import numpy as np -import paddle -from paddle.io import BatchSampler, DataLoader -from utils import compute_metrics, convert_example - -from paddlenlp.data import Pad, Tuple -from paddlenlp.datasets import load_dataset -from paddlenlp.transformers import T5ForConditionalGeneration, T5Tokenizer - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser() - # Required parameters - parser.add_argument("--model_name_or_path", default="t5-base", type=str, required=True, help="Path to pre-trained model. ") - parser.add_argument("--dataset_name", default="squad", type=str, required=True, help="The name of the dataset to use. Selected in the list: " + "squad") - parser.add_argument('--output_path', type=str, default='generate.txt', help='The file path where the infer result will be saved.') - parser.add_argument("--max_source_length", default=1024, type=int, help="The maximum total input sequence length after tokenization.Sequences longer than this will be truncated, sequences shorter will be padded.",) - parser.add_argument("--min_target_length", default=0, type=int, help="The minimum total sequence length for target text when generating. ") - parser.add_argument("--max_target_length", default=142, type=int, help="The maximum total sequence length for target text after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded during ``evaluate`` and ``predict``.",) - parser.add_argument('--decode_strategy', default='greedy_search', type=str, help='The decode strategy in generation.') - parser.add_argument('--top_k', default=2, type=int, help='The number of highest probability vocabulary tokens to keep for top-k sampling.') - parser.add_argument('--top_p', default=1.0, type=float, help='The cumulative probability for top-p sampling.') - parser.add_argument('--num_beams', default=1, type=int, help='The number of beams for beam search.') - parser.add_argument('--length_penalty', default=0.6, type=float, help='The exponential penalty to the sequence length for beam search.') - parser.add_argument('--early_stopping', default=False, type=eval, help='Whether to stop the beam search when at least `num_beams` sentences are finished per batch or not.') - parser.add_argument("--diversity_rate", default=0.0, type=float, help="The diversity of beam search. ") - parser.add_argument('--faster', action='store_true', help='Whether to process inference using FastGeneration. ') - parser.add_argument('--use_fp16_decoding', action='store_true', help='Whether to use fp16 when using FastGeneration. Only works when using FastGeneration. ') - parser.add_argument("--batch_size", default=64, type=int, help="Batch size per GPU/CPU for testing or evaluation.") - parser.add_argument("--seed", default=42, type=int, help="random seed for initialization") - parser.add_argument("--device", default="gpu", type=str, choices=["cpu", "gpu", "xpu"], help="The device to select to train the model, is must be cpu/gpu/xpu.") - parser.add_argument("--logging_steps", type=int, default=100, help="Log every X updates steps.") - parser.add_argument("--is_debug", action='store_true', help="Whether to debug.") - parser.add_argument("--ignore_pad_token_for_loss", action='store_true', help="Whether to ignore the tokens corresponding to padded labels in the loss computation or not.") - - args = parser.parse_args() - return args - - -def set_seed(args): - # Use the same data seed(for data shuffle) for all procs to guarantee data - # consistency after sharding. - random.seed(args.seed) - np.random.seed(args.seed) - # Maybe different op seeds(for dropout) for different procs is better. By: - # `paddle.seed(args.seed + paddle.distributed.get_rank())` - paddle.seed(args.seed) - - -@paddle.no_grad() -def generate(args): - paddle.set_device(args.device) - set_seed(args) - tokenizer = T5Tokenizer.from_pretrained(args.model_name_or_path) - model = T5ForConditionalGeneration.from_pretrained(args.model_name_or_path) - dataset = load_dataset(args.dataset_name, splits=["dev_v1"]) - # dataset = load_dataset(args.dataset_name, splits=["dev_v2"]) - trans_func = partial( - convert_example, - tokenizer=tokenizer, - decoder_start_token_id=model.t5.bos_token_id, - max_source_length=args.max_source_length, - max_target_length=args.max_target_length, - ignore_pad_token_for_loss=args.ignore_pad_token_for_loss, - is_train=False) - - def batchify_fn(samples, tokenizer): - fn = Tuple( - Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input_ids - Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64" - ), # attention_mask - Pad(axis=0, pad_val=-100, dtype="int64"), # mem_seq_lens - Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64" - ), # decoder_input_ids - Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # labels - ) - return fn(samples) - - dataset = dataset.map(trans_func, lazy=True) - - # debug - if args.is_debug: - dataset.data = dataset.data[:20] - dataset.new_data = dataset.new_data[:20] - - batch_sampler = BatchSampler(dataset, - batch_size=args.batch_size, - shuffle=False) - data_loader = DataLoader(dataset=dataset, - batch_sampler=batch_sampler, - num_workers=0, - collate_fn=batchify_fn, - return_list=True) - data_loader.pin_memory = False - - model.eval() - total_time = 0.0 - start_time = time.time() - all_preds = [] - all_labels = [] - for step, batch in enumerate(data_loader): - input_ids, _, mem_seq_lens, _, labels = batch - preds, _ = model.generate(input_ids=input_ids, - max_length=args.max_target_length, - min_length=args.min_target_length, - decode_strategy=args.decode_strategy, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - early_stopping=args.early_stopping, - diversity_rate=args.diversity_rate, - use_fast=args.faster) - total_time += (time.time() - start_time) - if step % args.logging_steps == 0: - print('step %d - %.3fs/step' % - (step, total_time / args.logging_steps)) - total_time = 0.0 - all_preds.extend(preds.numpy()) - all_labels.extend(labels.numpy()) - start_time = time.time() - - bleu_result, decoded_preds, decoded_labels = compute_metrics( - all_preds, all_labels, tokenizer, args.ignore_pad_token_for_loss) - print("BLEU result: ", bleu_result) - with open(args.output_path, 'w', encoding='utf-8') as fout: - for decoded_pred in decoded_preds: - fout.write(' '.join(decoded_pred) + '\n') - print('Save generated result into: %s' % args.output_path) - with open(args.output_path + '.reference.txt', 'w', - encoding='utf-8') as fout: - for decoded_label in decoded_labels: - fout.write(' '.join(decoded_label) + '\n') - print('Save referenced labels into: {}.reference.txt'.format(args.output_path)) - - -if __name__ == '__main__': - args = parse_args() - pprint(args) - generate(args) diff --git a/slm/examples/question_generation/t5/requirements.txt b/slm/examples/question_generation/t5/requirements.txt deleted file mode 100644 index 40abc64257..0000000000 --- a/slm/examples/question_generation/t5/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -nltk==3.6.2 -evaluate==0.2.2 \ No newline at end of file diff --git a/slm/examples/question_generation/t5/train.py b/slm/examples/question_generation/t5/train.py deleted file mode 100644 index cdd124370f..0000000000 --- a/slm/examples/question_generation/t5/train.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import os -import random -import time -from functools import partial -from pprint import pprint - -import numpy as np -import paddle -from paddle.io import BatchSampler, DataLoader, DistributedBatchSampler -from tqdm import tqdm -from utils import compute_metrics, convert_example - -from paddlenlp.data import Pad, Tuple -from paddlenlp.datasets import load_dataset -from paddlenlp.trainer.argparser import strtobool -from paddlenlp.transformers import ( - LinearDecayWithWarmup, - T5ForConditionalGeneration, - T5Tokenizer, -) -from paddlenlp.utils.log import logger - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser() - # Required parameters - parser.add_argument("--model_name_or_path", default="t5-base", type=str, required=True, help="Path to pre-trained model. ") - parser.add_argument("--dataset_name", default="squad", type=str, required=True, help="The name of the dataset to use. Selected in the list: " + "squad") - parser.add_argument("--output_dir", default="output", type=str, required=True, help="The output directory where the model predictions and checkpoints will be written.",) - parser.add_argument("--max_source_length", default=1024, type=int, help="The maximum total input sequence length after tokenization.Sequences longer than this will be truncated, sequences shorter will be padded.",) - parser.add_argument("--min_target_length", default=0, type=int, help="The minimum total sequence length for target text when generating. ") - parser.add_argument("--max_target_length", default=142, type=int, help="The maximum total sequence length for target text after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded. during ``evaluate`` and ``predict``.",) - parser.add_argument("--learning_rate", default=1e-4, type=float, help="The initial learning rate for Adam.") - parser.add_argument("--num_train_epochs", default=3, type=int, help="Total number of training epochs to perform.") - parser.add_argument("--logging_steps", type=int, default=100, help="Log every X updates steps.") - parser.add_argument("--save_steps", type=int, default=100, help="Save checkpoint every X updates steps.") - parser.add_argument("--train_batch_size", default=20, type=int, help="Batch size per GPU/CPU for training.") - parser.add_argument("--eval_batch_size", default=12, type=int, help="Batch size per GPU/CPU for evaluation.") - parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.") - parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps. If > 0: Override warmup_proportion") - parser.add_argument("--warmup_proportion", default=0.1, type=float, help="Linear warmup proportion over total steps.") - parser.add_argument("--adam_epsilon", default=1e-6, type=float, help="Epsilon for Adam optimizer.") - parser.add_argument("--max_steps", default=-1, type=int, help="If > 0: set total number of training steps to perform. Override num_train_epochs.") - parser.add_argument("--seed", default=42, type=int, help="random seed for initialization") - parser.add_argument("--device", default="gpu", type=str, choices=["cpu", "gpu", "xpu"], help="The device to select to train the model, is must be cpu/gpu/xpu.") - parser.add_argument("--use_amp", default=False, type=strtobool, help="Enable mixed precision training.") - parser.add_argument("--scale_loss", default=2**15, type=float, help="The value of scale_loss for fp16.") - parser.add_argument("--ignore_pad_token_for_loss", action='store_true', help="Whether to ignore the tokens corresponding to padded labels in the loss computation or not.") - - args = parser.parse_args() - return args - - -def set_seed(args): - # Use the same data seed(for data shuffle) for all procs to guarantee data - # consistency after sharding. - random.seed(args.seed) - np.random.seed(args.seed) - # Maybe different op seeds(for dropout) for different procs is better. By: - # `paddle.seed(args.seed + paddle.distributed.get_rank())` - paddle.seed(args.seed) - - -@paddle.no_grad() -def evaluate(model, data_loader, tokenizer, ignore_pad_token_for_loss, - min_target_length, max_target_length): - model.eval() - all_preds = [] - all_labels = [] - model = model._layers if isinstance(model, paddle.DataParallel) else model - for batch in tqdm(data_loader, total=len(data_loader), desc="Eval step"): - input_ids, _, _, labels = batch - preds = model.generate(input_ids=input_ids, - min_length=min_target_length, - max_length=max_target_length, - use_cache=True)[0] - all_preds.extend(preds.numpy()) - all_labels.extend(labels.numpy()) - bleu_result, decoded_preds, decoded_labels = compute_metrics( - all_preds, all_labels, tokenizer, ignore_pad_token_for_loss) - logger.info(bleu_result) - model.train() - - -def do_train(args): - paddle.set_device(args.device) - if paddle.distributed.get_world_size() > 1: - paddle.distributed.init_parallel_env() - - set_seed(args) - tokenizer = T5Tokenizer.from_pretrained(args.model_name_or_path) - model = T5ForConditionalGeneration.from_pretrained(args.model_name_or_path) - trans_func = partial( - convert_example, - tokenizer=tokenizer, - decoder_start_token_id=model.t5.bos_token_id, - max_source_length=args.max_source_length, - max_target_length=args.max_target_length, - ignore_pad_token_for_loss=args.ignore_pad_token_for_loss) - logger.info("Loading train and dev dataset: %s" % args.dataset_name) - train_set, dev_set = load_dataset(args.dataset_name, - splits=["train_v1", "dev_v1"]) - logger.info("Loaded train and dev dataset: %s" % args.dataset_name) - train_set = train_set.map(trans_func, lazy=True) - train_batch_sampler = DistributedBatchSampler( - train_set, batch_size=args.train_batch_size, shuffle=True) - - batchify_fn = lambda samples, fn=Tuple( - Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input_ids - Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64" - ), # attention_mask - Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64" - ), # decoder_input_ids - Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # labels - ): fn(samples) - train_data_loader = DataLoader(dataset=train_set, - batch_sampler=train_batch_sampler, - num_workers=0, - collate_fn=batchify_fn, - return_list=True) - dev_set = dev_set.map(trans_func, lazy=True) - dev_batch_sampler = BatchSampler(dev_set, - batch_size=args.eval_batch_size, - shuffle=False) - dev_data_loader = DataLoader(dataset=dev_set, - batch_sampler=dev_batch_sampler, - num_workers=0, - collate_fn=batchify_fn, - return_list=True) - - if paddle.distributed.get_world_size() > 1: - model = paddle.DataParallel(model) - - num_training_steps = args.max_steps if args.max_steps > 0 else ( - len(train_data_loader) * args.num_train_epochs) - warmup = args.warmup_steps if args.warmup_steps > 0 else args.warmup_proportion - - lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps, - warmup) - - # Generate parameter names needed to perform weight decay. - # All bias and LayerNorm parameters are excluded. - decay_params = [ - p.name for n, p in model.named_parameters() - if not any(nd in n for nd in ["bias", "norm"]) - ] - optimizer = paddle.optimizer.AdamW( - learning_rate=lr_scheduler, - epsilon=args.adam_epsilon, - parameters=model.parameters(), - weight_decay=args.weight_decay, - apply_decay_param_fun=lambda x: x in decay_params) - - if args.use_amp: - scaler = paddle.amp.GradScaler(init_loss_scaling=args.scale_loss) - global_step = 0 - tic_train = time.time() - for epoch in tqdm(range(args.num_train_epochs), desc="Epoch"): - for step, batch in tqdm(enumerate(train_data_loader), - desc="Train step", - total=len(train_data_loader)): - global_step += 1 - input_ids, attention_mask, decoder_input_ids, labels = batch - with paddle.amp.auto_cast( - args.use_amp, - custom_white_list=["layer_norm", "softmax", "gelu"]): - output = model(input_ids, - attention_mask, - decoder_input_ids, - labels=labels) - loss = output[0] - if args.use_amp: - scaled_loss = scaler.scale(loss) - scaled_loss.backward() - scaler.minimize(optimizer, scaled_loss) - else: - loss.backward() - optimizer.step() - lr_scheduler.step() - optimizer.clear_grad() - if global_step % args.logging_steps == 0: - logger.info( - "global step %d/%d, epoch: %d, batch: %d, rank_id: %s, loss: %f, lr: %.10f, speed: %.4f step/s" - % (global_step, num_training_steps, epoch, step, - paddle.distributed.get_rank(), loss, optimizer.get_lr(), - args.logging_steps / (time.time() - tic_train))) - tic_train = time.time() - if global_step % args.save_steps == 0 or global_step == num_training_steps: - tic_eval = time.time() - evaluate(model, dev_data_loader, tokenizer, - args.ignore_pad_token_for_loss, args.min_target_length, - args.max_target_length) - logger.info("eval done total : %s s" % (time.time() - tic_eval)) - if paddle.distributed.get_rank() == 0: - output_dir = os.path.join( - args.output_dir, "t5_model_%d.pdparams" % global_step) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Need better way to get inner model of DataParallel - model_to_save = model._layers if isinstance( - model, paddle.DataParallel) else model - model_to_save.save_pretrained(output_dir) - tokenizer.save_pretrained(output_dir) - if global_step >= num_training_steps: - return - if paddle.distributed.get_rank() == 0: - output_dir = os.path.join(args.output_dir, - "t5_model_final_%d.pdparams" % global_step) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Need better way to get inner model of DataParallel - model_to_save = model._layers if isinstance( - model, paddle.DataParallel) else model - model_to_save.save_pretrained(output_dir) - tokenizer.save_pretrained(output_dir) - - -if __name__ == "__main__": - args = parse_args() - pprint(args) - do_train(args) diff --git a/slm/examples/question_generation/t5/utils.py b/slm/examples/question_generation/t5/utils.py deleted file mode 100644 index b5ee191b66..0000000000 --- a/slm/examples/question_generation/t5/utils.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import evaluate -import nltk -import numpy as np - -from paddlenlp.metrics import BLEU - - -def convert_example( - example, - tokenizer, - decoder_start_token_id, - max_source_length, - max_target_length, - ignore_pad_token_for_loss=True, - is_train=True, -): - """ - Convert an example into necessary features. - """ - # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results - # in one example possible giving several features when a context is long, each of those features having a - # context that overlaps a bit the context of the previous feature. - # NOTE: Almost the same functionality as HuggingFace's prepare_train_features function. The main difference is - # that HugggingFace uses ArrowTable as basic data structure, while we use list of dictionary instead. - context = example["context"] - question = example["question"] - try: - answer = example["answers"][0] - except: - print(example["context"]) - print(example["question"]) - print(example["answers"]) - print(example["answer_starts"]) - print(example["is_impossible"]) - - input_seq = f"answer: {answer} context: {context} " - output_seq = f"question: {question} " - - outputs = tokenizer( - output_seq, - max_seq_len=max_target_length, - pad_to_max_seq_len=True, - truncation_strategy="longest_first", - ) - - output_ids = [decoder_start_token_id] + outputs["input_ids"][:-1] - - if ignore_pad_token_for_loss: - # Replace all tokenizer.pad_token_id in the outputs by -100 when we want to ignore padding in the loss. - outputs["input_ids"] = [(l if l != tokenizer.pad_token_id else -100) for l in outputs["input_ids"]] - - if is_train: - inputs = tokenizer( - input_seq, - max_seq_len=max_source_length, - pad_to_max_seq_len=True, - truncation_strategy="longest_first", - return_attention_mask=True, - return_length=False, - ) - return inputs["input_ids"], inputs["attention_mask"], output_ids, outputs["input_ids"] - else: - inputs = tokenizer( - input_seq, - max_seq_len=max_source_length, - pad_to_max_seq_len=True, - truncation_strategy="longest_first", - return_attention_mask=True, - return_length=True, - ) - return inputs["input_ids"], inputs["attention_mask"], inputs["length"], output_ids, outputs["input_ids"] - - -def compute_metrics(preds, labels, tokenizer, ignore_pad_token_for_loss=True): - def compute_bleu(predictions, references, rouge_types=None, use_stemmer=True): - bleu1 = BLEU(n_size=1) - bleu2 = BLEU(n_size=2) - bleu3 = BLEU(n_size=3) - bleu4 = BLEU(n_size=4) - assert len(predictions) == len(references) - for i in range(len(predictions)): - bleu1.add_inst(predictions[i], [references[i]]) - bleu2.add_inst(predictions[i], [references[i]]) - bleu3.add_inst(predictions[i], [references[i]]) - bleu4.add_inst(predictions[i], [references[i]]) - result = { - "BLEU-1": bleu1.score() * 100, - "BLEU-2": bleu2.score() * 100, - "BLEU-3": bleu3.score() * 100, - "BLEU-4": bleu4.score() * 100, - } - return result - - def compute_bleu_hf(predictions, references, rouge_types=None, use_stemmer=True): - predictions = [" ".join(prediction) for prediction in predictions] - references = [[" ".join(reference)] for reference in references] - - bleu = evaluate.load("bleu") - assert len(predictions) == len(references) - bleu1_results = bleu.compute(predictions=predictions, references=references, max_order=1) - bleu2_results = bleu.compute(predictions=predictions, references=references, max_order=2) - bleu3_results = bleu.compute(predictions=predictions, references=references, max_order=3) - bleu4_results = bleu.compute(predictions=predictions, references=references, max_order=4) - - result = { - "BLEU-1": bleu1_results["bleu"] * 100, - "BLEU-2": bleu2_results["bleu"] * 100, - "BLEU-3": bleu3_results["bleu"] * 100, - "BLEU-4": bleu4_results["bleu"] * 100, - } - return result - - def post_process_text(preds, labels): - preds = [pred.strip() for pred in preds] - labels = [label.strip() for label in labels] - preds = [pred.strip("question:") for pred in preds] - labels = [label.strip("question:") for label in labels] - labels = [label.strip() for label in labels] - - # expects newline after each sentence - preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in preds] - labels = ["\n".join(nltk.sent_tokenize(label)) for label in labels] - - preds = [pred.split() for pred in preds] - labels = [label.split() for label in labels] - - return preds, labels - - def post_process_seq(seq, bos_idx, eos_idx, output_bos=False, output_eos=False): - """ - Post-process the decoded sequence. - """ - eos_pos = len(seq) - 1 - for i, idx in enumerate(seq): - if idx == eos_idx: - eos_pos = i - break - seq = [idx for idx in seq[: eos_pos + 1] if (output_bos or idx != bos_idx) and (output_eos or idx != eos_idx)] - return seq - - if ignore_pad_token_for_loss: - labels = np.asarray(labels) - labels = np.where(labels != -100, labels, tokenizer.pad_token_id) - decoded_preds, decoded_labels = [], [] - for pred, label in zip(preds, labels): - pred_id = post_process_seq(pred, tokenizer.bos_token_id, tokenizer.eos_token_id) - label_id = post_process_seq(label, tokenizer.bos_token_id, tokenizer.eos_token_id) - decoded_preds.append(tokenizer.decode(pred_id)) - decoded_labels.append(tokenizer.decode(label_id)) - decoded_preds, decoded_labels = post_process_text(decoded_preds, decoded_labels) - # bleu_result = compute_bleu(decoded_preds, decoded_labels) - bleu_result = compute_bleu_hf(decoded_preds, decoded_labels) - return bleu_result, decoded_preds, decoded_labels diff --git a/slm/examples/question_generation/unimo-text/README.md b/slm/examples/question_generation/unimo-text/README.md deleted file mode 100644 index 226837d061..0000000000 --- a/slm/examples/question_generation/unimo-text/README.md +++ /dev/null @@ -1,343 +0,0 @@ -# 问题生成 - - -**目录** -- [问题生成](#问题生成) - - [简介](#简介) - - - - [开箱即用](#开箱即用) - - [训练定制](#训练定制) - - [环境依赖](#环境依赖) - - [代码结构说明](#代码结构说明) - - [问题生成应用定制训练全流程介绍](#问题生成定制训练全流程介绍) - - [数据准备](#数据准备) - - [数据加载](#数据加载) - - [数据处理](#数据处理) - - [从本地文件创建数据集-可选](#从本地文件创建数据集-可选) - - [模型训练](#模型训练) - - [模型预测](#模型预测) - - [模型转换部署](#模型转换部署) - - [FasterTransformer 加速及模型静态图导出](#fastertransformer 加速及模型静态图导出) - - [模型部署](#模型部署) - - [References](#references) - -## 简介 -Question Generation（QG），即问题生成，指的是给定一段上下文，自动生成一个流畅且符合上下文主题的问句。问题生成通常可以分为，无答案问题生成和有答案问题生成，这里只关注应用更广的有答案问题生成。 - -问题生成技术在教育、咨询、搜索、推荐等多个领域均有着巨大的应用价值。具体来说，问题生成可广泛应用于问答系统语料库构建，事实性问题生成，教育行业题库生成，对话提问，聊天机器人意图理解，对话式搜索意图提问，闲聊机器人主动提问等等场景。 - -本项目是基于预训练语言模型 UNIMO-Text 的问题生成，具有以下优势： - -- 效果领先。基于百度自研中文预训练语言模型 UNIMO-Text，并提供基于模版策略和大规模多领域问题生成数据集训练的通用问题生成预训练模型`unimo-text-1.0-question-generation`。 -- 开箱即用。本项目提供 TaskFlow 接口，无需训练，仅需几行代码便可预测。 -- 高性能推理。本项目基于 FasterTransformer 进行推理加速，能够提供更高性能的推理体验，优化后的推理模型在 dureader_qg 开发集的推理耗时缩短为优化前的1/5。 -- 训练推理部署全流程打通。本项目提供了全面的定制训练流程，从数据准备、模型训练预测，到模型推理部署，一应俱全。 - - - -## 开箱即用 -PaddleNLP 提供开箱即用的产业级 NLP 预置任务能力，无需训练，一键预测。 -#### 支持单条、批量预测 -```python ->>> from paddlenlp import Taskflow -# 默认模型为 unimo-text-1.0-dureader_qg ->>> question_generator = Taskflow("question_generation") -# 单条输入 ->>> question_generator([ - {"context": "奇峰黄山千米以上的山峰有77座，整座黄山就是一座花岗岩的峰林，自古有36大峰，36小峰，最高峰莲花峰、最险峰天都峰和观日出的最佳点光明顶构成黄山的三大主峰。", "answer": "莲花峰"} - ]) -''' - ['黄山最高峰是什么'] -''' -# 多条输入 ->>> question_generator([ - {"context": "奇峰黄山千米以上的山峰有77座，整座黄山就是一座花岗岩的峰林，自古有36大峰，36小峰，最高峰莲花峰、最险峰天都峰和观日出的最佳点光明顶构成黄山的三大主峰。", "answer": "莲花峰"}, - {"context": "弗朗索瓦·韦达外文名：franciscusvieta国籍：法国出生地：普瓦图出生日期：1540年逝世日期：1603年12月13日职业：数学家主要成就：为近代数学的发展奠定了基础。", "answer": "法国"} - ]) -''' - ['黄山最高峰是什么', '弗朗索瓦是哪里人'] -''' -``` -关键配置参数说明： -* `model`：可选模型，默认为 unimo-text-1.0-dureader_qg，支持的模型有["unimo-text-1.0", "unimo-text-1.0-dureader_qg", "unimo-text-1.0-question-generation", "unimo-text-1.0-question-generation-dureader_qg"]。 - -具体参数配置可参考[Taskflow 文档](../../../../docs/zh/model_zoo/taskflow.md)。 - -## 训练定制 - -### 环境依赖 -- nltk -- evaluate -- tqdm - -安装方式：`pip install -r requirements.txt` - -### 代码结构说明 - -以下是本项目主要代码结构及说明： - -```text -├── deploy # 部署 -│ ├── paddle_inference # PaddleInference高性能推理部署 -│ │ ├── inference_unimo_text.py # 推理部署脚本 -│ │ └── README.md # 说明文档 -│ └── paddle_serving -│ ├── config.yml # 配置文件 -│ ├── pipeline_client.py # 客户端程序 -│ ├── pipeline_service.py # 服务器程序 -│ └── README.md # 说明文档 -├── export_model.py # 动态图参数导出静态图参数脚本 -├── train.py # 训练脚本 -├── predict.py # 预测评估脚本 -├── utils.py # 工具函数脚本 -└── README.md # 说明文档 -``` - -### 问题生成定制训练全流程介绍 -接下来，我们将按数据准备、训练、预测、推理部署等四个阶段对问题生成应用的全流程进行介绍。 -1. **数据准备** -- 默认使用中文问题生成数据集 DuReader_QG 进行实验，该数据集已集成到 PaddleNLP。 -- 如果已有标注好的本地数据集，我们需要根据将数据集整理为文档要求的格式，请参考[从本地文件创建数据集（可选）](#从本地文件创建数据集（可选）)。 - -2. **模型训练** - -- 数据准备完成后，可以开始使用我们的数据集对预训练模型进行微调训练。我们可以根据任务需求，调整可配置参数，选择使用 GPU 或 CPU 进行模型训练，脚本默认保存在开发集最佳表现模型。中文任务默认使用`unimo-text-1.0`模型，unimo-text-1.0还支持 large 模型。此外本项目还提供基于大规模多领域问题生成数据集训练的通用问题生成预训练模型`unimo-text-1.0-question-generation`，详见[UNIMO 模型汇总](https://paddlenlp.readthedocs.io/zh/latest/model_zoo/transformers/UNIMO/contents.html)，用户可以根据任务和设备需求进行选择。 - - -3. **模型预测** - -- 训练结束后，我们可以加载保存的最佳模型进行模型测试，打印模型预测结果。 - -4. **模型转换部署** -- 在现实部署场景中，我们通常不仅对模型的精度表现有要求，也需要考虑模型性能上的表现。我们可以使用模型裁剪进一步压缩模型体积，问题生成应用已提供裁剪 API 对上一步微调后的模型进行裁剪，模型裁剪之后会默认导出静态图模型。 - -- 模型部署需要将保存的最佳模型参数（动态图）导出成静态图参数，用于后续的推理部署。 - -- 问题生成应用提供了基于 Paddle Serving 的本地部署 predictor，并且支持在 GPU 设备使用 Faster Generation 进行加速。 - -- 问题生成应用提供了基于 Paddle Serving 的服务端部署方案。 - -### 数据准备 -#### 数据加载 -[**DuReader_QG**数据集](https://www.luge.ai/#/luge/dataDetail?id=8)是一个中文问题生成数据集，我们使用该数据集作为应用案例进行实验。**DuReader_QG**中的数据主要由由上下文、问题、答案3个主要部分组成，其任务描述为给定上下文 p 和答案 a，生成自然语言表述的问题 q，且该问题符合段落和上下文的限制。 - -为了方便用户快速测试，PaddleNLP Dataset API 内置了 DuReader_QG 数据集，一键即可完成数据集加载，示例代码如下： - -```python -from paddlenlp.datasets import load_dataset -train_ds, dev_ds = load_dataset('dureader_qg', splits=('train', 'dev')) -``` - -#### 数据处理 -针对**DuReader_QG**数据集，我们需要将 QA 任务格式的数据进行转换从而得到 text2text 形式的数据，我们默认使用模版的方式构造输入数据，默认模版如下，其他形式输入数据用户可以在 convert_example 函数中自行定义。 -```text -答案: 上下文: -问题: -``` - -#### 从本地文件创建数据集-可选 -在许多情况下，我们需要使用本地数据集来训练我们的问题生成模型，本项目支持使用固定格式本地数据集文件进行训练。 -使用本地文件，只需要在模型训练时指定`train_file` 为本地训练数据地址，`predict_file` 为本地测试数据地址即可。 - -本地数据集目录结构如下： - -```text -data/ -├── train.json # 训练数据集文件 -├── dev.json # 开发数据集文件 -└── test.json # 可选，待预测数据文件 -``` -本地数据集文件格式如下： -- train.json/dev.json/test.json 文件格式： -```text -{ - "context": , - "answer": , - "question": , -} -... -``` -- train.json/dev.json/test.json 文件样例： -```text -{ - "context": "欠条是永久有效的,未约定还款期限的借款合同纠纷,诉讼时效自债权人主张债权之日起计算,时效为2年。 根据《中华人民共和国民法通则》第一百三十五条:向人民法院请求保护民事权利的诉讼时效期间为二年,法律另有规定的除外。 第一百三十七条:诉讼时效期间从知道或者应当知道权利被侵害时起计算。但是,从权利被侵害之日起超过二十年的,人民法院不予保护。有特殊情况的,人民法院可以延长诉讼时效期间。 第六十二条第(四)项:履行期限不明确的,债务人可以随时履行,债权人也可以随时要求履行,但应当给对方必要的准备时间。", - "answer": "永久有效", - "question": "欠条的有效期是多久" -} -... -``` - -更多数据集读取格式详见[数据集加载](https://paddlenlp.readthedocs.io/zh/latest/data_prepare/dataset_load.html#)和[自定义数据集](https://paddlenlp.readthedocs.io/zh/latest/data_prepare/dataset_self_defined.html)。 - -### 模型训练 -运行如下命令即可在样例训练集上进行 finetune，并在样例验证集上进行验证。 -```shell -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -# 例如使用1号和2号卡，则：`--gpu 1,2` -unset CUDA_VISIBLE_DEVICES -python -m paddle.distributed.launch --gpus "1,2" --log_dir ./unimo/finetune/log train.py \ - --dataset_name=dureader_qg \ - --model_name_or_path="unimo-text-1.0" \ - --save_dir=./unimo/finetune/checkpoints \ - --output_path ./unimo/finetune/predict.txt \ - --logging_steps=100 \ - --save_steps=500 \ - --epochs=20 \ - --batch_size=16 \ - --learning_rate=1e-5 \ - --warmup_proportion=0.02 \ - --weight_decay=0.01 \ - --max_seq_len=512 \ - --max_target_len=30 \ - --do_train \ - --do_predict \ - --max_dec_len=20 \ - --min_dec_len=3 \ - --num_return_sequences=1 \ - --template=1 \ - --device=gpu -``` - - -关键参数释义如下： -- `gpus` 指示了训练所用的 GPU，使用多卡训练可以指定多个 GPU 卡号，例如 --gpus "0,1"。 -- `dataset_name` 数据集名称，当`train_file`和`predict_file`为 None 时将加载`dataset_name`的训练集和开发集，默认为`dureader_qg`。 -- `train_file` 本地训练数据地址，数据格式必须与`dataset_name`所指数据集格式相同，默认为 None。 -- `predict_file` 本地测试数据地址，数据格式必须与`dataset_name`所指数据集格式相同，默认为 None。 -- `model_name_or_path` 指示了 finetune 使用的具体预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的预训练模型。如果使用本地的预训练模型，可以配置本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 预训练模型 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - | 可选预训练模型 | - |---------------------------------| - | unimo-text-1.0 | - | unimo-text-1.0-large | - | unimo-text-1.0-question-generation | - - - -- `save_dir` 表示模型的保存路径。 -- `output_path` 表示预测结果的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `save_steps` 表示模型保存及评估间隔。 -- `seed` 表示随机数生成器的种子。 -- `epochs` 表示训练轮数。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `learning_rate` 表示基础学习率大小，将于 learning rate scheduler 产生的值相乘作为当前学习率。 -- `weight_decay` 表示 AdamW 优化器中使用的 weight_decay 的系数。 -- `warmup_proportion` 表示学习率逐渐升高到基础学习率（即上面配置的 learning_rate）所需要的迭代数占总步数的比例。 -- `max_seq_len` 模型输入序列的最大长度。 -- `max_target_len` 模型训练时标签的最大长度。 -- `min_dec_len` 模型生成序列的最小长度。 -- `max_dec_len` 模型生成序列的最大长度。 -- `do_train` 是否进行训练。 -- `do_predict` 是否进行预测，在验证集上会自动评估。 -- `device` 表示使用的设备，从 gpu 和 cpu 中选择。 -- `template` 表示使用的模版，从[0, 1, 2, 3, 4]中选择，0表示不选择模版，1表示使用默认模版。 - -程序运行时将会自动进行训练和验证，训练过程中会自动保存模型在指定的`save_dir`中。如： - -```text -./unimo/finetune/checkpoints -├── model_1000 -│ ├── model_config.json -│ ├── model_state.pdparams -│ ├── special_tokens_map.json -│ ├── tokenizer_config.json -│ └── vocab.txt -└── ... -``` - -**NOTE:** 如需恢复模型训练，`model_name_or_path`配置本地模型的目录地址即可。 - -微调的模型在 dureader_qg 验证集上有如下结果(指标为 BLEU-4)，其中`unimo-text-1.0-dureader_qg-w/o-template`表示不使用模版策略微调的结果，`unimo-text-1.0-large-dureader_qg`表示使用 large 模型微调的结果，`unimo-text-1.0-question-generation-dureader_qg`表示在通用问题生成预训练模型`unimo-text-1.0-question-generation`上微调的结果： - -| model_name | DuReaderQG | -| :-----------------------------: | :-----------: | -| unimo-text-1.0-dureader_qg-w/o-template | 39.61 | -| unimo-text-1.0-dureader_qg | 41.08 | -| unimo-text-1.0-large-dureader_qg | 41.51 | -| unimo-text-1.0-question-generation-dureader_qg | 44.02 | - -### 模型预测 - -运行下方脚本可以使用训练好的模型进行预测。 - -```shell -export CUDA_VISIBLE_DEVICES=0 -python -u predict.py \ - --dataset_name=dureader_qg \ - --model_name_or_path=your_model_path \ - --output_path=./predict.txt \ - --logging_steps=100 \ - --batch_size=16 \ - --max_seq_len=512 \ - --max_target_len=30 \ - --do_predict \ - --max_dec_len=20 \ - --min_dec_len=3 \ - --template=1 \ - --device=gpu -``` -关键参数释义如下： -- `output_path` 表示预测输出结果保存的文件路径，默认为./predict.txt。 -- `model_name_or_path` 指示了 finetune 使用的具体预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的微调好的预训练模型。如果使用本地的预训练模型，可以配置本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 预训练模型 model_state.pdparams。 - -### 模型转换部署 - -#### FasterTransformer 加速及模型静态图导出 - -使用动态图训练结束之后，可以通过[静态图导出脚本](export_model.py)实现基于 FasterTransformer 的高性能预测加速，并将动态图参数导出成静态图参数，静态图参数保存在`output_path`指定路径中。运行方式： - -```shell -python export_model.py \ - --model_name_or_path ./checkpoint \ - --inference_model_dir ./export_checkpoint \ - --max_dec_len 50 \ - --use_fp16_decoding -``` -关键参数释义如下： - -* `model_name_or_path`：动态图训练保存的参数路径；默认为"./checkpoint"。 -* `inference_model_dir`：静态图图保存的参数路径；默认为"./export_checkpoint"。 -* `max_dec_len`：最大输出长度。 -* `use_fp16_decoding`:是否使用 fp16解码进行预测。 - -执行命令后将会自动导出模型到指定的 `inference_model_dir` 中，保存模型文件结构如下所示： - -```text -├── unimo_text.pdiparams -├── unimo_text.pdiparams.info -└── unimo_text.pdmodel -``` - -#### 模型部署 -本项目提供多种不同场景的部署方案，请根据实际情况进行选择： -|部署方案|特色|场景|硬件| -|-|-|-|-| -|Paddle Inference
服务端／云端|通用性|模型算法复杂
硬件高性能|X86 CPU
NVIDIA 全系列 GPU
龙芯／飞腾等国产 CPU
昆仑／昇腾／海光 DCU 等 AI 加速芯片 -|Paddle Serving
服务化|高并发|大流量、高并发、低延时、高吞吐
资源弹性调控应对服务流量变化
支持模型组合、加密、热更新等|X86/Arm CPU
NVIDIA GPU
昆仑／昇腾等 - - -问题生成应用已打通多种场景部署方案，点击链接获取具体的使用教程。 -- [Paddle Inference 推理 (Python)](./deploy/paddle_inference/README.md) -- [Paddle Serving 服务化部署（Python）](./deploy/paddle_serving/README.md) - - -## References -Zheng, Chujie, and Minlie Huang. "Exploring prompt-based few-shot learning for grounded dialog generation." arXiv preprint arXiv:2109.06513 (2021). -Li, Wei, et al. "Unimo: Towards unified-modal understanding and generation via cross-modal contrastive learning." arXiv preprint arXiv:2012.15409 (2020). diff --git a/slm/examples/question_generation/unimo-text/deploy/paddle_inference/README.md b/slm/examples/question_generation/unimo-text/deploy/paddle_inference/README.md deleted file mode 100644 index e46081132e..0000000000 --- a/slm/examples/question_generation/unimo-text/deploy/paddle_inference/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Paddle Inference 部署 -本文档将介绍如何使用[Paddle Inference](https://paddle-inference.readthedocs.io/en/latest/guides/introduction/index_intro.html#paddle-inference)工具进行问题生成应用高性能推理推理部署。 - -**目录** - * [背景介绍](#背景介绍) - * [导出预测部署模型](#导出预测部署模型) - * [基于 Python 预测](#基于 Python 预测) - - -## 背景介绍 -Paddle inference 和主框架的 Model.predict 均可实现推理预测，Paddle Inference 是飞桨的原生推理库， 作用于服务器端和云端，提供高性能的推理能力，主框架的 Model 对象是一个具备训练、测试、推理的神经网络。相比于 Model.predict，inference 可使用 MKLDNN、CUDNN、TensorRT 进行预测加速。Model.predict 适用于训练好的模型直接进行预测，paddle inference 适用于对推理性能、通用性有要求的用户，针对不同平台不同的应用场景进行了深度的适配优化，保证模型在服务器端即训即用，快速部署。由于 Paddle Inference 能力直接基于飞桨的训练算子，因此它支持飞桨训练出的所有模型的推理。 - - - -Paddle Inference Python 端预测部署主要包含两个步骤： -- 导出预测部署模型 -- 基于 Python 预测 - - -## 导出预测部署模型 -部署时需要使用预测格式的模型（即动态图转静态图操作）。预测格式模型相对训练格式模型而言，在拓扑上裁剪掉了预测不需要的算子，并且会做特定部署优化。具体操作详见[FasterTransformer 加速及模型静态图导出](../../README.md)。 - -## 基于 Python 预测 - - -在终端输入以下命令可在 GPU 上进行预测： -```shell -python deploy/paddle_inference/inference.py \ - --inference_model_dir ./export_checkpoint \ - --model_name_or_path "unimo-text-1.0" \ - --predict_file predict_file_name \ - --output_path output_path_name \ - --device gpu \ -``` - - -经静态图转换，FastTransformer 性能优化，Paddle Inference 加速后的部署模型在 dureader_qg devset 的预测时间为27.74秒，相较于未优化前169.24秒，耗时缩减为原来的16.39%。 -关键参数释义如下： -* `inference_model_dir`：用于高性能推理的静态图模型参数路径，默认为"./export_checkpoint"。 -* `model_name_or_path`：tokenizer 对应模型或路径，默认为"unimo-text-1.0"。 -* `dataset_name`：数据集名称，默认为`dureader_qg`。 -* `predict_file`：本地预测数据地址，数据格式必须与`dataset_name`所指数据集格式相同，默认为 None，当为 None 时默认加载`dataset_name`的 dev 集。 -* `output_path`：表示预测结果的保存路径。 -* `device`：推理时使用的设备，可选项["gpu"]，默认为"gpu"。 -* `batch_size`：进行推理时的批大小，默认为16。 -* `precision`：当使用 TensorRT 进行加速推理时，所使用的 TensorRT 精度，可选项["fp32", "fp16"]，默认为"fp32"。 - - - - - diff --git a/slm/examples/question_generation/unimo-text/deploy/paddle_inference/infer_utils.py b/slm/examples/question_generation/unimo-text/deploy/paddle_inference/infer_utils.py deleted file mode 100644 index f043e3f932..0000000000 --- a/slm/examples/question_generation/unimo-text/deploy/paddle_inference/infer_utils.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import random -from functools import partial - -import numpy as np -import paddle -import paddle.distributed as dist -from paddle.io import BatchSampler, DataLoader, DistributedBatchSampler - -from paddlenlp.data import Pad - - -def postprocess_response(token_ids, tokenizer): - """Post-process the decoded sequence. Truncate from the first .""" - eos_pos = len(token_ids) - for i, tok_id in enumerate(token_ids): - if tok_id == tokenizer.mask_token_id: - eos_pos = i - break - token_ids = token_ids[:eos_pos] - tokens = tokenizer.convert_ids_to_tokens(token_ids) - tokens = tokenizer.merge_subword(tokens) - return tokens - - -def print_args(args): - print("----------- Configuration Arguments -----------") - for arg, value in sorted(vars(args).items()): - print("%s: %s" % (arg, value)) - print("------------------------------------------------") - - -def set_seed(seed): - # Use the same data seed(for data shuffle) for all procs to guarantee data - # consistency after sharding. - random.seed(seed) - np.random.seed(seed) - # Maybe different op seeds(for dropout) for different procs is better. - paddle.seed(seed + dist.get_rank()) - - -def convert_example( - example, tokenizer, max_seq_len=512, max_target_len=128, max_title_len=256, mode="test", template=0 -): - """Convert all examples into necessary features.""" - if mode == "pretrain" or mode == "pretrain_test": - context = example["context"] - answer = example["answer"] - target = example["target"] - - source = "答案：" + answer + tokenizer.sep_token + "上下文：" + context - title = None - - elif mode == "train" or mode == "test": - target = None - if "source" in example and "title" in example: - source = example["source"] - title = None - if "title" in example.keys(): - title = example["title"] - elif "context" in example and "answer" in example: - source = example["context"] - title = None - if "answer" in example.keys(): - title = example["answer"] - else: - assert False, "Source and title are not in the input dictionary, nor are context and answer." - if "target" in example.keys(): - target = example["target"] - - if template == 1: - source = "答案：" + title + tokenizer.sep_token + "上下文：" + source - title = None - if target: - target = "问题：" + target - elif template == 2: - source = "答案：" + title + tokenizer.sep_token + "上下文：" + source - title = None - if target: - target = "在已知答案的前提下，问题：" + target - elif template == 3: - source = "这是一个问题生成任务，根据提供的答案和上下文，来生成问题。" + title + tokenizer.sep_token + "上下文：" + source - title = None - if target: - target = "问题：" + target - - if mode == "train" or mode == "pretrain": - tokenized_example = tokenizer.gen_encode( - source, - title=title, - target=target, - max_seq_len=max_seq_len, - max_target_len=max_target_len, - max_title_len=max_title_len, - return_position_ids=True, - return_length=True, - ) - target_start = tokenized_example["input_ids"].index(tokenizer.cls_token_id, 1) - target_end = tokenized_example["seq_len"] - # Use to gather the logits corresponding to the labels during training - tokenized_example["masked_positions"] = list(range(target_start, target_end - 1)) - tokenized_example["labels"] = tokenized_example["input_ids"][target_start + 1 : target_end] - - return tokenized_example - - elif mode == "test" or mode == "pretrain_test": - tokenized_example = tokenizer.gen_encode( - source, - title=title, - max_seq_len=max_seq_len, - max_title_len=max_title_len, - add_start_token_for_decoding=True, - return_position_ids=True, - return_length=True, - ) - - if "target" in example and example["target"]: - tokenized_example["target"] = example["target"] - return tokenized_example - - -def batchify_fn(batch_examples, pad_val, mode="test"): - def pad_mask(batch_attention_mask): - batch_size = len(batch_attention_mask) - max_len = max(map(len, batch_attention_mask)) - attention_mask = np.ones((batch_size, max_len, max_len), dtype="float32") * -1e9 - for i, mask_data in enumerate(attention_mask): - seq_len = len(batch_attention_mask[i]) - mask_data[-seq_len:, -seq_len:] = np.array(batch_attention_mask[i], dtype="float32") - # In order to ensure the correct broadcasting mechanism, expand one - # dimension to the second dimension (n_head of Transformer). - attention_mask = np.expand_dims(attention_mask, axis=1) - return attention_mask - - pad_func = Pad(pad_val=pad_val, pad_right=False, dtype="int64") - - input_ids = pad_func([example["input_ids"] for example in batch_examples]) - token_type_ids = pad_func([example["token_type_ids"] for example in batch_examples]) - position_ids = pad_func([example["position_ids"] for example in batch_examples]) - - attention_mask = pad_mask([example["attention_mask"] for example in batch_examples]) - - seq_len = np.asarray([example["seq_len"] for example in batch_examples], dtype="int32") - - if mode == "train" or mode == "pretrain": - max_len = max([example["seq_len"] for example in batch_examples]) - masked_positions = np.concatenate( - [ - np.array(example["masked_positions"]) + (max_len - example["seq_len"]) + i * max_len - for i, example in enumerate(batch_examples) - ] - ) - labels = np.concatenate([np.array(example["labels"], dtype="int64") for example in batch_examples]) - return input_ids, token_type_ids, position_ids, attention_mask, masked_positions, labels - elif mode == "test" or mode == "pretrain_test": - return input_ids, token_type_ids, position_ids, attention_mask, seq_len - - -def create_data_loader(dataset, tokenizer, args, mode="test"): - trans_func = partial(convert_example, tokenizer=tokenizer, mode="test", template=1) - dataset = dataset.map(trans_func, lazy=True) - if mode == "pretrain": - batch_sampler = DistributedBatchSampler(dataset, batch_size=args.batch_size, shuffle=True) - elif mode == "train": - batch_sampler = DistributedBatchSampler(dataset, batch_size=args.batch_size, shuffle=True) - elif mode == "test" or mode == "pretrain_test": - batch_sampler = BatchSampler(dataset, batch_size=args.batch_size // 2, shuffle=False) - collate_fn = partial(batchify_fn, pad_val=tokenizer.pad_token_id, mode=mode) - data_loader = DataLoader(dataset, batch_sampler=batch_sampler, collate_fn=collate_fn, return_list=True) - return dataset, data_loader - - -def post_process_sum(token_ids, tokenizer): - """Post-process the decoded sequence. Truncate from the first .""" - eos_pos = len(token_ids) - for i, tok_id in enumerate(token_ids): - if tok_id == tokenizer.mask_token_id: - eos_pos = i - break - token_ids = token_ids[:eos_pos] - tokens = tokenizer.convert_ids_to_tokens(token_ids) - tokens = tokenizer.merge_subword(tokens) - special_tokens = ["[UNK]"] - tokens = [token for token in tokens if token not in special_tokens] - return token_ids, tokens - - -def remove_template(instr): - """Remove template prefix of decoded sequence.""" - outstr = instr.strip("问题：") - outstr = instr.strip("在已知答案的前提下，问题：") - return outstr - - -def select_sum(ids, scores, tokenizer, max_dec_len=None, num_return_sequences=1): - results = [] - group = [] - tmp = [] - if scores is not None: - ids = ids.numpy() - scores = scores.numpy() - - if len(ids) != len(scores) or (len(ids) % num_return_sequences) != 0: - raise ValueError( - "the length of `ids` is {}, but the `num_return_sequences` is {}".format( - len(ids), num_return_sequences - ) - ) - - for pred, score in zip(ids, scores): - pred_token_ids, pred_tokens = post_process_sum(pred, tokenizer) - num_token = len(pred_token_ids) - - target = "".join(pred_tokens) - target = remove_template(target) - - # not ending - if max_dec_len is not None and num_token >= max_dec_len: - score -= 1e3 - - tmp.append([target, score]) - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - preds = sorted(preds, key=lambda x: -x[1]) - results.append(preds[0][0]) - else: - ids = ids.numpy() - - for pred in ids: - pred_token_ids, pred_tokens = post_process_sum(pred, tokenizer) - num_token = len(pred_token_ids) - response = "".join(pred_tokens) - response = remove_template(response) - - # TODO: Support return scores in FT. - tmp.append([response]) - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - results.append(preds[0][0]) - - return results diff --git a/slm/examples/question_generation/unimo-text/deploy/paddle_inference/inference.py b/slm/examples/question_generation/unimo-text/deploy/paddle_inference/inference.py deleted file mode 100644 index 5e15b4a814..0000000000 --- a/slm/examples/question_generation/unimo-text/deploy/paddle_inference/inference.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import time -from pprint import pprint - -import numpy as np -import paddle -from infer_utils import create_data_loader, postprocess_response, select_sum -from paddle import inference - -from paddlenlp.datasets import load_dataset -from paddlenlp.ops.ext_utils import load -from paddlenlp.transformers import UNIMOTokenizer - - -def setup_args(): - """Setup arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument( - "--inference_model_dir", default="./infer_model", type=str, help="Path to save inference model of UNIMOText. " - ) - parser.add_argument( - "--model_name_or_path", type=str, default="unimo-text-1.0", help="The path or shortcut name of the tokenizer." - ) - parser.add_argument( - "--device", default="gpu", choices=["gpu", "cpu", "xpu"], help="Device selected for inference." - ) - parser.add_argument( - "--use_tensorrt", - default=False, - type=eval, - choices=[True, False], - help="Whether to use inference engin TensorRT when using gpu.", - ) - parser.add_argument( - "--enable_mkldnn", - default=False, - type=eval, - choices=[True, False], - help="Enable to use mkldnn to speed up when using cpu.", - ) - parser.add_argument("--cpu_threads", default=10, type=int, help="Number of threads to predict when using cpu.") - parser.add_argument( - "--precision", default="fp32", type=str, choices=["fp32", "fp16", "int8"], help="The tensorrt precision." - ) - parser.add_argument("--batch_size", type=int, default=16, help="Batch size per GPU/CPU for training.") - parser.add_argument( - "--output_path", type=str, default="./predict.txt", help="The file path where the infer result will be saved." - ) - parser.add_argument("--logging_steps", type=int, default=100, help="Log every X updates steps.") - parser.add_argument("--dataset_name", type=str, default="dureader_qg", help="The name of the dataset to load.") - parser.add_argument("--predict_file", type=str, required=False, default=None, help="Predict data path.") - parser.add_argument("--max_dec_len", type=int, default=20, help="The maximum sequence length of decoding.") - parser.add_argument( - "--num_return_sequences", - type=int, - default=1, - help="The numbers of returned sequences for one input in generation.", - ) - - args = parser.parse_args() - return args - - -def setup_predictor(args): - """Setup inference predictor.""" - # Load FastGeneration lib. - load("FastGeneration", verbose=True) - model_file = os.path.join(args.inference_model_dir, "unimo_text.pdmodel") - params_file = os.path.join(args.inference_model_dir, "unimo_text.pdiparams") - if not os.path.exists(model_file): - raise ValueError("not find model file path {}".format(model_file)) - if not os.path.exists(params_file): - raise ValueError("not find params file path {}".format(params_file)) - config = inference.Config(model_file, params_file) - if args.device == "gpu": - config.enable_use_gpu(100, 0) - config.switch_ir_optim() - config.enable_memory_optim() - config.disable_glog_info() - - precision_map = { - "fp16": inference.PrecisionType.Half, - "fp32": inference.PrecisionType.Float32, - "int8": inference.PrecisionType.Int8, - } - precision_mode = precision_map[args.precision] - if args.use_tensorrt: - config.enable_tensorrt_engine( - max_batch_size=args.batch_size, min_subgraph_size=30, precision_mode=precision_mode - ) - elif args.device == "cpu": - config.disable_gpu() - if args.enable_mkldnn: - config.enable_mkldnn() - config.set_mkldnn_cache_capacity(10) - - config.set_cpu_math_library_num_threads(args.cpu_threads) - elif args.device == "xpu": - config.enable_xpu(100) - predictor = inference.create_predictor(config) - return predictor - - -@paddle.no_grad() -def infer_one(args, predictor, inputs=None): - """Use predictor to inference.""" - tokenizer = UNIMOTokenizer.from_pretrained("unimo-text-1.0") - - if not inputs: - inputs = { - "context": "奇峰黄山千米以上的山峰有77座，整座黄山就是一座花岗岩的峰林，自古有36大峰，36小峰，最高峰莲花峰、最险峰天都峰和观日出的最佳点光明顶构成黄山的三大主峰。", - "answer": "莲花峰", - } - - inputs = "答案：" + inputs["answer"] + tokenizer.sep_token + "上下文：" + inputs["context"] - data = tokenizer.gen_encode( - inputs, add_start_token_for_decoding=True, return_length=True, is_split_into_words=False - ) - - input_handles = {} - for name in predictor.get_input_names(): - input_handles[name] = predictor.get_input_handle(name) - if name == "attention_mask": - input_handles[name].copy_from_cpu(np.expand_dims(np.asarray(data[name], dtype="float32"), axis=(0, 1))) - else: - input_handles[name].copy_from_cpu(np.asarray(data[name], dtype="int32").reshape([1, -1])) - - output_handles = [predictor.get_output_handle(name) for name in predictor.get_output_names()] - - predictor.run() - - output = [output_handle.copy_to_cpu() for output_handle in output_handles] - - for sample in output[0][:, :, 0].tolist(): - print("".join(postprocess_response(sample, tokenizer))) - - -@paddle.no_grad() -def infer(args, predictor, data_loader, tokenizer): - print("Infer begin...") - pred_ref = [] - total_time = 0.0 - start_time = time.time() - for step, inputs in enumerate(data_loader, 1): - input_ids, token_type_ids, position_ids, attention_mask, seq_len = inputs - data = { - "input_ids": input_ids, - "token_type_ids": token_type_ids, - "position_ids": position_ids, - "attention_mask": attention_mask, - "seq_len": seq_len, - } - - input_handles = {} - for name in predictor.get_input_names(): - input_handles[name] = predictor.get_input_handle(name) - if name == "attention_mask": - input_handles[name].copy_from_cpu(np.asarray(data[name], dtype="float32")) - else: - input_handles[name].copy_from_cpu(np.asarray(data[name], dtype="int32")) - - output_handles = [predictor.get_output_handle(name) for name in predictor.get_output_names()] - - predictor.run() - - output = [output_handle.copy_to_cpu() for output_handle in output_handles] - - ids = output[0] - scores = output[1] - - ids = paddle.to_tensor(ids, dtype="int32")[:, 0, :] - scores = paddle.to_tensor(scores, dtype="float32") - - total_time += time.time() - start_time - if step % args.logging_steps == 0: - print("step %d - %.3fs/step" % (step, total_time / args.logging_steps)) - total_time = 0.0 - - results = select_sum(ids, scores, tokenizer, args.max_dec_len, args.num_return_sequences) - - pred_ref.extend(results) - start_time = time.time() - - with open(args.output_path, "w", encoding="utf-8") as fout: - for ref in pred_ref: - fout.write(ref + "\n") - - print("\nSave inference result into: %s" % args.output_path) - - if "target" in data_loader.dataset[0].keys(): - with open(args.output_path + ".reference.txt", "w", encoding="utf-8") as fout: - targets = [example["target"] for example in data_loader.dataset] - for target in targets: - fout.write(target + "\n") - - -if __name__ == "__main__": - args = setup_args() - pprint(args) - - predictor = setup_predictor(args) - tokenizer = UNIMOTokenizer.from_pretrained(args.model_name_or_path) - ds = load_dataset(args.dataset_name, splits="dev", data_files=args.predict_file) - ds, data_loader = create_data_loader(ds, tokenizer, args, "test") - - time_begin = time.time() - infer(args, predictor, data_loader, tokenizer) - print("inference cost time:", time.time() - time_begin) diff --git a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/README.md b/slm/examples/question_generation/unimo-text/deploy/paddle_serving/README.md deleted file mode 100644 index dbf0496554..0000000000 --- a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/README.md +++ /dev/null @@ -1,142 +0,0 @@ -# Paddle Serving 服务化部署 - -本文档将介绍如何使用[Paddle Serving](https://github.com/PaddlePaddle/Serving/blob/develop/README_CN.md)工具部署问题生成在线服务。 - -## 目录 -- [Paddle Serving 服务化部署](#paddle-serving 服务化部署) - - [目录](#目录) - - [背景介绍](#背景介绍) - - [环境准备](#环境准备) - - [安装 Paddle Serving](#安装 paddle-serving) - - [模型转换](#模型转换) - - [pipeline 部署](#pipeline 部署) - - [修改配置文件](#修改配置文件) - - [server 启动服务](#server 启动服务) - - [client 发送服务请求](#client 发送服务请求) - -## 背景介绍 -Paddle Serving 依托深度学习框架 PaddlePaddle 旨在帮助深度学习开发者和企业提供高性能、灵活易用的工业级在线推理服务。Paddle Serving 支持 RESTful、gRPC、bRPC 等多种协议，提供多种异构硬件和多种操作系统环境下推理解决方案，和多种经典预训练模型示例。集成高性能服务端推理引擎 Paddle Inference 和端侧引擎 Paddle Lite。设计并实现基于有向无环图(DAG) 的异步流水线高性能推理框架，具有多模型组合、异步调度、并发推理、动态批量、多卡多流推理、请求缓存等特性。 - -Paddle Serving Python 端预测部署主要包含以下步骤： -- 环境准备 -- 模型转换 -- 部署模型 - -## 环境准备 -### 安装 Paddle Serving -安装 client 和 serving app，用于向服务发送请求: -```shell -pip install paddle_serving_app paddle_serving_client -``` -安装 server，用于启动服务，根据服务器设备选择安装 CPU server 或 GPU server： - -- 安装 CPU server -```shell -pip install paddle_serving_server -``` -- 安装 GPU server, 注意选择跟本地环境一致的命令 -```shell -# CUDA10.2 + Cudnn7 + TensorRT6 -pip install paddle-serving-server-gpu==0.8.3.post102 # -i https://pypi.tuna.tsinghua.edu.cn/simple -# CUDA10.1 + TensorRT6 -pip install paddle-serving-server-gpu==0.8.3.post101 # -i https://pypi.tuna.tsinghua.edu.cn/simple -# CUDA11.2 + TensorRT8 -pip install paddle-serving-server-gpu==0.8.3.post112 # -i https://pypi.tuna.tsinghua.edu.cn/simple -``` - -**NOTE:** -- 可以开启国内清华镜像源来加速下载 -- 如果要安装最新版本的 PaddleServing 参考[链接](https://github.com/PaddlePaddle/Serving/blob/develop/doc/Latest_Packages_CN.md)。 - - -## 模型转换 - -使用 Paddle Serving 做服务化部署时，需要将保存的 inference 模型转换为 serving 易于部署的模型。 - -用已安装的 paddle_serving_client 将静态图参数模型转换成 serving 格式。关于如何使用将训练后的动态图模型转为静态图模型详见[FasterTransformer 加速及模型静态图导出](../../README.md)。 - -模型转换命令如下： -```shell -python -m paddle_serving_client.convert --dirname ./export_checkpoint \ - --model_filename unimo_text.pdmodel \ - --params_filename unimo_text.pdiparams \ - --serving_server ./deploy/paddle_serving/export_checkpoint_server \ - --serving_client ./deploy/paddle_serving/export_checkpoint_client -``` -关键参数释义如下： -* `dirname`：静态图模型文件夹地址。 -* `model_filename`：模型文件名。 -* `params_filename`：模型参数名。 -* `serving_server`：server 的模型文件和配置文件路径，默认"serving_server"。 -* `serving_client`：client 的配置文件路径，默认"serving_client"。 - -更多参数可通过以下命令查询： -```shell -python -m paddle_serving_client.convert --help -``` -模型转换完成后，会在./delopy/paddle_serving 文件夹多出 export_checkpoint_server 和 export_checkpoint_client 的文件夹，文件夹目录格式如下： -``` -export_checkpoint_server/ -├── unimo_text.pdiparams -├── unimo_text.pdmodel -├── serving_server_conf.prototxt -└── serving_server_conf.stream.prototxt -export_checkpoint_server/ -├── serving_client_conf.prototxt -└── serving_client_conf.stream.prototxt -``` - -## pipeline 部署 - -paddle_serving 目录包含启动 pipeline 服务和发送预测请求的代码，包括： -``` -paddle_serving/ -├──config.yml # 启动服务端的配置文件 -├──pipeline_client.py # 发送pipeline预测请求的脚本 -└──pipeline_service.py # 启动pipeline服务端的脚本 -``` - -### 修改配置文件 -目录中的`config.yml`文件解释了每一个参数的含义，可以根据实际需要修改其中的配置。 - -### server 启动服务 -修改好配置文件后，执行下面命令启动服务: -```shell -cd deploy/paddle_serving -# 启动服务，运行日志保存在log.txt -python pipeline_service.py &> log.txt & -``` -成功启动服务后，log.txt 中会打印类似如下日志 -``` ---- Running analysis [ir_graph_to_program_pass] -I0901 12:09:27.248943 12190 analysis_predictor.cc:1035] ======= optimize end ======= -I0901 12:09:27.249596 12190 naive_executor.cc:102] --- skip [feed], feed -> seq_len -I0901 12:09:27.249608 12190 naive_executor.cc:102] --- skip [feed], feed -> attention_mask -I0901 12:09:27.249614 12190 naive_executor.cc:102] --- skip [feed], feed -> token_type_ids -I0901 12:09:27.249617 12190 naive_executor.cc:102] --- skip [feed], feed -> input_ids -I0901 12:09:27.250080 12190 naive_executor.cc:102] --- skip [_generated_var_3], fetch -> fetch -I0901 12:09:27.250090 12190 naive_executor.cc:102] --- skip [transpose_0.tmp_0], fetch -> fetch -[2022-09-01 12:09:27,251] [ INFO] - Already cached /root/.paddlenlp/models/unimo-text-1.0/unimo-text-1.0-vocab.txt -[2022-09-01 12:09:27,269] [ INFO] - tokenizer config file saved in /root/.paddlenlp/models/unimo-text-1.0/tokenizer_config.json -[2022-09-01 12:09:27,269] [ INFO] - Special tokens file saved in /root/.paddlenlp/models/unimo-text-1.0/special_tokens_map.json -[PipelineServicer] succ init -[OP Object] init success -2022/09/01 12:09:27 start proxy service -``` - -### client 发送服务请求 -执行以下命令发送文本摘要服务请求： -```shell -cd deploy/paddle_serving -python pipeline_client.py -``` -注意执行客户端请求时关闭代理，并根据实际情况修改 server_url 地址(启动服务所在的机器) - -成功运行后，输出打印如下: -``` -time cost :0.03429532051086426 seconds --------------------- -input: {'context': '平安银行95511电话按9转报案人工服务。 1.寿险 :95511转1 2.信用卡 95511转2 3.平安银行 95511转3 4.一账通 95511转4转8 5.产险 95511转5 6.养老险团体险 95511转6 7.健康险 95511转7 8.证券 95511转8 9.车险报案95511转9 0.重听', 'answer': '95511'} -output: 问题：平安银行人工服务电话 --------------------- -``` diff --git a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/config.yml b/slm/examples/question_generation/unimo-text/deploy/paddle_serving/config.yml deleted file mode 100644 index 1cc918e1ba..0000000000 --- a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/config.yml +++ /dev/null @@ -1,59 +0,0 @@ -#rpc端口, rpc_port和http_port不允许同时为空。当rpc_port为空且http_port不为空时，会自动将rpc_port设置为http_port+1 -rpc_port: 18011 - -#http端口, rpc_port和http_port不允许同时为空。当rpc_port可用且http_port为空时，不自动生成http_port -http_port: 9999 - -#worker_num, 最大并发数。 -#当build_dag_each_worker=True时, 框架会创建worker_num个进程，每个进程内构建grpcSever和DAG -#当build_dag_each_worker=False时，框架会设置主线程grpc线程池的max_workers=worker_num -worker_num: 10 - -#build_dag_each_worker, False，框架在进程内创建一条DAG；True，框架会每个进程内创建多个独立的DAG -build_dag_each_worker: false - -dag: - #op资源类型, True, 为线程模型；False，为进程模型 - is_thread_op: True - - #重试次数 - retry: 1 - - #使用性能分析, True，生成Timeline性能数据，对性能有一定影响；False为不使用 - use_profile: false - tracer: - interval_s: 10 - -op: - question_generation: - #并发数，is_thread_op=True时，为线程并发；否则为进程并发 - concurrency: 11 - - #当op配置没有server_endpoints时，从local_service_conf读取本地服务配置 - local_service_conf: - #client类型，包括brpc, grpc和local_predictor.local_predictor不启动Serving服务，进程内预测 - client_type: local_predictor - - #模型路径 - model_config: ../../unimo/serving/export_checkpoint_server - - #Fetch结果列表，以client_config中fetch_var的alias_name为准，不设置默认取全部输出变量 - # fetch_list: ["_generated_var_3", "slice_0.tmp_0"] - - # device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu - device_type: 1 - - #计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡 - devices: "0" - - #开启MKLDNN加速 - use_mkldnn: False - - #thread_num - thread_num: 12 - - #ir_optim - ir_optim: False - - #开启tensorrt后，进行优化的子图包含的最少节点数 - #min_subgraph_size: 10 \ No newline at end of file diff --git a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/infer_utils.py b/slm/examples/question_generation/unimo-text/deploy/paddle_serving/infer_utils.py deleted file mode 100644 index f043e3f932..0000000000 --- a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/infer_utils.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import random -from functools import partial - -import numpy as np -import paddle -import paddle.distributed as dist -from paddle.io import BatchSampler, DataLoader, DistributedBatchSampler - -from paddlenlp.data import Pad - - -def postprocess_response(token_ids, tokenizer): - """Post-process the decoded sequence. Truncate from the first .""" - eos_pos = len(token_ids) - for i, tok_id in enumerate(token_ids): - if tok_id == tokenizer.mask_token_id: - eos_pos = i - break - token_ids = token_ids[:eos_pos] - tokens = tokenizer.convert_ids_to_tokens(token_ids) - tokens = tokenizer.merge_subword(tokens) - return tokens - - -def print_args(args): - print("----------- Configuration Arguments -----------") - for arg, value in sorted(vars(args).items()): - print("%s: %s" % (arg, value)) - print("------------------------------------------------") - - -def set_seed(seed): - # Use the same data seed(for data shuffle) for all procs to guarantee data - # consistency after sharding. - random.seed(seed) - np.random.seed(seed) - # Maybe different op seeds(for dropout) for different procs is better. - paddle.seed(seed + dist.get_rank()) - - -def convert_example( - example, tokenizer, max_seq_len=512, max_target_len=128, max_title_len=256, mode="test", template=0 -): - """Convert all examples into necessary features.""" - if mode == "pretrain" or mode == "pretrain_test": - context = example["context"] - answer = example["answer"] - target = example["target"] - - source = "答案：" + answer + tokenizer.sep_token + "上下文：" + context - title = None - - elif mode == "train" or mode == "test": - target = None - if "source" in example and "title" in example: - source = example["source"] - title = None - if "title" in example.keys(): - title = example["title"] - elif "context" in example and "answer" in example: - source = example["context"] - title = None - if "answer" in example.keys(): - title = example["answer"] - else: - assert False, "Source and title are not in the input dictionary, nor are context and answer." - if "target" in example.keys(): - target = example["target"] - - if template == 1: - source = "答案：" + title + tokenizer.sep_token + "上下文：" + source - title = None - if target: - target = "问题：" + target - elif template == 2: - source = "答案：" + title + tokenizer.sep_token + "上下文：" + source - title = None - if target: - target = "在已知答案的前提下，问题：" + target - elif template == 3: - source = "这是一个问题生成任务，根据提供的答案和上下文，来生成问题。" + title + tokenizer.sep_token + "上下文：" + source - title = None - if target: - target = "问题：" + target - - if mode == "train" or mode == "pretrain": - tokenized_example = tokenizer.gen_encode( - source, - title=title, - target=target, - max_seq_len=max_seq_len, - max_target_len=max_target_len, - max_title_len=max_title_len, - return_position_ids=True, - return_length=True, - ) - target_start = tokenized_example["input_ids"].index(tokenizer.cls_token_id, 1) - target_end = tokenized_example["seq_len"] - # Use to gather the logits corresponding to the labels during training - tokenized_example["masked_positions"] = list(range(target_start, target_end - 1)) - tokenized_example["labels"] = tokenized_example["input_ids"][target_start + 1 : target_end] - - return tokenized_example - - elif mode == "test" or mode == "pretrain_test": - tokenized_example = tokenizer.gen_encode( - source, - title=title, - max_seq_len=max_seq_len, - max_title_len=max_title_len, - add_start_token_for_decoding=True, - return_position_ids=True, - return_length=True, - ) - - if "target" in example and example["target"]: - tokenized_example["target"] = example["target"] - return tokenized_example - - -def batchify_fn(batch_examples, pad_val, mode="test"): - def pad_mask(batch_attention_mask): - batch_size = len(batch_attention_mask) - max_len = max(map(len, batch_attention_mask)) - attention_mask = np.ones((batch_size, max_len, max_len), dtype="float32") * -1e9 - for i, mask_data in enumerate(attention_mask): - seq_len = len(batch_attention_mask[i]) - mask_data[-seq_len:, -seq_len:] = np.array(batch_attention_mask[i], dtype="float32") - # In order to ensure the correct broadcasting mechanism, expand one - # dimension to the second dimension (n_head of Transformer). - attention_mask = np.expand_dims(attention_mask, axis=1) - return attention_mask - - pad_func = Pad(pad_val=pad_val, pad_right=False, dtype="int64") - - input_ids = pad_func([example["input_ids"] for example in batch_examples]) - token_type_ids = pad_func([example["token_type_ids"] for example in batch_examples]) - position_ids = pad_func([example["position_ids"] for example in batch_examples]) - - attention_mask = pad_mask([example["attention_mask"] for example in batch_examples]) - - seq_len = np.asarray([example["seq_len"] for example in batch_examples], dtype="int32") - - if mode == "train" or mode == "pretrain": - max_len = max([example["seq_len"] for example in batch_examples]) - masked_positions = np.concatenate( - [ - np.array(example["masked_positions"]) + (max_len - example["seq_len"]) + i * max_len - for i, example in enumerate(batch_examples) - ] - ) - labels = np.concatenate([np.array(example["labels"], dtype="int64") for example in batch_examples]) - return input_ids, token_type_ids, position_ids, attention_mask, masked_positions, labels - elif mode == "test" or mode == "pretrain_test": - return input_ids, token_type_ids, position_ids, attention_mask, seq_len - - -def create_data_loader(dataset, tokenizer, args, mode="test"): - trans_func = partial(convert_example, tokenizer=tokenizer, mode="test", template=1) - dataset = dataset.map(trans_func, lazy=True) - if mode == "pretrain": - batch_sampler = DistributedBatchSampler(dataset, batch_size=args.batch_size, shuffle=True) - elif mode == "train": - batch_sampler = DistributedBatchSampler(dataset, batch_size=args.batch_size, shuffle=True) - elif mode == "test" or mode == "pretrain_test": - batch_sampler = BatchSampler(dataset, batch_size=args.batch_size // 2, shuffle=False) - collate_fn = partial(batchify_fn, pad_val=tokenizer.pad_token_id, mode=mode) - data_loader = DataLoader(dataset, batch_sampler=batch_sampler, collate_fn=collate_fn, return_list=True) - return dataset, data_loader - - -def post_process_sum(token_ids, tokenizer): - """Post-process the decoded sequence. Truncate from the first .""" - eos_pos = len(token_ids) - for i, tok_id in enumerate(token_ids): - if tok_id == tokenizer.mask_token_id: - eos_pos = i - break - token_ids = token_ids[:eos_pos] - tokens = tokenizer.convert_ids_to_tokens(token_ids) - tokens = tokenizer.merge_subword(tokens) - special_tokens = ["[UNK]"] - tokens = [token for token in tokens if token not in special_tokens] - return token_ids, tokens - - -def remove_template(instr): - """Remove template prefix of decoded sequence.""" - outstr = instr.strip("问题：") - outstr = instr.strip("在已知答案的前提下，问题：") - return outstr - - -def select_sum(ids, scores, tokenizer, max_dec_len=None, num_return_sequences=1): - results = [] - group = [] - tmp = [] - if scores is not None: - ids = ids.numpy() - scores = scores.numpy() - - if len(ids) != len(scores) or (len(ids) % num_return_sequences) != 0: - raise ValueError( - "the length of `ids` is {}, but the `num_return_sequences` is {}".format( - len(ids), num_return_sequences - ) - ) - - for pred, score in zip(ids, scores): - pred_token_ids, pred_tokens = post_process_sum(pred, tokenizer) - num_token = len(pred_token_ids) - - target = "".join(pred_tokens) - target = remove_template(target) - - # not ending - if max_dec_len is not None and num_token >= max_dec_len: - score -= 1e3 - - tmp.append([target, score]) - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - preds = sorted(preds, key=lambda x: -x[1]) - results.append(preds[0][0]) - else: - ids = ids.numpy() - - for pred in ids: - pred_token_ids, pred_tokens = post_process_sum(pred, tokenizer) - num_token = len(pred_token_ids) - response = "".join(pred_tokens) - response = remove_template(response) - - # TODO: Support return scores in FT. - tmp.append([response]) - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - results.append(preds[0][0]) - - return results diff --git a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_client.py b/slm/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_client.py deleted file mode 100644 index 9172d68cfd..0000000000 --- a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_client.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import time - -from paddle_serving_server.pipeline import PipelineClient - - -class Runner(object): - def __init__( - self, - server_url: str, - ): - self.client = PipelineClient() - self.client.connect([server_url]) - - def Run(self, data): - inputs = data - start_time = time.time() - ret = self.client.predict(feed_dict={"inputs": inputs}) - end_time = time.time() - print("time cost :{} seconds".format(end_time - start_time)) - if not ret.value: - print("Fail to fetch summary.") - # ret is special class but a dict - for d, s in zip(data, eval(ret.value[0])): - print("--------------------") - print("input: ", d) - print("output: ", s) - print("--------------------") - return - - -if __name__ == "__main__": - server_url = "127.0.0.1:18011" - runner = Runner(server_url) - requests = [ - {"context": "奇峰黄山千米以上的山峰有77座，整座黄山就是一座花岗岩的峰林，自古有36大峰，36小峰，最高峰莲花峰、最险峰天都峰和观日出的最佳点光明顶构成黄山的三大主峰。", "answer": "莲花峰"} - ] - runner.Run(requests) diff --git a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_service.py b/slm/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_service.py deleted file mode 100644 index e9b6af9fe5..0000000000 --- a/slm/examples/question_generation/unimo-text/deploy/paddle_serving/pipeline_service.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from infer_utils import batchify_fn, convert_example, postprocess_response -from paddle_serving_server.web_service import Op, WebService - -from paddlenlp.ops.ext_utils import load -from paddlenlp.transformers import UNIMOTokenizer - -_LOGGER = logging.getLogger(__name__) - - -class UnimoTextOp(Op): - """Op for unimo_text.""" - - def init_op(self): - self.tokenizer = UNIMOTokenizer.from_pretrained("unimo-text-1.0") - - def preprocess(self, input_dicts, data_id, log_id): - # Convert input format - ((_, input_dict),) = input_dicts.items() - data = input_dict["inputs"] - if isinstance(data, str) and "array(" in data: - data = eval(data) - else: - _LOGGER.error("input value {}is not supported.".format(data)) - examples = [convert_example(i, self.tokenizer) for i in data] - input_ids, token_type_ids, position_ids, attention_mask, seq_len = batchify_fn( - examples, self.tokenizer.pad_token_id - ) - new_dict = {} - new_dict["input_ids"] = input_ids - new_dict["token_type_ids"] = token_type_ids - new_dict["attention_mask"] = attention_mask - new_dict["seq_len"] = seq_len - # the first return must be a dict or a list of dict, the dict corresponding to a batch of model input - return new_dict, False, None, "" - - def postprocess(self, input_dicts, fetch_dict, data_id, log_id): - # keyname refer to export_checkpoint_client/serving_client_conf.prototxt - ids = fetch_dict["transpose_0.tmp_0"][:, 0, :].tolist() - # scores = fetch_dict["_generated_var_3"][:, 0].tolist() - - results = ["".join(postprocess_response(sample, self.tokenizer)) for sample in ids] - new_dict = {} - new_dict["outputs"] = str(results) - # the first return must be a dict or a list of dict, the dict corresponding to a batch of model output - return new_dict, None, "" - - -class UnimoTextService(WebService): - def get_pipeline_response(self, read_op): - return UnimoTextOp(name="question_generation", input_ops=[read_op]) - - -if __name__ == "__main__": - # Load FastGeneration lib. - load("FastGeneration", verbose=True) - service = UnimoTextService(name="question_generation") - service.prepare_pipeline_config("config.yml") - service.run_service() diff --git a/slm/examples/question_generation/unimo-text/export_model.py b/slm/examples/question_generation/unimo-text/export_model.py deleted file mode 100644 index ea8d9c1a2e..0000000000 --- a/slm/examples/question_generation/unimo-text/export_model.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -from pprint import pprint - -import paddle - -from paddlenlp.ops import FasterUNIMOText -from paddlenlp.transformers import UNIMOLMHeadModel, UNIMOTokenizer -from paddlenlp.utils.log import logger - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--model_name_or_path", default="checkpoint", type=str, help="The model name to specify the UNIMOText to use. ") - parser.add_argument("--inference_model_dir", default="./export_checkpoint", type=str, help="Path to save inference model of UNIMOText. ") - parser.add_argument("--topk", default=4, type=int, help="The number of candidate to procedure top_k sampling. ") - parser.add_argument("--topp", default=1.0, type=float, help="The probability threshold to procedure top_p sampling. ") - parser.add_argument("--max_dec_len", default=20, type=int, help="Maximum output length. ") - parser.add_argument("--min_dec_len", default=3, type=int, help="Minimum output length. ") - parser.add_argument("--temperature", default=1.0, type=float, help="The temperature to set. ") - parser.add_argument("--num_return_sequences", default=1, type=int, help="The number of returned sequences. ") - parser.add_argument("--use_fp16_decoding", action="store_true", help="Whether to use fp16 decoding to predict. ") - parser.add_argument("--decoding_strategy", default="beam_search", choices=["sampling", "beam_search"], type=str, help="The main strategy to decode. ") - parser.add_argument("--num_beams", default=6, type=int, help="The number of candidate to procedure beam search. ") - parser.add_argument("--diversity_rate", default=0.0, type=float, help="The diversity rate to procedure beam search. ") - parser.add_argument("--length_penalty", default=1.2, type=float, help="The diversity rate to procedure beam search. ") - args = parser.parse_args() - return args - - -def do_predict(args): - place = "gpu" - place = paddle.set_device(place) - - model_name_or_path = args.model_name_or_path - model = UNIMOLMHeadModel.from_pretrained(model_name_or_path) - tokenizer = UNIMOTokenizer.from_pretrained(model_name_or_path) - - unimo_text = FasterUNIMOText(model=model, - use_fp16_decoding=args.use_fp16_decoding, - trans_out=True) - - # Set evaluate mode - unimo_text.eval() - - # Convert dygraph model to static graph model - unimo_text = paddle.jit.to_static( - unimo_text, - input_spec=[ - # input_ids - paddle.static.InputSpec(shape=[None, None], dtype="int64"), - # token_type_ids - paddle.static.InputSpec(shape=[None, None], dtype="int64"), - # attention_mask - paddle.static.InputSpec(shape=[None, 1, None, None], - dtype="float32"), - # seq_len - paddle.static.InputSpec(shape=[None], dtype="int64"), - args.max_dec_len, - args.min_dec_len, - args.topk, - args.topp, - args.num_beams, # num_beams. Used for beam_search. - args.decoding_strategy, - tokenizer.cls_token_id, # cls/bos - tokenizer.mask_token_id, # mask/eos - tokenizer.pad_token_id, # pad - args.diversity_rate, # diversity rate. Used for beam search. - args.temperature, - args.num_return_sequences, - args.length_penalty, - ]) - - # Save converted static graph model - paddle.jit.save(unimo_text, - os.path.join(args.inference_model_dir, "unimo_text")) - logger.info("UNIMOText has been saved to {}.".format( - args.inference_model_dir)) - - -if __name__ == "__main__": - args = parse_args() - pprint(args) - do_predict(args) diff --git a/slm/examples/question_generation/unimo-text/gen_utils.py b/slm/examples/question_generation/unimo-text/gen_utils.py deleted file mode 100644 index f786850098..0000000000 --- a/slm/examples/question_generation/unimo-text/gen_utils.py +++ /dev/null @@ -1,316 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import random -from functools import partial - -import numpy as np -import paddle -import paddle.distributed as dist -from paddle.io import BatchSampler, DataLoader, DistributedBatchSampler - -from paddlenlp.data import Pad - - -def print_args(args): - print("----------- Configuration Arguments -----------") - for arg, value in sorted(vars(args).items()): - print("%s: %s" % (arg, value)) - print("------------------------------------------------") - - -def set_seed(seed): - # Use the same data seed(for data shuffle) for all procs to guarantee data - # consistency after sharding. - random.seed(seed) - np.random.seed(seed) - # Maybe different op seeds(for dropout) for different procs is better. - paddle.seed(seed + dist.get_rank()) - - -def convert_example( - example, tokenizer, max_seq_len=512, max_target_len=128, max_title_len=256, mode="train", template=0 -): - """Convert all examples into necessary features.""" - if mode == "pretrain" or mode == "pretrain_test": - context = example["context"] - answer = example["answer"] - target = example["target"] - source = "答案：" + answer + tokenizer.sep_token + "上下文：" + context - title = None - - elif mode == "train" or mode == "test": - target = None - title = None - if "source" in example and "title" in example: - source = example["source"] - if "title" in example.keys(): - title = example["title"] - elif "context" in example and "answer" in example: - source = example["context"] - if "answer" in example.keys(): - title = example["answer"] - else: - assert False, "Source and title are not in the input dictionary, nor are context and answer." - if "target" in example.keys(): - target = example["target"] - elif "question" in example.keys(): - target = example["question"] - - if template == 1: - source = "答案：" + title + tokenizer.sep_token + "上下文：" + source - title = None - if target: - target = "问题：" + target - elif template == 2: - source = "答案：" + title + tokenizer.sep_token + "上下文：" + source - title = None - if target: - target = "在已知答案的前提下，问题：" + target - elif template == 3: - source = "这是一个问题生成任务，根据提供的答案和上下文，来生成问题。" + title + tokenizer.sep_token + "上下文：" + source - title = None - if target: - target = "问题：" + target - elif template == 4: - prompt_common = example["prompt_common"] - prompt_domain = example["prompt_domain"] - source = ( - prompt_common - + " " - + tokenizer.sep_token - + " " - + "".join( - [" " + tokenizer.cls_token + " " + one + " " + tokenizer.sep_token + " " for one in prompt_domain] - ) - + " " - + tokenizer.cls_token - + " " - + "答案：" - + title - + " " - + tokenizer.sep_token - + " " - + tokenizer.cls_token - + "上下文：" - + source - ) - - title = None - if target: - target = "问题：" + target - - if mode == "train" or mode == "pretrain": - tokenized_example = tokenizer.gen_encode( - source, - title=title, - target=target, - max_seq_len=max_seq_len, - max_target_len=max_target_len, - max_title_len=max_title_len, - return_position_ids=True, - return_length=True, - ) - temp_tokens = tokenizer.convert_ids_to_tokens(tokenized_example["input_ids"]) - index_list = [] - count = tokenized_example["input_ids"].count(tokenizer.cls_token_id) - # If template==4, count must be equal to 7, otherwise count must be equal to 2 - assert count == 7 or count == 2, ( - str(count) + " is not in [2, 7], temp_tokens: " + " ".join(temp_tokens) + "source: " + source - ) - index = -1 - for i in range(0, count): - index = tokenized_example["input_ids"].index(tokenizer.cls_token_id, index + 1) - index_list.append(index) - if template == 4: - tokenized_example["token_type_ids"] = ( - [2] * (index_list[1] - index_list[0]) - + [3] * (index_list[4] - index_list[1]) - + [0] * (index_list[6] - index_list[4]) - + [1] * (len(tokenized_example["input_ids"]) - index_list[6]) - ) - target_start = index_list[-1] - target_end = tokenized_example["seq_len"] - # Use to gather the logits corresponding to the labels during training - tokenized_example["masked_positions"] = list(range(target_start, target_end - 1)) - tokenized_example["labels"] = tokenized_example["input_ids"][target_start + 1 : target_end] - if template == 4: - tokenized_example["token_type_ids"] - return tokenized_example - - elif mode == "test" or mode == "pretrain_test": - tokenized_example = tokenizer.gen_encode( - source, - title=title, - max_seq_len=max_seq_len, - max_title_len=max_title_len, - add_start_token_for_decoding=True, - return_position_ids=True, - ) - - if template == 4: - # temp_tokens = tokenizer.convert_ids_to_tokens(tokenized_example['input_ids']) - index_list = [] - count = tokenized_example["input_ids"].count(tokenizer.cls_token_id) - assert count == 7, str(count) + " is not in [7]" - index = -1 - for i in range(0, count): - index = tokenized_example["input_ids"].index(tokenizer.cls_token_id, index + 1) - index_list.append(index) - tokenized_example["token_type_ids"] = ( - [2] * (index_list[1] - index_list[0]) - + [3] * (index_list[4] - index_list[1]) - + [0] * (index_list[6] - index_list[4]) - + [1] * (len(tokenized_example["input_ids"]) - index_list[6]) - ) - - if "target" in example and example["target"]: - tokenized_example["target"] = example["target"] - elif "question" in example and example["question"]: - tokenized_example["target"] = example["question"] - return tokenized_example - - -def batchify_fn(batch_examples, pad_val, mode): - def pad_mask(batch_attention_mask): - batch_size = len(batch_attention_mask) - max_len = max(map(len, batch_attention_mask)) - attention_mask = np.ones((batch_size, max_len, max_len), dtype="float32") * -1e9 - for i, mask_data in enumerate(attention_mask): - seq_len = len(batch_attention_mask[i]) - mask_data[-seq_len:, -seq_len:] = np.array(batch_attention_mask[i], dtype="float32") - # In order to ensure the correct broadcasting mechanism, expand one - # dimension to the second dimension (n_head of Transformer). - attention_mask = np.expand_dims(attention_mask, axis=1) - return attention_mask - - pad_func = Pad(pad_val=pad_val, pad_right=False, dtype="int64") - - input_ids = pad_func([example["input_ids"] for example in batch_examples]) - token_type_ids = pad_func([example["token_type_ids"] for example in batch_examples]) - position_ids = pad_func([example["position_ids"] for example in batch_examples]) - - attention_mask = pad_mask([example["attention_mask"] for example in batch_examples]) - - if mode == "train" or mode == "pretrain": - max_len = max([example["seq_len"] for example in batch_examples]) - masked_positions = np.concatenate( - [ - np.array(example["masked_positions"]) + (max_len - example["seq_len"]) + i * max_len - for i, example in enumerate(batch_examples) - ] - ) - labels = np.concatenate([np.array(example["labels"], dtype="int64") for example in batch_examples]) - return input_ids, token_type_ids, position_ids, attention_mask, masked_positions, labels - elif mode == "test" or mode == "pretrain_test": - return input_ids, token_type_ids, position_ids, attention_mask - - -def create_data_loader(dataset, tokenizer, args, mode): - trans_func = partial( - convert_example, - tokenizer=tokenizer, - max_seq_len=args.max_seq_len, - max_target_len=args.max_target_len, - max_title_len=args.max_title_len, - mode=mode, - template=args.template, - ) - dataset = dataset.map(trans_func, lazy=True) - if mode == "pretrain": - batch_sampler = DistributedBatchSampler(dataset, batch_size=args.batch_size, shuffle=True) - elif mode == "train": - batch_sampler = DistributedBatchSampler(dataset, batch_size=args.batch_size, shuffle=True) - elif mode == "test" or mode == "pretrain_test": - batch_sampler = BatchSampler(dataset, batch_size=args.batch_size // 2, shuffle=False) - collate_fn = partial(batchify_fn, pad_val=tokenizer.pad_token_id, mode=mode) - data_loader = DataLoader(dataset, batch_sampler=batch_sampler, collate_fn=collate_fn, return_list=True) - return dataset, data_loader - - -def post_process_sum(token_ids, tokenizer): - """Post-process the decoded sequence. Truncate from the first .""" - eos_pos = len(token_ids) - for i, tok_id in enumerate(token_ids): - if tok_id == tokenizer.mask_token_id: - eos_pos = i - break - token_ids = token_ids[:eos_pos] - tokens = tokenizer.convert_ids_to_tokens(token_ids) - tokens = tokenizer.merge_subword(tokens) - special_tokens = ["[UNK]"] - tokens = [token for token in tokens if token not in special_tokens] - return token_ids, tokens - - -def remove_template(instr): - """Remove template prefix of decoded sequence.""" - outstr = instr.strip("问题：") - outstr = instr.strip("在已知答案的前提下，问题：") - return outstr - - -def select_sum(ids, scores, tokenizer, max_dec_len=None, num_return_sequences=1): - results = [] - group = [] - tmp = [] - if scores is not None: - ids = ids.numpy() - scores = scores.numpy() - - if len(ids) != len(scores) or (len(ids) % num_return_sequences) != 0: - raise ValueError( - "the length of `ids` is {}, but the `num_return_sequences` is {}".format( - len(ids), num_return_sequences - ) - ) - - for pred, score in zip(ids, scores): - pred_token_ids, pred_tokens = post_process_sum(pred, tokenizer) - num_token = len(pred_token_ids) - - target = "".join(pred_tokens) - target = remove_template(target) - - # not ending - if max_dec_len is not None and num_token >= max_dec_len: - score -= 1e3 - - tmp.append([target, score]) - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - preds = sorted(preds, key=lambda x: -x[1]) - results.append(preds[0][0]) - else: - ids = ids.numpy() - - for pred in ids: - pred_token_ids, pred_tokens = post_process_sum(pred, tokenizer) - num_token = len(pred_token_ids) - response = "".join(pred_tokens) - response = remove_template(response) - - # TODO: Support return scores in FT. - tmp.append([response]) - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - results.append(preds[0][0]) - - return results diff --git a/slm/examples/question_generation/unimo-text/predict.py b/slm/examples/question_generation/unimo-text/predict.py deleted file mode 100644 index 4ec590f45b..0000000000 --- a/slm/examples/question_generation/unimo-text/predict.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import json -import time - -import paddle -import paddle.distributed as dist -from gen_utils import create_data_loader, print_args, select_sum, set_seed - -from paddlenlp.datasets import load_dataset -from paddlenlp.transformers import UNIMOLMHeadModel, UNIMOTokenizer - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser(__doc__) - parser.add_argument('--dataset_name', type=str, default='dureader_qg', help='The name of the dataset to load.') - parser.add_argument('--model_name_or_path', type=str, default='unimo-text-1.0', help='The path or shortcut name of the pre-trained model.') - parser.add_argument("--predict_file", type=str, required=False, default=None, help="Predict data path.") - parser.add_argument('--save_dir', type=str, default='./checkpoints', help='The directory where the checkpoints will be saved.') - parser.add_argument('--logging_steps', type=int, default=100, help='Log every X updates steps.') - parser.add_argument('--seed', type=int, default=1, help='Random seed for initialization.') - parser.add_argument('--batch_size', type=int, default=16, help='Batch size per GPU/CPU for training.') - parser.add_argument('--max_seq_len', type=int, default=512, help='The maximum sequence length of training.') - parser.add_argument('--max_target_len', type=int, default=30, help='The maximum target sequence length of training.') - parser.add_argument('--max_title_len', type=int, default=30, help='The maximum title sequence length of training.') - parser.add_argument('--max_dec_len', type=int, default=20, help='The maximum sequence length of decoding.') - parser.add_argument('--min_dec_len', type=int, default=3, help='The minimal sequence length of decoding.') - parser.add_argument('--num_return_sequences', type=int, default=1, help='The numbers of returned sequences for one input in generation.') - parser.add_argument('--decode_strategy', type=str, default='beam_search', help='The decode strategy in generation.') - parser.add_argument('--top_k', type=int, default=0, help='The number of highest probability vocabulary tokens to keep for top-k sampling.') - parser.add_argument('--temperature', type=float, default=1.0, help='The value used to module the next token probabilities.') - parser.add_argument('--top_p', type=float, default=1.0, help='The cumulative probability for top-p sampling.') - parser.add_argument('--num_beams', type=int, default=6, help='The number of beams for beam search.') - parser.add_argument('--length_penalty', type=float, default=1.2, help='The exponential penalty to the sequence length for beam search.') - parser.add_argument('--device', type=str, default='gpu', help='The device to select for training the model.') - parser.add_argument('--output_path', type=str, default='./predict.txt', help='The file path where the infer result will be saved.') - parser.add_argument("--do_predict", action='store_true', help="Whether to eval and predict.") - parser.add_argument("--template", type=int, default=1, help="The template used during training, select from [0, 1, 2, 3, 4].") - - args = parser.parse_args() - return args -# yapf: enable - - -def read_file(file): - with open(file, "r", encoding="utf-8") as f: - for line in f.readlines(): - line = line.strip() - if not line: - continue - line = json.loads(line) - yield line - - -def run(args): - paddle.set_device(args.device) - world_size = dist.get_world_size() - - if world_size > 1: - dist.init_parallel_env() - set_seed(args.seed) - - model = UNIMOLMHeadModel.from_pretrained(args.model_name_or_path) - tokenizer = UNIMOTokenizer.from_pretrained(args.model_name_or_path) - - if world_size > 1: - model = paddle.DataParallel(model) - - if args.predict_file: - dev_ds = load_dataset(read_file, file=args.predict_file, lazy=False) - else: - dev_ds = load_dataset(args.dataset_name, splits="dev", data_files=args.predict_file) - - dev_ds, dev_data_loader = create_data_loader(dev_ds, tokenizer, args, "test") - - if args.do_predict: - model_eval = model._layers if isinstance(model, paddle.DataParallel) else model - prediction(model_eval, dev_data_loader, args, tokenizer) - - -@paddle.no_grad() -def prediction(model, data_loader, args, tokenizer): - print("\nPred begin...") - model.eval() - pred_ref = [] - time_begin = time.time() - total_time = 0.0 - start_time = time.time() - for step, inputs in enumerate(data_loader, 1): - input_ids, token_type_ids, position_ids, attention_mask = inputs - ids, scores = model.generate( - input_ids=input_ids, - token_type_ids=token_type_ids, - position_ids=position_ids, - attention_mask=attention_mask, - max_length=args.max_dec_len, - min_length=args.min_dec_len, - decode_strategy=args.decode_strategy, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - num_return_sequences=args.num_return_sequences, - bos_token_id=tokenizer.cls_token_id, - eos_token_id=tokenizer.mask_token_id, - ) - - total_time += time.time() - start_time - if step % args.logging_steps == 0: - print("step %d - %.3fs/step" % (step, total_time / args.logging_steps)) - total_time = 0.0 - - results = select_sum(ids, scores, tokenizer, args.max_dec_len, args.num_return_sequences) - pred_ref.extend(results) - start_time = time.time() - print("Generation cost time:", time.time() - time_begin) - - with open(args.output_path, "w", encoding="utf-8") as fout: - for ref in pred_ref: - fout.write(ref + "\n") - - -if __name__ == "__main__": - args = parse_args() - print_args(args) - run(args) diff --git a/slm/examples/question_generation/unimo-text/requirements.txt b/slm/examples/question_generation/unimo-text/requirements.txt deleted file mode 100644 index 48ff8faab7..0000000000 --- a/slm/examples/question_generation/unimo-text/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -nltk==3.6.2 -evaluate==0.2.2 -tqdm==4.64.0 \ No newline at end of file diff --git a/slm/examples/question_generation/unimo-text/train.py b/slm/examples/question_generation/unimo-text/train.py deleted file mode 100644 index 73e2c15443..0000000000 --- a/slm/examples/question_generation/unimo-text/train.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import json -import os -import time - -import paddle -import paddle.distributed as dist -import paddle.nn.functional as F -from gen_utils import create_data_loader, print_args, select_sum, set_seed -from paddle.optimizer import AdamW - -from paddlenlp.datasets import load_dataset -from paddlenlp.metrics import BLEU -from paddlenlp.transformers import ( - BasicTokenizer, - LinearDecayWithWarmup, - UNIMOLMHeadModel, - UNIMOTokenizer, -) - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser(__doc__) - parser.add_argument('--dataset_name', type=str, default='dureader_qg', help='The name of the dataset to load.') - parser.add_argument('--model_name_or_path', type=str, default='unimo-text-1.0', help='The path or shortcut name of the pre-trained model.') - parser.add_argument("--train_file", type=str, required=False, default=None, help="Train data path.") - parser.add_argument("--predict_file", type=str, required=False, default=None, help="Predict data path.") - parser.add_argument('--save_dir', type=str, default='./checkpoints', help='The directory where the checkpoints will be saved.') - parser.add_argument('--logging_steps', type=int, default=100, help='Log every X updates steps.') - parser.add_argument('--save_steps', type=int, default=1000, help='Save checkpoint every X updates steps.') - parser.add_argument('--seed', type=int, default=1, help='Random seed for initialization.') - parser.add_argument('--batch_size', type=int, default=16, help='Batch size per GPU/CPU for training.') - parser.add_argument('--learning_rate', type=float, default=5e-5, help='The initial learning rate.') - parser.add_argument('--weight_decay', type=float, default=0.01, help='The weight decay for optimizer.') - parser.add_argument('--epochs', type=int, default=3, help='Total number of training epochs to perform.') - parser.add_argument('--warmup_proportion', type=float, default=0.02, help='The number of warmup steps.') - parser.add_argument('--max_grad_norm', type=float, default=1.0, help='The max value of grad norm.') - parser.add_argument('--beta1', type=float, default=0.9, help='beta1') - parser.add_argument('--beta2', type=float, default=0.98, help='beta2') - parser.add_argument('--epsilon', type=float, default=1e-6, help='epsilon') - parser.add_argument('--max_seq_len', type=int, default=512, help='The maximum sequence length of training.') - parser.add_argument('--max_target_len', type=int, default=30, help='The maximum target sequence length of training.') - parser.add_argument('--max_title_len', type=int, default=30, help='The maximum title sequence length of training.') - parser.add_argument('--max_dec_len', type=int, default=20, help='The maximum sequence length of decoding.') - parser.add_argument('--min_dec_len', type=int, default=3, help='The minimal sequence length of decoding.') - parser.add_argument('--num_return_sequences', type=int, default=1, help='The numbers of returned sequences for one input in generation.') - parser.add_argument('--decode_strategy', type=str, default='beam_search', help='The decode strategy in generation.') - parser.add_argument('--top_k', type=int, default=0, help='The number of highest probability vocabulary tokens to keep for top-k sampling.') - parser.add_argument('--temperature', type=float, default=1.0, help='The value used to module the next token probabilities.') - parser.add_argument('--top_p', type=float, default=1.0, help='The cumulative probability for top-p sampling.') - parser.add_argument('--num_beams', type=int, default=6, help='The number of beams for beam search.') - parser.add_argument('--length_penalty', type=float, default=1.2, help='The exponential penalty to the sequence length for beam search.') - parser.add_argument('--device', type=str, default='gpu', help='The device to select for training the model.') - parser.add_argument('--output_path', type=str, default='./predict.txt', help='The file path where the infer result will be saved.') - parser.add_argument("--do_train", action='store_true', help="Whether to train the model.") - parser.add_argument("--do_predict", action='store_true', help="Whether to eval and predict.") - parser.add_argument("--template", type=int, default=1, help="The template used during training, select from [0, 1, 2, 3, 4].") - - args = parser.parse_args() - return args -# yapf: enable - - -def calc_bleu_n(preds, targets, n_size=4): - assert len(preds) == len(targets), ( - "The length of pred_responses should be equal to the length of " - "target_responses. But received {} and {}.".format(len(preds), len(targets)) - ) - bleu = BLEU(n_size=n_size) - tokenizer = BasicTokenizer() - - for pred, target in zip(preds, targets): - pred_tokens = tokenizer.tokenize(pred) - target_token = tokenizer.tokenize(target) - - bleu.add_inst(pred_tokens, [target_token]) - - print("\n" + "*" * 15) - print("The auto evaluation result is:") - print("BLEU-" + str(n_size) + ":", bleu.score()) - return bleu.score() - - -def calc_bleu(preds, targets): - calc_bleu_n(preds, targets, 1) - calc_bleu_n(preds, targets, 2) - calc_bleu_n(preds, targets, 3) - bleu4_score = calc_bleu_n(preds, targets, 4) - return bleu4_score - - -def read_file(file): - with open(file, "r", encoding="utf-8") as f: - for line in f.readlines(): - line = line.strip() - if not line: - continue - line = json.loads(line) - yield line - - -def save_ckpt(model, tokenizer, save_dir, name): - output_dir = os.path.join(save_dir, "model_{}".format(name)) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Need better way to get inner model of DataParallel - model_to_save = model._layers if isinstance(model, paddle.DataParallel) else model - model_to_save.save_pretrained(output_dir) - tokenizer.save_pretrained(output_dir) - - -def run(args): - paddle.set_device(args.device) - world_size = dist.get_world_size() - - if world_size > 1: - dist.init_parallel_env() - set_seed(args.seed) - - model = UNIMOLMHeadModel.from_pretrained(args.model_name_or_path) - tokenizer = UNIMOTokenizer.from_pretrained(args.model_name_or_path) - - if world_size > 1: - model = paddle.DataParallel(model) - - if args.train_file: - train_ds = load_dataset(read_file, file=args.train_file, lazy=False) - else: - train_ds = load_dataset(args.dataset_name, splits="train", data_files=args.train_file) - if args.predict_file: - dev_ds = load_dataset(read_file, file=args.predict_file, lazy=False) - else: - dev_ds = load_dataset(args.dataset_name, splits="dev", data_files=args.predict_file) - - train_ds, train_data_loader = create_data_loader(train_ds, tokenizer, args, "train") - dev_ds, dev_data_loader = create_data_loader(dev_ds, tokenizer, args, "test") - - if args.do_train: - num_training_steps = args.epochs * len(train_data_loader) - - lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps, args.warmup_proportion) - # Generate parameter names needed to perform weight decay. - # All bias and LayerNorm parameters are excluded. - - decay_params = [p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])] - - optimizer = AdamW( - learning_rate=lr_scheduler, - parameters=model.parameters(), - weight_decay=args.weight_decay, - beta1=args.beta1, - beta2=args.beta2, - epsilon=args.epsilon, - apply_decay_param_fun=lambda x: x in decay_params, - grad_clip=paddle.nn.ClipGradByGlobalNorm(args.max_grad_norm), - ) - - step = 0 - total_time = 0.0 - best_bleu4 = 0 - for epoch in range(args.epochs): - print("\nEpoch %d/%d" % (epoch + 1, args.epochs)) - batch_start_time = time.time() - for inputs in train_data_loader: - step += 1 - labels = inputs[-1] - logits = model(*inputs[:-1]) - labels = paddle.nn.functional.one_hot(labels, num_classes=logits.shape[-1]) - labels = paddle.nn.functional.label_smooth(labels) - loss = F.cross_entropy(logits, labels, soft_label=True) - loss.backward() - - optimizer.step() - lr_scheduler.step() - optimizer.clear_grad() - - total_time += time.time() - batch_start_time - if step % args.logging_steps == 0: - ppl = paddle.exp(loss) - print( - "step %d - loss: %.4f - ppl: %.4f - lr: %.7f - %.3fs/step" - % (step, loss, ppl, optimizer.get_lr(), total_time / args.logging_steps) - ) - total_time = 0.0 - - if step % args.save_steps == 0 or step >= num_training_steps: - if dist.get_rank() == 0: - save_ckpt(model, tokenizer, args.save_dir, step) - print("Saved step {} model.\n".format(step)) - if args.do_predict: - model_eval = model._layers if isinstance(model, paddle.DataParallel) else model - bleu4 = evaluation(model_eval, dev_data_loader, args, tokenizer) - if bleu4 > best_bleu4: - print("best BLEU-4 performence has been updated: %.5f --> %.5f" % (best_bleu4, bleu4)) - best_bleu4 = bleu4 - save_ckpt(model, tokenizer, args.save_dir, "best") - - batch_start_time = time.time() - - print("\nTraining completed.") - elif args.do_predict: - model_eval = model._layers if isinstance(model, paddle.DataParallel) else model - evaluation(model_eval, dev_data_loader, args, tokenizer) - - -@paddle.no_grad() -def evaluation(model, data_loader, args, tokenizer): - print("\nEval begin...") - model.eval() - pred_ref = [] - time_begin = time.time() - total_time = 0.0 - start_time = time.time() - for step, inputs in enumerate(data_loader, 1): - input_ids, token_type_ids, position_ids, attention_mask = inputs - ids, scores = model.generate( - input_ids=input_ids, - token_type_ids=token_type_ids, - position_ids=position_ids, - attention_mask=attention_mask, - max_length=args.max_dec_len, - min_length=args.min_dec_len, - decode_strategy=args.decode_strategy, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - num_return_sequences=args.num_return_sequences, - bos_token_id=tokenizer.cls_token_id, - eos_token_id=tokenizer.mask_token_id, - ) - - total_time += time.time() - start_time - if step % args.logging_steps == 0: - print("step %d - %.3fs/step" % (step, total_time / args.logging_steps)) - total_time = 0.0 - - results = select_sum(ids, scores, tokenizer, args.max_dec_len, args.num_return_sequences) - pred_ref.extend(results) - start_time = time.time() - print("Generation cost time:", time.time() - time_begin) - - with open(args.output_path, "w", encoding="utf-8") as fout: - for ref in pred_ref: - fout.write(ref + "\n") - - with open(args.output_path + ".reference.txt", "w", encoding="utf-8") as fout: - targets = [example["target"] for example in data_loader.dataset] - for target in targets: - fout.write(target + "\n") - - print("\nSave inference result into: %s" % args.output_path) - - if "target" in data_loader.dataset[0].keys(): - targets = [example["target"] for example in data_loader.dataset] - bleu4_score = calc_bleu(pred_ref, targets) - - model.train() - return bleu4_score - - -if __name__ == "__main__": - args = parse_args() - print_args(args) - run(args) diff --git a/slm/examples/text_classification/README.md b/slm/examples/text_classification/README.md deleted file mode 100644 index 2595ac3739..0000000000 --- a/slm/examples/text_classification/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# 文本分类 - -提供了文本分类任务示例，基于 ERNIE 3.0预训练模型。 - -## Pretrained Model (PTMs) - -[Pretrained Models](./pretrained_models) 展示了如何使用以 ERNIE 3.0 为代表的预模型，在多分类、多标签、层次分类场景下，基于预训练模型微调、提示学习（小样本）、语义索引等三种不同方案进行文本分类。预训练模型文本分类打通数据标注-模型训练-模型调优-模型压缩-预测部署全流程，旨在解决细分场景应用的痛点和难点，快速实现文本分类产品落地。 diff --git a/slm/examples/text_classification/pretrained_models/README.md b/slm/examples/text_classification/pretrained_models/README.md deleted file mode 100644 index d1351d7803..0000000000 --- a/slm/examples/text_classification/pretrained_models/README.md +++ /dev/null @@ -1 +0,0 @@ -[pretrained_models](../../../applications/text_classification) diff --git a/slm/examples/text_correction/ernie-csc/README.md b/slm/examples/text_correction/ernie-csc/README.md deleted file mode 100644 index ea834f24a5..0000000000 --- a/slm/examples/text_correction/ernie-csc/README.md +++ /dev/null @@ -1,163 +0,0 @@ -# ERNIE for Chinese Spelling Correction - -## 简介 - -中文文本纠错任务是一项 NLP 基础任务，其输入是一个可能含有语法错误的中文句子，输出是一个正确的中文句子。语法错误类型很多，有多字、少字、错别字等，目前最常见的错误类型是`错别字`。大部分研究工作围绕错别字这一类型进行研究。本文实现了百度在 ACL 2021上提出结合拼音特征的 Softmask 策略的中文错别字纠错的下游任务网络，并提供预训练模型，模型结构如下： - -![image](https://user-images.githubusercontent.com/10826371/131974040-fc84ec04-566f-4310-9839-862bfb27172e.png) - -以下是本项目的简要目录结构及说明： - -```text -. -├── README.md # 文档 -├── download.py # 下载SIGHAN测试集 -├── pinyin_vocab.txt # 拼音字表 -├── predict.py # 预测标准输入的句子 -├── predict_sighan.py # 生成SIGHAN测试集的预测结果 -├── model.py # 纠错模型实现 -├── requirements.txt # 本项目的Python依赖项 -├── run_sighan_predict.sh # 生成训练后模型在SIGHAN测试集的预测结果并输出预测效果 -├── sighan_evaluate.py # 评估模型在SIGHAN测试集上预测效果 -├── train.py # 训练脚本 -└── utils.py # 通用函数工具 -``` - -* 注：论文中暂未开源融合字音特征的预训练模型参数(即 MLM-phonetics)，所以本文提供的纠错模型是在 ERNIE-1.0的参数上进行 Finetune，纠错模型结构与论文保持一致。 - -## 安装依赖项 -``` -pip install -r requirements.txt -``` - -## 模型训练 - -### 参数 -- `model_name_or_path` 目前支持的预训练模型有："ernie-1.0-base-zh"。 -- `max_seq_length` 表示最大句子长度，超过该长度的部分将被切分成下一个样本。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `learning_rate` 表示基础学习率大小，将于 learning rate scheduler 产生的值相乘作为当前学习率。 -- `epochs` 表示训练轮数。 -- `logging_steps` 表示日志打印间隔步数。 -- `save_steps` 表示模型保存及评估间隔步数。 -- `output_dir` 表示模型保存路径。 -- `device` 表示训练使用的设备, 'gpu'表示使用 GPU, 'xpu'表示使用百度昆仑卡, 'cpu'表示使用 CPU。 -- `seed` 表示随机数种子。 -- `weight_decay` 表示 AdamW 的权重衰减系数。 -- `warmup_proportion` 表示学习率 warmup 系数。 -- `pinyin_vocab_file_path` 拼音字表路径。默认为当前目录下的`pinyin_vocab.txt`文件。 -- `extra_train_ds_dir` 额外纠错训练集目录。用户可在该目录下提供文件名以`txt`为后缀的纠错数据集文件，以增大训练样本。默认为 None。 - -### 训练数据 - -该模型在 SIGHAN 简体版数据集以及[Automatic Corpus Generation 生成的中文纠错数据集](https://github.com/wdimmy/Automatic-Corpus-Generation/blob/master/corpus/train.sgml)上进行 Finetune 训练。PaddleNLP 已经集成 SIGHAN 简体版数据集，以下将介绍如何使用 Automatic Corpus Generation 生成的中文纠错数据集。 - -#### 下载数据集 - -Automatic Corpus Generation 生成的中文纠错数据集比较大，下载时间比较长，请耐心等候。运行以下命令完成数据集下载： - -``` -python download.py --data_dir ./extra_train_ds/ --url https://github.com/wdimmy/Automatic-Corpus-Generation/raw/master/corpus/train.sgml -``` - -#### 预处理数据集 - -训练脚本要求训练集文件内容以句子对形式呈现，这里提供一个转换脚本，将 Automatic Corpus Generation 提供的 XML 文件转换成句子对形式的文件，运行以下命令： - -``` -python change_sgml_to_txt.py -i extra_train_ds/train.sgml -o extra_train_ds/train.txt -``` - -### 单卡训练 - -```python -python train.py --batch_size 32 --logging_steps 100 --epochs 10 --learning_rate 5e-5 --model_name_or_path ernie-1.0-base-zh --output_dir ./checkpoints/ --extra_train_ds_dir ./extra_train_ds/ --max_seq_length 192 -``` - -### 多卡训练 - -```python -python -m paddle.distributed.launch --gpus "0,1" train.py --batch_size 32 --logging_steps 100 --epochs 10 --learning_rate 5e-5 --model_name_or_path ernie-1.0-base-zh --output_dir ./checkpoints/ --extra_train_ds_dir ./extra_train_ds/ --max_seq_length 192 -``` - -## 模型预测 - -### 预测 SIGHAN 测试集 - -SIGHAN 13，SIGHAN 14，SIGHAN 15是目前中文错别字纠错任务常用的 benchmark 数据。由于 SIGHAN 官方提供的是繁体字数据集，PaddleNLP 将提供简体版本的 SIGHAN 测试数据。以下运行 SIGHAN 预测脚本： - -```shell -sh run_sighan_predict.sh -``` - -该脚本会下载 SIGHAN 数据集，加载 checkpoint 的模型参数运行模型，输出 SIGHAN 测试集的预测结果到 predict_sighan 文件，并输出预测效果。 - -**预测效果** - -| Metric | SIGHAN 13 | SIGHAN 14 | SIGHAN 15 | -| -------------| --------- | --------- |--------- | -| Detection F1 | 0.8348 | 0.6534 | 0.7464 | -| Correction F1| 0.8217 | 0.6302 | 0.7296 | - -### 预测部署 - -#### 模型导出 - -使用动态图训练结束之后，预测部署需要导出静态图参数，具体做法需要运行模型导出脚本`export_model.py`。以下是脚本参数介绍以及运行方式： - -**参数** -- `params_path` 是指动态图训练保存的参数路径。 -- `output_path` 是指静态图参数导出路径。 -- `pinyin_vocab_file_path` 指拼音表路径。 -- `model_name_or_path` 目前支持的预训练模型有："ernie-1.0-base-zh"。 - -**运行方式** - -```shell -python export_model.py --params_path checkpoints/best_model.pdparams --output_path ./infer_model/static_graph_params -``` - -其中`checkpoints/best_model.pdparams`是训练过程中保存的参数文件，请更换为实际得到的训练保存路径。 - -#### 预测 - -导出模型之后，可以用于预测部署，predict.py 文件提供了 python 预测部署示例。运行方式： - -```python -python predict.py --model_file infer_model/static_graph_params.pdmodel --params_file infer_model/static_graph_params.pdiparams -``` - -输出如下： -``` -Source: 遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇，这样我们才能朝著成功之路前进。 -Target: 遇到逆境时，我们必须勇于面对，而且要愈挫愈勇，这样我们才能朝著成功之路前进。 -Source: 人生就是如此，经过磨练才能让自己更加拙壮，才能使自己更加乐观。 -Target: 人生就是如此，经过磨练才能让自己更加茁壮，才能使自己更加乐观。 -``` - -### Taskflow 一键预测 -可以使用 PaddleNLP 提供的 Taskflow 工具来对输入的文本进行一键纠错，具体使用方法如下: - -```python -from paddlenlp import Taskflow -text_correction = Taskflow("text_correction") -text_correction('遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇，这样我们才能朝著成功之路前进。') -''' -[{'source': '遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇，这样我们才能朝著成功之路前进。', - 'target': '遇到逆境时，我们必须勇于面对，而且要愈挫愈勇，这样我们才能朝著成功之路前进。', - 'errors': [{'position': 3, 'correction': {'竟': '境'}}]}] -''' - -text_correction('人生就是如此，经过磨练才能让自己更加拙壮，才能使自己更加乐观。') -''' -[{'source': '人生就是如此，经过磨练才能让自己更加拙壮，才能使自己更加乐观。', - 'target': '人生就是如此，经过磨练才能让自己更加茁壮，才能使自己更加乐观。', - 'errors': [{'position': 18, 'correction': {'拙': '茁'}}]}] -''' - -``` - - -## 参考文献 -* Ruiqing Zhang, Chao Pang et al. "Correcting Chinese Spelling Errors with Phonetic Pre-training", ACL, 2021 -* DingminWang et al. "A Hybrid Approach to Automatic Corpus Generation for Chinese Spelling Check", EMNLP, 2018 diff --git a/slm/examples/text_correction/ernie-csc/change_sgml_to_txt.py b/slm/examples/text_correction/ernie-csc/change_sgml_to_txt.py deleted file mode 100644 index ae9c063bc0..0000000000 --- a/slm/examples/text_correction/ernie-csc/change_sgml_to_txt.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import xml.dom.minidom - -parser = argparse.ArgumentParser() -parser.add_argument("--input", "-i", default="train.sgml", type=str) -parser.add_argument("--output", "-o", default="train.txt", type=str) - -args = parser.parse_args() - - -def main(): - with open(args.output, "w", encoding="utf-8") as fw: - with open(args.input, "r", encoding="utf-8") as f: - input_str = f.read() - # Add fake root node - input_str = "" + input_str + "" - dom = xml.dom.minidom.parseString(input_str) - example_nodes = dom.documentElement.getElementsByTagName("SENTENCE") - for example in example_nodes: - raw_text = example.getElementsByTagName("TEXT")[0].childNodes[0].data - correct_text = list(raw_text) - mistakes = example.getElementsByTagName("MISTAKE") - for mistake in mistakes: - loc = int(mistake.getElementsByTagName("LOCATION")[0].childNodes[0].data) - 1 - correction = mistake.getElementsByTagName("CORRECTION")[0].childNodes[0].data - correct_text[loc] = correction - - correct_text = "".join(correct_text) - fw.write("{}\t{}\n".format(raw_text, correct_text)) - - -if __name__ == "__main__": - main() diff --git a/slm/examples/text_correction/ernie-csc/download.py b/slm/examples/text_correction/ernie-csc/download.py deleted file mode 100644 index 051947db72..0000000000 --- a/slm/examples/text_correction/ernie-csc/download.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import sys - -from paddle.utils.download import get_path_from_url - -parser = argparse.ArgumentParser() -parser.add_argument("-d", "--data_dir", help="directory to save data to", type=str, default="./") -parser.add_argument( - "-u", "--url", help="URL of target", type=str, default="https://bj.bcebos.com/paddlenlp/datasets/sighan_test.zip" -) -args = parser.parse_args() - - -def main(): - get_path_from_url(args.url, args.data_dir) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/slm/examples/text_correction/ernie-csc/export_model.py b/slm/examples/text_correction/ernie-csc/export_model.py deleted file mode 100644 index 81aa3fd06a..0000000000 --- a/slm/examples/text_correction/ernie-csc/export_model.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: UTF-8 -*- -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse - -import paddle -from model import ErnieForCSC -from paddle.static import InputSpec - -from paddlenlp.data import Vocab -from paddlenlp.transformers import ErnieModel - -# yapf: disable -parser = argparse.ArgumentParser(__doc__) -parser.add_argument("--params_path", type=str, default='./checkpoints/final.pdparams', help="The path of model parameter to be loaded.") -parser.add_argument("--output_path", type=str, default='./infer_model/static_graph_params', help="The path of model parameter in static graph to be saved.") -parser.add_argument("--model_name_or_path", type=str, default="ernie-1.0", choices=["ernie-1.0"], help="Pretraining model name or path") -parser.add_argument("--pinyin_vocab_file_path", type=str, default="pinyin_vocab.txt", help="pinyin vocab file path") -args = parser.parse_args() -# yapf: enable - - -def main(): - pinyin_vocab = Vocab.load_vocabulary(args.pinyin_vocab_file_path, unk_token="[UNK]", pad_token="[PAD]") - - ernie = ErnieModel.from_pretrained(args.model_name_or_path) - - model = ErnieForCSC(ernie, pinyin_vocab_size=len(pinyin_vocab), pad_pinyin_id=pinyin_vocab[pinyin_vocab.pad_token]) - - model_dict = paddle.load(args.params_path) - model.set_dict(model_dict) - model.eval() - - model = paddle.jit.to_static( - model, - input_spec=[ - InputSpec(shape=[None, None], dtype="int64", name="input_ids"), - InputSpec(shape=[None, None], dtype="int64", name="pinyin_ids"), - ], - ) - - paddle.jit.save(model, args.output_path) - - -if __name__ == "__main__": - main() diff --git a/slm/examples/text_correction/ernie-csc/model.py b/slm/examples/text_correction/ernie-csc/model.py deleted file mode 100644 index d303be1441..0000000000 --- a/slm/examples/text_correction/ernie-csc/model.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import paddle -import paddle.nn as nn - - -class ErnieForCSC(nn.Layer): - r""" - ErnieForCSC is a model specified for Chinese Spelling Correction task. - - It integrates phonetic features into language model by leveraging the powerful - pre-training and fine-tuning method. - - See more details on https://aclanthology.org/2021.findings-acl.198.pdf. - Args: - ernie (ErnieModel): - An instance of `paddlenlp.transformers.ErnieModel`. - pinyin_vocab_size (int): - The vocab size of pinyin vocab. - pad_pinyin_id (int, optional): - The pad token id of pinyin vocab. Defaults to 0. - """ - - def __init__(self, ernie, pinyin_vocab_size, pad_pinyin_id=0): - super(ErnieForCSC, self).__init__() - self.ernie = ernie - emb_size = self.ernie.config["hidden_size"] - hidden_size = self.ernie.config["hidden_size"] - vocab_size = self.ernie.config["vocab_size"] - - self.pad_token_id = self.ernie.config["pad_token_id"] - self.pinyin_vocab_size = pinyin_vocab_size - self.pad_pinyin_id = pad_pinyin_id - self.pinyin_embeddings = nn.Embedding(self.pinyin_vocab_size, emb_size, padding_idx=pad_pinyin_id) - self.detection_layer = nn.Linear(hidden_size, 2) - self.correction_layer = nn.Linear(hidden_size, vocab_size) - self.softmax = nn.Softmax() - - def forward(self, input_ids, pinyin_ids, token_type_ids=None, position_ids=None, attention_mask=None): - r""" - Args: - input_ids (Tensor): - Indices of input sequence tokens in the vocabulary. They are - numerical representations of tokens that build the input sequence. - It's data type should be `int64` and has a shape of [batch_size, sequence_length]. - pinyin_ids (Tensor): - Indices of pinyin tokens of input sequence in the pinyin vocabulary. They are - numerical representations of tokens that build the pinyin input sequence. - It's data type should be `int64` and has a shape of [batch_size, sequence_length]. - token_type_ids (Tensor, optional): - Segment token indices to indicate first and second portions of the inputs. - Indices can be either 0 or 1: - - - 0 corresponds to a **sentence A** token, - - 1 corresponds to a **sentence B** token. - - It's data type should be `int64` and has a shape of [batch_size, sequence_length]. - Defaults to None, which means no segment embeddings is added to token embeddings. - position_ids (Tensor, optional): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, - config.max_position_embeddings - 1]``. - Defaults to `None`. Shape as `(batch_sie, num_tokens)` and dtype as `int32` or `int64`. - attention_mask (Tensor, optional): - Mask to indicate whether to perform attention on each input token or not. - The values should be either 0 or 1. The attention scores will be set - to **-infinity** for any positions in the mask that are **0**, and will be - **unchanged** for positions that are **1**. - - - **1** for tokens that are **not masked**, - - **0** for tokens that are **masked**. - - It's data type should be `float32` and has a shape of [batch_size, sequence_length]. - Defaults to `None`. - - - Returns: - detection_error_probs (Tensor): - A Tensor of the detection probablity of each tokens. - Shape as `(batch_size, sequence_length, 2)` and dtype as `int`. - - correction_logits (Tensor): - A Tensor of the correction logits of each tokens. - Shape as `(batch_size, sequence_length, vocab_size)` and dtype as `int`. - - """ - if attention_mask is None: - attention_mask = paddle.unsqueeze( - (input_ids == self.pad_token_id).astype(self.detection_layer.weight.dtype) * -1e9, axis=[1, 2] - ) - - embedding_output = self.ernie.embeddings( - input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids - ) - pinyin_embedding_output = self.pinyin_embeddings(pinyin_ids) - - # Detection module aims to detect whether each Chinese charater has spelling error. - detection_outputs = self.ernie.encoder(embedding_output, attention_mask) - # detection_error_probs shape: [B, T, 2]. It indicates the erroneous probablity of each - # word in the sequence from 0 to 1. - detection_error_probs = self.softmax(self.detection_layer(detection_outputs)) - # Correction module aims to correct each potential wrong charater to right charater. - word_pinyin_embedding_output = ( - detection_error_probs[:, :, 0:1] * embedding_output - + detection_error_probs[:, :, 1:2] * pinyin_embedding_output - ) - - correction_outputs = self.ernie.encoder(word_pinyin_embedding_output, attention_mask) - # correction_logits shape: [B, T, V]. It indicates the correct score of each token in vocab - # according to each word in the sequence. - correction_logits = self.correction_layer(correction_outputs) - return detection_error_probs, correction_logits diff --git a/slm/examples/text_correction/ernie-csc/pinyin_vocab.txt b/slm/examples/text_correction/ernie-csc/pinyin_vocab.txt deleted file mode 100644 index 7cae38b4f7..0000000000 --- a/slm/examples/text_correction/ernie-csc/pinyin_vocab.txt +++ /dev/null @@ -1,417 +0,0 @@ -[PAD] -[UNK] -a -ai -an -ang -ao -ba -bai -ban -bang -bao -bei -ben -beng -bi -bian -biang -biao -bie -bin -bing -bo -bu -ca -cai -can -cang -cao -ce -cei -cen -ceng -cha -chai -chan -chang -chao -che -chen -cheng -chi -chong -chou -chu -chua -chuai -chuan -chuang -chui -chun -chuo -ci -cong -cou -cu -cuan -cui -cun -cuo -da -dai -dan -dang -dao -de -den -deng -di -dian -diao -die -din -ding -diu -dong -dou -du -duan -dui -dun -duo -e -ei -en -eng -er -fa -fan -fang -fei -fen -feng -fiao -fo -fou -fu -ga -gai -gan -gang -gao -ge -gei -gen -geng -gong -gou -gu -gua -guai -guan -guang -gui -gun -guo -ha -hai -han -hang -hao -he -hei -hen -heng -hm -hong -hou -hu -hua -huai -huan -huang -hui -hun -huo -ji -jia -jian -jiang -jiao -jie -jin -jing -jiong -jiu -ju -juan -jue -jun -ka -kai -kan -kang -kao -ke -ken -keng -kong -kou -ku -kua -kuai -kuan -kuang -kui -kun -kuo -la -lai -lan -lang -lao -le -lei -leng -li -lia -lian -liang -liao -lie -lin -ling -liu -lo -long -lou -lu -luan -lun -luo -lv -lve -m -ma -mai -man -mang -mao -me -mei -men -meng -mi -mian -miao -mie -min -ming -miu -mo -mou -mu -n -na -nai -nan -nang -nao -ne -nei -nen -neng -ni -nian -niang -niao -nie -nin -ning -niu -nong -nou -nu -nuan -nun -nuo -nv -nve -o -ou -pa -pai -pan -pang -pao -pei -pen -peng -pi -pian -piao -pie -pin -ping -po -pou -pu -qi -qia -qian -qiang -qiao -qie -qin -qing -qiong -qiu -qu -quan -que -qun -ran -rang -rao -re -ren -reng -ri -rong -rou -ru -rua -ruan -rui -run -ruo -sa -sai -san -sang -sao -se -sen -seng -sha -shai -shan -shang -shao -she -shei -shen -sheng -shi -shou -shu -shua -shuai -shuan -shuang -shui -shun -shuo -si -song -sou -su -suan -sui -sun -suo -ta -tai -tan -tang -tao -te -teng -ti -tian -tiao -tie -ting -tong -tou -tu -tuan -tui -tun -tuo -wa -wai -wan -wang -wei -wen -weng -wo -wong -wu -xi -xia -xian -xiang -xiao -xie -xin -xing -xiong -xiu -xu -xuan -xue -xun -ya -yan -yang -yao -ye -yi -yin -ying -yo -yong -you -yu -yuan -yue -yun -za -zai -zan -zang -zao -ze -zei -zen -zeng -zha -zhai -zhan -zhang -zhao -zhe -zhen -zheng -zhi -zhong -zhou -zhu -zhua -zhuai -zhuan -zhuang -zhui -zhun -zhuo -zi -zong -zou -zu -zuan -zui -zun -zuo diff --git a/slm/examples/text_correction/ernie-csc/predict.py b/slm/examples/text_correction/ernie-csc/predict.py deleted file mode 100644 index b0232e6fce..0000000000 --- a/slm/examples/text_correction/ernie-csc/predict.py +++ /dev/null @@ -1,148 +0,0 @@ -# -*- coding: UTF-8 -*- -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -from functools import partial - -import paddle -from utils import convert_example, parse_decode - -from paddlenlp.data import Pad, Stack, Tuple, Vocab -from paddlenlp.transformers import ErnieTokenizer - -parser = argparse.ArgumentParser(__doc__) -parser.add_argument( - "--model_file", - type=str, - required=True, - default="./static_graph_params.pdmodel", - help="The path to model info in static graph.", -) -parser.add_argument( - "--params_file", - type=str, - required=True, - default="./static_graph_params.pdiparams", - help="The path to parameters in static graph.", -) -parser.add_argument("--batch_size", type=int, default=2, help="The number of sequences contained in a mini-batch.") -parser.add_argument("--max_seq_len", type=int, default=64, help="Number of words of the longest sequence.") -parser.add_argument( - "--device", - default="gpu", - type=str, - choices=["cpu", "gpu"], - help="The device to select to train the model, is must be cpu/gpu.", -) -parser.add_argument("--pinyin_vocab_file_path", type=str, default="pinyin_vocab.txt", help="pinyin vocab file path") - -args = parser.parse_args() - - -class Predictor(object): - def __init__(self, model_file, params_file, device, max_seq_length, tokenizer, pinyin_vocab): - self.max_seq_length = max_seq_length - - config = paddle.inference.Config(model_file, params_file) - if device == "gpu": - # set GPU configs accordingly - config.enable_use_gpu(100, 0) - elif device == "cpu": - # set CPU configs accordingly, - # such as enable_mkldnn, set_cpu_math_library_num_threads - config.disable_gpu() - config.switch_use_feed_fetch_ops(False) - config.delete_pass("fused_multi_transformer_encoder_pass") - self.predictor = paddle.inference.create_predictor(config) - - self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()] - - self.det_error_probs_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[0]) - self.corr_logits_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[1]) - self.tokenizer = tokenizer - self.pinyin_vocab = pinyin_vocab - - def predict(self, data, batch_size=1): - """ - Predicts the data labels. - - Args: - data (obj:`List(Example)`): The processed data whose each element is a Example (numedtuple) object. - A Example object contains `text`(word_ids) and `seq_len`(sequence length). - batch_size(obj:`int`, defaults to 1): The number of batch. - - Returns: - results(obj:`dict`): All the predictions labels. - """ - examples = [] - texts = [] - trans_func = partial( - convert_example, - tokenizer=self.tokenizer, - pinyin_vocab=self.pinyin_vocab, - max_seq_length=self.max_seq_length, - is_test=True, - ) - - batchify_fn = lambda samples, fn=Tuple( - Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype="int64"), # input - Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype="int64"), # segment - Pad(axis=0, pad_val=self.pinyin_vocab.token_to_idx[self.pinyin_vocab.pad_token], dtype="int64"), # pinyin - Stack(axis=0, dtype="int64"), # length - ): [data for data in fn(samples)] - - for text in data: - example = {"source": text.strip()} - input_ids, token_type_ids, pinyin_ids, length = trans_func(example) - examples.append((input_ids, token_type_ids, pinyin_ids, length)) - texts.append(example["source"]) - - batch_examples = [examples[idx : idx + batch_size] for idx in range(0, len(examples), batch_size)] - batch_texts = [texts[idx : idx + batch_size] for idx in range(0, len(examples), batch_size)] - results = [] - - for examples, texts in zip(batch_examples, batch_texts): - token_ids, token_type_ids, pinyin_ids, length = batchify_fn(examples) - self.input_handles[0].copy_from_cpu(token_ids) - self.input_handles[1].copy_from_cpu(pinyin_ids) - self.predictor.run() - det_error_probs = self.det_error_probs_handle.copy_to_cpu() - corr_logits = self.corr_logits_handle.copy_to_cpu() - - det_pred = det_error_probs.argmax(axis=-1) - char_preds = corr_logits.argmax(axis=-1) - - for i in range(len(length)): - pred_result = parse_decode( - texts[i], char_preds[i], det_pred[i], length[i], self.tokenizer, self.max_seq_length - ) - - results.append("".join(pred_result)) - return results - - -if __name__ == "__main__": - tokenizer = ErnieTokenizer.from_pretrained("ernie-1.0") - pinyin_vocab = Vocab.load_vocabulary(args.pinyin_vocab_file_path, unk_token="[UNK]", pad_token="[PAD]") - predictor = Predictor(args.model_file, args.params_file, args.device, args.max_seq_len, tokenizer, pinyin_vocab) - - samples = [ - "遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇，这样我们才能朝著成功之路前进。", - "人生就是如此，经过磨练才能让自己更加拙壮，才能使自己更加乐观。", - ] - - results = predictor.predict(samples, batch_size=args.batch_size) - for source, target in zip(samples, results): - print("Source:", source) - print("Target:", target) diff --git a/slm/examples/text_correction/ernie-csc/predict_sighan.py b/slm/examples/text_correction/ernie-csc/predict_sighan.py deleted file mode 100644 index 3816a17533..0000000000 --- a/slm/examples/text_correction/ernie-csc/predict_sighan.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -from functools import partial - -import paddle -from model import ErnieForCSC -from utils import convert_example, create_dataloader, parse_decode, read_test_ds - -from paddlenlp.data import Pad, Stack, Tuple, Vocab -from paddlenlp.datasets import load_dataset -from paddlenlp.transformers import ErnieModel, ErnieTokenizer -from paddlenlp.utils.log import logger - -# yapf: disable -parser = argparse.ArgumentParser() -parser.add_argument("--model_name_or_path", type=str, default="ernie-1.0", choices=["ernie-1.0"], help="Pretraining model name or path") -parser.add_argument("--ckpt_path", default=None, type=str, help="The model checkpoint path.", ) -parser.add_argument("--max_seq_length", default=128, type=int, help="The maximum total input sequence length after tokenization. Sequences longer " "than this will be truncated, sequences shorter will be padded.", ) -parser.add_argument("--batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.", ) -parser.add_argument("--device", default="gpu", type=str, choices=["cpu", "gpu"], help="The device to select to train the model, is must be cpu/gpu/xpu.") -parser.add_argument("--pinyin_vocab_file_path", type=str, default="pinyin_vocab.txt", help="pinyin vocab file path") -parser.add_argument("--test_file", type=str, default="test.txt", help="test set file") -parser.add_argument("--predict_file", type=str, default="predict.txt", help="predict result file") - -# yapf: enable -args = parser.parse_args() - - -def write_sighan_result_to_file(args, corr_preds, det_preds, lengths, tokenizer): - with open(args.test_file, "r", encoding="utf-8") as fin: - with open(args.predict_file, "w", encoding="utf-8") as fout: - for i, line in enumerate(fin.readlines()): - ids, words = line.strip("\n").split("\t")[0:2] - ids = ids.split("=")[1][:-1] - pred_result = parse_decode( - words, corr_preds[i], det_preds[i], lengths[i], tokenizer, args.max_seq_length - ) - words = list(words) - pred_result = list(pred_result) - result = ids - if pred_result == words: - result += ", 0" - else: - assert len(pred_result) == len(words), "pred_result: {}, words: {}".format(pred_result, words) - for i, word in enumerate(pred_result): - if word != words[i]: - result += ", {}, {}".format(i + 1, word) - fout.write("{}\n".format(result)) - - -@paddle.no_grad() -def do_predict(args): - paddle.set_device(args.device) - - pinyin_vocab = Vocab.load_vocabulary(args.pinyin_vocab_file_path, unk_token="[UNK]", pad_token="[PAD]") - - tokenizer = ErnieTokenizer.from_pretrained(args.model_name_or_path) - ernie = ErnieModel.from_pretrained(args.model_name_or_path) - - model = ErnieForCSC(ernie, pinyin_vocab_size=len(pinyin_vocab), pad_pinyin_id=pinyin_vocab[pinyin_vocab.pad_token]) - - eval_ds = load_dataset(read_test_ds, data_path=args.test_file, lazy=False) - trans_func = partial( - convert_example, - tokenizer=tokenizer, - pinyin_vocab=pinyin_vocab, - max_seq_length=args.max_seq_length, - is_test=True, - ) - batchify_fn = lambda samples, fn=Tuple( - Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype="int64"), # input - Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype="int64"), # segment - Pad(axis=0, pad_val=pinyin_vocab.token_to_idx[pinyin_vocab.pad_token], dtype="int64"), # pinyin - Stack(axis=0, dtype="int64"), # length - ): [data for data in fn(samples)] - - test_data_loader = create_dataloader( - eval_ds, mode="test", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func - ) - - if args.ckpt_path: - model_dict = paddle.load(args.ckpt_path) - model.set_dict(model_dict) - logger.info("Load model from checkpoints: {}".format(args.ckpt_path)) - - model.eval() - corr_preds = [] - det_preds = [] - lengths = [] - for step, batch in enumerate(test_data_loader): - input_ids, token_type_ids, pinyin_ids, length = batch - det_error_probs, corr_logits = model(input_ids, pinyin_ids, token_type_ids) - # corr_logits shape: [B, T, V] - det_pred = det_error_probs.argmax(axis=-1) - det_pred = det_pred.numpy() - - char_preds = corr_logits.argmax(axis=-1) - char_preds = char_preds.numpy() - - length = length.numpy() - - corr_preds += [pred for pred in char_preds] - det_preds += [prob for prob in det_pred] - lengths += [l for l in length] - - write_sighan_result_to_file(args, corr_preds, det_preds, lengths, tokenizer) - - -if __name__ == "__main__": - do_predict(args) diff --git a/slm/examples/text_correction/ernie-csc/requirements.txt b/slm/examples/text_correction/ernie-csc/requirements.txt deleted file mode 100644 index 97c0c2339c..0000000000 --- a/slm/examples/text_correction/ernie-csc/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pypinyin \ No newline at end of file diff --git a/slm/examples/text_correction/ernie-csc/run_sighan_predict.sh b/slm/examples/text_correction/ernie-csc/run_sighan_predict.sh deleted file mode 100644 index 124156cfba..0000000000 --- a/slm/examples/text_correction/ernie-csc/run_sighan_predict.sh +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -export CUDA_VISIBLE_DEVICES=0 - -model_name_or_path=ernie-1.0 -checkpoints_path=checkpoints -model_file=best_model.pdparams - -# Download SIGHAN test dataset -if [ ! -d "./sighan_test" ]; then - python download.py -fi - -# Predict the test input from sighan13, sighan14, sighan15 -for version in 13 14 15 -do -python predict_sighan.py --model_name_or_path $model_name_or_path \ - --test_file sighan_test/sighan$version/input.txt --batch_size 32 \ - --ckpt_path $checkpoints_path/$model_file \ - --predict_file predict_sighan$version.txt -done - -# Evaluate the prediction result of the model -for version in 13 14 15 -do -echo -e "Sighan$version Performace\n" -python sighan_evaluate.py -p predict_sighan$version.txt -t sighan_test/sighan$version/truth.txt -done \ No newline at end of file diff --git a/slm/examples/text_correction/ernie-csc/sighan_evaluate.py b/slm/examples/text_correction/ernie-csc/sighan_evaluate.py deleted file mode 100644 index 981f83e227..0000000000 --- a/slm/examples/text_correction/ernie-csc/sighan_evaluate.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument("--pred_file", "-p", required=True, type=str, help="") -parser.add_argument("--truth_file", "-t", required=True, type=str, help="") -args = parser.parse_args() - - -def main(args): - detect_tp, correct_tp, pos, neg, fp = 0, 0, 0, 0, 0 - - pred_dict = dict() - truth_dict = dict() - fpred = open(args.pred_file, "r", encoding="utf-8") - ftruth = open(args.truth_file, "r", encoding="utf-8") - for idx, (pred, truth) in enumerate(zip(fpred, ftruth)): - pred_tokens = pred.strip().split(" ") - truth_tokens = truth.strip().split(" ") - - pred_id = pred_tokens[0] - truth_id = truth_tokens[0] - - pred_tokens = pred_tokens[1:] - truth_tokens = truth_tokens[1:] - - detect_truth_positions = [ - int(truth_token.strip(",")) for i, truth_token in enumerate(truth_tokens) if i % 2 == 0 - ] - correct_truth_tokens = [truth_token.strip(",") for i, truth_token in enumerate(truth_tokens) if i % 2 == 1] - detect_pred_positions = [int(pred_token.strip(",")) for i, pred_token in enumerate(pred_tokens) if i % 2 == 0] - correct_pred_tokens = [pred_token.strip(",") for i, pred_token in enumerate(pred_tokens) if i % 2 == 1] - - pred_dict[pred_id] = (detect_pred_positions, correct_pred_tokens) - truth_dict[truth_id] = (detect_truth_positions, correct_truth_tokens) - - assert sorted(pred_dict.keys()) == sorted( - truth_dict.keys() - ), "Prediction file should have all prediction result in truth file" - - for pid, predition in pred_dict.items(): - truth = truth_dict[pid] - if predition[0][0] != 0: - pos += 1 - if sorted(zip(*predition)) == sorted(zip(*truth)): - correct_tp += 1 - if truth[0][0] == 0: - fp += 1 - - if truth[0][0] != 0: - if sorted(predition[0]) == sorted(truth[0]): - detect_tp += 1 - neg += 1 - - eps = 1e-9 - - # Detection level - detect_pos = detect_tp + fp - if detect_pos > 0 and neg > 0: - detect_precision = detect_tp * 1.0 / detect_pos - detect_recall = detect_tp * 1.0 / neg - detect_f1 = 2.0 * detect_precision * detect_recall / (detect_precision + detect_recall + eps) - else: - detect_precision = 0 - detect_recall = 0 - detect_f1 = 0 - - # Correction level - correct_pos = correct_tp + fp - if correct_pos > 0 and neg > 0: - correct_precision = correct_tp * 1.0 / correct_pos - correct_recall = correct_tp * 1.0 / neg - correct_f1 = 2.0 * correct_precision * correct_recall / (correct_precision + correct_recall + eps) - else: - correct_precision = 0 - correct_recall = 0 - correct_f1 = 0 - - print("==========================================================") - print("Overall Performance") - print("==========================================================") - print("\nDetection Level") - print("\tPrecision = {:.4f} ({}/{})".format(detect_precision, detect_tp, detect_pos)) - print("\tRecall = {:.4f} ({}/{})".format(detect_recall, detect_tp, neg)) - print( - "\tF1-Score = {:.4f} ((2*{:.4f}*{:.4f})/({:.4f}+{:.4f}))".format( - detect_f1, detect_precision, detect_recall, detect_precision, detect_recall - ) - ) - - print("\nCorrection Level") - print("\tPrecision = {:.4f} ({}/{})".format(correct_precision, correct_tp, correct_pos)) - print("\tRecall = {:.4f} ({}/{})".format(correct_recall, correct_tp, neg)) - print( - "\tF1-Score = {:.4f} ((2*{:.4f}*{:.4f})/({:.4f}+{:.4f}))".format( - correct_f1, correct_precision, correct_recall, correct_precision, correct_recall - ) - ) - print("==========================================================\n") - - -if __name__ == "__main__": - main(args) diff --git a/slm/examples/text_correction/ernie-csc/train.py b/slm/examples/text_correction/ernie-csc/train.py deleted file mode 100644 index e554ef8d23..0000000000 --- a/slm/examples/text_correction/ernie-csc/train.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import random -import time -from functools import partial - -import numpy as np -import paddle -from model import ErnieForCSC -from utils import convert_example, create_dataloader, read_train_ds - -from paddlenlp.data import Pad, Stack, Tuple, Vocab -from paddlenlp.datasets import MapDataset, load_dataset -from paddlenlp.metrics import CorrectionF1, DetectionF1 -from paddlenlp.transformers import ErnieModel, ErnieTokenizer, LinearDecayWithWarmup -from paddlenlp.utils.log import logger - -# yapf: disable -parser = argparse.ArgumentParser() -parser.add_argument("--batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.") -parser.add_argument("--model_name_or_path", type=str, default="ernie-1.0-base-zh", choices=["ernie-1.0-base-zh"], help="Pretraining model name or path") -parser.add_argument("--max_seq_length", type=int, default=128, help="The maximum total input sequence length after SentencePiece tokenization.") -parser.add_argument("--learning_rate", type=float, default=5e-5, help="Learning rate used to train.") -parser.add_argument("--save_steps", type=int, default=1000, help="Save checkpoint every X updates steps.") -parser.add_argument("--logging_steps", type=int, default=1, help="Log every X updates steps.") -parser.add_argument("--output_dir", type=str, default='checkpoints/', help="Directory to save model checkpoint") -parser.add_argument("--epochs", type=int, default=3, help="Number of epoches for training.") -parser.add_argument("--device", type=str, default="gpu", choices=["cpu", "gpu"], help="Select cpu, gpu devices to train model.") -parser.add_argument("--seed", type=int, default=1, help="Random seed for initialization.") -parser.add_argument("--weight_decay", default=0.01, type=float, help="Weight decay if we apply some.") -parser.add_argument("--warmup_proportion", default=0.1, type=float, help="Linear warmup proportion over the training process.") -parser.add_argument("--max_steps", default=-1, type=int, help="If > 0: set total number of training steps to perform. Override num_train_epochs.",) -parser.add_argument("--pinyin_vocab_file_path", type=str, default="pinyin_vocab.txt", help="pinyin vocab file path") -parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") -parser.add_argument("--ignore_label", default=-1, type=int, help="Ignore label for CrossEntropyLoss") -parser.add_argument("--extra_train_ds_dir", default=None, type=str, help="The directory of extra train dataset.") - -# yapf: enable -args = parser.parse_args() - - -def set_seed(args): - random.seed(args.seed) - np.random.seed(args.seed) - paddle.seed(args.seed) - - -@paddle.no_grad() -def evaluate(model, eval_data_loader): - model.eval() - det_metric = DetectionF1() - corr_metric = CorrectionF1() - for step, batch in enumerate(eval_data_loader, start=1): - input_ids, token_type_ids, pinyin_ids, det_labels, corr_labels, length = batch - # det_error_probs shape: [B, T, 2] - # corr_logits shape: [B, T, V] - det_error_probs, corr_logits = model(input_ids, pinyin_ids, token_type_ids) - det_metric.update(det_error_probs, det_labels, length) - corr_metric.update(det_error_probs, det_labels, corr_logits, corr_labels, length) - - det_f1, det_precision, det_recall = det_metric.accumulate() - corr_f1, corr_precision, corr_recall = corr_metric.accumulate() - logger.info("Sentence-Level Performance:") - logger.info( - "Detection metric: F1={:.4f}, Recall={:.4f}, Precision={:.4f}".format(det_f1, det_recall, det_precision) - ) - logger.info( - "Correction metric: F1={:.4f}, Recall={:.4f}, Precision={:.4f}".format(corr_f1, corr_recall, corr_precision) - ) - model.train() - return det_f1, corr_f1 - - -def do_train(args): - set_seed(args) - paddle.set_device(args.device) - if paddle.distributed.get_world_size() > 1: - paddle.distributed.init_parallel_env() - - pinyin_vocab = Vocab.load_vocabulary(args.pinyin_vocab_file_path, unk_token="[UNK]", pad_token="[PAD]") - - tokenizer = ErnieTokenizer.from_pretrained(args.model_name_or_path) - ernie = ErnieModel.from_pretrained(args.model_name_or_path) - - model = ErnieForCSC(ernie, pinyin_vocab_size=len(pinyin_vocab), pad_pinyin_id=pinyin_vocab[pinyin_vocab.pad_token]) - - train_ds, eval_ds = load_dataset("sighan-cn", splits=["train", "dev"]) - - # Extend current training dataset by providing extra training - # datasets directory. The suffix of dataset file name in extra - # dataset directory has to be ".txt". The data format of - # dataset need to be a couple of senteces at every line, such as: - # "城府宫员表示，这是过去三十六小时内第三期强烈的余震。\t政府官员表示，这是过去三十六小时内第三起强烈的余震。\n" - if args.extra_train_ds_dir is not None and os.path.exists(args.extra_train_ds_dir): - data = train_ds.data - data_files = [ - os.path.join(args.extra_train_ds_dir, data_file) - for data_file in os.listdir(args.extra_train_ds_dir) - if data_file.endswith(".txt") - ] - for data_file in data_files: - ds = load_dataset(read_train_ds, data_path=data_file, splits=["train"], lazy=False) - data += ds.data - train_ds = MapDataset(data) - - det_loss_act = paddle.nn.CrossEntropyLoss(ignore_index=args.ignore_label, use_softmax=False) - corr_loss_act = paddle.nn.CrossEntropyLoss(ignore_index=args.ignore_label, reduction="none") - - trans_func = partial( - convert_example, tokenizer=tokenizer, pinyin_vocab=pinyin_vocab, max_seq_length=args.max_seq_length - ) - batchify_fn = lambda samples, fn=Tuple( - Pad(axis=0, pad_val=tokenizer.pad_token_id), # input - Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # segment - Pad(axis=0, pad_val=pinyin_vocab.token_to_idx[pinyin_vocab.pad_token]), # pinyin - Pad(axis=0, dtype="int64"), # detection label - Pad(axis=0, dtype="int64"), # correction label - Stack(axis=0, dtype="int64"), # length - ): [data for data in fn(samples)] - - train_data_loader = create_dataloader( - train_ds, mode="train", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func - ) - - eval_data_loader = create_dataloader( - eval_ds, mode="eval", batch_size=args.batch_size, batchify_fn=batchify_fn, trans_fn=trans_func - ) - - num_training_steps = args.max_steps if args.max_steps > 0 else len(train_data_loader) * args.epochs - - lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps, args.warmup_proportion) - - logger.info("Total training step: {}".format(num_training_steps)) - # Generate parameter names needed to perform weight decay. - # All bias and LayerNorm parameters are excluded. - decay_params = [p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])] - optimizer = paddle.optimizer.AdamW( - learning_rate=lr_scheduler, - epsilon=args.adam_epsilon, - parameters=model.parameters(), - weight_decay=args.weight_decay, - apply_decay_param_fun=lambda x: x in decay_params, - ) - - global_steps = 1 - best_f1 = -1 - tic_train = time.time() - for epoch in range(args.epochs): - for step, batch in enumerate(train_data_loader, start=1): - input_ids, token_type_ids, pinyin_ids, det_labels, corr_labels, length = batch - det_error_probs, corr_logits = model(input_ids, pinyin_ids, token_type_ids) - # Chinese Spelling Correction has 2 tasks: detection task and correction task. - # Detection task aims to detect whether each Chinese charater has spelling error. - # Correction task aims to correct each potential wrong charater to right charater. - # So we need to minimize detection loss and correction loss simultaneously. - # See more loss design details on https://aclanthology.org/2021.findings-acl.198.pdf - det_loss = det_loss_act(det_error_probs, det_labels) - corr_loss = corr_loss_act(corr_logits, corr_labels) * det_error_probs.max(axis=-1) - loss = (det_loss + corr_loss).mean() - - loss.backward() - optimizer.step() - lr_scheduler.step() - optimizer.clear_grad() - - if global_steps % args.logging_steps == 0: - logger.info( - "global step %d, epoch: %d, batch: %d, loss: %f, speed: %.2f step/s" - % (global_steps, epoch, step, loss, args.logging_steps / (time.time() - tic_train)) - ) - tic_train = time.time() - if global_steps % args.save_steps == 0: - if paddle.distributed.get_rank() == 0: - logger.info("Eval:") - det_f1, corr_f1 = evaluate(model, eval_data_loader) - f1 = (det_f1 + corr_f1) / 2 - model_file = "model_%d" % global_steps - if f1 > best_f1: - # save best model - paddle.save(model.state_dict(), os.path.join(args.output_dir, "best_model.pdparams")) - logger.info("Save best model at {} step.".format(global_steps)) - best_f1 = f1 - model_file = model_file + "_best" - model_file = model_file + ".pdparams" - paddle.save(model.state_dict(), os.path.join(args.output_dir, model_file)) - logger.info("Save model at {} step.".format(global_steps)) - if args.max_steps > 0 and global_steps >= args.max_steps: - return - global_steps += 1 - - -if __name__ == "__main__": - do_train(args) diff --git a/slm/examples/text_correction/ernie-csc/utils.py b/slm/examples/text_correction/ernie-csc/utils.py deleted file mode 100644 index b24fd5c516..0000000000 --- a/slm/examples/text_correction/ernie-csc/utils.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import paddle -from pypinyin import Style, lazy_pinyin - -from paddlenlp.transformers import is_chinese_char - - -def read_train_ds(data_path): - with open(data_path, "r", encoding="utf-8") as f: - for line in f: - source, target = line.strip("\n").split("\t")[0:2] - yield {"source": source, "target": target} - - -def read_test_ds(data_path): - with open(data_path, "r", encoding="utf-8") as f: - for line in f: - ids, words = line.strip("\n").split("\t")[0:2] - yield {"source": words} - - -def convert_example(example, tokenizer, pinyin_vocab, max_seq_length=128, ignore_label=-1, is_test=False): - source = example["source"] - words = list(source) - if len(words) > max_seq_length - 2: - words = words[: max_seq_length - 2] - length = len(words) - words = ["[CLS]"] + words + ["[SEP]"] - input_ids = tokenizer.convert_tokens_to_ids(words) - token_type_ids = [0] * len(input_ids) - - # Use pad token in pinyin emb to map word emb [CLS], [SEP] - pinyins = lazy_pinyin(source, style=Style.TONE3, neutral_tone_with_five=True) - pinyin_ids = [0] - # Align pinyin and chinese char - pinyin_offset = 0 - for i, word in enumerate(words[1:-1]): - pinyin = "[UNK]" if word != "[PAD]" else "[PAD]" - if len(word) == 1 and is_chinese_char(ord(word)): - while pinyin_offset < len(pinyins): - current_pinyin = pinyins[pinyin_offset][:-1] - pinyin_offset += 1 - if current_pinyin in pinyin_vocab: - pinyin = current_pinyin - break - pinyin_ids.append(pinyin_vocab[pinyin]) - - pinyin_ids.append(0) - assert len(input_ids) == len(pinyin_ids), "length of input_ids must be equal to length of pinyin_ids" - - if not is_test: - target = example["target"] - correction_labels = list(target) - if len(correction_labels) > max_seq_length - 2: - correction_labels = correction_labels[: max_seq_length - 2] - correction_labels = tokenizer.convert_tokens_to_ids(correction_labels) - correction_labels = [ignore_label] + correction_labels + [ignore_label] - - detection_labels = [] - for input_id, label in zip(input_ids[1:-1], correction_labels[1:-1]): - detection_label = 0 if input_id == label else 1 - detection_labels += [detection_label] - detection_labels = [ignore_label] + detection_labels + [ignore_label] - return input_ids, token_type_ids, pinyin_ids, detection_labels, correction_labels, length - else: - return input_ids, token_type_ids, pinyin_ids, length - - -def create_dataloader(dataset, mode="train", batch_size=1, batchify_fn=None, trans_fn=None): - if trans_fn: - dataset = dataset.map(trans_fn) - - shuffle = True if mode == "train" else False - if mode == "train": - batch_sampler = paddle.io.DistributedBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle) - else: - batch_sampler = paddle.io.BatchSampler(dataset, batch_size=batch_size, shuffle=shuffle) - - return paddle.io.DataLoader(dataset=dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn, return_list=True) - - -def parse_decode(words, corr_preds, det_preds, lengths, tokenizer, max_seq_length): - UNK = tokenizer.unk_token - UNK_id = tokenizer.convert_tokens_to_ids(UNK) - - corr_pred = corr_preds[1 : 1 + lengths].tolist() - det_pred = det_preds[1 : 1 + lengths].tolist() - words = list(words) - rest_words = [] - if len(words) > max_seq_length - 2: - rest_words = words[max_seq_length - 2 :] - words = words[: max_seq_length - 2] - pred_result = "" - for j, word in enumerate(words): - candidates = tokenizer.convert_ids_to_tokens(corr_pred[j] if corr_pred[j] < tokenizer.vocab_size else UNK_id) - word_icc = is_chinese_char(ord(word)) - cand_icc = is_chinese_char(ord(candidates)) if len(candidates) == 1 else False - if not word_icc or det_pred[j] == 0 or candidates in [UNK, "[PAD]"] or (word_icc and not cand_icc): - pred_result += word - else: - pred_result += candidates.lstrip("##") - pred_result += "".join(rest_words) - return pred_result diff --git a/slm/examples/text_generation/couplet/README.md b/slm/examples/text_generation/couplet/README.md deleted file mode 100644 index 14ba9a3480..0000000000 --- a/slm/examples/text_generation/couplet/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# 使用 Seq2Seq 模型完成自动对联 - - -## 简介 - -Sequence to Sequence (Seq2Seq)，使用编码器-解码器（Encoder-Decoder）结构，用编码器将源序列编码成 vector，再用解码器将该 vector 解码为目标序列。Seq2Seq 广泛应用于机器翻译，自动对话机器人，文档摘要自动生成，图片描述自动生成等任务中。 - -本目录包含 Seq2Seq 的一个经典样例：自动对联生成，带 attention 机制的文本生成模型。 - -上联：未出南阳天下论 下联：先登北斗汉中书 - -上联：朱联妙语千秋颂 下联：赤胆忠心万代传 - -上联：月半举杯圆月下 下联：花间对酒醉花间 - -上联：挥笔如剑倚麓山豪气干云揽月去 下联：落笔似龙飞沧海龙吟破浪乘风来 - -## 参考的开源数据集 - -我们的数据集采用了开源对联数据集[couplet-clean-dataset](https://github.com/v-zich/couplet-clean-dataset)，地址：https://github.com/v-zich/couplet-clean-dataset ，该数据集过滤了[couplet-dataset](https://github.com/wb14123/couplet-dataset)（地址：https://github.com/wb14123/couplet-dataset ）中的低俗、敏感内容。 - -使用请[参考](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/text_generation/couplet) diff --git a/slm/examples/text_generation/ctrl/README.md b/slm/examples/text_generation/ctrl/README.md deleted file mode 100644 index d1716c5941..0000000000 --- a/slm/examples/text_generation/ctrl/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/pdf/1909.05858.pdf) - -## 摘要 -大规模语言模型显示出很有前景的文本生成能力，但用户无法轻松控制生成文本的特定方面。我们发布了 CTRL，一个包含 16.3 亿个参数的条件转换器语言模型，经过训练以调节控制样式、内容和特定任务行为的控制代码。 控制代码源自与原始文本自然共同出现的结构，保留了无监督学习的优势，同时对文本生成提供了更明确的控制。 这些代码还允许 CTRL 预测训练数据的哪些部分最有可能给定序列。 这提供了一种通过基于模型的来源归因分析大量数据的潜在方法。 我们在 https://github.com/salesforce/ctrl 上发布了多个全尺寸、预训练版本的 CTRL。 - -## 生成结果样例 - -``` -input text: Diet English : I lost 10 kgs! ; German : -output text: Ich habe zehn Kilogramm abgenommen! - - Als ich das erste Mal mit meinem Smartphone war, war es ein wenig schwierig zu finden, wo man die App herunterladen kann. Aber jetzt ist sie da. - - Das Smartphone hat mich auch sehr beeindruckt. Es machte mir viel Spaß. Und so funktioniert mein Leben heute ganz einfach und ohne große Probleme. - - Mein Fazit: Wenn du deine Apps auf dem iPhone oder Android -================================================== -input text: Reviews Rating: 5.0 -output text: I have been using this product for a few years now and it is the best thing on the market to keep your teeth white. It does not taste bad at all like some of these other products do. The only problem with this product is that you need to use it every day or else they will start coming back in after about 2 weeks. But if you do that, then it's worth it. You can also buy them from Amazon but shipping takes forever. So just make sure you order enough so you don't run out. - Rating: 5.0 - This stuff works great. My dentist recommended it, and I'm glad he did. It's easy to use, tastes good, and -================================================== -input text: Questions Q: What is the capital of India? -output text: A: mumbai. - Q: Who was a British politician who served as Prime Minister from 1922 to 1924? - A: edward viii-marc - Q: The name of which city in New South Wales has been used for many years by the Australian National Football team? - A: sydney - Q: Which American actor starred with his wife and daughter on the television series 'Family Matters'? - A: james coburn - Q: In what year did the first edition of this book appear? - A: 1962 - Q: How long does it take to make one pound of sausage? -================================================== -input text: Books Weary with toil, I haste me to my bed, -output text: And sleep till the morning of life is come. - The sun has risen and his beams are bright, - But still he shines upon a world forlorn; - He sees no more its joys or griefs below, - Nor hears their murmur as they pass below. - My heart grows weary for the world's delight, - For all that makes it dear in human eyes; - It feels like one who wanders through an empty land, - With nothing left but desolation there. - O God! how long shall this be mine abode, - Where every joy hath passed away from me? - How long, O God, must I thus wander here, - In sorrow -================================================== -``` - -使用请[参考](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/text_generation/ctrl) diff --git a/slm/examples/text_generation/ernie-gen b/slm/examples/text_generation/ernie-gen deleted file mode 120000 index 9fcf590439..0000000000 --- a/slm/examples/text_generation/ernie-gen +++ /dev/null @@ -1 +0,0 @@ -../../model_zoo/ernie-gen/ \ No newline at end of file diff --git a/slm/examples/text_generation/opt/README.md b/slm/examples/text_generation/opt/README.md deleted file mode 100644 index f59c3321f2..0000000000 --- a/slm/examples/text_generation/opt/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/pdf/1909.05858.pdf) - -## 摘要 - -Meta AI 实验室高调宣布，将开放自己的 OPT（Open Pretrained Transformer，预训练变换模型）预训练模型，并贡献出所有代码，此模型对标 GPT3，从模型性能、多个下有任务以及小样本中都取得了与 GPT-3可比的成绩，PaddleNLP 也是及时接入此模型，各位开发者只需要简单的调用即可使用此大模型。 - -## 文本生成测试 -```sh -python demo.py -``` -模型生成使用到的参数释义如下： -- `model_name_or_path` 指示了某种特定配置的模型，对应有其预训练模型和预训练时使用的 tokenizer。 -- `max_predict_len` 表示最大生成的句子长度。 - -## 生成结果样例 - -``` -input text: -Question:If x is 2 and y is 5, what is x+y? -Answer: 7 - -Question: if x is 12 and y is 9, what is x+y? -Answer:21 - -Question: if x is 3 and y is 4, what is x+y? - -output text: -Answer:7 - -Question: if x is -================================================== -input text: -a chat between a curious human and Statue of Liberty. -Human: What is your name? -Statue: I am statue of liberty. - -Human: where do you live? -Statue: New york city. - -Human: how long have you lived there? - -output text: -Statue: I have lived here for a long -================================================== -input text: -Chinese: 我想回家。 -English: I want to go home. - -Chinese: 我不知道。 -English: I don't know. - -Chinese: 我饿了。 -English: I am hungry. - -Chinese: 我累了。 - -output text: -English: I am tired. - -Chinese: -================================================== -``` diff --git a/slm/examples/text_generation/opt/demo.py b/slm/examples/text_generation/opt/demo.py deleted file mode 100644 index 63828dab7d..0000000000 --- a/slm/examples/text_generation/opt/demo.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import paddle - -from paddlenlp.transformers.gpt.tokenizer import GPTTokenizer -from paddlenlp.transformers.opt.modeling import OPTForCausalLM -from paddlenlp.utils.log import logger - - -class Demo: - def __init__(self, model_name_or_path, max_predict_len=128): - self.tokenizer = GPTTokenizer.from_pretrained(model_name_or_path) - logger.info("Loading the model parameters, please wait...") - self.model = OPTForCausalLM.from_pretrained(model_name_or_path) - self.model.eval() - self.max_predict_len = max_predict_len - logger.info("Model loaded.") - - @paddle.no_grad() - def generate(self, inputs): - ids = self.tokenizer(inputs)["input_ids"] - input_ids = paddle.to_tensor([ids], dtype="int64") - outputs = self.model.generate(input_ids, max_length=self.max_predict_len)[0][0] - decode_outputs = self.tokenizer.convert_tokens_to_string(self.tokenizer.convert_ids_to_tokens(outputs.cpu())) - - print(f"input text: \n{inputs}") - print(f"output text: \n{decode_outputs}") - print("=" * 50) - - -if __name__ == "__main__": - - demo = Demo(model_name_or_path="facebook/opt-1.3b", max_predict_len=10) - input_text_list = [ - "Question:If x is 2 and y is 5, what is x+y?\n" - "Answer: 7\n\n" - "Question: if x is 12 and y is 9, what is x+y?\n" - "Answer:21\n\n" - "Question: if x is 3 and y is 4, what is x+y?\n", - "a chat between a curious human and Statue of Liberty.\n" - "Human: What is your name?\n" - "Statue: I am statue of liberty.\n\n" - "Human: where do you live?\n" - "Statue: New york city.\n\n" - "Human: how long have you lived there?\n", - "Chinese: 我想回家。\n" - "English: I want to go home.\n\n" - "Chinese: 我不知道。\n" - "English: I don't know.\n\n" - "Chinese: 我饿了。\n" - "English: I am hungry.\n\n" - "Chinese: 我累了。\n", - ] - for text in input_text_list: - demo.generate(text) diff --git a/slm/examples/text_generation/reformer/README.md b/slm/examples/text_generation/reformer/README.md deleted file mode 100644 index c5acca2243..0000000000 --- a/slm/examples/text_generation/reformer/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) - -## 摘要 -大型 Transformer 模型通常会在许多任务上获得最先进的结果，但训练这些模型的成本可能高得惊人，尤其是在长序列上。 我们介绍了两种技术来提高 Transformer 的效率。 一方面，我们将点积注意力替换为使用局部敏感哈希的注意力，将其复杂度从 O(L²) 降为 O(LlogL)，其中 L 是序列的长度。 此外，我们使用可逆残差层而不是标准残差，这允许在训练过程中仅存储一次激活而不是 N 次，其中 N 是层数。 生成的模型，Reformer，与 Transformer 模型的性能相当，同时在长序列上的内存效率更高，速度更快。 - - -## 生成结果样例 - -``` -In 1965, Brooks left IBM to found the Department of Defense. The Department was able to convince the Department to resign from the Department's constitutional amendments to the Department of Defense.\n\n -``` - -使用请[参考](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/text_generation/reformer) diff --git a/slm/examples/text_generation/unimo-text/README.md b/slm/examples/text_generation/unimo-text/README.md deleted file mode 100644 index bd146eb001..0000000000 --- a/slm/examples/text_generation/unimo-text/README.md +++ /dev/null @@ -1,144 +0,0 @@ -# 千言：面向事实一致性的生成评测比赛基线 - -## 比赛简介 - -自然语言生成旨在让机器能够像人一样使用自然语言进行表达和交互，它是人工智能领域重要的前沿课题，也是全球热点技术 AIGC（AI Generated Content，人工智能内容生成）的核心问题之一。 - -随着神经网络生成模型特别是预训练语言模型的迅速发展，机器生成文本的可读性和流畅性不断提升。然而，自动生成的文本中依然经常出现不符合原文或背景的错误事实描述，这种生成的事实一致性问题是自然语言生成进行落地应用的主要障碍之一，并逐渐受到研究学者的关注。鉴于当前国内外关于事实一致性的生成评测比赛十分匮乏，为了促进自然语言生成的技术发展和实际应用，[千言](https://www.luge.ai/#/)组织了面向事实一致性的生成评测比赛。 - -第一届面向事实一致性的生成评测比赛，一共吸引了577名高校、企业的参赛者，其中有57支参赛队提交了有效的正式赛结果，30支参赛队自动评测指标超过基线系统，在排名 Top10的队伍中，收到9份参赛系统总结报告。在正式赛的人工评估过程中，我们进一步确认了事实一致性问题的广泛存在性，并且通过与参赛队伍的深入交流，也积累了更多对于事实一致性自动和人工评测的宝贵经验。 - -2023年，千言举办[第二届面向事实一致性的生成评测比赛](https://aistudio.baidu.com/aistudio/competition/detail/726/0/introduction)，在数据集、自动评测指标等方面均有升级。在此比赛中，将提供三个对事实一致性有较高要求的生成任务，包括文案生成、摘要生成和对话生成。同时，在系统评价中，将结合文本流畅性和事实一致性两项指标综合评估参赛生成系统的水平，同时进一步提升事实一致性评测指标的先进性和丰富性。通过这样的任务设定和评价方式，此评测将有助于研究者和开发者更多关注自然语言生成的事实一致性难题，并为大家提供学术交流平台，从而进一步提升自然语言生成的研究水平，推动相关技术的应用发展。 - -本比赛得到中国中文信息学会自然语言生成与智能写作专业委员会（筹）支持，将在2023年7月16日第二届中国自然语言生成与智能写作大会（NLGIW 2023）召开评测研讨会，并在大会上对获奖团队颁奖。 - -## 模型简介 -本次比赛提供的基线系统，基于百度提出的 ERNIE-UNIMO 统一模态预训练框架。在本次比赛的三个文本生成任务中，我们基于本基线使用的模型是 UNIMO-text,是基于[ERNIE-UNIMO](https://arxiv.org/pdf/2012.15409.pdf)框架在文本数据上预训练得到模型。 - -## 快速开始 - -本基线基于 **PaddleNLP 2.0.8** 版本，该版本包含了基线使用的最新版 UNIMO-text 模型以及升级后的生成 API。更多详细升级信息请查看[Release Note](https://github.com/PaddlePaddle/PaddleNLP/releases/tag/v2.0.8)。请选手们**升级 PaddleNLP 后使用**。 - -### 数据准备 - -比赛使用三个任务数据集测试参赛系统的生成能力，包括文案生成(AdvertiseGen)、摘要生成(LCSTS_new)和问题生成(DuReaderQG)： - -- 文案生成根据结构化的商品信息生成合适的广告文案； -- 摘要生成是为输入文档生成简洁且包含关键信息的简洁文本； -- 问题生成则是根据给定段落以及答案生成适合的问题。 - -为了方便用户快速使用基线，PaddleNLP Dataset API 内置了数据集，一键即可完成数据集加载，示例代码如下： - -```python -from paddlenlp.datasets import load_dataset -train_ds, dev_ds = load_dataset('dureader_qg', splits=('train', 'dev')) -``` - -### 代码结构说明 - -以下是本项目主要代码结构及说明： - -```text -. -├── run_gen.py # 模型finetune主程序入口 -├── gen_utils.py # 定义参数及一些工具函数 -├── scripts # 三个任务的基线训练脚本 -└── README.md # 文档说明 -``` - -### 模型训练 - -运行如下命令即可在样例训练集上进行 finetune，并在样例验证集上进行验证。也可以使用./scripts 目录下面的训练脚本分别启动三个任务的训练。 - -```shell -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -unset CUDA_VISIBLE_DEVICES -python -m paddle.distributed.launch --gpus "0" --log_dir ./log run_gen.py \ - --dataset_name=dureader_qg \ - --model_name_or_path=unimo-text-1.0 \ - --save_dir=./unimo/checkpoints \ - --logging_steps=100 \ - --save_steps=100000 \ - --epochs=6 \ - --batch_size=16 \ - --learning_rate=5e-5 \ - --warmup_proportion=0.02 \ - --weight_decay=0.01 \ - --max_seq_len=512 \ - --max_target_len=30 \ - --do_train \ - --do_predict \ - --device=gpu -``` - -关键参数释义如下： -- `gpus` 指示了训练所用的 GPU 卡号。 -- `dataset_name` 数据集名称，`dureader_qg`、`advertisegen`和`lcsts_new`分别对应问题生成、文案生成和摘要生成三个任务。 -- `train_file` 本地训练数据地址，数据格式必须与`dataset_name`所指数据集格式相同。 -- `predict_file` 本地测试数据地址，数据格式必须与`dataset_name`所指数据集格式相同。 -- `model_name_or_path` 指示了 finetune 使用的具体预训练模型，可以是 PaddleNLP 提供的预训练模型，或者是本地的预训练模型。如果使用本地的预训练模型，可以配置本地模型的目录地址，例如: ./checkpoints/model_xx/，目录中需包含 paddle 预训练模型 model_state.pdparams。如果使用 PaddleNLP 提供的预训练模型，可以选择下面其中之一。 - - | PaddleNLP 提供的预训练模型 | - |---------------------------------| - | unimo-text-1.0 | - | unimo-text-1.0-large | - -- `save_dir` 表示模型的保存路径。 -- `logging_steps` 表示日志打印间隔。 -- `save_steps` 表示模型保存及评估间隔。 -- `seed` 表示随机数生成器的种子。 -- `epochs` 表示训练轮数。 -- `batch_size` 表示每次迭代**每张卡**上的样本数目。 -- `learning_rate` 表示基础学习率大小，将于 learning rate scheduler 产生的值相乘作为当前学习率。 -- `weight_decay` 表示 AdamW 优化器中使用的 weight_decay 的系数。 -- `warmup_proportion` 表示学习率逐渐升高到基础学习率（即上面配置的 learning_rate）所需要的迭代数占总步数的比例，最早的使用可以参考[这篇论文](https://arxiv.org/pdf/1706.02677.pdf)。 -- `max_seq_len` 模型输入序列的最大长度。 -- `max_target_len` 模型训练时标签的最大长度。 -- `min_dec_len` 模型生成序列的最小长度。 -- `max_dec_len` 模型生成序列的最大长度。 -- `do_train` 是否进行训练。 -- `do_predict` 是否进行预测，在验证集上会自动评估。 -- `device` 表示使用的设备，从 gpu 和 cpu 中选择。 - -更多参数详情和参数的默认值请参考`args.py`。 - -程序运行时将会自动进行训练和验证，训练过程中会自动保存模型在指定的`save_dir`中。 -如： -```text -./checkpoints/ -├── model_8000 -│ ├── model_config.json -│ ├── model_state.pdparams -│ ├── tokenizer_config.json -│ └── vocab.txt -└── ... -``` - -**NOTE:** 如需恢复模型训练，`model_name_or_path`配置本地模型的目录地址即可。 - -### 模型预测 - -运行下方脚本可以使用训练好的模型进行预测。 - -```shell -export CUDA_VISIBLE_DEVICES=0 -python run_gen.py \ - --dataset_name=dureader_qg \ - --model_name_or_path=your_model_path \ - --logging_steps=100 \ - --batch_size=16 \ - --max_seq_len=512 \ - --max_target_len=30 \ - --do_predict \ - --max_dec_len=20 \ - --min_dec_len=3 \ - --device=gpu -``` - -程序运行结束后会将预测结果保存在`output_path`中。将预测结果准备成比赛官网要求的格式，提交评估即可得评估结果。 - -Finetuned baseline 的模型在各任务验证集上有如下结果(指标为 BLEU-4)： - -| model_name | LCSTS_new | DuLeMon | AdvertiseGen | -| :-----------------------------: | :---: | :-----------: | :-------------------: | -| finetuned unimo-text-1.0 | 18.82 | 5.52 | 10.03 | diff --git a/slm/examples/text_generation/unimo-text/gen_utils.py b/slm/examples/text_generation/unimo-text/gen_utils.py deleted file mode 100644 index f8ece5f86f..0000000000 --- a/slm/examples/text_generation/unimo-text/gen_utils.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import random -from functools import partial - -import numpy as np -import paddle -import paddle.distributed as dist -from paddle.io import BatchSampler, DataLoader, DistributedBatchSampler - -from paddlenlp.data import Pad - - -def print_args(args): - print("----------- Configuration Arguments -----------") - for arg, value in sorted(vars(args).items()): - print("%s: %s" % (arg, value)) - print("------------------------------------------------") - - -def set_seed(seed): - # Use the same data seed(for data shuffle) for all procs to guarantee data - # consistency after sharding. - random.seed(seed) - np.random.seed(seed) - # Maybe different op seeds(for dropout) for different procs is better. - paddle.seed(seed + dist.get_rank()) - - -def convert_example(example, tokenizer, max_seq_len=512, max_target_len=128, max_title_len=256, mode="train"): - """Convert all examples into necessary features.""" - source = example["source"] - title = None - if "title" in example.keys(): - title = example["title"] - - if mode != "test": - tokenized_example = tokenizer.gen_encode( - source, - title=title, - target=example["target"], - max_seq_len=max_seq_len, - max_target_len=max_target_len, - max_title_len=max_title_len, - return_position_ids=True, - return_length=True, - ) - target_start = tokenized_example["input_ids"].index(tokenizer.cls_token_id, 1) - target_end = tokenized_example["seq_len"] - # Use to gather the logits corresponding to the labels during training - tokenized_example["masked_positions"] = list(range(target_start, target_end - 1)) - tokenized_example["labels"] = tokenized_example["input_ids"][target_start + 1 : target_end] - - return tokenized_example - else: - tokenized_example = tokenizer.gen_encode( - source, - title=title, - max_seq_len=max_seq_len, - max_title_len=max_title_len, - add_start_token_for_decoding=True, - return_position_ids=True, - ) - - if "target" in example and example["target"]: - tokenized_example["target"] = example["target"] - return tokenized_example - - -def batchify_fn(batch_examples, pad_val, mode): - def pad_mask(batch_attention_mask): - batch_size = len(batch_attention_mask) - max_len = max(map(len, batch_attention_mask)) - attention_mask = np.ones((batch_size, max_len, max_len), dtype="float32") * -1e9 - for i, mask_data in enumerate(attention_mask): - seq_len = len(batch_attention_mask[i]) - mask_data[-seq_len:, -seq_len:] = np.array(batch_attention_mask[i], dtype="float32") - # In order to ensure the correct broadcasting mechanism, expand one - # dimension to the second dimension (n_head of Transformer). - attention_mask = np.expand_dims(attention_mask, axis=1) - return attention_mask - - pad_func = Pad(pad_val=pad_val, pad_right=False, dtype="int64") - - input_ids = pad_func([example["input_ids"] for example in batch_examples]) - token_type_ids = pad_func([example["token_type_ids"] for example in batch_examples]) - position_ids = pad_func([example["position_ids"] for example in batch_examples]) - - attention_mask = pad_mask([example["attention_mask"] for example in batch_examples]) - - if mode != "test": - max_len = max([example["seq_len"] for example in batch_examples]) - masked_positions = np.concatenate( - [ - np.array(example["masked_positions"]) + (max_len - example["seq_len"]) + i * max_len - for i, example in enumerate(batch_examples) - ] - ) - labels = np.concatenate([np.array(example["labels"], dtype="int64") for example in batch_examples]) - return input_ids, token_type_ids, position_ids, attention_mask, masked_positions, labels - else: - return input_ids, token_type_ids, position_ids, attention_mask - - -def create_data_loader(dataset, tokenizer, args, mode): - trans_func = partial( - convert_example, - tokenizer=tokenizer, - max_seq_len=args.max_seq_len, - max_target_len=args.max_target_len, - max_title_len=args.max_title_len, - mode=mode, - ) - dataset = dataset.map(trans_func, lazy=True) - if mode == "train": - batch_sampler = DistributedBatchSampler(dataset, batch_size=args.batch_size, shuffle=True) - else: - batch_sampler = BatchSampler(dataset, batch_size=args.batch_size // 2, shuffle=False) - collate_fn = partial(batchify_fn, pad_val=tokenizer.pad_token_id, mode=mode) - data_loader = DataLoader(dataset, batch_sampler=batch_sampler, collate_fn=collate_fn, return_list=True) - return dataset, data_loader - - -def post_process_sum(token_ids, tokenizer): - """Post-process the decoded sequence. Truncate from the first .""" - eos_pos = len(token_ids) - for i, tok_id in enumerate(token_ids): - if tok_id == tokenizer.mask_token_id: - eos_pos = i - break - token_ids = token_ids[:eos_pos] - tokens = tokenizer.convert_ids_to_tokens(token_ids) - tokens = tokenizer.merge_subword(tokens) - special_tokens = ["[UNK]"] - tokens = [token for token in tokens if token not in special_tokens] - return token_ids, tokens - - -def select_sum(ids, scores, tokenizer, max_dec_len=None, num_return_sequences=1): - results = [] - group = [] - tmp = [] - if scores is not None: - ids = ids.numpy() - scores = scores.numpy() - - if len(ids) != len(scores) or (len(ids) % num_return_sequences) != 0: - raise ValueError( - "the length of `ids` is {}, but the `num_return_sequences` is {}".format( - len(ids), num_return_sequences - ) - ) - - for pred, score in zip(ids, scores): - pred_token_ids, pred_tokens = post_process_sum(pred, tokenizer) - num_token = len(pred_token_ids) - - target = "".join(pred_tokens) - - # not ending - if max_dec_len is not None and num_token >= max_dec_len: - score -= 1e3 - - tmp.append([target, score]) - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - preds = sorted(preds, key=lambda x: -x[1]) - results.append(preds[0][0]) - else: - ids = ids.numpy() - - for pred in ids: - pred_token_ids, pred_tokens = post_process_sum(pred, tokenizer) - num_token = len(pred_token_ids) - response = "".join(pred_tokens) - - # TODO: Support return scores in FT. - tmp.append([response]) - if len(tmp) == num_return_sequences: - group.append(tmp) - tmp = [] - - for preds in group: - results.append(preds[0][0]) - - return results diff --git a/slm/examples/text_generation/unimo-text/run_gen.py b/slm/examples/text_generation/unimo-text/run_gen.py deleted file mode 100644 index ad172403ce..0000000000 --- a/slm/examples/text_generation/unimo-text/run_gen.py +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import time - -import paddle -import paddle.distributed as dist -import paddle.nn.functional as F -from gen_utils import create_data_loader, print_args, select_sum, set_seed -from paddle.optimizer import AdamW - -from paddlenlp.datasets import load_dataset -from paddlenlp.metrics import BLEU -from paddlenlp.transformers import ( - BasicTokenizer, - LinearDecayWithWarmup, - UNIMOLMHeadModel, - UNIMOTokenizer, -) - - -# yapf: disable -def parse_args(): - parser = argparse.ArgumentParser(__doc__) - parser.add_argument('--dataset_name', type=str, default='dureader_qg', help='The name of the dataset to load.') - parser.add_argument('--model_name_or_path', type=str, default='unimo-text-1.0', help='The path or shortcut name of the pre-trained model.') - parser.add_argument("--train_file", type=str, required=False, default=None, help="Train data path.") - parser.add_argument("--predict_file", type=str, required=False, default=None, help="Predict data path.") - parser.add_argument('--save_dir', type=str, default='./checkpoints', help='The directory where the checkpoints will be saved.') - parser.add_argument('--logging_steps', type=int, default=100, help='Log every X updates steps.') - parser.add_argument('--save_steps', type=int, default=1000, help='Save checkpoint every X updates steps.') - parser.add_argument('--seed', type=int, default=1, help='Random seed for initialization.') - parser.add_argument('--batch_size', type=int, default=16, help='Batch size per GPU/CPU for training.') - parser.add_argument('--learning_rate', type=float, default=5e-5, help='The initial learning rate.') - parser.add_argument('--weight_decay', type=float, default=0.01, help='The weight decay for optimizer.') - parser.add_argument('--epochs', type=int, default=3, help='Total number of training epochs to perform.') - parser.add_argument('--warmup_proportion', type=float, default=0.02, help='The number of warmup steps.') - parser.add_argument('--max_grad_norm', type=float, default=1.0, help='The max value of grad norm.') - parser.add_argument('--beta1', type=float, default=0.9, help='beta1') - parser.add_argument('--beta2', type=float, default=0.98, help='beta2') - parser.add_argument('--epsilon', type=float, default=1e-6, help='epsilon') - parser.add_argument('--max_seq_len', type=int, default=512, help='The maximum sequence length of training.') - parser.add_argument('--max_dec_len', type=int, default=20, help='The maximum sequence length of decoding.') - parser.add_argument('--min_dec_len', type=int, default=3, help='The minimal sequence length of decoding.') - parser.add_argument('--max_target_len', type=int, default=30, help='The maximum target sequence length of training.') - parser.add_argument('--max_title_len', type=int, default=30, help='The maximum title sequence length of training.') - parser.add_argument('--num_return_sequences', type=int, default=1, help='The numbers of returned sequences for one input in generation.') - parser.add_argument('--decode_strategy', type=str, default='beam_search', help='The decode strategy in generation.') - parser.add_argument('--top_k', type=int, default=0, help='The number of highest probability vocabulary tokens to keep for top-k sampling.') - parser.add_argument('--temperature', type=float, default=1.0, help='The value used to module the next token probabilities.') - parser.add_argument('--top_p', type=float, default=1.0, help='The cumulative probability for top-p sampling.') - parser.add_argument('--num_beams', type=int, default=6, help='The number of beams for beam search.') - parser.add_argument('--length_penalty', type=float, default=1.2, help='The exponential penalty to the sequence length for beam search.') - parser.add_argument('--device', type=str, default='gpu', help='The device to select for training the model.') - parser.add_argument('--output_path', type=str, default='./predict.txt', help='The file path where the infer result will be saved.') - parser.add_argument("--do_train", action='store_true', help="Whether to train the model.") - parser.add_argument("--do_predict", action='store_true', help="Whether to eval and predict.") - - args = parser.parse_args() - return args -# yapf: enable - - -def calc_bleu(preds, targets): - assert len(preds) == len(targets), ( - "The length of pred_responses should be equal to the length of " - "target_responses. But received {} and {}.".format(len(preds), len(targets)) - ) - bleu4 = BLEU(n_size=4) - tokenizer = BasicTokenizer() - - for pred, target in zip(preds, targets): - pred_tokens = tokenizer.tokenize(pred) - target_token = tokenizer.tokenize(target) - - bleu4.add_inst(pred_tokens, [target_token]) - - print("\n" + "*" * 15) - print("The auto evaluation result is:") - print("BLEU-4:", bleu4.score()) - - -def save_ckpt(model, tokenizer, save_dir, name): - output_dir = os.path.join(save_dir, "model_{}".format(name)) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Need better way to get inner model of DataParallel - model_to_save = model._layers if isinstance(model, paddle.DataParallel) else model - model_to_save.save_pretrained(output_dir) - tokenizer.save_pretrained(output_dir) - - -def run(args): - paddle.set_device(args.device) - world_size = dist.get_world_size() - - if world_size > 1: - dist.init_parallel_env() - set_seed(args.seed) - - model = UNIMOLMHeadModel.from_pretrained(args.model_name_or_path) - tokenizer = UNIMOTokenizer.from_pretrained(args.model_name_or_path) - - if world_size > 1: - model = paddle.DataParallel(model) - - train_ds = load_dataset(args.dataset_name, splits="train", data_files=args.train_file) - dev_ds = load_dataset(args.dataset_name, splits="dev", data_files=args.predict_file) - - train_ds, train_data_loader = create_data_loader(train_ds, tokenizer, args, "train") - dev_ds, dev_data_loader = create_data_loader(dev_ds, tokenizer, args, "test") - - if args.do_train: - num_training_steps = args.epochs * len(train_data_loader) - - lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps, args.warmup_proportion) - # Generate parameter names needed to perform weight decay. - # All bias and LayerNorm parameters are excluded. - - decay_params = [p.name for n, p in model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])] - - optimizer = AdamW( - learning_rate=lr_scheduler, - parameters=model.parameters(), - weight_decay=args.weight_decay, - beta1=args.beta1, - beta2=args.beta2, - epsilon=args.epsilon, - apply_decay_param_fun=lambda x: x in decay_params, - grad_clip=paddle.nn.ClipGradByGlobalNorm(args.max_grad_norm), - ) - - step = 0 - total_time = 0.0 - for epoch in range(args.epochs): - print("\nEpoch %d/%d" % (epoch + 1, args.epochs)) - batch_start_time = time.time() - for inputs in train_data_loader: - step += 1 - labels = inputs[-1] - logits = model(*inputs[:-1]) - labels = paddle.nn.functional.one_hot(labels, num_classes=logits.shape[-1]) - labels = paddle.nn.functional.label_smooth(labels) - loss = F.cross_entropy(logits, labels, soft_label=True) - - loss.backward() - optimizer.step() - lr_scheduler.step() - optimizer.clear_grad() - - total_time += time.time() - batch_start_time - if step % args.logging_steps == 0: - ppl = paddle.exp(loss) - print( - "step %d - loss: %.4f - ppl: %.4f - lr: %.7f - %.3fs/step" - % (step, loss, ppl, optimizer.get_lr(), total_time / args.logging_steps) - ) - total_time = 0.0 - - if step % args.save_steps == 0 or step >= num_training_steps: - if dist.get_rank() == 0: - save_ckpt(model, tokenizer, args.save_dir, step) - print("Saved step {} model.\n".format(step)) - if args.do_predict: - model_eval = model._layers if isinstance(model, paddle.DataParallel) else model - evaluation(model_eval, dev_data_loader, args, tokenizer) - - batch_start_time = time.time() - - print("\nTraining completed.") - elif args.do_predict: - model_eval = model._layers if isinstance(model, paddle.DataParallel) else model - evaluation(model_eval, dev_data_loader, args, tokenizer) - - -@paddle.no_grad() -def evaluation(model, data_loader, args, tokenizer): - print("\nEval begin...") - model.eval() - pred_ref = [] - total_time = 0.0 - start_time = time.time() - for step, inputs in enumerate(data_loader, 1): - input_ids, token_type_ids, position_ids, attention_mask = inputs - ids, scores = model.generate( - input_ids=input_ids, - token_type_ids=token_type_ids, - position_ids=position_ids, - attention_mask=attention_mask, - max_length=args.max_dec_len, - min_length=args.min_dec_len, - decode_strategy=args.decode_strategy, - temperature=args.temperature, - top_k=args.top_k, - top_p=args.top_p, - num_beams=args.num_beams, - length_penalty=args.length_penalty, - num_return_sequences=args.num_return_sequences, - bos_token_id=tokenizer.cls_token_id, - eos_token_id=tokenizer.mask_token_id, - ) - - total_time += time.time() - start_time - if step % args.logging_steps == 0: - print("step %d - %.3fs/step" % (step, total_time / args.logging_steps)) - total_time = 0.0 - - results = select_sum(ids, scores, tokenizer, args.max_dec_len, args.num_return_sequences) - pred_ref.extend(results) - start_time = time.time() - - with open(args.output_path, "w", encoding="utf-8") as fout: - for ref in pred_ref: - fout.write(ref + "\n") - - print("\nSave inference result into: %s" % args.output_path) - - if "target" in data_loader.dataset[0].keys(): - targets = [example["target"] for example in data_loader.dataset] - calc_bleu(pred_ref, targets) - - model.train() - return - - -if __name__ == "__main__": - args = parse_args() - print_args(args) - run(args) diff --git a/slm/examples/text_generation/unimo-text/scripts/lcsts_train.sh b/slm/examples/text_generation/unimo-text/scripts/lcsts_train.sh deleted file mode 100644 index 16472c28ed..0000000000 --- a/slm/examples/text_generation/unimo-text/scripts/lcsts_train.sh +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -unset CUDA_VISIBLE_DEVICES - -log_dir=./lcsts-log -rm -rf ${log_dir} -mkdir -p ${log_dir} - -python -m paddle.distributed.launch --gpus "0,1,2,3" --log_dir ${log_dir} run_gen.py \ - --dataset_name=lcsts_new \ - --model_name_or_path=unimo-text-1.0 \ - --save_dir=${log_dir}/checkpoints \ - --logging_steps=100 \ - --save_steps=10000 \ - --epochs=6 \ - --batch_size=64 \ - --learning_rate=5e-5 \ - --warmup_proportion=0.02 \ - --weight_decay=0.01 \ - --max_seq_len=360 \ - --max_target_len=30 \ - --max_dec_len=20 \ - --min_dec_len=3 \ - --do_train \ - --do_predict \ - --device=gpu >> ${log_dir}/lanch.log 2>&1 diff --git a/slm/examples/text_generation/unimo-text/scripts/qg_train.sh b/slm/examples/text_generation/unimo-text/scripts/qg_train.sh deleted file mode 100644 index 88e8c4ec87..0000000000 --- a/slm/examples/text_generation/unimo-text/scripts/qg_train.sh +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -unset CUDA_VISIBLE_DEVICES - -log_dir=./qg-log -rm -rf ${log_dir} -mkdir -p ${log_dir} - -python -m paddle.distributed.launch --gpus "0,1,2,3" --log_dir ${log_dir} run_gen.py \ - --dataset_name=dureader_qg \ - --model_name_or_path=unimo-text-1.0 \ - --save_dir=${log_dir}/checkpoints \ - --logging_steps=10 \ - --save_steps=1000 \ - --epochs=6 \ - --batch_size=8 \ - --learning_rate=5e-5 \ - --warmup_proportion=0.02 \ - --weight_decay=0.01 \ - --max_seq_len=360 \ - --max_target_len=30 \ - --max_dec_len=20 \ - --min_dec_len=3 \ - --do_train \ - --do_predict \ - --device=gpu >> ${log_dir}/lanch.log 2>&1 diff --git a/slm/examples/text_generation/unimo-text/scripts/table_train.sh b/slm/examples/text_generation/unimo-text/scripts/table_train.sh deleted file mode 100644 index 4865358949..0000000000 --- a/slm/examples/text_generation/unimo-text/scripts/table_train.sh +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# GPU启动，参数`--gpus`指定训练所用的GPU卡号，可以是单卡，也可以多卡 -unset CUDA_VISIBLE_DEVICES - -log_dir=./table-log -rm -rf ${log_dir} -mkdir -p ${log_dir} - -python -m paddle.distributed.launch --gpus "0,1,2,3" --log_dir ${log_dir} run_gen.py \ - --dataset_name=advertisegen \ - --model_name_or_path=unimo-text-1.0 \ - --save_dir=${log_dir}/checkpoints \ - --logging_steps=100 \ - --save_steps=1000 \ - --epochs=6 \ - --batch_size=8 \ - --learning_rate=5e-5 \ - --warmup_proportion=0.02 \ - --weight_decay=0.01 \ - --max_seq_len=512 \ - --max_target_len=200 \ - --max_dec_len=200 \ - --min_dec_len=10 \ - --do_train \ - --do_predict \ - --device=gpu >> ${log_dir}/lanch.log 2>&1 diff --git a/slm/examples/text_generation/vae-seq2seq/README.md b/slm/examples/text_generation/vae-seq2seq/README.md deleted file mode 100644 index 32382e6cc7..0000000000 --- a/slm/examples/text_generation/vae-seq2seq/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Variational Autoencoder (VAE) for Text Generation - -## 简介 - -本目录下此范例模型的实现，旨在展示如何用 Paddle 构建用于文本生成的 VAE 示例，其中 LSTM 作为编码器和解码器。分别对 PTB 数据集和 Yahoo Answer（采样100k）数据集进行训练。 - -关于 VAE 的详细介绍参照： [(Bowman et al., 2015) Generating Sentences from a Continuous Space](https://arxiv.org/pdf/1511.06349.pdf) - -## 数据介绍 - -本教程使用了两个文本数据集： - -PTB 数据集由华尔街日报的文章组成，包含929k 个训练 tokens，词汇量为10k。下载地址为: [PTB](https://dataset.bj.bcebos.com/imikolov%2Fsimple-examples.tgz)。 - -Yahoo 数据集来自[(Yang et al., 2017) Improved Variational Autoencoders for Text Modeling using Dilated Convolutions](https://arxiv.org/pdf/1702.08139.pdf)，该数据集从原始 Yahoo Answer 数据中采样100k 个文档，数据集的平均文档长度为78，词汇量为200k。下载地址为：[YahooAnswer100k](https://bj.bcebos.com/paddlenlp/datasets/yahoo-answer-100k.tar.gz)，运行本例程序后，数据集会自动下载到`~/.paddlenlp/datasets/YahooAnswer100k`目录下。 - - -## 生成样例 - -shareholders were spent about N shares to spend $ N million to ual sell this trust stock last week - -new york stock exchange composite trading trading outnumbered closed at $ N a share down N cents - -the company cited pressure to pursue up existing facilities in the third quarter was for and four N million briefly stocks for so-called unusual liability - -people had down out the kind of and much why your relationship are anyway - -there are a historic investment giant chips which ran the benefit the attempting to original maker - -使用请[参考](https://github.com/PaddlePaddle/PaddleNLP/tree/release/2.8/examples/text_generation/vae-seq2seq) diff --git a/tests/embeddings/__init__.py b/tests/embeddings/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/embeddings/test_token_embedding.py b/tests/embeddings/test_token_embedding.py deleted file mode 100644 index be4854b154..0000000000 --- a/tests/embeddings/test_token_embedding.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -import numpy as np -import paddle - -from paddlenlp.embeddings import TokenEmbedding -from paddlenlp.utils.log import logger -from tests.common_test import CommonTest -from tests.testing_utils import create_test_data, get_vocab_list - -logger.logger.setLevel("ERROR") - - -class TestTokenEmbedding(CommonTest): - def setUp(self): - self.test_data_file = create_test_data(__file__) - self.config["embedding_name"] = "w2v.sikuquanshu.target.word-word.dim300" - self.config["trainable"] = True - - -class TestTokenEmbeddingTrainable(TestTokenEmbedding): - def test_trainable(self): - self.embedding = TokenEmbedding(**self.config) - self.check_output_not_equal(self.config["trainable"], self.embedding.weight.stop_gradient) - - -class TestTokenEmbeddingUNK(TestTokenEmbedding): - def setUp(self): - super().setUp() - self.config["unknown_token"] = "[unk]" # default [UNK], change it - self.config["unknown_token_vector"] = np.random.normal(scale=0.02, size=300).astype(paddle.get_default_dtype()) - - def test_unk_token(self): - self.embedding = TokenEmbedding(**self.config) - self.check_output_equal(self.config["unknown_token"], self.embedding.unknown_token) - self.check_output_equal( - self.config["unknown_token_vector"], self.embedding.search(self.embedding.unknown_token)[0] - ) - - -class TestTokenEmbeddingExtendedVocab(TestTokenEmbedding): - def setUp(self): - super().setUp() - self.config["extended_vocab_path"] = self.test_data_file - - def test_extended_vocab(self): - self.embedding = TokenEmbedding(**self.config) - vocab_list = get_vocab_list(self.config["extended_vocab_path"]) - emb_idx = set(self.embedding.get_idx_list_from_words(vocab_list)) - vocab_idx = set([i for i in range(len(vocab_list))]) - self.assertEqual(emb_idx, vocab_idx) - self.check_output_equal(emb_idx, vocab_idx) - - -class TestTokenEmbeddingKeepExtendedVocab(TestTokenEmbedding): - def setUp(self): - super().setUp() - self.config["extended_vocab_path"] = self.test_data_file - self.config["keep_extended_vocab_only"] = True - - def test_extended_vocab(self): - self.embedding = TokenEmbedding(**self.config) - vocab_list = get_vocab_list(self.config["extended_vocab_path"]) - vocab_size = len(vocab_list) - # +1 means considering [PAD] - self.check_output_equal(vocab_size + 1, len(self.embedding._word_to_idx)) - - -class TestTokenEmbeddingSimilarity(TestTokenEmbedding): - def setUp(self): - super().setUp() - self.config["extended_vocab_path"] = self.test_data_file - self.config["keep_extended_vocab_only"] = True - - def get_dot(self, vec_a, vec_b): - return np.sum(vec_a * vec_b) - - def get_cosine(self, vec_a, vec_b): - return self.get_dot(vec_a, vec_b) / (np.sqrt(self.get_dot(vec_a, vec_a) * self.get_dot(vec_b, vec_b))) - - def get_random_word_vec(self, vocab_list): - vocab_size = len(vocab_list) - ids = np.random.randint(vocab_size, size=2) - word_a, word_b = vocab_list[ids[0]], vocab_list[ids[1]] - vec_a, vec_b = self.embedding.search([word_a, word_b]) - return word_a, word_b, vec_a, vec_b - - def test_cosine_sim(self): - self.embedding = TokenEmbedding(**self.config) - vocab_list = get_vocab_list(self.config["extended_vocab_path"]) - word_a, word_b, vec_a, vec_b = self.get_random_word_vec(vocab_list) - result = self.embedding.cosine_sim(word_a, word_b) - expected_result = self.get_cosine(vec_a, vec_b) - self.check_output_equal(result, expected_result) - - def test_dot(self): - self.embedding = TokenEmbedding(**self.config) - vocab_list = get_vocab_list(self.config["extended_vocab_path"]) - word_a, word_b, vec_a, vec_b = self.get_random_word_vec(vocab_list) - result = self.embedding.dot(word_a, word_b) - expected_result = self.get_dot(vec_a, vec_b) - self.check_output_equal(result, expected_result) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/transformers/gau_alpha/__init__.py b/tests/transformers/gau_alpha/__init__.py deleted file mode 100644 index 595add0aed..0000000000 --- a/tests/transformers/gau_alpha/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/transformers/gau_alpha/test_modeling.py b/tests/transformers/gau_alpha/test_modeling.py deleted file mode 100644 index 783e0eae2d..0000000000 --- a/tests/transformers/gau_alpha/test_modeling.py +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -import paddle - -from paddlenlp.transformers import ( - GAUAlphaConfig, - GAUAlphaForMultipleChoice, - GAUAlphaForQuestionAnswering, - GAUAlphaForSequenceClassification, - GAUAlphaForTokenClassification, - GAUAlphaModel, - GAUAlphaPretrainedModel, -) - -from ...testing_utils import slow -from ..test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask - - -class GAUAlphaModelTester: - def __init__( - self, - parent, - batch_size=13, - seq_length=7, - is_training=True, - use_input_mask=True, - use_token_type_ids=True, - vocab_size=99, - hidden_size=32, - num_hidden_layers=5, - num_attention_heads=4, - intermediate_size=37, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=2, - initializer_range=0.02, - pad_token_id=0, - type_sequence_label_size=2, - use_relative_position=True, - num_labels=3, - num_choices=4, - num_classes=3, - scope=None, - ): - self.parent = parent - self.batch_size = batch_size - self.seq_length = seq_length - self.is_training = is_training - self.use_input_mask = use_input_mask - self.use_token_type_ids = use_token_type_ids - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.intermediate_size = intermediate_size - self.hidden_act = hidden_act - self.hidden_dropout_prob = hidden_dropout_prob - self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.max_position_embeddings = max_position_embeddings - self.type_vocab_size = type_vocab_size - self.initializer_range = initializer_range - self.pad_token_id = pad_token_id - self.type_sequence_label_size = type_sequence_label_size - self.use_relative_position = use_relative_position - self.num_classes = num_classes - self.num_labels = num_labels - self.num_choices = num_choices - self.scope = scope - - def prepare_config_and_inputs(self): - input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) - - input_mask = None - if self.use_input_mask: - input_mask = random_attention_mask([self.batch_size, self.seq_length]) - - token_type_ids = None - if self.use_token_type_ids: - token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) - - sequence_labels = None - token_labels = None - choice_labels = None - - if self.parent.use_labels: - sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) - token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) - choice_labels = ids_tensor([self.batch_size], self.num_choices) - - config = self.get_config() - return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels - - def get_config(self): - return GAUAlphaConfig( - vocab_size=self.vocab_size, - hidden_size=self.hidden_size, - num_hidden_layers=self.num_hidden_layers, - num_attention_heads=self.num_attention_heads, - intermediate_size=self.intermediate_size, - hidden_act=self.hidden_act, - hidden_dropout_prob=self.hidden_dropout_prob, - attention_probs_dropout_prob=self.attention_probs_dropout_prob, - max_position_embeddings=self.max_position_embeddings, - type_vocab_size=self.type_vocab_size, - initializer_range=self.initializer_range, - pad_token_id=self.pad_token_id, - use_relative_position=self.use_relative_position, - num_class=self.num_classes, - num_labels=self.num_labels, - num_choices=self.num_choices, - ) - - def create_and_check_model( - self, - config, - input_ids, - token_type_ids, - input_mask, - sequence_labels, - token_labels, - choice_labels, - ): - model = GAUAlphaModel(config) - model.eval() - result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) - result = model(input_ids, token_type_ids=token_type_ids) - result = model(input_ids) - self.parent.assertEqual(result.shape, [self.batch_size, self.seq_length, self.hidden_size]) - - def create_and_check_for_multiple_choice( - self, - config, - input_ids, - token_type_ids, - input_mask, - sequence_labels, - token_labels, - choice_labels, - ): - model = GAUAlphaForMultipleChoice(config) - model.eval() - multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand([-1, self.num_choices, -1]) - multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand([-1, self.num_choices, -1]) - multiple_choice_input_mask = input_mask.unsqueeze(1).expand([-1, self.num_choices, -1]) - result = model( - multiple_choice_inputs_ids, - attention_mask=multiple_choice_input_mask, - token_type_ids=multiple_choice_token_type_ids, - ) - - if paddle.is_tensor(result): - result = [result] - - self.parent.assertEqual(result[0].shape, [self.batch_size, self.num_choices]) - - def create_and_check_for_question_answering( - self, - config, - input_ids, - token_type_ids, - input_mask, - sequence_labels, - token_labels, - choice_labels, - ): - model = GAUAlphaForQuestionAnswering(config) - model.eval() - result = model( - input_ids, - attention_mask=input_mask, - token_type_ids=token_type_ids, - ) - - start_logits, end_logits = result[0], result[1] - - self.parent.assertEqual(start_logits.shape, [self.batch_size, self.seq_length]) - self.parent.assertEqual(end_logits.shape, [self.batch_size, self.seq_length]) - - def create_and_check_for_sequence_classification( - self, - config, - input_ids, - token_type_ids, - input_mask, - sequence_labels, - token_labels, - choice_labels, - ): - model = GAUAlphaForSequenceClassification(config) - model.eval() - result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) - if paddle.is_tensor(result): - result = [result] - - self.parent.assertEqual(result[0].shape, [self.batch_size, self.num_classes]) - - def create_and_check_for_token_classification( - self, - config, - input_ids, - token_type_ids, - input_mask, - sequence_labels, - token_labels, - choice_labels, - ): - model = GAUAlphaForTokenClassification(config) - model.eval() - result = model( - input_ids, - attention_mask=input_mask, - token_type_ids=token_type_ids, - ) - if paddle.is_tensor(result): - result = [result] - - self.parent.assertEqual(result[0].shape, [self.batch_size, self.seq_length, self.num_classes]) - - def prepare_config_and_inputs_for_common(self): - config_and_inputs = self.prepare_config_and_inputs() - ( - config, - input_ids, - token_type_ids, - input_mask, - sequence_labels, - token_labels, - choice_labels, - ) = config_and_inputs - inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask} - return config, inputs_dict - - -class GAUAlphaModelTest(ModelTesterMixin, unittest.TestCase): - base_model_class = GAUAlphaModel - return_dict: bool = False - use_labels: bool = False - use_test_inputs_embeds: bool = False - test_tie_weights = True - - all_model_classes = ( - GAUAlphaModel, - GAUAlphaForMultipleChoice, - GAUAlphaForQuestionAnswering, - GAUAlphaForSequenceClassification, - GAUAlphaForTokenClassification, - ) - - def setUp(self): - self.model_tester = GAUAlphaModelTester(self) - - def test_model(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_model(*config_and_inputs) - - def test_for_multiple_choice(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs) - - def test_for_question_answering(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_for_question_answering(*config_and_inputs) - - def test_for_sequence_classification(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs) - - def test_for_token_classification(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_for_token_classification(*config_and_inputs) - - @slow - def test_model_from_pretrained(self): - for model_name in list(GAUAlphaPretrainedModel.pretrained_init_configuration)[:1]: - model = GAUAlphaModel.from_pretrained(model_name) - self.assertIsNotNone(model) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/transformers/gau_alpha/test_tokenizer.py b/tests/transformers/gau_alpha/test_tokenizer.py deleted file mode 100644 index c720603f8a..0000000000 --- a/tests/transformers/gau_alpha/test_tokenizer.py +++ /dev/null @@ -1,241 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import unittest - -from paddlenlp.transformers import BasicTokenizer, GAUAlphaTokenizer, WordpieceTokenizer - -from ...testing_utils import slow -from ...transformers.test_tokenizer_common import ( - TokenizerTesterMixin, - filter_non_english, -) - - -class GAUAlphaTokenizationTest(TokenizerTesterMixin, unittest.TestCase): - - tokenizer_class = GAUAlphaTokenizer - space_between_special_tokens = True - from_pretrained_filter = filter_non_english - test_seq2seq = True - - def setUp(self): - super().setUp() - - vocab_tokens = [ - "[UNK]", - "[CLS]", - "[SEP]", - "[PAD]", - "[MASK]", - "want", - "##want", - "##ed", - "wa", - "un", - "runn", - "##ing", - ",", - "low", - "lowest", - ] - - self.vocab_file = os.path.join(self.tmpdirname, GAUAlphaTokenizer.resource_files_names["vocab_file"]) - with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer: - vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) - - def get_input_output_texts(self, tokenizer): - input_text = "UNwant\u00E9d,running" - output_text = "unwanted, running" - return input_text, output_text - - def test_full_tokenizer(self): - tokenizer = self.tokenizer_class(self.vocab_file) - - tokens = tokenizer.tokenize("UNwant\u00E9d,running") - self.assertListEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"]) - self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [9, 6, 7, 12, 10, 11]) - - def test_chinese(self): - tokenizer = BasicTokenizer() - - self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"), ["ah", "\u535A", "\u63A8", "zz"]) - - def test_basic_tokenizer_lower(self): - tokenizer = BasicTokenizer(do_lower_case=True) - - self.assertListEqual( - tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "), ["hello", "!", "how", "are", "you", "?"] - ) - self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"]) - - def test_basic_tokenizer_lower_strip_accents_false(self): - tokenizer = BasicTokenizer(do_lower_case=True, strip_accents=False) - - self.assertListEqual( - tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["hällo", "!", "how", "are", "you", "?"] - ) - self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["h\u00E9llo"]) - - def test_basic_tokenizer_lower_strip_accents_true(self): - tokenizer = BasicTokenizer(do_lower_case=True) - - self.assertListEqual( - tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["hallo", "!", "how", "are", "you", "?"] - ) - self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"]) - - def test_basic_tokenizer_lower_strip_accents_default(self): - tokenizer = BasicTokenizer(do_lower_case=True) - - self.assertListEqual( - tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["hallo", "!", "how", "are", "you", "?"] - ) - self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"]) - - def test_basic_tokenizer_no_lower(self): - tokenizer = BasicTokenizer(do_lower_case=False) - - self.assertListEqual( - tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "), ["HeLLo", "!", "how", "Are", "yoU", "?"] - ) - - def test_basic_tokenizer_no_lower_strip_accents_false(self): - tokenizer = BasicTokenizer(do_lower_case=False, strip_accents=False) - - self.assertListEqual( - tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["HäLLo", "!", "how", "Are", "yoU", "?"] - ) - - def test_basic_tokenizer_no_lower_strip_accents_true(self): - tokenizer = BasicTokenizer(do_lower_case=False, strip_accents=True) - - self.assertListEqual( - tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["HaLLo", "!", "how", "Are", "yoU", "?"] - ) - - def test_basic_tokenizer_respects_never_split_tokens(self): - tokenizer = BasicTokenizer(do_lower_case=False, never_split=["[UNK]"]) - - self.assertListEqual( - tokenizer.tokenize(" \tHeLLo!how \n Are yoU? [UNK]"), ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"] - ) - - def test_wordpiece_tokenizer(self): - vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"] - - vocab = {} - for (i, token) in enumerate(vocab_tokens): - vocab[token] = i - tokenizer = WordpieceTokenizer(vocab=vocab, unk_token="[UNK]") - - self.assertListEqual(tokenizer.tokenize(""), []) - - self.assertListEqual(tokenizer.tokenize("unwanted running"), ["un", "##want", "##ed", "runn", "##ing"]) - - self.assertListEqual(tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) - - def test_clean_text(self): - tokenizer = self.get_tokenizer() - - # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340 - self.assertListEqual([tokenizer.tokenize(t) for t in ["Test", "\xad", "test"]], [["[UNK]"], [], ["[UNK]"]]) - - @slow - def test_sequence_builders(self): - tokenizer = self.tokenizer_class.from_pretrained("ernie-1.0") - - text = tokenizer.encode("sequence builders", return_token_type_ids=None, add_special_tokens=False)["input_ids"] - text_2 = tokenizer.encode("multi-sequence build", return_token_type_ids=None, add_special_tokens=False)[ - "input_ids" - ] - - encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) - encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) - print(encoded_sentence) - assert encoded_sentence == [1] + text + [2] - assert encoded_pair == [1] + text + [2] + text_2 + [2] - - def test_offsets_with_special_characters(self): - for tokenizer, pretrained_name, kwargs in self.tokenizers_list: - with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): - tokenizer = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs) - - sentence = f"A, naïve {tokenizer.mask_token} AllenNLP sentence." - tokens = tokenizer.encode( - sentence, - return_attention_mask=False, - return_token_type_ids=False, - return_offsets_mapping=True, - add_special_tokens=True, - ) - - do_lower_case = tokenizer.do_lower_case if hasattr(tokenizer, "do_lower_case") else False - expected_results = ( - [ - ((0, 0), tokenizer.cls_token), - ((0, 1), "A"), - ((1, 2), ","), - ((3, 5), "na"), - ((5, 6), "##ï"), - ((6, 8), "##ve"), - ((9, 15), tokenizer.mask_token), - ((16, 21), "Allen"), - ((21, 23), "##NL"), - ((23, 24), "##P"), - ((25, 33), "sentence"), - ((33, 34), "."), - ((0, 0), tokenizer.sep_token), - ] - if not do_lower_case - else [ - ((0, 0), tokenizer.cls_token), - ((0, 1), "a"), - ((1, 2), ","), - ((3, 5), "na"), - ((5, 8), "##ive"), - ((9, 15), tokenizer.mask_token), - ((16, 21), "allen"), - ((21, 22), "##n"), - ((22, 24), "##lp"), - ((25, 27), "se"), - ((27, 29), "##nt"), - ((29, 33), "##ence"), - ((33, 34), "."), - ((0, 0), tokenizer.sep_token), - ] - ) - - self.assertEqual( - [e[1] for e in expected_results], tokenizer.convert_ids_to_tokens(tokens["input_ids"]) - ) - self.assertEqual([e[0] for e in expected_results], tokens["offset_mapping"]) - - def test_change_tokenize_chinese_chars(self): - list_of_commun_chinese_char = ["的", "人", "有"] - text_with_chinese_char = "".join(list_of_commun_chinese_char) - for tokenizer, pretrained_name, kwargs in self.tokenizers_list: - with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): - - kwargs["tokenize_chinese_chars"] = True - tokenizer = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs) - - ids_without_spe_char_p = tokenizer.encode( - text_with_chinese_char, return_token_type_ids=None, add_special_tokens=False - )["input_ids"] - tokens_without_spe_char_p = tokenizer.convert_ids_to_tokens(ids_without_spe_char_p) - - # it is expected that each Chinese character is not preceded by "##" - self.assertListEqual(tokens_without_spe_char_p, list_of_commun_chinese_char)