From cedff3ed812144f5f00f9c0e64de643d5dc44b06 Mon Sep 17 00:00:00 2001 From: ForFishes <2282912238@qq.com> Date: Mon, 12 Sep 2022 13:09:26 +0000 Subject: [PATCH 1/3] add yaml --- ppfleetx/configs/nlp/gpt/gpt_base.yaml | 6 ++++ .../configs/nlp/gpt/pretrain_gpt_1.3B.yaml | 31 +++++++++++++++++++ .../nlp/gpt/pretrain_gpt_1.3B_dp8.yaml | 3 -- .../nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml | 31 +++++++++++++++++++ .../configs/nlp/gpt/pretrain_gpt_345M.yaml | 31 +++++++++++++++++++ .../nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml | 31 +++++++++++++++++++ .../models/language_model/gpt/__init__.py | 2 +- ppfleetx/models/language_model/utils.py | 2 -- ppfleetx/utils/env.py | 15 +++++---- tools/train.py | 7 +++-- tools/train.sh | 1 + 11 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B.yaml create mode 100644 ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml create mode 100644 ppfleetx/configs/nlp/gpt/pretrain_gpt_345M.yaml create mode 100644 ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml diff --git a/ppfleetx/configs/nlp/gpt/gpt_base.yaml b/ppfleetx/configs/nlp/gpt/gpt_base.yaml index 5af9e71a1..dda2fedf4 100644 --- a/ppfleetx/configs/nlp/gpt/gpt_base.yaml +++ b/ppfleetx/configs/nlp/gpt/gpt_base.yaml @@ -23,6 +23,12 @@ Engine: output_dir: ./output ckpt_dir: +Model: + module: "GPTModule" + name: "GPT" + fused_linear: False + + Data: Train: dataset: diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B.yaml new file mode 100644 index 000000000..714d98ba8 --- /dev/null +++ b/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B.yaml @@ -0,0 +1,31 @@ +_base_: ./gpt_base.yaml + +Global: + global_batch_size: 8 + local_batch_size: 8 + micro_batch_size: 8 + + +Model: + vocab_size: 50304 + hidden_size: 2048 + num_layers: 24 + num_attention_heads: 16 + ffn_hidden_size: + hidden_dropout_prob: 0.1 + attention_probs_dropout_prob: 0.1 + max_position_embeddings: 1024 + type_vocab_size: 16 + initializer_range: 0.02 + use_recompute: True + recompute_granularity: + + +Distributed: + dp_degree: 1 + mp_degree: 1 + pp_degree: 1 + sharding: + sharding_degree: 1 + sharding_stage: 1 + sharding_offload: False diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_dp8.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_dp8.yaml index 10f218407..80f45f0fa 100644 --- a/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_dp8.yaml +++ b/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_dp8.yaml @@ -7,8 +7,6 @@ Global: Model: - module: "GPTModule" - name: "GPT" vocab_size: 50304 hidden_size: 2048 num_layers: 24 @@ -21,7 +19,6 @@ Model: initializer_range: 0.02 use_recompute: True recompute_granularity: - fused_linear: False Distributed: diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml new file mode 100644 index 000000000..2313509ba --- /dev/null +++ b/ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml @@ -0,0 +1,31 @@ +_base_: ./gpt_base.yaml + +Global: + global_batch_size: + local_batch_size: 1536 + micro_batch_size: 1 + + +Model: + vocab_size: 51200 + hidden_size: 12288 + num_layers: 96 + num_attention_heads: 96 + ffn_hidden_size: + hidden_dropout_prob: 0.1 + attention_probs_dropout_prob: 0.1 + max_position_embeddings: 1024 + type_vocab_size: 16 + initializer_range: 0.02 + use_recompute: True + recompute_granularity: + + +Distributed: + dp_degree: + mp_degree: 8 + pp_degree: 16 + sharding: + sharding_degree: 1 + sharding_stage: 1 + sharding_offload: False diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_345M.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_345M.yaml new file mode 100644 index 000000000..d0277b7a7 --- /dev/null +++ b/ppfleetx/configs/nlp/gpt/pretrain_gpt_345M.yaml @@ -0,0 +1,31 @@ +_base_: ./gpt_base.yaml + +Global: + global_batch_size: 8 + local_batch_size: 8 + micro_batch_size: 8 + + +Model: + vocab_size: 50304 + hidden_size: 1024 + num_layers: 24 + num_attention_heads: 16 + ffn_hidden_size: 4096 + hidden_dropout_prob: 0.1 + attention_probs_dropout_prob: 0.1 + max_position_embeddings: 1024 + type_vocab_size: 16 + initializer_range: 0.02 + use_recompute: False + recompute_granularity: + + +Distributed: + dp_degree: 1 + mp_degree: 1 + pp_degree: 1 + sharding: + sharding_degree: 1 + sharding_stage: 1 + sharding_offload: False diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml new file mode 100644 index 000000000..570b2fc83 --- /dev/null +++ b/ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml @@ -0,0 +1,31 @@ +_base_: ./gpt_base.yaml + +Global: + global_batch_size: + local_batch_size: 8 + micro_batch_size: 8 + + +Model: + vocab_size: 50304 + hidden_size: 4096 + num_layers: 32 + num_attention_heads: 32 + ffn_hidden_size: + hidden_dropout_prob: 0.1 + attention_probs_dropout_prob: 0.1 + max_position_embeddings: 1024 + type_vocab_size: 16 + initializer_range: 0.02 + use_recompute: True + recompute_granularity: + + +Distributed: + dp_degree: + mp_degree: 1 + pp_degree: 1 + sharding: + sharding_degree: 16 + sharding_stage: 2 + sharding_offload: False diff --git a/ppfleetx/models/language_model/gpt/__init__.py b/ppfleetx/models/language_model/gpt/__init__.py index dcfecdd37..8d11f8ee9 100644 --- a/ppfleetx/models/language_model/gpt/__init__.py +++ b/ppfleetx/models/language_model/gpt/__init__.py @@ -16,4 +16,4 @@ GPTPretrainingCriterionHybird, GPTForPretrainingHybrid) -from .single_model import GPTForPretraining, GPTPretrainingCriterion +from .single_model import GPTForPretraining, GPTPretrainingCriterion, GPTModel diff --git a/ppfleetx/models/language_model/utils.py b/ppfleetx/models/language_model/utils.py index c204e5277..205721886 100644 --- a/ppfleetx/models/language_model/utils.py +++ b/ppfleetx/models/language_model/utils.py @@ -120,8 +120,6 @@ def process_engine_configs(config): def process_configs(config): - - # process_dist_configs(config) process_data_configs(config) process_fused_configs(config) process_model_configs(config) diff --git a/ppfleetx/utils/env.py b/ppfleetx/utils/env.py index 1c6aaa2e5..264d8a66c 100644 --- a/ppfleetx/utils/env.py +++ b/ppfleetx/utils/env.py @@ -24,12 +24,15 @@ __all__ = ['init_dist_env'] -def set_dist_seed(seed): - # obtain rank message of hybrid parallel - hcg = fleet.get_hybrid_communicate_group() - mp_rank = hcg.get_model_parallel_rank() - pp_rank = hcg.get_stage_id() - data_world_rank = get_data_world_rank() +def set_seed(seed): + if dist.get_world_size() > 1: + # obtain rank message of hybrid parallel + hcg = fleet.get_hybrid_communicate_group() + mp_rank = hcg.get_model_parallel_rank() + pp_rank = hcg.get_stage_id() + data_world_rank = get_data_world_rank() + else: + mp_rank, pp_rank, data_world_rank = 1, 1, 1 random.seed(seed + data_world_rank) np.random.seed(seed + data_world_rank) diff --git a/tools/train.py b/tools/train.py index 022f9d9c4..14af48ae2 100644 --- a/tools/train.py +++ b/tools/train.py @@ -29,6 +29,7 @@ # from ppfleetx.data import build_dataloader from ppfleetx.models import build_module from ppfleetx.optims import build_lr_scheduler, build_optimizer +import paddle.distributed as dist init_logger() @@ -36,8 +37,10 @@ args = config.parse_args() cfg = config.get_config(args.config, overrides=args.override, show=False) - fleet.init(is_collective=True, strategy=env.init_dist_env(cfg)) - env.set_dist_seed(cfg.Global.seed) + if dist.get_world_size() > 1: + fleet.init(is_collective=True, strategy=env.init_dist_env(cfg)) + + env.set_seed(cfg.Global.seed) module = build_module(cfg) config.print_config(cfg) diff --git a/tools/train.sh b/tools/train.sh index cc178ff40..b24bdb84f 100644 --- a/tools/train.sh +++ b/tools/train.sh @@ -20,3 +20,4 @@ # for multi-cards train export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python3.7 -m paddle.distributed.launch --gpus="0,1,2,3,4,5,6,7" train.py -c ../ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_dp8.yaml +# python train.py -c ../ppfleetx/configs/nlp/gpt/pretrain_gpt_345M.yaml From 0c59d45563549a837b1a568eec5a38ae560af50e Mon Sep 17 00:00:00 2001 From: ForFishes <2282912238@qq.com> Date: Mon, 12 Sep 2022 13:11:03 +0000 Subject: [PATCH 2/3] add yaml --- projects/gpt/pretrain_gpt_1.3B.sh | 35 +++++++++++++++ projects/gpt/pretrain_gpt_1.3B_dp8.sh | 23 ++++++++++ projects/gpt/pretrain_gpt_175B_mp8_pp16.sh | 46 ++++++++++++++++++++ projects/gpt/pretrain_gpt_345M.sh | 35 +++++++++++++++ projects/gpt/pretrain_gpt_6.7B_sharding16.sh | 23 ++++++++++ 5 files changed, 162 insertions(+) create mode 100644 projects/gpt/pretrain_gpt_1.3B.sh create mode 100644 projects/gpt/pretrain_gpt_1.3B_dp8.sh create mode 100644 projects/gpt/pretrain_gpt_175B_mp8_pp16.sh create mode 100644 projects/gpt/pretrain_gpt_345M.sh create mode 100644 projects/gpt/pretrain_gpt_6.7B_sharding16.sh diff --git a/projects/gpt/pretrain_gpt_1.3B.sh b/projects/gpt/pretrain_gpt_1.3B.sh new file mode 100644 index 000000000..9fac470c1 --- /dev/null +++ b/projects/gpt/pretrain_gpt_1.3B.sh @@ -0,0 +1,35 @@ + +#! /bin/bash + +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs the "1.3B" parameter model + +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +python ../../tools/train.py -c ../../ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B.yaml diff --git a/projects/gpt/pretrain_gpt_1.3B_dp8.sh b/projects/gpt/pretrain_gpt_1.3B_dp8.sh new file mode 100644 index 000000000..cb8a5873b --- /dev/null +++ b/projects/gpt/pretrain_gpt_1.3B_dp8.sh @@ -0,0 +1,23 @@ +#! /bin/bash +# Runs the "1.3B" parameter model +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log_dir=log_hybrid +rm -rf $log_dir + +# 1.3B+dp8 run_pretrain +python -m paddle.distributed.launch --log_dir $log_dir --devices "0,1,2,3,4,5,6,7" \ + ../../tools/train.py \ + -c ../../ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_dp8.yaml diff --git a/projects/gpt/pretrain_gpt_175B_mp8_pp16.sh b/projects/gpt/pretrain_gpt_175B_mp8_pp16.sh new file mode 100644 index 000000000..d23f680c9 --- /dev/null +++ b/projects/gpt/pretrain_gpt_175B_mp8_pp16.sh @@ -0,0 +1,46 @@ +#! /bin/bash + +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### + # @Author: your name + # @Date: 2022-09-12 21:06:53 + # @LastEditTime: 2022-09-12 21:07:38 + # @LastEditors: yq01-sys-hic-k8s-v100-box-a225-0281.yq01.baidu.com + # @Description: In User Settings Edit + # @FilePath: /FleetX/projects/gpt/pretrain_gpt_6.7B_sharding16 copy.sh +### +# Runs the "1.3B" parameter model +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log_dir=log_hybrid +rm -rf $log_dir + +# 175B+mp8_pp16 run_pretrain +python -m paddle.distributed.launch --log_dir $log_dir --devices "0,1,2,3,4,5,6,7" \ + ../../tools/train.py \ + -c ../../ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml diff --git a/projects/gpt/pretrain_gpt_345M.sh b/projects/gpt/pretrain_gpt_345M.sh new file mode 100644 index 000000000..02516ca57 --- /dev/null +++ b/projects/gpt/pretrain_gpt_345M.sh @@ -0,0 +1,35 @@ + +#! /bin/bash + +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs the "345M" parameter model + +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +python ../../tools/train.py -c ../../ppfleetx/configs/nlp/gpt/pretrain_gpt_345M.yaml diff --git a/projects/gpt/pretrain_gpt_6.7B_sharding16.sh b/projects/gpt/pretrain_gpt_6.7B_sharding16.sh new file mode 100644 index 000000000..55e549ec8 --- /dev/null +++ b/projects/gpt/pretrain_gpt_6.7B_sharding16.sh @@ -0,0 +1,23 @@ +#! /bin/bash +# Runs the "1.3B" parameter model +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log_dir=log_hybrid +rm -rf $log_dir + +# 6.7B+sharding16 run_pretrain +python -m paddle.distributed.launch --log_dir $log_dir --devices "0,1,2,3,4,5,6,7" \ + ../../tools/train.py \ + -c ../../ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml From dd24968ddf65c8dba08dd6687d07b7f4d25ea197 Mon Sep 17 00:00:00 2001 From: ForFishes <2282912238@qq.com> Date: Mon, 12 Sep 2022 13:12:53 +0000 Subject: [PATCH 3/3] add yaml --- projects/gpt/pretrain_gpt_1.3B.sh | 16 --------------- projects/gpt/pretrain_gpt_1.3B_dp8.sh | 2 +- projects/gpt/pretrain_gpt_175B_mp8_pp16.sh | 23 ---------------------- projects/gpt/pretrain_gpt_345M.sh | 15 -------------- 4 files changed, 1 insertion(+), 55 deletions(-) diff --git a/projects/gpt/pretrain_gpt_1.3B.sh b/projects/gpt/pretrain_gpt_1.3B.sh index 9fac470c1..109626e96 100644 --- a/projects/gpt/pretrain_gpt_1.3B.sh +++ b/projects/gpt/pretrain_gpt_1.3B.sh @@ -15,21 +15,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Runs the "1.3B" parameter model - -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - python ../../tools/train.py -c ../../ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B.yaml diff --git a/projects/gpt/pretrain_gpt_1.3B_dp8.sh b/projects/gpt/pretrain_gpt_1.3B_dp8.sh index cb8a5873b..2905cdc84 100644 --- a/projects/gpt/pretrain_gpt_1.3B_dp8.sh +++ b/projects/gpt/pretrain_gpt_1.3B_dp8.sh @@ -1,5 +1,5 @@ #! /bin/bash -# Runs the "1.3B" parameter model + # Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/projects/gpt/pretrain_gpt_175B_mp8_pp16.sh b/projects/gpt/pretrain_gpt_175B_mp8_pp16.sh index d23f680c9..d3bda96cf 100644 --- a/projects/gpt/pretrain_gpt_175B_mp8_pp16.sh +++ b/projects/gpt/pretrain_gpt_175B_mp8_pp16.sh @@ -14,29 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -### - # @Author: your name - # @Date: 2022-09-12 21:06:53 - # @LastEditTime: 2022-09-12 21:07:38 - # @LastEditors: yq01-sys-hic-k8s-v100-box-a225-0281.yq01.baidu.com - # @Description: In User Settings Edit - # @FilePath: /FleetX/projects/gpt/pretrain_gpt_6.7B_sharding16 copy.sh -### -# Runs the "1.3B" parameter model -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - log_dir=log_hybrid rm -rf $log_dir diff --git a/projects/gpt/pretrain_gpt_345M.sh b/projects/gpt/pretrain_gpt_345M.sh index 02516ca57..3b5f17ea4 100644 --- a/projects/gpt/pretrain_gpt_345M.sh +++ b/projects/gpt/pretrain_gpt_345M.sh @@ -15,21 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Runs the "345M" parameter model - -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. python ../../tools/train.py -c ../../ppfleetx/configs/nlp/gpt/pretrain_gpt_345M.yaml