From 141d198b9f2b5f171924d1f550aaedcb16f168fc Mon Sep 17 00:00:00 2001 From: wangshuai09 <391746016@qq.com> Date: Wed, 22 Nov 2023 09:34:52 +0800 Subject: [PATCH 1/2] fix tokenizer.pad_token attribute error --- fastchat/train/train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastchat/train/train.py b/fastchat/train/train.py index 4997ca76b..e04244e22 100644 --- a/fastchat/train/train.py +++ b/fastchat/train/train.py @@ -277,7 +277,8 @@ def train(): padding_side="right", use_fast=False, ) - tokenizer.pad_token = tokenizer.unk_token + if tokenizer.pad_token and tokenizer.pad_token != tokenizer.unk_token: + tokenizer.pad_token = tokenizer.unk_token # Load data data_module = make_supervised_data_module(tokenizer=tokenizer, data_args=data_args) From 34d5c9004b7654c82181c0783388c4462b08d132 Mon Sep 17 00:00:00 2001 From: wangshuai09 <391746016@qq.com> Date: Wed, 22 Nov 2023 17:17:16 +0800 Subject: [PATCH 2/2] fix bug --- fastchat/train/train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastchat/train/train.py b/fastchat/train/train.py index e04244e22..8b94004d4 100644 --- a/fastchat/train/train.py +++ b/fastchat/train/train.py @@ -277,7 +277,8 @@ def train(): padding_side="right", use_fast=False, ) - if tokenizer.pad_token and tokenizer.pad_token != tokenizer.unk_token: + + if tokenizer.pad_token != tokenizer.unk_token: tokenizer.pad_token = tokenizer.unk_token # Load data