File “./ds_train.py”, line 438, in
main()
File “./ds_train.py”, line 378, in main
train_result = trainer.train(resume_from_checkpoint=checkpoint)
File “/home/hechenghua/anaconda3/envs/swiftsage/lib/python3.8/site-packages/transformers/trainer.py”, line 1539, in train
return inner_training_loop(
File “/home/hechenghua/anaconda3/envs/swiftsage/lib/python3.8/site-packages/transformers/trainer.py”, line 1659, in _inner_training_loop
model, self.optimizer, self.lr_scheduler = self.accelerator.prepare(
File “/home/hechenghua/anaconda3/envs/swiftsage/lib/python3.8/site-packages/accelerate/accelerator.py”, line 1178, in prepare
result = self._prepare_deepspeed(*args)
File “/home/hechenghua/anaconda3/envs/swiftsage/lib/python3.8/site-packages/accelerate/accelerator.py”, line 1486, in _prepare_deepspeed
deepspeed_plugin.deepspeed_config_process(must_match=False, **config_kwargs)
File “/home/hechenghua/anaconda3/envs/swiftsage/lib/python3.8/site-packages/accelerate/utils/dataclasses.py”, line 624, in deepspeed_config_process
self.deepspeed_config_process(
File “/home/hechenghua/anaconda3/envs/swiftsage/lib/python3.8/site-packages/accelerate/utils/dataclasses.py”, line 628, in deepspeed_config_process
self.fill_match(prefix + key, mismatches, must_match=must_match, **kwargs)
File “/home/hechenghua/anaconda3/envs/swiftsage/lib/python3.8/site-packages/accelerate/utils/dataclasses.py”, line 603, in fill_match
raise ValueError(
ValueError: bfloat16.enabled not found in kwargs. Please specify bfloat16.enabled without auto(set to correct value) in the DeepSpeed config file or pass it in kwargs.