Merge pull request #964 from QwenLM/fix-finetune

Fix bug of low_cpu_mem_usage in finetune.py.
1 year ago · 204c2c59f4
parent ab109ced9f 73b34d4a9d
commit 204c2c59f4
1 changed files with 3 additions and 3 deletions
--- a/finetune.py
+++ b/finetune.py
@ -291,9 +291,9 @@ def train():
    ):
        raise RuntimeError("ZeRO3 is incompatible with LoRA when finetuning on base model.")
-    model_load_kwargs = {}
+    model_load_kwargs = {
-    if deepspeed.is_deepspeed_zero3_enabled():
+        'low_cpu_mem_usage': not deepspeed.is_deepspeed_zero3_enabled(),
-        model_load_kwargs['low_cpu_mem_usage'] = False
+    }
    # Set RoPE scaling factor
    config = transformers.AutoConfig.from_pretrained(