From 73f4040abb8e4cf3f92bc577d161611cea78b652 Mon Sep 17 00:00:00 2001 From: JustinLin610 Date: Tue, 12 Sep 2023 23:10:59 +0800 Subject: [PATCH] add gradient checkpointing --- finetune/finetune_lora_ds.sh | 1 + finetune/finetune_lora_single_gpu.sh | 1 + finetune/finetune_qlora_ds.sh | 1 + finetune/finetune_qlora_single_gpu.sh | 1 + 4 files changed, 4 insertions(+) diff --git a/finetune/finetune_lora_ds.sh b/finetune/finetune_lora_ds.sh index 12efc6e..d877030 100644 --- a/finetune/finetune_lora_ds.sh +++ b/finetune/finetune_lora_ds.sh @@ -44,4 +44,5 @@ torchrun $DISTRIBUTED_ARGS finetune.py \ --model_max_length 2048 \ --lazy_preprocess True \ --use_lora \ + --gradient_checkpointing \ --deepspeed finetune/ds_config_zero2.json \ No newline at end of file diff --git a/finetune/finetune_lora_single_gpu.sh b/finetune/finetune_lora_single_gpu.sh index 47b7237..9333b7d 100644 --- a/finetune/finetune_lora_single_gpu.sh +++ b/finetune/finetune_lora_single_gpu.sh @@ -32,4 +32,5 @@ python finetune.py \ --report_to "none" \ --model_max_length 2048 \ --lazy_preprocess True \ + --gradient_checkpointing \ --use_lora \ No newline at end of file diff --git a/finetune/finetune_qlora_ds.sh b/finetune/finetune_qlora_ds.sh index debea4f..58c05e9 100644 --- a/finetune/finetune_qlora_ds.sh +++ b/finetune/finetune_qlora_ds.sh @@ -46,4 +46,5 @@ torchrun $DISTRIBUTED_ARGS finetune.py \ --lazy_preprocess True \ --use_lora \ --q_lora \ + --gradient_checkpointing \ --deepspeed finetune/ds_config_zero2.json \ No newline at end of file diff --git a/finetune/finetune_qlora_single_gpu.sh b/finetune/finetune_qlora_single_gpu.sh index 3b89ee4..bf65b9f 100644 --- a/finetune/finetune_qlora_single_gpu.sh +++ b/finetune/finetune_qlora_single_gpu.sh @@ -32,5 +32,6 @@ python finetune.py \ --report_to "none" \ --model_max_length 2048 \ --lazy_preprocess True \ + --gradient_checkpointing \ --use_lora \ --q_lora \ No newline at end of file