hyperzlib
/
Isekai-Qwen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add gradient checkpointing

Browse Source
main
JustinLin610 1 year ago
parent c5f7fa9487
commit 73f4040abb
4 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File

1
finetune/finetune_lora_ds.sh
Unescape Escape View File

@ -44,4 +44,5 @@ torchrun $DISTRIBUTED_ARGS finetune.py \
--model_max_length 2048 \ --model_max_length 2048 \
--lazy_preprocess True \ --lazy_preprocess True \
--use_lora \ --use_lora \
--gradient_checkpointing \
--deepspeed finetune/ds_config_zero2.json --deepspeed finetune/ds_config_zero2.json

1
finetune/finetune_lora_single_gpu.sh
Unescape Escape View File

@ -32,4 +32,5 @@ python finetune.py \
--report_to "none" \ --report_to "none" \
--model_max_length 2048 \ --model_max_length 2048 \
--lazy_preprocess True \ --lazy_preprocess True \
--gradient_checkpointing \
--use_lora --use_lora

1
finetune/finetune_qlora_ds.sh
Unescape Escape View File

@ -46,4 +46,5 @@ torchrun $DISTRIBUTED_ARGS finetune.py \
--lazy_preprocess True \ --lazy_preprocess True \
--use_lora \ --use_lora \
--q_lora \ --q_lora \
--gradient_checkpointing \
--deepspeed finetune/ds_config_zero2.json --deepspeed finetune/ds_config_zero2.json

1
finetune/finetune_qlora_single_gpu.sh
Unescape Escape View File

@ -32,5 +32,6 @@ python finetune.py \
--report_to "none" \ --report_to "none" \
--model_max_length 2048 \ --model_max_length 2048 \
--lazy_preprocess True \ --lazy_preprocess True \
--gradient_checkpointing \
--use_lora \ --use_lora \
--q_lora --q_lora
Write Preview
Loading…
Cancel
Save