add finetuning
JustinLin610
1 year ago
parent
7ba36aad16
commit
af22d5e0ce
@ -0,0 +1,353 @@
|
||||
# This code is based on the revised code from fastchat based on tatsu-lab/stanford_alpaca.
|
||||
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import json
|
||||
import math
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Optional, List
|
||||
import torch
|
||||
from torch.utils.data import Dataset
|
||||
from deepspeed import zero
|
||||
from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus
|
||||
import transformers
|
||||
from transformers import Trainer, GPTQConfig, deepspeed
|
||||
from transformers.trainer_pt_utils import LabelSmoother
|
||||
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
|
||||
|
||||
|
||||
IGNORE_TOKEN_ID = LabelSmoother.ignore_index
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelArguments:
|
||||
model_name_or_path: Optional[str] = field(default="Qwen/Qwen-7B")
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataArguments:
|
||||
data_path: str = field(
|
||||
default=None, metadata={"help": "Path to the training data."}
|
||||
)
|
||||
eval_data_path: str = field(
|
||||
default=None, metadata={"help": "Path to the evaluation data."}
|
||||
)
|
||||
lazy_preprocess: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrainingArguments(transformers.TrainingArguments):
|
||||
cache_dir: Optional[str] = field(default=None)
|
||||
optim: str = field(default="adamw_torch")
|
||||
model_max_length: int = field(
|
||||
default=8192,
|
||||
metadata={
|
||||
"help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)."
|
||||
},
|
||||
)
|
||||
use_lora: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoraArguments:
|
||||
lora_r: int = 64
|
||||
lora_alpha: int = 16
|
||||
lora_dropout: float = 0.05
|
||||
lora_target_modules: List[str] = field(
|
||||
default_factory=lambda: ["c_attn", "c_proj", "w1", "w2"]
|
||||
)
|
||||
lora_weight_path: str = ""
|
||||
lora_bias: str = "none"
|
||||
q_lora: bool = False
|
||||
|
||||
|
||||
def maybe_zero_3(param):
|
||||
if hasattr(param, "ds_id"):
|
||||
assert param.ds_status == ZeroParamStatus.NOT_AVAILABLE
|
||||
with zero.GatheredParameters([param]):
|
||||
param = param.data.detach().cpu().clone()
|
||||
else:
|
||||
param = param.detach().cpu().clone()
|
||||
return param
|
||||
|
||||
|
||||
# Borrowed from peft.utils.get_peft_model_state_dict
|
||||
def get_peft_state_maybe_zero_3(named_params, bias):
|
||||
if bias == "none":
|
||||
to_return = {k: t for k, t in named_params if "lora_" in k}
|
||||
elif bias == "all":
|
||||
to_return = {k: t for k, t in named_params if "lora_" in k or "bias" in k}
|
||||
elif bias == "lora_only":
|
||||
to_return = {}
|
||||
maybe_lora_bias = {}
|
||||
lora_bias_names = set()
|
||||
for k, t in named_params:
|
||||
if "lora_" in k:
|
||||
to_return[k] = t
|
||||
bias_name = k.split("lora_")[0] + "bias"
|
||||
lora_bias_names.add(bias_name)
|
||||
elif "bias" in k:
|
||||
maybe_lora_bias[k] = t
|
||||
for k, t in maybe_lora_bias:
|
||||
if bias_name in lora_bias_names:
|
||||
to_return[bias_name] = t
|
||||
else:
|
||||
raise NotImplementedError
|
||||
to_return = {k: maybe_zero_3(v) for k, v in to_return.items()}
|
||||
return to_return
|
||||
|
||||
local_rank = None
|
||||
|
||||
def rank0_print(*args):
|
||||
if local_rank == 0:
|
||||
print(*args)
|
||||
|
||||
|
||||
def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, output_dir: str, bias="none"):
|
||||
"""Collects the state dict and dump to disk."""
|
||||
# check if zero3 mode enabled
|
||||
if deepspeed.is_deepspeed_zero3_enabled():
|
||||
state_dict = trainer.model_wrapped._zero3_consolidated_16bit_state_dict()
|
||||
else:
|
||||
if trainer.args.use_lora:
|
||||
state_dict = get_peft_state_maybe_zero_3(
|
||||
trainer.model.named_parameters(), bias
|
||||
)
|
||||
else:
|
||||
state_dict = trainer.model.state_dict()
|
||||
if trainer.args.should_save and trainer.args.local_rank == 0:
|
||||
trainer._save(output_dir, state_dict=state_dict)
|
||||
|
||||
|
||||
def preprocess(
|
||||
sources,
|
||||
tokenizer: transformers.PreTrainedTokenizer,
|
||||
max_len: int,
|
||||
system_message: str = "You are a helpful assistant."
|
||||
) -> Dict:
|
||||
roles = {"user": "<|im_start|>user", "assistant": "<|im_start|>assistant"}
|
||||
|
||||
im_start = tokenizer.im_start_id
|
||||
im_end = tokenizer.im_end_id
|
||||
nl_tokens = tokenizer('\n').input_ids
|
||||
_system = tokenizer('system').input_ids + nl_tokens
|
||||
_user = tokenizer('user').input_ids + nl_tokens
|
||||
_assistant = tokenizer('assistant').input_ids + nl_tokens
|
||||
|
||||
# Apply prompt templates
|
||||
input_ids, targets = [], []
|
||||
for i, source in enumerate(sources):
|
||||
if roles[source[0]["from"]] != roles["user"]:
|
||||
source = source[1:]
|
||||
|
||||
input_id, target = [], []
|
||||
system = [im_start] + _system + tokenizer(system_message).input_ids + [im_end] + nl_tokens
|
||||
input_id += system
|
||||
target += [im_start] + [IGNORE_TOKEN_ID] * (len(system)-3) + [im_end] + nl_tokens
|
||||
assert len(input_id) == len(target)
|
||||
for j, sentence in enumerate(source):
|
||||
role = roles[sentence["from"]]
|
||||
_input_id = tokenizer(role).input_ids + nl_tokens + \
|
||||
tokenizer(sentence["value"]).input_ids + [im_end] + nl_tokens
|
||||
input_id += _input_id
|
||||
if role == '<|im_start|>user':
|
||||
_target = [im_start] + [IGNORE_TOKEN_ID] * (len(_input_id)-3) + [im_end] + nl_tokens
|
||||
elif role == '<|im_start|>assistant':
|
||||
_target = [im_start] + [IGNORE_TOKEN_ID] * len(tokenizer(role).input_ids) + \
|
||||
_input_id[len(tokenizer(role).input_ids)+1:-2] + [im_end] + nl_tokens
|
||||
else:
|
||||
raise NotImplementedError
|
||||
target += _target
|
||||
assert len(input_id) == len(target)
|
||||
input_id += [tokenizer.pad_token_id] * (max_len - len(input_id))
|
||||
target += [IGNORE_TOKEN_ID] * (max_len - len(target))
|
||||
input_ids.append(input_id[:max_len])
|
||||
targets.append(target[:max_len])
|
||||
input_ids = torch.tensor(input_ids, dtype=torch.int)
|
||||
targets = torch.tensor(targets, dtype=torch.int)
|
||||
|
||||
return dict(
|
||||
input_ids=input_ids,
|
||||
labels=targets,
|
||||
attention_mask=input_ids.ne(tokenizer.pad_token_id),
|
||||
)
|
||||
|
||||
|
||||
class SupervisedDataset(Dataset):
|
||||
"""Dataset for supervised fine-tuning."""
|
||||
|
||||
def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, max_len: int):
|
||||
super(SupervisedDataset, self).__init__()
|
||||
|
||||
rank0_print("Formatting inputs...")
|
||||
sources = [example["conversations"] for example in raw_data]
|
||||
data_dict = preprocess(sources, tokenizer, max_len)
|
||||
|
||||
self.input_ids = data_dict["input_ids"]
|
||||
self.labels = data_dict["labels"]
|
||||
self.attention_mask = data_dict["attention_mask"]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.input_ids)
|
||||
|
||||
def __getitem__(self, i) -> Dict[str, torch.Tensor]:
|
||||
return dict(
|
||||
input_ids=self.input_ids[i],
|
||||
labels=self.labels[i],
|
||||
attention_mask=self.attention_mask[i],
|
||||
)
|
||||
|
||||
|
||||
class LazySupervisedDataset(Dataset):
|
||||
"""Dataset for supervised fine-tuning."""
|
||||
|
||||
def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, max_len: int):
|
||||
super(LazySupervisedDataset, self).__init__()
|
||||
self.tokenizer = tokenizer
|
||||
self.max_len = max_len
|
||||
|
||||
rank0_print("Formatting inputs...Skip in lazy mode")
|
||||
self.tokenizer = tokenizer
|
||||
self.raw_data = raw_data
|
||||
self.cached_data_dict = {}
|
||||
|
||||
def __len__(self):
|
||||
return len(self.raw_data)
|
||||
|
||||
def __getitem__(self, i) -> Dict[str, torch.Tensor]:
|
||||
if i in self.cached_data_dict:
|
||||
return self.cached_data_dict[i]
|
||||
|
||||
ret = preprocess([self.raw_data[i]["conversations"]], self.tokenizer, self.max_len)
|
||||
ret = dict(
|
||||
input_ids=ret["input_ids"][0],
|
||||
labels=ret["labels"][0],
|
||||
attention_mask=ret["attention_mask"][0],
|
||||
)
|
||||
self.cached_data_dict[i] = ret
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def make_supervised_data_module(
|
||||
tokenizer: transformers.PreTrainedTokenizer, data_args, max_len,
|
||||
) -> Dict:
|
||||
"""Make dataset and collator for supervised fine-tuning."""
|
||||
dataset_cls = (
|
||||
LazySupervisedDataset if data_args.lazy_preprocess else SupervisedDataset
|
||||
)
|
||||
rank0_print("Loading data...")
|
||||
|
||||
train_json = json.load(open(data_args.data_path, "r"))
|
||||
train_dataset = dataset_cls(train_json, tokenizer=tokenizer, max_len=max_len)
|
||||
|
||||
if data_args.eval_data_path:
|
||||
eval_json = json.load(open(data_args.eval_data_path, "r"))
|
||||
eval_dataset = dataset_cls(eval_json, tokenizer=tokenizer, max_len=max_len)
|
||||
else:
|
||||
eval_dataset = None
|
||||
|
||||
return dict(train_dataset=train_dataset, eval_dataset=eval_dataset)
|
||||
|
||||
|
||||
def train():
|
||||
global local_rank
|
||||
|
||||
parser = transformers.HfArgumentParser(
|
||||
(ModelArguments, DataArguments, TrainingArguments, LoraArguments)
|
||||
)
|
||||
(
|
||||
model_args,
|
||||
data_args,
|
||||
training_args,
|
||||
lora_args,
|
||||
) = parser.parse_args_into_dataclasses()
|
||||
|
||||
compute_dtype = (
|
||||
torch.float16
|
||||
if training_args.fp16
|
||||
else (torch.bfloat16 if training_args.bf16 else torch.float32)
|
||||
)
|
||||
|
||||
local_rank = training_args.local_rank
|
||||
|
||||
device_map = None
|
||||
world_size = int(os.environ.get("WORLD_SIZE", 1))
|
||||
ddp = world_size != 1
|
||||
if lora_args.q_lora:
|
||||
device_map = {"": int(os.environ.get("LOCAL_RANK") or 0)} if ddp else None
|
||||
if len(training_args.fsdp) > 0 or deepspeed.is_deepspeed_zero3_enabled():
|
||||
logging.warning(
|
||||
"FSDP or ZeRO3 are not incompatible with QLoRA."
|
||||
)
|
||||
|
||||
# Set RoPE scaling factor
|
||||
config = transformers.AutoConfig.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
cache_dir=training_args.cache_dir,
|
||||
trust_remote_code=True,
|
||||
)
|
||||
config.use_cache = False
|
||||
|
||||
# Load model and tokenizer
|
||||
model = transformers.AutoModelForCausalLM.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
config=config,
|
||||
cache_dir=training_args.cache_dir,
|
||||
device_map=device_map,
|
||||
trust_remote_code=True,
|
||||
quantization_config=GPTQConfig(
|
||||
bits=4, disable_exllama=True
|
||||
)
|
||||
if training_args.use_lora and lora_args.q_lora
|
||||
else None,
|
||||
)
|
||||
tokenizer = transformers.AutoTokenizer.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
cache_dir=training_args.cache_dir,
|
||||
model_max_length=training_args.model_max_length,
|
||||
padding_side="right",
|
||||
use_fast=False,
|
||||
trust_remote_code=True,
|
||||
)
|
||||
tokenizer.pad_token_id = tokenizer.eod_id
|
||||
|
||||
if training_args.use_lora:
|
||||
lora_config = LoraConfig(
|
||||
r=lora_args.lora_r,
|
||||
lora_alpha=lora_args.lora_alpha,
|
||||
target_modules=lora_args.lora_target_modules,
|
||||
lora_dropout=lora_args.lora_dropout,
|
||||
bias=lora_args.lora_bias,
|
||||
task_type="CAUSAL_LM",
|
||||
modules_to_save=["wte", "lm_head"] # This argument serves for adding new tokens.
|
||||
)
|
||||
if lora_args.q_lora:
|
||||
model = prepare_model_for_kbit_training(
|
||||
model, use_gradient_checkpointing=training_args.gradient_checkpointing
|
||||
)
|
||||
|
||||
model = get_peft_model(model, lora_config)
|
||||
|
||||
if training_args.gradient_checkpointing:
|
||||
model.enable_input_require_grads()
|
||||
|
||||
# Load data
|
||||
data_module = make_supervised_data_module(
|
||||
tokenizer=tokenizer, data_args=data_args, max_len=training_args.model_max_length
|
||||
)
|
||||
|
||||
# Start trainner
|
||||
trainer = Trainer(
|
||||
model=model, tokenizer=tokenizer, args=training_args, **data_module
|
||||
)
|
||||
|
||||
trainer.train()
|
||||
trainer.save_state()
|
||||
|
||||
safe_save_model_for_hf_trainer(trainer=trainer, output_dir=training_args.output_dir, bias=lora_args.lora_bias)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
train()
|
||||
@ -0,0 +1,52 @@
|
||||
{
|
||||
"fp16": {
|
||||
"enabled": "auto",
|
||||
"loss_scale": 0,
|
||||
"loss_scale_window": 1000,
|
||||
"initial_scale_power": 16,
|
||||
"hysteresis": 2,
|
||||
"min_loss_scale": 1
|
||||
},
|
||||
"bf16": {
|
||||
"enabled": "auto"
|
||||
},
|
||||
"optimizer": {
|
||||
"type": "AdamW",
|
||||
"params": {
|
||||
"lr": "auto",
|
||||
"betas": "auto",
|
||||
"eps": "auto",
|
||||
"weight_decay": "auto"
|
||||
}
|
||||
},
|
||||
|
||||
"scheduler": {
|
||||
"type": "WarmupLR",
|
||||
"params": {
|
||||
"warmup_min_lr": "auto",
|
||||
"warmup_max_lr": "auto",
|
||||
"warmup_num_steps": "auto"
|
||||
}
|
||||
},
|
||||
|
||||
"zero_optimization": {
|
||||
"stage": 2,
|
||||
"offload_optimizer": {
|
||||
"device": "none",
|
||||
"pin_memory": true
|
||||
},
|
||||
"allgather_partitions": true,
|
||||
"allgather_bucket_size": 2e8,
|
||||
"overlap_comm": true,
|
||||
"reduce_scatter": true,
|
||||
"reduce_bucket_size": 2e8,
|
||||
"contiguous_gradients": true
|
||||
},
|
||||
|
||||
"gradient_accumulation_steps": "auto",
|
||||
"gradient_clipping": "auto",
|
||||
"steps_per_print": 100,
|
||||
"train_batch_size": "auto",
|
||||
"train_micro_batch_size_per_gpu": "auto",
|
||||
"wall_clock_breakdown": false
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
{
|
||||
"fp16": {
|
||||
"enabled": "auto",
|
||||
"loss_scale": 0,
|
||||
"loss_scale_window": 1000,
|
||||
"initial_scale_power": 16,
|
||||
"hysteresis": 2,
|
||||
"min_loss_scale": 1
|
||||
},
|
||||
"bf16": {
|
||||
"enabled": "auto"
|
||||
},
|
||||
"optimizer": {
|
||||
"type": "AdamW",
|
||||
"params": {
|
||||
"lr": "auto",
|
||||
"betas": "auto",
|
||||
"eps": "auto",
|
||||
"weight_decay": "auto"
|
||||
}
|
||||
},
|
||||
|
||||
"scheduler": {
|
||||
"type": "WarmupLR",
|
||||
"params": {
|
||||
"warmup_min_lr": "auto",
|
||||
"warmup_max_lr": "auto",
|
||||
"warmup_num_steps": "auto"
|
||||
}
|
||||
},
|
||||
|
||||
"zero_optimization": {
|
||||
"stage": 3,
|
||||
"offload_optimizer": {
|
||||
"device": "none",
|
||||
"pin_memory": true
|
||||
},
|
||||
"offload_param": {
|
||||
"device": "none",
|
||||
"pin_memory": true
|
||||
},
|
||||
"overlap_comm": true,
|
||||
"contiguous_gradients": true,
|
||||
"sub_group_size": 1e9,
|
||||
"reduce_bucket_size": "auto",
|
||||
"stage3_prefetch_bucket_size": "auto",
|
||||
"stage3_param_persistence_threshold": "auto",
|
||||
"stage3_max_live_parameters": 1e9,
|
||||
"stage3_max_reuse_distance": 1e9,
|
||||
"stage3_gather_16bit_weights_on_model_save": true
|
||||
},
|
||||
|
||||
"gradient_accumulation_steps": "auto",
|
||||
"gradient_clipping": "auto",
|
||||
"steps_per_print": 100,
|
||||
"train_batch_size": "auto",
|
||||
"train_micro_batch_size_per_gpu": "auto",
|
||||
"wall_clock_breakdown": false
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
export CUDA_DEVICE_MAX_CONNECTIONS=1
|
||||
DIR=`pwd`
|
||||
|
||||
GPUS_PER_NODE=8
|
||||
NNODES=1
|
||||
NODE_RANK=0
|
||||
MASTER_ADDR=localhost
|
||||
MASTER_PORT=6001
|
||||
|
||||
MODEL="Qwen/Qwen-7B" # Set the path if you do not want to load from huggingface directly
|
||||
# ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations.
|
||||
# See the section for finetuning in README for more information.
|
||||
DATA="path_to_data"
|
||||
|
||||
DISTRIBUTED_ARGS="
|
||||
--nproc_per_node $GPUS_PER_NODE \
|
||||
--nnodes $NNODES \
|
||||
--node_rank $NODE_RANK \
|
||||
--master_addr $MASTER_ADDR \
|
||||
--master_port $MASTER_PORT
|
||||
"
|
||||
|
||||
torchrun $DISTRIBUTED_ARGS finetune.py \
|
||||
--model_name_or_path $MODEL \
|
||||
--data_path $DATA \
|
||||
--bf16 True \
|
||||
--output_dir output_qwen \
|
||||
--num_train_epochs 5 \
|
||||
--per_device_train_batch_size 1 \
|
||||
--per_device_eval_batch_size 1 \
|
||||
--gradient_accumulation_steps 16 \
|
||||
--evaluation_strategy "no" \
|
||||
--save_strategy "steps" \
|
||||
--save_steps 1000 \
|
||||
--save_total_limit 10 \
|
||||
--learning_rate 1e-5 \
|
||||
--weight_decay 0.1 \
|
||||
--adam_beta2 0.95 \
|
||||
--warmup_ratio 0.01 \
|
||||
--lr_scheduler_type "cosine" \
|
||||
--logging_steps 1 \
|
||||
--report_to "none" \
|
||||
--model_max_length 2048 \
|
||||
--gradient_checkpointing True \
|
||||
--lazy_preprocess True \
|
||||
--deepspeed finetune/ds_config_zero3.json
|
||||
@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
export CUDA_DEVICE_MAX_CONNECTIONS=1
|
||||
DIR=`pwd`
|
||||
|
||||
GPUS_PER_NODE=8
|
||||
NNODES=1
|
||||
NODE_RANK=0
|
||||
MASTER_ADDR=localhost
|
||||
MASTER_PORT=6001
|
||||
|
||||
MODEL="Qwen/Qwen-7B" # Set the path if you do not want to load from huggingface directly
|
||||
# ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations.
|
||||
# See the section for finetuning in README for more information.
|
||||
DATA="path_to_data"
|
||||
|
||||
DISTRIBUTED_ARGS="
|
||||
--nproc_per_node $GPUS_PER_NODE \
|
||||
--nnodes $NNODES \
|
||||
--node_rank $NODE_RANK \
|
||||
--master_addr $MASTER_ADDR \
|
||||
--master_port $MASTER_PORT
|
||||
"
|
||||
|
||||
torchrun $DISTRIBUTED_ARGS finetune.py \
|
||||
--model_name_or_path $MODEL \
|
||||
--data_path $DATA \
|
||||
--bf16 True \
|
||||
--output_dir output_qwen \
|
||||
--num_train_epochs 5 \
|
||||
--per_device_train_batch_size 2 \
|
||||
--per_device_eval_batch_size 1 \
|
||||
--gradient_accumulation_steps 8 \
|
||||
--evaluation_strategy "no" \
|
||||
--save_strategy "steps" \
|
||||
--save_steps 1000 \
|
||||
--save_total_limit 10 \
|
||||
--learning_rate 1e-5 \
|
||||
--weight_decay 0.1 \
|
||||
--adam_beta2 0.95 \
|
||||
--warmup_ratio 0.01 \
|
||||
--lr_scheduler_type "cosine" \
|
||||
--logging_steps 1 \
|
||||
--report_to "none" \
|
||||
--model_max_length 2048 \
|
||||
--lazy_preprocess True \
|
||||
--use_lora \
|
||||
--deepspeed finetune/ds_config_zero2.json
|
||||
@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
export CUDA_DEVICE_MAX_CONNECTIONS=1
|
||||
DIR=`pwd`
|
||||
|
||||
|
||||
MODEL="Qwen/Qwen-7B" # Set the path if you do not want to load from huggingface directly
|
||||
# ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations.
|
||||
# See the section for finetuning in README for more information.
|
||||
DATA="path_to_data"
|
||||
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
|
||||
python finetune.py \
|
||||
--model_name_or_path $MODEL \
|
||||
--data_path $DATA \
|
||||
--bf16 True \
|
||||
--output_dir output_qwen \
|
||||
--num_train_epochs 5 \
|
||||
--per_device_train_batch_size 2 \
|
||||
--per_device_eval_batch_size 1 \
|
||||
--gradient_accumulation_steps 8 \
|
||||
--evaluation_strategy "no" \
|
||||
--save_strategy "steps" \
|
||||
--save_steps 1000 \
|
||||
--save_total_limit 10 \
|
||||
--learning_rate 1e-5 \
|
||||
--weight_decay 0.1 \
|
||||
--adam_beta2 0.95 \
|
||||
--warmup_ratio 0.01 \
|
||||
--lr_scheduler_type "cosine" \
|
||||
--logging_steps 1 \
|
||||
--report_to "none" \
|
||||
--model_max_length 2048 \
|
||||
--lazy_preprocess True \
|
||||
--use_lora
|
||||
@ -0,0 +1,49 @@
|
||||
#!/bin/bash
|
||||
export CUDA_DEVICE_MAX_CONNECTIONS=1
|
||||
DIR=`pwd`
|
||||
|
||||
GPUS_PER_NODE=8
|
||||
NNODES=1
|
||||
NODE_RANK=0
|
||||
MASTER_ADDR=localhost
|
||||
MASTER_PORT=6001
|
||||
|
||||
MODEL="Qwen/Qwen-7B-Chat-Int4" # Set the path if you do not want to load from huggingface directly
|
||||
# ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations.
|
||||
# See the section for finetuning in README for more information.
|
||||
DATA="path_to_data"
|
||||
|
||||
DISTRIBUTED_ARGS="
|
||||
--nproc_per_node $GPUS_PER_NODE \
|
||||
--nnodes $NNODES \
|
||||
--node_rank $NODE_RANK \
|
||||
--master_addr $MASTER_ADDR \
|
||||
--master_port $MASTER_PORT
|
||||
"
|
||||
|
||||
# Remember to use --fp16 instead of --bf16 due to autogptq
|
||||
torchrun $DISTRIBUTED_ARGS finetune.py \
|
||||
--model_name_or_path $MODEL \
|
||||
--data_path $DATA \
|
||||
--fp16 True \
|
||||
--output_dir output_qwen \
|
||||
--num_train_epochs 5 \
|
||||
--per_device_train_batch_size 2 \
|
||||
--per_device_eval_batch_size 1 \
|
||||
--gradient_accumulation_steps 8 \
|
||||
--evaluation_strategy "no" \
|
||||
--save_strategy "steps" \
|
||||
--save_steps 1000 \
|
||||
--save_total_limit 10 \
|
||||
--learning_rate 1e-5 \
|
||||
--weight_decay 0.1 \
|
||||
--adam_beta2 0.95 \
|
||||
--warmup_ratio 0.01 \
|
||||
--lr_scheduler_type "cosine" \
|
||||
--logging_steps 1 \
|
||||
--report_to "none" \
|
||||
--model_max_length 2048 \
|
||||
--lazy_preprocess True \
|
||||
--use_lora \
|
||||
--q_lora \
|
||||
--deepspeed finetune/ds_config_zero2.json
|
||||
@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
export CUDA_DEVICE_MAX_CONNECTIONS=1
|
||||
DIR=`pwd`
|
||||
|
||||
MODEL="Qwen/Qwen-7B-Chat-Int4" # Set the path if you do not want to load from huggingface directly
|
||||
# ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations.
|
||||
# See the section for finetuning in README for more information.
|
||||
DATA="path_to_data"
|
||||
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
|
||||
# Remember to use --fp16 instead of --bf16 due to autogptq
|
||||
python finetune.py \
|
||||
--model_name_or_path $MODEL \
|
||||
--data_path $DATA \
|
||||
--fp16 True \
|
||||
--output_dir output_qwen \
|
||||
--num_train_epochs 5 \
|
||||
--per_device_train_batch_size 2 \
|
||||
--per_device_eval_batch_size 1 \
|
||||
--gradient_accumulation_steps 8 \
|
||||
--evaluation_strategy "no" \
|
||||
--save_strategy "steps" \
|
||||
--save_steps 1000 \
|
||||
--save_total_limit 10 \
|
||||
--learning_rate 1e-5 \
|
||||
--weight_decay 0.1 \
|
||||
--adam_beta2 0.95 \
|
||||
--warmup_ratio 0.01 \
|
||||
--lr_scheduler_type "cosine" \
|
||||
--logging_steps 1 \
|
||||
--report_to "none" \
|
||||
--model_max_length 2048 \
|
||||
--lazy_preprocess True \
|
||||
--use_lora \
|
||||
--q_lora
|
||||
Loading…
Reference in New Issue