add 72B and 1.8B Qwen models, add Ascend 910 and Hygon DCU support, add docker support
yangapku
1 year ago
parent
981c89b2a9
commit
e8e15962d8
@ -0,0 +1,14 @@
|
||||
__pycache__
|
||||
*.so
|
||||
build
|
||||
.coverage_*
|
||||
*.egg-info
|
||||
*~
|
||||
.vscode/
|
||||
.idea/
|
||||
.git/
|
||||
.github/
|
||||
.DS_Store
|
||||
|
||||
/private/
|
||||
/README-docker.md
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
|
||||
IMAGE_NAME=qwenllm/qwen-mindspore:v23.0.RC3
|
||||
CONTAINER_NAME=qwen-mindspore
|
||||
CHECKPOINT_PATH='NOT_SET'
|
||||
|
||||
DOCKER_CHECKPOINT_PATH=/data/qwen/models/Qwen-7B-Chat
|
||||
|
||||
function usage() {
|
||||
echo '
|
||||
Usage: bash ascend-support/docker_qwen.sh [-i IMAGE_NAME] -c [/path/to/Qwen-7B-Chat] [-n CONTAINER_NAME]
|
||||
'
|
||||
}
|
||||
|
||||
while [[ "$1" != "" ]]; do
|
||||
case $1 in
|
||||
-i | --image )
|
||||
shift
|
||||
IMAGE_NAME=$1
|
||||
;;
|
||||
-c | --checkpoint )
|
||||
shift
|
||||
CHECKPOINT_PATH=$1
|
||||
;;
|
||||
-n | --name )
|
||||
shift
|
||||
CONTAINER_NAME=$1
|
||||
;;
|
||||
-h )
|
||||
usage
|
||||
exit
|
||||
;;
|
||||
* )
|
||||
echo "Unknown argument ${1}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
docker run -it --rm -u root --network=host --ipc=host \
|
||||
--device=/dev/davinci0 \
|
||||
--device=/dev/davinci1 \
|
||||
--device=/dev/davinci2 \
|
||||
--device=/dev/davinci3 \
|
||||
--device=/dev/davinci4 \
|
||||
--device=/dev/davinci5 \
|
||||
--device=/dev/davinci6 \
|
||||
--device=/dev/davinci7 \
|
||||
--name=${CONTAINER_NAME} \
|
||||
--device=/dev/davinci_manager \
|
||||
--device=/dev/devmm_svm \
|
||||
--device=/dev/hisi_hdc \
|
||||
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
|
||||
-v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \
|
||||
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
|
||||
-v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \
|
||||
-v /etc/ascend_install.info:/etc/ascend_install.info \
|
||||
-v ${CHECKPOINT_PATH}:${DOCKER_CHECKPOINT_PATH} \
|
||||
-v /var/log/npu/:/usr/slog \
|
||||
${IMAGE_NAME} /bin/bash
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB After Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 473 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 205 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 396 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 176 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 182 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 824 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 426 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 433 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 466 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 403 KiB |
@ -0,0 +1,39 @@
|
||||
import argparse
|
||||
from fastllm_pytools import llm
|
||||
import time
|
||||
|
||||
def args_parser():
|
||||
parser = argparse.ArgumentParser(description = 'fastllm_chat_demo')
|
||||
parser.add_argument('-p', '--path', type = str, required = True, default = '', help = '模型文件的路径')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = args_parser()
|
||||
|
||||
model_path = args.path
|
||||
|
||||
prompts = ["深圳有什么好玩的", "上海有什么好玩的", "晚上睡不着怎么办", "南京有什么好吃的"] * 2
|
||||
print(prompts)
|
||||
|
||||
responses, historys = [], []
|
||||
|
||||
model = llm.model(model_path)
|
||||
|
||||
t0 = time.time()
|
||||
responses, historys = model.response_batch(prompts)
|
||||
t1 = time.time()
|
||||
|
||||
token_output_count = 0
|
||||
word_len = 0
|
||||
for i, res in enumerate(responses):
|
||||
tokens = model.tokenizer_encode_string(res)
|
||||
token_output_count += len(tokens)
|
||||
word_len += len(res)
|
||||
|
||||
print("batch index: ", i)
|
||||
print(res)
|
||||
print("")
|
||||
|
||||
print("\ntoken/s: {:.2f}, character/s: {:.2f}".format(token_output_count/(t1-t0), word_len/(t1-t0)))
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
# 模型唯一标识
|
||||
modelCode = 411
|
||||
# 模型名称
|
||||
modelName=qwen-7b_fastllm
|
||||
# 模型描述
|
||||
modelDescription=qwen-7b是阿里云研发的通义千问大模型系列的70亿参数规模的模型
|
||||
# 应用场景
|
||||
appScenario=推理,对话问答,医疗,科研,金融,教育
|
||||
# 框架类型
|
||||
frameType=fastllm
|
||||
@ -0,0 +1 @@
|
||||
__all__ = ["llm"]
|
||||
@ -0,0 +1,154 @@
|
||||
from fastllm_pytools import llm;
|
||||
import torch;
|
||||
import ctypes;
|
||||
import numpy as np;
|
||||
|
||||
fastllm_data_type_dict = {
|
||||
"int4": 8,
|
||||
"int8": 3,
|
||||
"float16": 7
|
||||
}
|
||||
fastllm_weight_type_dict = {
|
||||
"linear": 1,
|
||||
"embedding": 2,
|
||||
"QuantizedLinear": 111
|
||||
}
|
||||
|
||||
def create(model,
|
||||
tokenizer = None,
|
||||
pre_prompt = None,
|
||||
user_role = None,
|
||||
bot_role = None,
|
||||
history_sep = None,
|
||||
dtype = "float16"):
|
||||
if (dtype not in fastllm_data_type_dict):
|
||||
print("dtype should in ", list(fastllm_data_type_dict.keys()));
|
||||
exit(0);
|
||||
|
||||
# 0.1 model info
|
||||
if model.config.model_type == "chatglm" and model.config.transformers_version == "4.30.2":
|
||||
model.config.model_type = "chatglm3"
|
||||
modelInfo = model.config.__dict__
|
||||
if model.generation_config is not None:
|
||||
modelInfo.update(model.generation_config.__dict__)
|
||||
if (pre_prompt):
|
||||
modelInfo["pre_prompt"] = pre_prompt;
|
||||
if (user_role):
|
||||
modelInfo["user_role"] = user_role;
|
||||
if (bot_role):
|
||||
modelInfo["bot_role"] = bot_role;
|
||||
if (history_sep):
|
||||
modelInfo["history_sep"] = history_sep;
|
||||
if (modelInfo["model_type"] == "baichuan" and hasattr(model, "model") and hasattr(model.model, "get_alibi_mask")):
|
||||
# Baichuan 2代
|
||||
modelInfo["use_alibi"] = "1";
|
||||
modelInfo["pre_prompt"] = "";
|
||||
modelInfo["user_role"] = ("<FLM_FIX_TOKEN_" + str(model.generation_config.user_token_id) + "> ") if hasattr(model.generation_config, "user_token_id") else "";
|
||||
modelInfo["bot_role"] = ("<FLM_FIX_TOKEN_" + str(model.generation_config.assistant_token_id) + ">") if hasattr(model.generation_config, "assistant_token_id") else "";
|
||||
modelInfo["history_sep"] = "";
|
||||
if (modelInfo["model_type"] == "qwen"):
|
||||
if modelInfo["chat_format"] == "chatml":
|
||||
modelInfo["im_end_id"] = tokenizer.im_end_id
|
||||
modelInfo["im_start_id"] = tokenizer.im_start_id
|
||||
|
||||
|
||||
weight_type_dict = {};
|
||||
module_dict = {};
|
||||
weight_bits = {};
|
||||
for key, m in model.named_modules():
|
||||
if (str(type(m)).find("QuantizedLinear") != -1):
|
||||
weight_type_dict[key + ".weight"] = "QuantizedLinear";
|
||||
weight_bits[key + ".weight"] = m.weight_bit_width;
|
||||
if (isinstance(m, torch.nn.Linear)):
|
||||
weight_type_dict[key + ".weight"] = "linear";
|
||||
module_dict[key + ".weight"] = m;
|
||||
if (isinstance(m, torch.nn.Embedding)):
|
||||
weight_type_dict[key] = "embedding";
|
||||
|
||||
peft_config = {}
|
||||
active_adapter = ""
|
||||
if hasattr(model, "peft_config"):
|
||||
peft_config = model.peft_config
|
||||
if hasattr(model, "active_adapter") and isinstance(model.active_adapter, str):
|
||||
# in transformers >= 4.33.0, active_adapter is a funtion in model, ignore it now
|
||||
active_adapter = model.active_adapter
|
||||
|
||||
model = model.cpu();
|
||||
dict = model.state_dict();
|
||||
model_type = model.config.__dict__["model_type"];
|
||||
model = llm.fastllm_lib.create_empty_llm_model(model_type.encode());
|
||||
for it in modelInfo.keys():
|
||||
llm.fastllm_lib.add_dict_llm_model(model, str(it).encode(), str(modelInfo[it]).encode());
|
||||
|
||||
for adapter_name in peft_config.keys():
|
||||
adapter_dict = peft_config[adapter_name].__dict__
|
||||
for it in adapter_dict.keys():
|
||||
llm.fastllm_lib.add_adapter_dict_llm_model(model, str(adapter_name).encode(), str(it).encode(), str(adapter_dict[it]).encode())
|
||||
if len(active_adapter) != 0:
|
||||
llm.fastllm_lib.set_adapter(model, str(active_adapter).encode())
|
||||
|
||||
# 1. vocab
|
||||
if (tokenizer):
|
||||
if (hasattr(tokenizer, "tokenizer")):
|
||||
if modelInfo["model_type"] == "qwen":
|
||||
pass
|
||||
else:
|
||||
tokenizer = tokenizer.tokenizer;
|
||||
if (hasattr(tokenizer, "sp_model")):
|
||||
piece_size = tokenizer.sp_model.piece_size();
|
||||
for i in range(piece_size):
|
||||
llm.fastllm_lib.add_tokenizer_word_llm_model(model, tokenizer.sp_model.id_to_piece(i).encode(),
|
||||
i, ctypes.c_float(tokenizer.sp_model.get_score(i)));
|
||||
else:
|
||||
vocab = tokenizer.get_vocab();
|
||||
for v in vocab.keys():
|
||||
if (modelInfo["model_type"] == "moss"):
|
||||
vv = [(ord(c) if c not in tokenizer.byte_decoder else tokenizer.byte_decoder[c]) for c in v];
|
||||
llm.fastllm_lib.add_tokenizer_word_llm_model(model, vv, vocab[v], ctypes.c_float(1.0));
|
||||
elif (modelInfo["model_type"] == "qwen"):
|
||||
llm.fastllm_lib.add_tokenizer_word_llm_model(model, v, vocab[v], ctypes.c_float(1.0));
|
||||
else:
|
||||
llm.fastllm_lib.add_tokenizer_word_llm_model(model, v.encode(), vocab[v], ctypes.c_float(1.0));
|
||||
tot = 0;
|
||||
for key in dict:
|
||||
ori_data_type = 0;
|
||||
ori_np_data_type = np.float32;
|
||||
cur_weight_type = 0;
|
||||
if (key in weight_type_dict and weight_type_dict[key] in fastllm_weight_type_dict):
|
||||
cur_weight_type = fastllm_weight_type_dict[weight_type_dict[key]];
|
||||
to_data_type = 0;
|
||||
|
||||
if (cur_weight_type == 1):
|
||||
to_data_type = fastllm_data_type_dict[dtype];
|
||||
if (to_data_type == 7):
|
||||
ori_data_type = 7;
|
||||
ori_np_data_type = np.float16;
|
||||
elif (cur_weight_type == 2):
|
||||
# TODO bfloat
|
||||
to_data_type = 0;
|
||||
|
||||
weight_name = key
|
||||
if peft_config is not None:
|
||||
weight_name = weight_name.replace('base_model.model.', '')
|
||||
if (cur_weight_type == 111):
|
||||
llm.fastllm_lib.add_qlinear_weight_llm_model(model, weight_name.encode(),
|
||||
len(dict[key].shape),
|
||||
(ctypes.c_int * len(dict[key].shape))(*list(dict[key].shape)),
|
||||
weight_bits[key],
|
||||
dict[key + "_scale"].numpy().astype(np.float32).ctypes.data_as(ctypes.c_void_p),
|
||||
dict[key].numpy().ctypes.data_as(ctypes.c_void_p));
|
||||
else:
|
||||
llm.fastllm_lib.add_weight_llm_model(model, weight_name.encode(),
|
||||
len(dict[key].shape),
|
||||
(ctypes.c_int * len(dict[key].shape))(*list(dict[key].shape)),
|
||||
to_data_type, cur_weight_type, ori_data_type,
|
||||
dict[key].numpy().astype(ori_np_data_type).ctypes.data_as(ctypes.c_void_p));
|
||||
tot += 1;
|
||||
print("convert (", tot, "/", len(dict), end = " )\r");
|
||||
|
||||
print("");
|
||||
llm.fastllm_lib.init_params_llm_model(model);
|
||||
llm.fastllm_lib.warmup_llm_model(model);
|
||||
ret = llm.model("", id = model);
|
||||
return ret;
|
||||
|
||||
@ -0,0 +1,218 @@
|
||||
import struct
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
def writeString(fo, s):
|
||||
fo.write(struct.pack('i', len(s)))
|
||||
fo.write(s.encode())
|
||||
|
||||
def writeKeyValue(fo, key, value):
|
||||
writeString(fo, key)
|
||||
writeString(fo, value)
|
||||
|
||||
fastllm_data_type_dict = {
|
||||
"int4": 8,
|
||||
"int8": 3,
|
||||
"float16": 7,
|
||||
"float32": 0,
|
||||
}
|
||||
fastllm_weight_type_dict = {
|
||||
"linear": 1,
|
||||
"embedding": 2
|
||||
}
|
||||
|
||||
v = np.random.randint(-127, 127, [10, 20]);
|
||||
temp = v;
|
||||
c_max = np.expand_dims(np.abs(v).max(axis = -1), -1)
|
||||
c_scale = c_max / 127.0
|
||||
v = (v / c_scale + 128.5).clip(1, 255).astype(np.uint8)
|
||||
|
||||
def write_int8(fo, v):
|
||||
c_max = np.expand_dims(np.abs(v).max(axis = -1), -1).clip(0.1, 1e100)
|
||||
c_scale = c_max / 127.0
|
||||
v = (v / c_scale + 128.5).clip(1, 255).astype(np.uint8)
|
||||
fo.write(struct.pack('i', 3))
|
||||
fo.write(struct.pack('i', 0))
|
||||
for i in range(c_max.shape[0]):
|
||||
fo.write(struct.pack('f', -c_max[i][0]));
|
||||
fo.write(struct.pack('f', c_max[i][0]));
|
||||
fo.write(v.data)
|
||||
|
||||
def write_int4(fo, v):
|
||||
# c_min = np.expand_dims(-np.abs(v).max(axis = -1), -1)
|
||||
# c_max = np.expand_dims(np.abs(v).max(axis = -1), -1)
|
||||
# c_scale = c_max / 7.0
|
||||
# c_min = c_scale * -8.0
|
||||
|
||||
c_min = np.expand_dims(v.min(axis = -1), -1)
|
||||
c_max = np.expand_dims(v.max(axis = -1), -1)
|
||||
c_scale = (c_max - c_min) / 15.0
|
||||
c_zero = np.round(0.0 - c_min / c_scale)
|
||||
c_zero = c_zero.clip(0, 15)
|
||||
c_min = -c_scale * c_zero
|
||||
|
||||
v = (v - c_min) / c_scale
|
||||
v = (v + 0.5).astype(np.int8).clip(0, 15).astype(np.uint8)
|
||||
v = v[:, 0::2] * 16 + v[:, 1::2]
|
||||
fo.write(struct.pack('i', 8))
|
||||
fo.write(struct.pack('i', 0))
|
||||
for i in range(c_min.shape[0]):
|
||||
fo.write(struct.pack('f', c_min[i][0]));
|
||||
fo.write(struct.pack('f', c_max[i][0]));
|
||||
fo.write(v.data)
|
||||
|
||||
def tofile(exportPath,
|
||||
model,
|
||||
tokenizer = None,
|
||||
pre_prompt = None,
|
||||
user_role = None,
|
||||
bot_role = None,
|
||||
history_sep = None,
|
||||
dtype = "float16"):
|
||||
if (dtype not in fastllm_data_type_dict):
|
||||
print("dtype should in ", list(fastllm_data_type_dict.keys()))
|
||||
exit(0)
|
||||
|
||||
dict = model.state_dict()
|
||||
fo = open(exportPath, "wb")
|
||||
|
||||
# 0. version id
|
||||
fo.write(struct.pack('i', 2))
|
||||
|
||||
# 0.1 model info
|
||||
if model.config.model_type == "chatglm" and model.config.transformers_version == "4.30.2":
|
||||
model.config.model_type = "chatglm3"
|
||||
modelInfo = model.config.__dict__
|
||||
if model.generation_config is not None:
|
||||
modelInfo.update(model.generation_config.__dict__)
|
||||
if ("model_type" not in modelInfo):
|
||||
print("unknown model_type.")
|
||||
exit(0)
|
||||
|
||||
if (pre_prompt):
|
||||
modelInfo["pre_prompt"] = pre_prompt
|
||||
if (user_role):
|
||||
modelInfo["user_role"] = user_role
|
||||
if (bot_role):
|
||||
modelInfo["bot_role"] = bot_role
|
||||
if (history_sep):
|
||||
modelInfo["history_sep"] = history_sep
|
||||
if (modelInfo["model_type"] == "baichuan" and hasattr(model, "model") and hasattr(model.model, "get_alibi_mask")):
|
||||
# Baichuan 2代
|
||||
modelInfo["use_alibi"] = "1"
|
||||
modelInfo["pre_prompt"] = ""
|
||||
modelInfo["user_role"] = ("<FLM_FIX_TOKEN_" + str(model.generation_config.user_token_id) + ">") if hasattr(model.generation_config, "user_token_id") else "";
|
||||
modelInfo["bot_role"] = ("<FLM_FIX_TOKEN_" + str(model.generation_config.assistant_token_id) + ">") if hasattr(model.generation_config, "assistant_token_id") else "";
|
||||
modelInfo["history_sep"] = ""
|
||||
if (modelInfo["model_type"] == "baichuan" and modelInfo["vocab_size"] == 125696):
|
||||
# Baichuan 2代 7B
|
||||
modelInfo["pre_prompt"] = ""
|
||||
modelInfo["user_role"] = ("<FLM_FIX_TOKEN_" + str(model.generation_config.user_token_id) + ">") if hasattr(model.generation_config, "user_token_id") else "";
|
||||
modelInfo["bot_role"] = ("<FLM_FIX_TOKEN_" + str(model.generation_config.assistant_token_id) + ">") if hasattr(model.generation_config, "assistant_token_id") else "";
|
||||
modelInfo["history_sep"] = ""
|
||||
if modelInfo["model_type"] == "qwen":
|
||||
if modelInfo["chat_format"] == "chatml":
|
||||
modelInfo["im_end_id"] = tokenizer.im_end_id
|
||||
modelInfo["im_start_id"] = tokenizer.im_start_id
|
||||
|
||||
modelInfo["tokenizer_use_score"] = "1" # 分词带分数
|
||||
|
||||
if hasattr(model, "peft_config"):
|
||||
adapter_size = len(model.peft_config)
|
||||
modelInfo["peft_size"] = adapter_size
|
||||
|
||||
fo.write(struct.pack('i', len(modelInfo)))
|
||||
for it in modelInfo.keys():
|
||||
writeKeyValue(fo, str(it), str(modelInfo[it]))
|
||||
|
||||
if hasattr(model, "peft_config"):
|
||||
for adapter_name in model.peft_config.keys():
|
||||
adapter_dict = model.peft_config[adapter_name].__dict__
|
||||
writeString(fo, adapter_name)
|
||||
fo.write(struct.pack('i', len(adapter_dict)))
|
||||
for it in adapter_dict.keys():
|
||||
writeKeyValue(fo, str(it), str(adapter_dict[it]))
|
||||
|
||||
# 1. vocab
|
||||
if (tokenizer):
|
||||
if (hasattr(tokenizer, "tokenizer")):
|
||||
if (modelInfo['model_type'] == "qwen"):
|
||||
pass
|
||||
else:
|
||||
tokenizer = tokenizer.tokenizer
|
||||
if (hasattr(tokenizer, "sp_model")):
|
||||
piece_size = tokenizer.sp_model.piece_size()
|
||||
fo.write(struct.pack('i', piece_size))
|
||||
for i in range(piece_size):
|
||||
s = tokenizer.sp_model.id_to_piece(i).encode()
|
||||
fo.write(struct.pack('i', len(s)))
|
||||
for c in s:
|
||||
fo.write(struct.pack('i', c))
|
||||
fo.write(struct.pack('i', i))
|
||||
fo.write(struct.pack('f', float(tokenizer.sp_model.get_score(i))))
|
||||
else:
|
||||
vocab = tokenizer.get_vocab()
|
||||
fo.write(struct.pack('i', len(vocab)))
|
||||
for v in vocab.keys():
|
||||
if (modelInfo['model_type'] == "qwen"):
|
||||
s = v
|
||||
elif (modelInfo["model_type"] == "moss"):
|
||||
s = [(ord(c) if c not in tokenizer.byte_decoder else tokenizer.byte_decoder[c]) for c in v]
|
||||
else:
|
||||
s = v.encode()
|
||||
fo.write(struct.pack('i', len(s)))
|
||||
for c in s:
|
||||
fo.write(struct.pack('i', c))
|
||||
fo.write(struct.pack('i', vocab[v]))
|
||||
fo.write(struct.pack('f', 1.0))
|
||||
else:
|
||||
fo.write(struct.pack('i', 0))
|
||||
|
||||
weight_type_dict = {}
|
||||
module_dict = {}
|
||||
for key, m in model.named_modules():
|
||||
if (isinstance(m, torch.nn.Linear)):
|
||||
weight_type_dict[key + ".weight"] = "linear"
|
||||
module_dict[key + ".weight"] = m
|
||||
if (isinstance(m, torch.nn.Embedding)):
|
||||
weight_type_dict[key] = "embedding"
|
||||
|
||||
# 2. weight
|
||||
fo.write(struct.pack('i', len(dict)))
|
||||
tot = 0
|
||||
for key in dict:
|
||||
ori_data_type = 0
|
||||
ori_np_data_type = np.float32
|
||||
cur_weight_type = 0
|
||||
if (key in weight_type_dict and weight_type_dict[key] in fastllm_weight_type_dict):
|
||||
cur_weight_type = fastllm_weight_type_dict[weight_type_dict[key]]
|
||||
to_data_type = 0
|
||||
if (cur_weight_type == 1):
|
||||
to_data_type = fastllm_data_type_dict[dtype]
|
||||
if (to_data_type == 7):
|
||||
ori_data_type = 7
|
||||
ori_np_data_type = np.float16
|
||||
|
||||
cur = dict[key].numpy().astype(ori_np_data_type)
|
||||
|
||||
if hasattr(model, "peft_config"):
|
||||
weight_name = key.replace('base_model.model.', '')
|
||||
fo.write(struct.pack('i', len(weight_name)))
|
||||
fo.write(weight_name.encode())
|
||||
else:
|
||||
fo.write(struct.pack('i', len(key)))
|
||||
fo.write(key.encode())
|
||||
fo.write(struct.pack('i', len(cur.shape)))
|
||||
for i in cur.shape:
|
||||
fo.write(struct.pack('i', i))
|
||||
if (to_data_type == 3):
|
||||
write_int8(fo, cur)
|
||||
elif (to_data_type == 8):
|
||||
write_int4(fo, cur)
|
||||
else:
|
||||
fo.write(struct.pack('i', to_data_type))
|
||||
fo.write(cur.data)
|
||||
tot += 1
|
||||
print("output (", tot, "/", len(dict), end = " )\r")
|
||||
print("\nfinish.")
|
||||
fo.close()
|
||||
@ -0,0 +1,12 @@
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
setup (
|
||||
name = "fastllm_pytools",
|
||||
version = "0.0.1",
|
||||
description = "Fastllm pytools",
|
||||
packages = ['fastllm_pytools'],
|
||||
url = "https://developer.hpccube.com/codes/aicomponent/fastllm",
|
||||
package_data = {
|
||||
'': ['*.dll', '*.so']
|
||||
}
|
||||
)
|
||||
@ -0,0 +1,13 @@
|
||||
import sys
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
from transformers.generation import GenerationConfig
|
||||
from fastllm_pytools import torch2flm
|
||||
|
||||
if __name__ == "__main__":
|
||||
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-7B-Chat", trust_remote_code=True)
|
||||
model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen-7B-Chat", device_map="cpu", trust_remote_code=True, fp32=True).eval()
|
||||
model.generation_config = GenerationConfig.from_pretrained("Qwen/Qwen-7B-Chat", trust_remote_code=True) # 可指定不同的生成长度、top_p等相关超参
|
||||
|
||||
dtype = sys.argv[2] if len(sys.argv) >= 3 else "float16"
|
||||
exportPath = sys.argv[1] if len(sys.argv) >= 2 else "qwen-7b-" + dtype + ".flm"
|
||||
torch2flm.tofile(exportPath, model, tokenizer, dtype = dtype)
|
||||
@ -0,0 +1,9 @@
|
||||
transformers==4.32.0
|
||||
tiktoken
|
||||
streamlit>=1.24.0
|
||||
sentencepiece
|
||||
urllib3==1.26.16
|
||||
transformers_stream_generator==0.0.4
|
||||
accelerate
|
||||
einops
|
||||
#scipy
|
||||
@ -0,0 +1,37 @@
|
||||
import streamlit as st
|
||||
from streamlit_chat import message
|
||||
from fastllm_pytools import llm
|
||||
import sys
|
||||
|
||||
st.set_page_config(
|
||||
page_title="fastllm web demo",
|
||||
page_icon=":robot:"
|
||||
)
|
||||
|
||||
@st.cache_resource
|
||||
def get_model():
|
||||
model = llm.model(sys.argv[1])
|
||||
return model
|
||||
|
||||
if "messages" not in st.session_state:
|
||||
st.session_state.messages = []
|
||||
|
||||
for i, (prompt, response) in enumerate(st.session_state.messages):
|
||||
with st.chat_message("user"):
|
||||
st.markdown(prompt)
|
||||
with st.chat_message("assistant"):
|
||||
st.markdown(response)
|
||||
|
||||
if prompt := st.chat_input("请开始对话"):
|
||||
model = get_model()
|
||||
with st.chat_message("user"):
|
||||
st.markdown(prompt)
|
||||
|
||||
with st.chat_message("assistant"):
|
||||
message_placeholder = st.empty()
|
||||
full_response = ""
|
||||
for chunk in model.stream_response(prompt, st.session_state.messages, one_by_one = True):
|
||||
full_response += chunk
|
||||
message_placeholder.markdown(full_response + "▌")
|
||||
message_placeholder.markdown(full_response)
|
||||
st.session_state.messages.append((prompt, full_response))
|
||||
@ -0,0 +1,109 @@
|
||||
ARG CUDA_VERSION=11.7.1
|
||||
ARG from=nvidia/cuda:${CUDA_VERSION}-cudnn8-devel-ubuntu20.04
|
||||
|
||||
FROM ${from} as base
|
||||
|
||||
ARG from
|
||||
|
||||
RUN <<EOF
|
||||
apt update -y && apt upgrade -y && apt install -y --no-install-recommends \
|
||||
git \
|
||||
git-lfs \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
wget \
|
||||
vim \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
EOF
|
||||
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
RUN git lfs install
|
||||
|
||||
FROM base as dev
|
||||
|
||||
WORKDIR /
|
||||
|
||||
RUN mkdir -p /data/shared/Qwen
|
||||
|
||||
WORKDIR /data/shared/Qwen/
|
||||
|
||||
# Users can also mount '/data/shared/Qwen/' to keep the data
|
||||
COPY ../requirements.txt ./
|
||||
COPY ../requirements_web_demo.txt ./
|
||||
|
||||
FROM dev as bundle_req
|
||||
|
||||
ARG BUNDLE_REQUIREMENTS=true
|
||||
|
||||
RUN <<EOF
|
||||
if [ "$BUNDLE_REQUIREMENTS" = "true" ]; then
|
||||
cd /data/shared/Qwen
|
||||
pip3 install torch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install -r requirements_web_demo.txt
|
||||
fi
|
||||
EOF
|
||||
|
||||
FROM bundle_req as bundle_flash_attention
|
||||
ARG BUNDLE_FLASH_ATTENTION=true
|
||||
|
||||
RUN <<EOF
|
||||
if [ "$BUNDLE_FLASH_ATTENTION" = "true" ]; then
|
||||
cd /data/shared/Qwen
|
||||
test -d flash-attention || git clone -b v2.3.3 https://github.com/Dao-AILab/flash-attention
|
||||
cd /data/shared/Qwen/flash-attention &&
|
||||
pip3 install . &&
|
||||
pip3 install csrc/layer_norm
|
||||
fi
|
||||
EOF
|
||||
|
||||
FROM bundle_flash_attention as bundle_finetune
|
||||
ARG BUNDLE_FINETUNE=true
|
||||
|
||||
RUN <<EOF
|
||||
if [ "$BUNDLE_FINETUNE" = "true" ]; then
|
||||
cd /data/shared/Qwen
|
||||
|
||||
# Full-finetune / LoRA.
|
||||
pip3 install deepspeed peft
|
||||
|
||||
# Q-LoRA.
|
||||
apt update -y && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
|
||||
libopenmpi-dev openmpi-bin \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
pip3 install optimum auto-gptq mpi4py
|
||||
fi
|
||||
EOF
|
||||
|
||||
FROM bundle_finetune as bundle_openai_api
|
||||
ARG BUNDLE_OPENAI_API=true
|
||||
|
||||
RUN <<EOF
|
||||
if [ "$BUNDLE_OPENAI_API" = "true" ]; then
|
||||
cd /data/shared/Qwen
|
||||
|
||||
pip3 install fastapi uvicorn "openai<1.0.0" sse_starlette "pydantic<=1.10.13"
|
||||
fi
|
||||
EOF
|
||||
|
||||
FROM bundle_openai_api as final
|
||||
ARG from
|
||||
|
||||
COPY ../requirements.txt ./
|
||||
COPY ../requirements_web_demo.txt ./
|
||||
COPY ../cli_demo.py ./
|
||||
COPY ../web_demo.py ./
|
||||
COPY ../openai_api.py ./
|
||||
COPY ../finetune.py ./
|
||||
COPY ../utils.py ./
|
||||
COPY ./examples/* ./examples/
|
||||
COPY ./eval/* ./eval/
|
||||
COPY ./finetune/* ./finetune/
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
WORKDIR /data/shared/Qwen/
|
||||
|
||||
CMD ["python3", "web_demo.py", "--server-port", "80", "--server-name", "0.0.0.0", "-c", "/data/shared/Qwen/Qwen-Chat/"]
|
||||
@ -0,0 +1,105 @@
|
||||
ARG CUDA_VERSION=11.4.3
|
||||
ARG from=nvidia/cuda:${CUDA_VERSION}-cudnn8-devel-ubuntu20.04
|
||||
|
||||
FROM ${from} as base
|
||||
|
||||
ARG from
|
||||
|
||||
RUN <<EOF
|
||||
apt update -y && apt upgrade -y && apt install -y --no-install-recommends \
|
||||
git \
|
||||
git-lfs \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
wget \
|
||||
vim \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
EOF
|
||||
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
RUN git lfs install
|
||||
|
||||
FROM base as dev
|
||||
|
||||
WORKDIR /
|
||||
|
||||
RUN mkdir -p /data/shared/Qwen
|
||||
|
||||
WORKDIR /data/shared/Qwen/
|
||||
|
||||
# Users can also mount '/data/shared/Qwen/' to keep the data
|
||||
COPY ../requirements.txt ./
|
||||
COPY ../requirements_web_demo.txt ./
|
||||
|
||||
FROM dev as bundle_req
|
||||
|
||||
ARG BUNDLE_REQUIREMENTS=true
|
||||
|
||||
RUN <<EOF
|
||||
if [ "$BUNDLE_REQUIREMENTS" = "true" ]; then
|
||||
cd /data/shared/Qwen
|
||||
pip3 install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install -r requirements_web_demo.txt
|
||||
fi
|
||||
EOF
|
||||
|
||||
FROM bundle_req as bundle_flash_attention
|
||||
ARG BUNDLE_FLASH_ATTENTION=true
|
||||
|
||||
RUN <<EOF
|
||||
if [ "$BUNDLE_FLASH_ATTENTION" = "true" ]; then
|
||||
echo "CUDA 11.4 does not support flash-attention, please try other images."
|
||||
fi
|
||||
EOF
|
||||
|
||||
FROM bundle_flash_attention as bundle_finetune
|
||||
ARG BUNDLE_FINETUNE=true
|
||||
|
||||
RUN <<EOF
|
||||
if [ "$BUNDLE_FINETUNE" = "true" ]; then
|
||||
cd /data/shared/Qwen
|
||||
|
||||
# Full-finetune / LoRA.
|
||||
pip3 install deepspeed peft
|
||||
|
||||
# Q-LoRA.
|
||||
apt update -y && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
|
||||
libopenmpi-dev openmpi-bin \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
pip3 install optimum auto-gptq mpi4py
|
||||
fi
|
||||
EOF
|
||||
|
||||
FROM bundle_finetune as bundle_openai_api
|
||||
ARG BUNDLE_OPENAI_API=true
|
||||
|
||||
RUN <<EOF
|
||||
if [ "$BUNDLE_OPENAI_API" = "true" ]; then
|
||||
cd /data/shared/Qwen
|
||||
|
||||
pip3 install fastapi uvicorn "openai<1.0.0" sse_starlette "pydantic<=1.10.13"
|
||||
fi
|
||||
EOF
|
||||
|
||||
FROM bundle_openai_api as final
|
||||
ARG from
|
||||
|
||||
COPY ../requirements.txt ./
|
||||
COPY ../requirements_web_demo.txt ./
|
||||
COPY ../cli_demo.py ./
|
||||
COPY ../web_demo.py ./
|
||||
COPY ../openai_api.py ./
|
||||
COPY ../finetune.py ./
|
||||
COPY ../utils.py ./
|
||||
COPY ./examples/* ./examples/
|
||||
COPY ./eval/* ./eval/
|
||||
COPY ./finetune/* ./finetune/
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
WORKDIR /data/shared/Qwen/
|
||||
|
||||
CMD ["python3", "web_demo.py", "--server-port", "80", "--server-name", "0.0.0.0", "-c", "/data/shared/Qwen/Qwen-Chat/"]
|
||||
@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# This script will automatically pull docker image from DockerHub, and start a container to run the Qwen-Chat cli-demo.
|
||||
|
||||
IMAGE_NAME=qwenllm/qwen:cu117
|
||||
QWEN_CHECKPOINT_PATH=/path/to/Qwen-Chat
|
||||
CONTAINER_NAME=qwen
|
||||
|
||||
function usage() {
|
||||
echo '
|
||||
Usage: bash docker/docker_cli_demo.sh [-i IMAGE_NAME] -c [/path/to/Qwen-Chat] [-n CONTAINER_NAME]
|
||||
'
|
||||
}
|
||||
|
||||
while [[ "$1" != "" ]]; do
|
||||
case $1 in
|
||||
-i | --image-name )
|
||||
shift
|
||||
IMAGE_NAME=$1
|
||||
;;
|
||||
-c | --checkpoint )
|
||||
shift
|
||||
QWEN_CHECKPOINT_PATH=$1
|
||||
;;
|
||||
-n | --container-name )
|
||||
shift
|
||||
CONTAINER_NAME=$1
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
* )
|
||||
echo "Unknown argument ${1}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ ! -e ${QWEN_CHECKPOINT_PATH}/config.json ]; then
|
||||
echo "Checkpoint config.json file not found in ${QWEN_CHECKPOINT_PATH}, exit."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo docker pull ${IMAGE_NAME} || {
|
||||
echo "Pulling image ${IMAGE_NAME} failed, exit."
|
||||
exit 1
|
||||
}
|
||||
|
||||
sudo docker run --gpus all --rm --name ${CONTAINER_NAME} \
|
||||
--mount type=bind,source=${QWEN_CHECKPOINT_PATH},target=/data/shared/Qwen/Qwen-Chat \
|
||||
-it ${IMAGE_NAME} \
|
||||
python cli_demo.py -c /data/shared/Qwen/Qwen-Chat/
|
||||
@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# This script will automatically pull docker image from DockerHub, and start a daemon container to run the Qwen-Chat OpenAI API.
|
||||
|
||||
IMAGE_NAME=qwenllm/qwen:cu117
|
||||
QWEN_CHECKPOINT_PATH=/path/to/Qwen-Chat
|
||||
PORT=8000
|
||||
CONTAINER_NAME=qwen
|
||||
|
||||
function usage() {
|
||||
echo '
|
||||
Usage: bash docker/docker_openai_api.sh [-i IMAGE_NAME] -c [/path/to/Qwen-Chat] [-n CONTAINER_NAME] [--port PORT]
|
||||
'
|
||||
}
|
||||
|
||||
while [[ "$1" != "" ]]; do
|
||||
case $1 in
|
||||
-i | --image-name )
|
||||
shift
|
||||
IMAGE_NAME=$1
|
||||
;;
|
||||
-c | --checkpoint )
|
||||
shift
|
||||
QWEN_CHECKPOINT_PATH=$1
|
||||
;;
|
||||
-n | --container-name )
|
||||
shift
|
||||
CONTAINER_NAME=$1
|
||||
;;
|
||||
--port )
|
||||
shift
|
||||
PORT=$1
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
* )
|
||||
echo "Unknown argument ${1}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ ! -e ${QWEN_CHECKPOINT_PATH}/config.json ]; then
|
||||
echo "Checkpoint config.json file not found in ${QWEN_CHECKPOINT_PATH}, exit."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo docker pull ${IMAGE_NAME} || {
|
||||
echo "Pulling image ${IMAGE_NAME} failed, exit."
|
||||
exit 1
|
||||
}
|
||||
|
||||
sudo docker run --gpus all -d --restart always --name ${CONTAINER_NAME} \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock -p ${PORT}:80 \
|
||||
--mount type=bind,source=${QWEN_CHECKPOINT_PATH},target=/data/shared/Qwen/Qwen-Chat \
|
||||
-it ${IMAGE_NAME} \
|
||||
python openai_api.py --server-port 80 --server-name 0.0.0.0 -c /data/shared/Qwen/Qwen-Chat/ && {
|
||||
echo "Successfully started OpenAI API server. Access 'http://localhost:${PORT}/v1' to try!
|
||||
Run \`docker logs ${CONTAINER_NAME}\` to check server status.
|
||||
Run \`docker rm -f ${CONTAINER_NAME}\` to stop and remove the server."
|
||||
}
|
||||
@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# This script will automatically pull docker image from DockerHub, and start a daemon container to run the Qwen-Chat web-demo.
|
||||
|
||||
IMAGE_NAME=qwenllm/qwen:cu117
|
||||
QWEN_CHECKPOINT_PATH=/path/to/Qwen-7B-Chat
|
||||
PORT=8901
|
||||
CONTAINER_NAME=qwen
|
||||
|
||||
function usage() {
|
||||
echo '
|
||||
Usage: bash docker/docker_web_demo.sh [-i IMAGE_NAME] -c [/path/to/Qwen-Chat] [-n CONTAINER_NAME] [--port PORT]
|
||||
'
|
||||
}
|
||||
|
||||
while [[ "$1" != "" ]]; do
|
||||
case $1 in
|
||||
-i | --image-name )
|
||||
shift
|
||||
IMAGE_NAME=$1
|
||||
;;
|
||||
-c | --checkpoint )
|
||||
shift
|
||||
QWEN_CHECKPOINT_PATH=$1
|
||||
;;
|
||||
-n | --container-name )
|
||||
shift
|
||||
CONTAINER_NAME=$1
|
||||
;;
|
||||
--port )
|
||||
shift
|
||||
PORT=$1
|
||||
;;
|
||||
-h | --help )
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
* )
|
||||
echo "Unknown argument ${1}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ ! -e ${QWEN_CHECKPOINT_PATH}/config.json ]; then
|
||||
echo "Checkpoint config.json file not found in ${QWEN_CHECKPOINT_PATH}, exit."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo docker pull ${IMAGE_NAME} || {
|
||||
echo "Pulling image ${IMAGE_NAME} failed, exit."
|
||||
exit 1
|
||||
}
|
||||
|
||||
sudo docker run --gpus all -d --restart always --name ${CONTAINER_NAME} \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock -p ${PORT}:80 \
|
||||
--mount type=bind,source=${QWEN_CHECKPOINT_PATH},target=/data/shared/Qwen/Qwen-Chat \
|
||||
-it ${IMAGE_NAME} \
|
||||
python web_demo.py --server-port 80 --server-name 0.0.0.0 -c /data/shared/Qwen/Qwen-Chat/ && {
|
||||
echo "Successfully started web demo. Open 'http://localhost:${PORT}' to try!
|
||||
Run \`docker logs ${CONTAINER_NAME}\` to check demo status.
|
||||
Run \`docker rm -f ${CONTAINER_NAME}\` to stop and remove the demo."
|
||||
}
|
||||
Loading…
Reference in New Issue