release the evaluation benchmark for tool use; update tool use results to that of the hf version
兼欣
1 year ago
parent
fa33db2a26
commit
9139fbdf99
@ -0,0 +1,308 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import pprint
|
||||
|
||||
import json5
|
||||
import jsonlines
|
||||
from rouge_score import rouge_scorer
|
||||
from tqdm import tqdm
|
||||
from transformers import Agent, AutoModelForCausalLM, AutoTokenizer
|
||||
from transformers.generation import GenerationConfig
|
||||
from transformers.tools.evaluate_agent import evaluate_agent
|
||||
from transformers.trainer_utils import set_seed
|
||||
|
||||
data_root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
'data')
|
||||
|
||||
|
||||
def is_callable(response, golden):
|
||||
return response['action'].strip().lower() == golden['action'].strip(
|
||||
).lower()
|
||||
|
||||
|
||||
def process_res(response):
|
||||
# parse response
|
||||
response += '\n' # fix not-find bug
|
||||
thought = response[:response.find('Action:')].strip()
|
||||
action = response[response.find('Action:') +
|
||||
len('Action:'):response.find('Action Input:')].strip()
|
||||
action_input = response[response.find('Action Input:') +
|
||||
len('Action Input:'):response.find('Observation:'
|
||||
)].strip()
|
||||
#TODO: This parsing result is incorrect if the response contains multiple Actions. To be fixed in the future.
|
||||
observation = response[response.find('Observation:') +
|
||||
len('Observation:'):response.rfind('Thought:'
|
||||
)].strip()
|
||||
thought_last = response[response.rfind('Thought:') +
|
||||
len('Thought:'):response.find('Final Answer:'
|
||||
)].strip()
|
||||
final_answer = response[response.find('Final Answer:') +
|
||||
len('Final Answer:'):].strip()
|
||||
try:
|
||||
action_input = json.dumps(json5.loads(action_input),
|
||||
ensure_ascii=False,
|
||||
sort_keys=True)
|
||||
except:
|
||||
# print("JSON Load Error:", action_input)
|
||||
pass
|
||||
res_dict = {
|
||||
'thought': thought,
|
||||
'action': action,
|
||||
'action_input': action_input,
|
||||
'observation': observation,
|
||||
'thought_last': thought_last,
|
||||
'final_answer': final_answer
|
||||
}
|
||||
return res_dict
|
||||
|
||||
|
||||
class _DummyTokenizer:
|
||||
def tokenize(self, text: str):
|
||||
return text.split()
|
||||
|
||||
|
||||
def _get_tokenized_string(tokenizer, text_list):
|
||||
token_ids_list, tokenized_string_list = [], []
|
||||
for text in text_list:
|
||||
assert tokenizer is not None
|
||||
token_ids = tokenizer.encode(text)
|
||||
tokens_bytes = tokenizer.convert_ids_to_tokens(token_ids)
|
||||
tokens = [
|
||||
token.decode('utf-8', errors='replace') for token in tokens_bytes
|
||||
]
|
||||
tokenized_string = ' '.join(tokens)
|
||||
token_ids_list.append(token_ids)
|
||||
tokenized_string_list.append(tokenized_string)
|
||||
return token_ids_list, tokenized_string_list
|
||||
|
||||
|
||||
def eval_action(job):
|
||||
response = job['gen'][0]
|
||||
golden = job['response']
|
||||
|
||||
if 'Action:' in response:
|
||||
response, golden = process_res(response), process_res(golden)
|
||||
if is_callable(response, golden):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def eval_action_input(job, tokenizer):
|
||||
response = job['gen'][0]
|
||||
golden = job['response']
|
||||
response, golden = process_res(response), process_res(golden)
|
||||
query = job['prompt']
|
||||
|
||||
job = {}
|
||||
job['prompt'] = query
|
||||
job['gen'] = response['action_input']
|
||||
job['response'] = golden['action_input']
|
||||
|
||||
job['_gen_tok'], job['_gen_tok_str'] = _get_tokenized_string(
|
||||
tokenizer, [response['action_input']])
|
||||
job['_reference_tok'], job['_reference_tok_str'] = _get_tokenized_string(
|
||||
tokenizer, [golden['action_input']])
|
||||
|
||||
scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'],
|
||||
tokenizer=_DummyTokenizer())
|
||||
score = scorer.score(job['_reference_tok_str'][0], job['_gen_tok_str'][0])
|
||||
|
||||
rouge = score['rougeL'].fmeasure
|
||||
|
||||
return rouge
|
||||
|
||||
|
||||
class QWenAgent(Agent):
|
||||
"""
|
||||
Agent that uses QWen model and tokenizer to generate code.
|
||||
|
||||
Example:
|
||||
|
||||
```py
|
||||
agent = QWenAgent()
|
||||
agent.run("Draw me a picture of rivers and lakes.")
|
||||
```
|
||||
"""
|
||||
def __init__(self,
|
||||
chat_prompt_template=None,
|
||||
run_prompt_template=None,
|
||||
additional_tools=None,
|
||||
tokenizer=None,
|
||||
model=None):
|
||||
if tokenizer and model:
|
||||
self.tokenizer = tokenizer
|
||||
self.model = model
|
||||
else:
|
||||
checkpoint = 'Qwen/Qwen-7B-Chat'
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
checkpoint, trust_remote_code=True)
|
||||
self.model = AutoModelForCausalLM.from_pretrained(
|
||||
checkpoint, device_map='auto',
|
||||
trust_remote_code=True).cuda().eval()
|
||||
self.model.generation_config = GenerationConfig.from_pretrained(
|
||||
checkpoint, trust_remote_code=True) # 可指定不同的生成长度、top_p等相关超参
|
||||
self.model.generation_config.do_sample = False # greedy
|
||||
|
||||
super().__init__(
|
||||
chat_prompt_template=chat_prompt_template,
|
||||
run_prompt_template=run_prompt_template,
|
||||
additional_tools=additional_tools,
|
||||
)
|
||||
|
||||
def generate_one(self, prompt, stop):
|
||||
# "Human:" 和 "Assistant:" 曾为通义千问的特殊保留字，需要替换为 "_HUMAN_:" 和 "_ASSISTANT_:"。这一问题将在未来版本修复。
|
||||
prompt = prompt.replace('Human:',
|
||||
'_HUMAN_:').replace('Assistant:',
|
||||
'_ASSISTANT_:')
|
||||
stop = [
|
||||
item.replace('Human:', '_HUMAN_:').replace('Assistant:',
|
||||
'_ASSISTANT_:')
|
||||
for item in stop
|
||||
]
|
||||
|
||||
result, _ = self.model.chat(self.tokenizer, prompt, history=None)
|
||||
for stop_seq in stop:
|
||||
if result.endswith(stop_seq):
|
||||
result = result[:-len(stop_seq)]
|
||||
|
||||
result = result.replace('_HUMAN_:',
|
||||
'Human:').replace('_ASSISTANT_:', 'Assistant:')
|
||||
return result
|
||||
|
||||
|
||||
def load_models_tokenizer(args):
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.checkpoint_path,
|
||||
trust_remote_code=True)
|
||||
model = AutoModelForCausalLM.from_pretrained(args.checkpoint_path,
|
||||
device_map='auto',
|
||||
trust_remote_code=True,
|
||||
bf16=True,
|
||||
use_flash_attn=True).eval()
|
||||
model.generation_config = GenerationConfig.from_pretrained(
|
||||
args.checkpoint_path, trust_remote_code=True)
|
||||
model.generation_config.do_sample = False # use greedy decoding
|
||||
return model, tokenizer
|
||||
|
||||
|
||||
def load_jobs(filename):
|
||||
jobs = []
|
||||
with jsonlines.open(os.path.join(data_root_path, filename),
|
||||
mode='r') as reader:
|
||||
for job in reader:
|
||||
jobs.append(job)
|
||||
return jobs
|
||||
|
||||
|
||||
def react_inference(filename, model, tokenizer):
|
||||
filename_cache = filename + '.cache'
|
||||
if os.path.exists(os.path.join(data_root_path, filename_cache)):
|
||||
jobs = load_jobs(filename=filename_cache)
|
||||
print('Loaded from', filename_cache)
|
||||
else:
|
||||
with open(os.path.join(data_root_path, filename_cache), 'w') as f:
|
||||
jobs = load_jobs(filename=filename)
|
||||
print('Inference:', filename)
|
||||
for job in tqdm(jobs):
|
||||
response, history = model.chat(tokenizer,
|
||||
job['prompt'],
|
||||
history=None)
|
||||
job['gen'] = [response]
|
||||
f.writelines(json.dumps(job, ensure_ascii=False) + '\n')
|
||||
print(filename_cache, 'is saved.')
|
||||
return jobs
|
||||
|
||||
|
||||
def main(args):
|
||||
print('loading model weights')
|
||||
if args.checkpoint_path is not None:
|
||||
model, tokenizer = load_models_tokenizer(args)
|
||||
else:
|
||||
model, tokenizer = None, None
|
||||
print('model loaded')
|
||||
|
||||
result = {}
|
||||
# eval react positive
|
||||
if args.eval_react_positive:
|
||||
print('eval react positive ...')
|
||||
acc_count = 0
|
||||
rouge_mean = 0
|
||||
jobs = react_inference(filename=args.eval_react_positive_filename,
|
||||
model=model,
|
||||
tokenizer=tokenizer)
|
||||
for job in jobs:
|
||||
if eval_action(job):
|
||||
acc_count += 1
|
||||
rouge = eval_action_input(job, tokenizer)
|
||||
rouge_mean += (rouge / len(jobs))
|
||||
|
||||
scores = {
|
||||
'action_right_rate': acc_count / len(jobs),
|
||||
'action_input_rouge': rouge_mean,
|
||||
}
|
||||
|
||||
result.update({'react_positive': scores})
|
||||
|
||||
# eval react negative
|
||||
if args.eval_react_negative:
|
||||
print('eval react negative ...')
|
||||
bad_count = 0
|
||||
jobs = react_inference(filename=args.eval_react_negative_filename,
|
||||
model=model,
|
||||
tokenizer=tokenizer)
|
||||
for job in jobs:
|
||||
if '\nAction:' in job['gen'][0]:
|
||||
bad_count += 1
|
||||
scores = {'bad_rate': bad_count / len(jobs)}
|
||||
result.update({'react_negative': scores})
|
||||
|
||||
# eval hfagent
|
||||
if args.eval_hfagent:
|
||||
print('eval hfagent ...')
|
||||
agent = QWenAgent(model=model, tokenizer=tokenizer)
|
||||
scores = evaluate_agent(agent, verbose=False, return_errors=False)
|
||||
result.update({'hfagent': scores})
|
||||
|
||||
pp = pprint.PrettyPrinter(indent=4)
|
||||
pp.pprint(result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Test HF checkpoint.')
|
||||
parser.add_argument('-c',
|
||||
'--checkpoint-path',
|
||||
type=str,
|
||||
help='Checkpoint path',
|
||||
default='Qwen/Qwen-7B-Chat')
|
||||
parser.add_argument('-s',
|
||||
'--seed',
|
||||
type=int,
|
||||
default=1234,
|
||||
help='Random seed')
|
||||
"""Provide extra arguments required for tasks."""
|
||||
group = parser.add_argument_group(title='Evaluation options')
|
||||
group.add_argument('--eval-react-positive',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Eval react positive.')
|
||||
group.add_argument('--eval-react-positive-filename',
|
||||
type=str,
|
||||
default='exam_plugin_v1_react_positive.jsonl',
|
||||
help='Eval react positive filename.')
|
||||
group.add_argument('--eval-react-negative',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Eval react negative.')
|
||||
group.add_argument('--eval-react-negative-filename',
|
||||
type=str,
|
||||
default='exam_plugin_v1_react_negative.jsonl',
|
||||
help='Eval react negative filename.')
|
||||
group.add_argument('--eval-hfagent',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Eval hfagent.')
|
||||
|
||||
args = parser.parse_args()
|
||||
set_seed(args.seed)
|
||||
|
||||
main(args)
|
||||
Loading…
Reference in New Issue