From bff91b330517b410eee6bfcf1b255d004d29ad51 Mon Sep 17 00:00:00 2001 From: qinxy3 Date: Mon, 14 Aug 2023 14:47:12 +0800 Subject: [PATCH 1/2] fix bug for ceval --- eval/evaluate_ceval.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eval/evaluate_ceval.py b/eval/evaluate_ceval.py index 265af55..65423e8 100644 --- a/eval/evaluate_ceval.py +++ b/eval/evaluate_ceval.py @@ -80,7 +80,7 @@ def eval_subject( score = [] few_shot_prompt = generate_few_shot_prompt( - k, subject_name, dev_df) if few_shot else [] + k, subject_name, dev_df) if few_shot else '' all_probs = {'prob_A': [], 'prob_B': [], 'prob_C': [], 'prob_D': []} if args.debug: print(f"few_shot_prompt: {few_shot_prompt}") @@ -95,10 +95,10 @@ def eval_subject( softval = torch.nn.functional.softmax( torch.tensor( [ - logits[tokenizer("A")['input_ids']], - logits[tokenizer("B")['input_ids']], - logits[tokenizer("C")['input_ids']], - logits[tokenizer("D")['input_ids']], + logits[tokenizer("A")['input_ids'][-1]], + logits[tokenizer("B")['input_ids'][-1]], + logits[tokenizer("C")['input_ids'][-1]], + logits[tokenizer("D")['input_ids'][-1]], ] ), dim=0, From 543ffaf6170b057c20aa6934bea928a8665fd98a Mon Sep 17 00:00:00 2001 From: qinxy3 Date: Tue, 15 Aug 2023 11:03:24 +0800 Subject: [PATCH 2/2] fix code --- eval/evaluate_ceval.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eval/evaluate_ceval.py b/eval/evaluate_ceval.py index 65423e8..e1616a5 100644 --- a/eval/evaluate_ceval.py +++ b/eval/evaluate_ceval.py @@ -95,10 +95,10 @@ def eval_subject( softval = torch.nn.functional.softmax( torch.tensor( [ - logits[tokenizer("A")['input_ids'][-1]], - logits[tokenizer("B")['input_ids'][-1]], - logits[tokenizer("C")['input_ids'][-1]], - logits[tokenizer("D")['input_ids'][-1]], + logits[tokenizer("A")['input_ids']], + logits[tokenizer("B")['input_ids']], + logits[tokenizer("C")['input_ids']], + logits[tokenizer("D")['input_ids']], ] ), dim=0,