-
Notifications
You must be signed in to change notification settings - Fork 6
/
scorer.py
154 lines (108 loc) · 6.04 KB
/
scorer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
import six
def check_predicted_evidence_format(instance):
if 'predicted_evidence' in instance.keys() and len(instance['predicted_evidence']):
assert all(isinstance(prediction, list)
for prediction in instance["predicted_evidence"]), \
"Predicted evidence must be a list of (page,line) lists"
assert all(len(prediction) == 2
for prediction in instance["predicted_evidence"]), \
"Predicted evidence must be a list of (page,line) lists"
assert all(isinstance(prediction[0], six.string_types)
for prediction in instance["predicted_evidence"]), \
"Predicted evidence must be a list of (page<string>,line<int>) lists"
assert all(isinstance(prediction[1], int)
for prediction in instance["predicted_evidence"]), \
"Predicted evidence must be a list of (page<string>,line<int>) lists"
def is_correct_label(instance):
return instance["label"].upper() == instance["predicted_label"].upper()
def is_strictly_correct(instance, max_evidence=None):
# Strict evidence matching is only for NEI class
check_predicted_evidence_format(instance)
if instance["label"].upper() != "NOT ENOUGH INFO" and is_correct_label(instance):
assert 'predicted_evidence' in instance, "Predicted evidence must be provided for strict scoring"
if max_evidence is None:
max_evidence = len(instance["predicted_evidence"])
for evience_group in instance["evidence"]:
# Filter out the annotation ids. We just want the evidence page and line number
actual_sentences = [[e[2], e[3]] for e in evience_group]
# Only return true if an entire group of actual sentences is in the predicted sentences
if all([actual_sent in instance["predicted_evidence"][:max_evidence] for actual_sent in actual_sentences]):
return True
# If the class is NEI, we don't score the evidence retrieval component
elif instance["label"].upper() == "NOT ENOUGH INFO" and is_correct_label(instance):
return True
return False
def evidence_macro_precision(instance, max_evidence=None):
this_precision = 0.0
this_precision_hits = 0.0
if instance["label"].upper() != "NOT ENOUGH INFO":
all_evi = [[e[2], e[3]] for eg in instance["evidence"] for e in eg if e[3] is not None]
predicted_evidence = instance["predicted_evidence"] if max_evidence is None else \
instance["predicted_evidence"][:max_evidence]
for prediction in predicted_evidence:
if prediction in all_evi:
this_precision += 1.0
this_precision_hits += 1.0
return (this_precision / this_precision_hits) if this_precision_hits > 0 else 1.0, 1.0
return 0.0, 0.0
def evidence_macro_recall(instance, max_evidence=None):
# We only want to score F1/Precision/Recall of recalled evidence for NEI claims
if instance["label"].upper() != "NOT ENOUGH INFO":
# If there's no evidence to predict, return 1
if len(instance["evidence"]) == 0 or all([len(eg) == 0 for eg in instance]):
return 1.0, 1.0
predicted_evidence = instance["predicted_evidence"] if max_evidence is None else \
instance["predicted_evidence"][:max_evidence]
for evidence_group in instance["evidence"]:
evidence = [[e[2], e[3]] for e in evidence_group]
if all([item in predicted_evidence for item in evidence]):
# We only want to score complete groups of evidence. Incomplete groups are worthless.
return 1.0, 1.0
return 0.0, 1.0
return 0.0, 0.0
# Micro is not used. This code is just included to demostrate our model of macro/micro
def evidence_micro_precision(instance):
this_precision = 0
this_precision_hits = 0
# We only want to score Macro F1/Precision/Recall of recalled evidence for NEI claims
if instance["label"].upper() != "NOT ENOUGH INFO":
all_evi = [[e[2], e[3]] for eg in instance["evidence"] for e in eg if e[3] is not None]
for prediction in instance["predicted_evidence"]:
if prediction in all_evi:
this_precision += 1.0
this_precision_hits += 1.0
return this_precision, this_precision_hits
def fever_score(predictions, actual=None, max_evidence=5):
correct = 0
strict = 0
macro_precision = 0
macro_precision_hits = 0
macro_recall = 0
macro_recall_hits = 0
for idx, instance in enumerate(predictions):
assert 'predicted_evidence' in instance.keys(), 'evidence must be provided for the prediction'
# If it's a blind test set, we need to copy in the values from the actual data
if 'evidence' not in instance or 'label' not in instance:
assert actual is not None, 'in blind evaluation mode, actual data must be provided'
assert len(actual) == len(predictions), 'actual data and predicted data length must match'
assert 'evidence' in actual[idx].keys(), 'evidence must be provided for the actual evidence'
instance['evidence'] = actual[idx]['evidence']
instance['label'] = actual[idx]['label']
assert 'evidence' in instance.keys(), 'gold evidence must be provided'
if is_correct_label(instance):
correct += 1.0
if is_strictly_correct(instance, max_evidence):
strict += 1.0
macro_prec = evidence_macro_precision(instance, max_evidence)
macro_precision += macro_prec[0]
macro_precision_hits += macro_prec[1]
macro_rec = evidence_macro_recall(instance, max_evidence)
macro_recall += macro_rec[0]
macro_recall_hits += macro_rec[1]
total = len(predictions)
strict_score = strict / total
acc_score = correct / total
pr = (macro_precision / macro_precision_hits) if macro_precision_hits > 0 else 1.0
rec = (macro_recall / macro_recall_hits) if macro_recall_hits > 0 else 0.0
f1 = 2.0 * pr * rec / (pr + rec)
return strict_score, acc_score, pr, rec, f1