-
Notifications
You must be signed in to change notification settings - Fork 0
/
utils.py
56 lines (39 loc) · 1.49 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
import os
import sys
import logging
import ast
import pandas as pd
from collections import defaultdict
def load_train_file(filename):
train_df = pd.read_csv(filename, header=0, sep=',', encoding='gbk')
texts = []; labels = []; label_tag_dict = {}; tag_label_dict = {}
temp_dict = defaultdict(float)
for i, text in enumerate(train_df['norm_text']):
text = ast.literal_eval(text)
texts.append(text)
label = ast.literal_eval(train_df['labels'][i])
for j in range(len(label)):
temp_dict[label[j]] += 1
for i, label in enumerate(temp_dict.keys()):
label_tag_dict[label] = i
tag_label_dict[i] = label
for i, text in enumerate(train_df['norm_text']):
label = ast.literal_eval(train_df['labels'][i])
line_label = []
for j in range(len(label)):
line_label.append(label_tag_dict[label[j]])
labels.append(line_label)
print("text length: " + str(len(texts)))
print("label length: " + str(len(labels)))
print(temp_dict)
print(label_tag_dict)
print(tag_label_dict)
return texts, labels, label_tag_dict, tag_label_dict
def load_test_file(filename):
train_df = pd.read_csv(filename, header=0, sep=',', encoding='gbk')
texts = []
for i, text in enumerate(train_df['norm_text']):
text = ast.literal_eval(text)
texts.append(text)
print("text length: " + str(len(texts)))
return texts