-
Notifications
You must be signed in to change notification settings - Fork 0
/
NERAndNetworks.py
81 lines (66 loc) · 2.32 KB
/
NERAndNetworks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
#Import the requisite library
import spacy
import re
import networkx as nx
import matplotlib.pyplot as plt
from fuzzywuzzy.process import dedupe
from collections import deque
import pandas as pd
def normalize(string):
text = re.sub(r"\n.*", "", string)
text = re.sub(r"'s$", "", text)
return text
def retrieveEntities(file, type):
#Build upon the spaCy Large Model
nlp = spacy.load("en_core_web_lg")
#Sample text
with open (file, "r", encoding="utf-8") as f:
text = f.read()
doc = nlp(text)
person_all = []
variants = {}
if type == "PERSON":
#extract entities
for ent in doc.ents:
if ent.label_ == type:
text = normalize(ent.text)
person_all.append(text)
person_cleaned = set(dedupe(person_all))
for person in person_cleaned:
names = person.split(" ")
for name in names:
if name in person_all:
if name not in variants:
variants[name] = person
network = nx.Graph()
#extract entities
entitiesRating = {}
for sent in doc.sents:
entities = deque()
for ent in sent.ents:
if ent.label_ == type:
text = normalize(ent.text)
if text in variants:
text = variants[text]
entities.append(text)
if text not in entitiesRating:
entitiesRating[text] = 1
else:
entitiesRating[text] += 1
for i in range(len(entities)):
source = entities.popleft()
for entity in list(entities):
network.add_edge(source, entity)
entities.append(source)
nx.draw_spring(network, with_labels = True)
plt.show()
#rating of cited entities
print(entitiesRating)
df = pd.DataFrame(list(entitiesRating.items()),columns = ['Entity','Count']).sort_values(by=["Count"], ascending=False)
plt.barh(df["Entity"].head(10), df["Count"].head(10))
# setting x-label as entity
plt.xlabel("OCCURRENCES")
# setting y_label as price
plt.ylabel(type)
plt.title("Horizontal bar graph of entities occurrences")
plt.show()