/
token_type.py
150 lines (137 loc) · 5.46 KB
/
token_type.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
"""
token_type.py
Basic class and needed dictories of tokens
"""
class TokenTypeDict:
""" needed dictories of tokens """
token_dict = {}
# token EOF
token_dict["TOKEN_EOF"] = 0
# token #
token_dict["TOKEN_HASH"] = token_dict["TOKEN_EOF"] + 1
# token :
token_dict["TOKEN_COLON"] = token_dict["TOKEN_HASH"] + 1
# token ;
token_dict["TOKEN_SEMICOLON"] = token_dict["TOKEN_COLON"] + 1
#token (
token_dict["TOKEN_LEFTPAREN"] = token_dict["TOKEN_SEMICOLON"] + 1
#token )
token_dict["TOKEN_RIGHTPAREN"] = token_dict["TOKEN_LEFTPAREN"] + 1
#token {
token_dict["TOKEN_LEFTBRACE"] = token_dict["TOKEN_RIGHTPAREN"] + 1
#token }
token_dict["TOKEN_RIGHTBRACE"] = token_dict["TOKEN_LEFTBRACE"] + 1
#token [
token_dict["TOKEN_LEFTBRACKET"] = token_dict["TOKEN_RIGHTBRACE"] + 1
#token ]
token_dict["TOKEN_RIGHTBRACKET"] = token_dict["TOKEN_LEFTBRACKET"] + 1
#token"] =
token_dict["TOKEN_EQUAL"] = token_dict["TOKEN_RIGHTBRACKET"] + 1
#token .
token_dict["TOKEN_DOT"] = token_dict["TOKEN_EQUAL"] + 1
#token `
token_dict["TOKEN_GRAVE"] = token_dict["TOKEN_DOT"] + 1
#token '
token_dict["TOKEN_SINGLEQUOTE"] = token_dict["TOKEN_GRAVE"] + 1
#token "
token_dict["TOKEN_DOUBLEQUOTE"] = token_dict["TOKEN_SINGLEQUOTE"] + 1
#token @
token_dict["TOKEN_AT"] = token_dict["TOKEN_DOUBLEQUOTE"] + 1
#token +
token_dict["TOKEN_PLUS"] = token_dict["TOKEN_AT"] + 1
#token -
token_dict["TOKEN_MINUS"] = token_dict["TOKEN_PLUS"] + 1
#token /
token_dict["TOKEN_SLASH"] = token_dict["TOKEN_MINUS"] + 1
#token \
token_dict["TOKEN_BACKSLASH"] = token_dict["TOKEN_SLASH"] + 1
#token ?
token_dict["TOKEN_QUESTION"] = token_dict["TOKEN_BACKSLASH"] + 1
#token !
token_dict["TOKEN_EXCLAMATION"] = token_dict["TOKEN_QUESTION"] + 1
#token &
token_dict["TOKEN_AND"] = token_dict["TOKEN_EXCLAMATION"] + 1
#token |
token_dict["TOKEN_OR"] = token_dict["TOKEN_AND"] + 1
#token ~
token_dict["TOKEN_NOT"] = token_dict["TOKEN_OR"] + 1
#token number
token_dict["TOKEN_NUMBER"] = token_dict["TOKEN_NOT"] + 1
#token variable
token_dict["TOKEN_VARIABLE"] = token_dict["TOKEN_NUMBER"] + 1
#token ,
token_dict["TOKEN_COMMA"] = token_dict["TOKEN_VARIABLE"] + 1
#token keyunit, ms/us/ns/ps/fs
token_dict["TOKEN_KEYUNIT"] = token_dict["TOKEN_COMMA"] + 1
#token keydir, input/output/inout
token_dict["TOKEN_KEYDIR"] = token_dict["TOKEN_KEYUNIT"] + 1
#token keytype, wire/reg
token_dict["TOKEN_KEYTYPE"] = token_dict["TOKEN_KEYDIR"] + 1
#token keyop, +, -, &, |, ~, !
token_dict["TOKEN_KEYOP"] = token_dict["TOKEN_KEYTYPE"] + 1
#token keyword, module, endmodule, define, timescale
token_dict["TOKEN_KEYWORD"] = token_dict["TOKEN_KEYOP"] + 1
token_backward_dict = {}
for key, value in token_dict.items():
token_backward_dict[value] = key
symbol_dict = {}
symbol_dict["#"] = "TOKEN_HASH"
symbol_dict[":"] = "TOKEN_COLON"
symbol_dict[";"] = "TOKEN_SEMICOLON"
symbol_dict["("] = "TOKEN_LEFTPAREN"
symbol_dict[")"] = "TOKEN_RIGHTPAREN"
symbol_dict["{"] = "TOKEN_LEFTBRACE"
symbol_dict["}"] = "TOKEN_RIGHTBRACE"
symbol_dict["["] = "TOKEN_LEFTBRACKET"
symbol_dict["]"] = "TOKEN_RIGHTBRACKET"
symbol_dict["="] = "TOKEN_EQUAL"
symbol_dict["."] = "TOKEN_DOT"
symbol_dict["`"] = "TOKEN_GRAVE"
symbol_dict["'"] = "TOKEN_SINGLEQUOTE"
symbol_dict['"'] = "TOKEN_DOUBLEQUOTE"
symbol_dict["@"] = "TOKEN_AT"
symbol_dict["+"] = "TOKEN_PLUS"
symbol_dict["-"] = "TOKEN_MINUS"
symbol_dict["/"] = "TOKEN_SLASH"
symbol_dict["\\"] = "TOKEN_BACKSLASH"
symbol_dict["?"] = "TOKEN_QUESTION"
symbol_dict["!"] = "TOKEN_EXCLAMATION"
symbol_dict["&"] = "TOKEN_AND"
symbol_dict["|"] = "TOKEN_OR"
symbol_dict["~"] = "TOKEN_NOT"
symbol_dict[","] = "TOKEN_COMMA"
def get_symbol_type(self, symbol):
""" Return the number of the token type, or return 999 if it is not a token """
symbol_type = -1
if symbol in self.symbol_dict:
if symbol in ["+", "-", "&", "|", "~", "!"]:
symbol_type = self.token_dict["TOKEN_KEYOP"]
else:
symbol_type = self.token_dict[self.symbol_dict[symbol]]
elif symbol in ["ms", "us", "ns", "ps", "fs"]:
symbol_type = self.token_dict["TOKEN_KEYUNIT"]
elif symbol in ["input", "output", "inout"]:
symbol_type = self.token_dict["TOKEN_KEYDIR"]
elif symbol in ["reg", "wire"]:
symbol_type = self.token_dict["TOKEN_KEYTYPE"]
elif symbol in ["timescale", "define", "module", "endmodule"]:
symbol_type = self.token_dict["TOKEN_KEYWORD"]
return symbol_type
def get_token_string(self, type_number):
""" Return the string of the token """
if type_number in self.token_backward_dict:
return self.token_backward_dict[type_number]
print("ERROR!! token type number is wrong", type_number)
quit()
class BasicToken:
""" The base class of token """
def __init__(self, token_text="", token_type=0, line_number=0):
self.token_text = token_text
self.token_type = token_type
self.line_number = line_number
def get_token_type(self):
""" return the number of the token type """
return self.token_type
def get_token_text(self):
""" return the string of the token text """
return self.token_text