-
Notifications
You must be signed in to change notification settings - Fork 137
/
IntellisenseTokenizer.cs
78 lines (71 loc) · 2.43 KB
/
IntellisenseTokenizer.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
namespace SwqlStudio.Intellisense
{
internal class IntellisenseTokenizer
{
public enum Token
{
Special,
Number,
Identifier,
String,
EOF
}
private readonly string _input;
private static readonly IEnumerable<Regex> _ignoredRegexes;
private static readonly IEnumerable<Tuple<Regex, Token>> _regexes;
static IntellisenseTokenizer()
{
_ignoredRegexes = new[]
{
new Regex(@"\G--.*\n"), // comment
new Regex(@"\G\s+") // whitespace
};
_regexes = new[]
{
Tuple.Create(new Regex(@"\G[0-9]+.[0-9]*([eE][0-9]+)?"), Token.Number), // number
Tuple.Create(new Regex(@"\G@[a-zA-Z_]\w*"), Token.Identifier), // ident
Tuple.Create(new Regex(@"\G\[[^\]]*\]"), Token.Identifier), // quoted ident
Tuple.Create(new Regex(@"\G""(?:[^""\\]|\\.)*"""), Token.String), // quoted string
Tuple.Create(new Regex(@"\G[a-zA-Z_][a-zA-Z_0-9]*"), Token.Identifier) // ident
};
}
public IntellisenseTokenizer(string input)
{
_input = input;
}
public IEnumerator<Tuple<int, int, Token>> GetEnumerator()
{
int position = 0;
while (position < _input.Length)
{
foreach (var ignored in _ignoredRegexes)
{
var m = ignored.Match(_input, position);
if (m.Success)
{
position += m.Groups[0].Length;
goto end;
}
}
foreach (var rx in _regexes)
{
var m = rx.Item1.Match(_input, position);
if (m.Success)
{
yield return Tuple.Create(position, m.Groups[0].Length, rx.Item2);
position += m.Groups[0].Length;
goto end;
}
}
yield return Tuple.Create(position, 1, Token.Special);
position++;
end:
;
}
yield return Tuple.Create(position, 0, Token.EOF);
}
}
}