/
TokenStream.ts
100 lines (76 loc) · 2.13 KB
/
TokenStream.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import Token from "./Token";
class TokenStream {
private cursor = 0;
constructor(public tokens: Token[] = []) {}
currentToken(moveCursor: boolean = false): Token {
const token = this.tokens[this.cursor];
if (moveCursor) {
this.cursor++;
}
return token;
}
*filter(predicate: string | ((token: Token) => boolean)): Iterable<Token> {
if (typeof predicate === "string") {
yield* this.filter(t => t.is(predicate));
return;
}
for (let token of this.tokens) {
if (predicate(token)) {
yield token;
}
}
}
hasNext(): boolean {
return !!this.peekToken();
}
isCurrent(predicate: string | ((token: Token) => boolean)): boolean {
return this.currentToken().is(predicate);
}
joinAll(separator: string = ""): string {
return this.tokens.map(t => t.value).join(separator);
}
joinUntil(predicate: string | ((token: Token) => boolean)): string {
return [...this.takeUntil(predicate)].map(t => t.value).join("");
}
nextUntil(predicate: string | ((token: Token) => boolean)): void {
if (typeof predicate === "string") {
this.nextUntil(t => !t.is(predicate));
return;
}
let currentToken = this.currentToken();
while (predicate(currentToken)) {
currentToken = this.nextToken();
}
}
nextToken(): Token {
return this.tokens[++this.cursor];
}
peekToken(): Token {
return this.tokens[this.cursor + 1];
}
pushToken(token: Token): void {
this.tokens.push(token);
}
reset() {
this.cursor = 0;
}
*takeUntil(predicate: string | ((token: Token) => boolean)): Iterable<Token> {
if (typeof predicate === "string") {
yield* this.takeUntil(t => t.is(predicate));
return;
}
yield* this.takeWhile(t => !predicate(t));
}
*takeWhile(predicate: string | ((token: Token) => boolean)): Iterable<Token> {
if (typeof predicate === "string") {
yield* this.takeWhile(t => t.is(predicate));
return;
}
let current = this.currentToken();
while (predicate(current)) {
yield current;
current = this.nextToken();
}
}
}
export default TokenStream;