tokenizer: Fix identifier regular expression.

This blocks w3c/csswg-drafts#9686
emilio · Dec 11, 2023 · eeadd49 · eeadd49
1 parent 76ef630
commit eeadd49
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/widlparser/tokenizer.py b/widlparser/tokenizer.py
@@ -102,9 +102,10 @@ def __repr__(self) -> str:
 		return '[' + self.type.value + ':' + self.text + ']'
 
 
+# https://webidl.spec.whatwg.org/#idl-grammar
 TOKEN_FLOAT_RE = re.compile(r'(-?(([0-9]+\.[0-9]*|[0-9]*\.[0-9]+)([Ee][+-]?[0-9]+)?|[0-9]+[Ee][+-]?[0-9]+))(.*)', re.DOTALL)
 TOKEN_INTEGER_RE = re.compile(r'(-?(0[Xx][0-9A-Fa-f]+|0[0-7]*|[1-9][0-9]*))(.*)', re.DOTALL)
-TOKEN_IDENT_RE = re.compile(r'(_?[A-Z_a-z][0-9A-Z_a-z]*)(.*)', re.DOTALL)
+TOKEN_IDENT_RE = re.compile(r'([_-]?[A-Z_a-z][0-9A-Z_a-z-]*)(.*)', re.DOTALL)
 TOKEN_STRING_RE = re.compile(r'("[^"]*")(.*)', re.DOTALL)
 TOKEN_WHITESPACE_RE = re.compile(r'((\s+|//[^\n\r]*|/\*.*?\*/)+)(.*)', re.DOTALL)
 TOKEN_SYMBOL_RE = re.compile(r'(-Infinity|-|,|;|:|\?|\.\.\.|\.|\(|\)|\[|\]|\{|\}|\<|=|\>)(.*)', re.DOTALL)