From da899192f5d70cee717c1bd43bc0e76a40ef72f0 Mon Sep 17 00:00:00 2001 From: Michael Sippel Date: Mon, 2 Oct 2023 01:30:10 +0200 Subject: [PATCH] lexer: improve whitespace handling --- src/lexer.rs | 5 +++-- src/parser.rs | 2 +- src/test/lexer.rs | 18 +++++++++--------- src/test/parser.rs | 18 +++++++++--------- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 23d6c25..d03b2f5 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -77,9 +77,10 @@ where It: Iterator '>' => { self.chars.next(); return Some(Ok(LadderTypeToken::Close)); }, '~' => { self.chars.next(); return Some(Ok(LadderTypeToken::Ladder)); }, '\'' => { self.chars.next(); state = LexerState::Char(None); }, - ' ' => { self.chars.next(); }, c => { - if c.is_alphabetic() { + if c.is_whitespace() { + self.chars.next(); + } else if c.is_alphabetic() { state = LexerState::Sym( String::new() ); } else if c.is_digit(10) { state = LexerState::Num( 0 ); diff --git a/src/parser.rs b/src/parser.rs index 7c7c1fc..1b996bf 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -121,7 +121,7 @@ impl TypeDict { { match self.parse_partial(tokens) { Ok(t) => { - if let Some(tok) = tokens.peek() { + if let Some(_tok) = tokens.peek() { Err(ParseError::UnexpectedToken) } else { Ok(t) diff --git a/src/test/lexer.rs b/src/test/lexer.rs index 8017262..7cba4db 100644 --- a/src/test/lexer.rs +++ b/src/test/lexer.rs @@ -83,15 +83,15 @@ fn test_lexer_app_space() { #[test] fn test_lexer_large() { let mut lex = LadderTypeLexer::from( - " \ - ~ \ - ~ℕ \ - ~ \ - ~< Seq ~Unicode > > \ - ~ \ - ~ \ - ~UTF-8 \ + " + ~ + ~ℕ + ~ + ~< Seq ~Unicode > > + ~ + ~ + ~UTF-8 ~".chars()); assert_eq!( lex.next(), Some(Ok(LadderTypeToken::Open))); diff --git a/src/test/parser.rs b/src/test/parser.rs index 16c9744..41fd800 100644 --- a/src/test/parser.rs +++ b/src/test/parser.rs @@ -134,15 +134,15 @@ fn test_parser_ladder_between() { fn test_parser_ladder_large() { assert_eq!( TypeTerm::from_str( - " \ - ~ \ - ~ℕ \ - ~ \ - ~< Seq ~Unicode > > \ - ~ \ - ~ \ - ~UTF-8 \ + " + ~ + ~ℕ + ~ + ~< Seq ~Unicode > > + ~ + ~ + ~UTF-8 ~"), Ok(