Skip to main content

ironcalc_base/expressions/lexer/
mod.rs

1//! A tokenizer for spreadsheet formulas.
2//!
3//! This is meant to feed a formula parser.
4//!
5//! You will need to instantiate it with a language and a locale.
6//!
7//! It supports two working modes:
8//!
9//! 1. A1 or display mode
10//!    This is for user formulas. References are like `D4`, `D$4` or `F5:T10`
11//! 2. R1C1, internal or runtime mode
12//!    A reference like R1C1 refers to $A$1 and R3C4 to $D$4
13//!    `R[2]C[5]` refers to a cell two rows below and five columns to the right
14//!    It uses the 'en' locale and language.
15//!    This is used internally at runtime.
16//!
17//! Formulas look different in different locales:
18//!
19//! =IF(A1, B1, NA()) versus =IF(A1; B1; NA())
20//!
21//! Also numbers are different:
22//!
23//! 1,123.45 versus 1.123,45
24//!
25//! The names of the errors and functions are different in different languages,
26//! but they stay the same in different locales.
27//!
28//! Note that in IronCalc if you are using a locale different from 'en' or a language different from 'en'
29//! you will still need the 'en' locale and language because formulas are stored in that language and locale
30//!
31//! # Examples:
32//! ```
33//! use ironcalc_base::expressions::lexer::{Lexer, LexerMode};
34//! use ironcalc_base::expressions::token::{TokenType, OpCompare};
35//! use ironcalc_base::locale::get_locale;
36//! use ironcalc_base::language::get_language;
37//!
38//! let locale = get_locale("en").unwrap();
39//! let language = get_language("en").unwrap();
40//! let mut lexer = Lexer::new("=A1*SUM(Sheet2!C3:D5)", LexerMode::A1, &locale, &language);
41//! assert_eq!(lexer.next_token(), TokenType::Compare(OpCompare::Equal));
42//! assert!(matches!(lexer.next_token(), TokenType::Reference { .. }));
43//! ```
44
45use std::mem;
46
47use serde::{Deserialize, Serialize};
48
49use crate::expressions::token::{OpCompare, OpProduct, OpSum};
50use crate::number_format::parse_finite_number;
51
52use crate::language::Language;
53use crate::locale::Locale;
54
55use super::token::{Error, TokenType};
56use super::types::*;
57use super::utils;
58
59pub mod util;
60
61#[cfg(test)]
62mod test;
63
64mod ranges;
65mod structured_references;
66
67#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
68pub struct LexerError {
69    pub position: usize,
70    pub message: String,
71}
72
73pub(super) type Result<T> = std::result::Result<T, LexerError>;
74
75/// Decides whether `name` should be accepted as an identifier in R1C1 mode, once
76/// the reference-parsing paths have already failed.
77///
78/// This uses the A1 identifier rules (`is_valid_a1_identifier`) rather than
79/// `is_valid_identifier`, so the single-character names "R"/"C" are accepted: they
80/// are valid LAMBDA parameters and LET variables. This matters on reload, because
81/// formulas are stored internally in R1C1 format, so a formula like
82/// `=BYROW(A1:C1, LAMBDA(c, MAX(c)))` is re-parsed in R1C1 mode and must not reject
83/// the `c` parameter.
84///
85/// The one exception is a bare "R"/"C" immediately followed by '[': that is a
86/// malformed R1C1 reference (e.g. `R[`), not an identifier, so it is rejected.
87/// Genuine R1C1 references (`R[n]C[n]`, `RnCn`) are consumed before this fallback.
88fn is_valid_r1c1_identifier(name: &str, next_char: Option<char>) -> bool {
89    if !utils::is_valid_a1_identifier(name) {
90        return false;
91    }
92    // `is_valid_identifier` differs from `is_valid_a1_identifier` only for "R"/"C".
93    // For those, only reject when they start a (malformed) reference, i.e. `R[`.
94    utils::is_valid_identifier(name) || next_char != Some('[')
95}
96
97#[derive(Clone, PartialEq, Eq)]
98pub enum LexerMode {
99    A1,
100    R1C1,
101}
102
103/// Tokenize an input
104#[derive(Clone)]
105pub struct Lexer<'a> {
106    position: usize,
107    next_token_position: Option<usize>,
108    len: usize,
109    chars: Vec<char>,
110    mode: LexerMode,
111    locale: &'a Locale,
112    language: &'a Language,
113}
114
115impl<'a> Lexer<'a> {
116    /// Creates a new `Lexer` that returns the tokens of a formula.
117    pub fn new(
118        formula: &str,
119        mode: LexerMode,
120        locale: &'a Locale,
121        language: &'a Language,
122    ) -> Lexer<'a> {
123        let chars: Vec<char> = formula.chars().collect();
124        let len = chars.len();
125        Lexer {
126            chars,
127            position: 0,
128            next_token_position: None,
129            len,
130            mode,
131            locale,
132            language,
133        }
134    }
135
136    /// Changes the lexer mode
137    pub fn set_lexer_mode(&mut self, mode: LexerMode) {
138        self.mode = mode;
139    }
140
141    /// Sets the locale
142    pub fn set_locale(&mut self, locale: &'a Locale) {
143        self.locale = locale;
144    }
145
146    /// Sets the language
147    pub fn set_language(&mut self, language: &'a Language) {
148        self.language = language;
149    }
150
151    // FIXME: I don't think we should have `is_a1_mode` and   `get_formula`.
152    // The caller already knows those two
153
154    /// Returns true if mode is A1
155    pub fn is_a1_mode(&self) -> bool {
156        self.mode == LexerMode::A1
157    }
158
159    /// Returns the formula
160    pub fn get_formula(&self) -> String {
161        self.chars.iter().collect()
162    }
163
164    // FIXME: This is used to get the "marked tokens"
165    // I think a better API would be to return the marked tokens
166    /// Returns the position of the lexer
167    pub fn get_position(&self) -> i32 {
168        self.position as i32
169    }
170
171    /// Resets the formula
172    pub fn set_formula(&mut self, content: &str) {
173        self.chars = content.chars().collect();
174        self.len = self.chars.len();
175        self.position = 0;
176        self.next_token_position = None;
177    }
178
179    /// Returns an error if the token is not the expected one.
180    pub fn expect(&mut self, tk: TokenType) -> Result<()> {
181        let nt = self.next_token();
182        if mem::discriminant(&nt) != mem::discriminant(&tk) {
183            return Err(self.set_error(&format!("Error, expected {tk:?}"), self.position));
184        }
185        Ok(())
186    }
187
188    /// Checks the next token without advancing position
189    /// See also [advance_token](Self::advance_token)
190    pub fn peek_token(&mut self) -> TokenType {
191        let position = self.position;
192        let tk = self.next_token();
193        self.next_token_position = Some(self.position);
194        self.position = position;
195        tk
196    }
197
198    /// Advances position. This is used in conjunction with [`peek_token`](Self::peek_token)
199    /// It is a noop if the has not been a previous peek_token
200    pub fn advance_token(&mut self) {
201        if let Some(position) = self.next_token_position {
202            self.position = position;
203            self.next_token_position = None;
204        }
205    }
206
207    /// Returns the next token
208    pub fn next_token(&mut self) -> TokenType {
209        self.next_token_position = None;
210        self.consume_whitespace();
211
212        match self.read_next_char() {
213            Some(char) => {
214                match char {
215                    '+' => TokenType::Addition(OpSum::Add),
216                    '-' => TokenType::Addition(OpSum::Minus),
217                    '*' => TokenType::Product(OpProduct::Times),
218                    '/' => TokenType::Product(OpProduct::Divide),
219                    '(' => TokenType::LeftParenthesis,
220                    ')' => TokenType::RightParenthesis,
221                    '=' => TokenType::Compare(OpCompare::Equal),
222                    '{' => TokenType::LeftBrace,
223                    '}' => TokenType::RightBrace,
224                    '[' => TokenType::LeftBracket,
225                    ']' => TokenType::RightBracket,
226                    ':' => TokenType::Colon,
227                    ';' => TokenType::Semicolon,
228                    '@' => TokenType::At,
229                    '\\' => TokenType::Backslash,
230                    ',' => {
231                        if self.locale.numbers.symbols.decimal == "," {
232                            match self.consume_number(',') {
233                                Ok(number) => TokenType::Number(number),
234                                Err(error) => TokenType::Illegal(error),
235                            }
236                        } else {
237                            TokenType::Comma
238                        }
239                    }
240                    '.' => {
241                        if self.locale.numbers.symbols.decimal == "." {
242                            match self.consume_number('.') {
243                                Ok(number) => TokenType::Number(number),
244                                Err(error) => TokenType::Illegal(error),
245                            }
246                        } else {
247                            // There is no TokenType::PERIOD
248                            TokenType::Illegal(self.set_error("Expecting a number", self.position))
249                        }
250                    }
251                    '!' => TokenType::Bang,
252                    '^' => TokenType::Power,
253                    '%' => TokenType::Percent,
254                    '&' => TokenType::And,
255                    '$' => self.consume_absolute_reference(),
256                    '<' => {
257                        let next_token = self.peek_char();
258                        if next_token == Some('=') {
259                            self.position += 1;
260                            TokenType::Compare(OpCompare::LessOrEqualThan)
261                        } else if next_token == Some('>') {
262                            self.position += 1;
263                            TokenType::Compare(OpCompare::NonEqual)
264                        } else {
265                            TokenType::Compare(OpCompare::LessThan)
266                        }
267                    }
268                    '>' => {
269                        if self.peek_char() == Some('=') {
270                            self.position += 1;
271                            TokenType::Compare(OpCompare::GreaterOrEqualThan)
272                        } else {
273                            TokenType::Compare(OpCompare::GreaterThan)
274                        }
275                    }
276                    '#' => self.consume_error(),
277                    '"' => match self.consume_string() {
278                        Ok(s) => TokenType::String(s),
279                        Err(error) => TokenType::Illegal(error),
280                    },
281                    '\'' => self.consume_quoted_sheet_reference(),
282                    '0'..='9' => {
283                        let position = self.position - 1;
284                        match self.consume_number(char) {
285                            Ok(number) => {
286                                if self.peek_token() == TokenType::Colon
287                                    && self.mode == LexerMode::A1
288                                {
289                                    // Its a row range  3:5
290                                    // FIXME: There are faster ways of parsing this
291                                    // Like checking that 'number' is integer and that the next token is integer
292                                    self.position = position;
293                                    match self.consume_range_a1() {
294                                        Ok(ParsedRange { left, right }) => {
295                                            if let Some(right) = right {
296                                                TokenType::Range {
297                                                    sheet: None,
298                                                    left,
299                                                    right,
300                                                }
301                                            } else {
302                                                TokenType::Illegal(
303                                                    self.set_error("Expecting row range", position),
304                                                )
305                                            }
306                                        }
307                                        Err(error) => {
308                                            // Examples:
309                                            //   * 'Sheet 1'!3.4:5
310                                            //   * 'Sheet 1'!3:A2
311                                            //   * 'Sheet 1'!3:
312                                            TokenType::Illegal(error)
313                                        }
314                                    }
315                                } else {
316                                    TokenType::Number(number)
317                                }
318                            }
319                            Err(error) => {
320                                // tried to read a number but failed
321                                self.position = self.len;
322                                TokenType::Illegal(error)
323                            }
324                        }
325                    }
326                    _ => {
327                        if char.is_alphabetic() || char == '_' {
328                            // At this point is one of the following:
329                            //   1. A range with sheet: Sheet3!A3:D7
330                            //   2. A boolean: TRUE or FALSE (dependent on the language)
331                            //   3. A reference like WS34 or R3C5
332                            //   4. A range without sheet ER4:ER7
333                            //   5. A column range E:E
334                            //   6. An identifier like a function name or a defined name
335                            //   7. A range operator A1:OFFSET(...)
336                            //   8. An Invalid token
337                            let position = self.position;
338                            self.position -= 1;
339                            let name = self.consume_identifier();
340                            let position_indent = self.position;
341
342                            let peek_char = self.peek_char();
343                            let next_char_is_colon = self.peek_char() == Some(':');
344
345                            if peek_char == Some('!') {
346                                // reference
347                                self.position += 1;
348                                return self.consume_range(Some(name));
349                            } else if peek_char == Some('$') {
350                                self.position = position - 1;
351                                return self.consume_range(None);
352                            }
353                            let name_upper = name.to_uppercase();
354                            if name_upper == self.language.booleans.r#true {
355                                return TokenType::Boolean(true);
356                            } else if name_upper == self.language.booleans.r#false {
357                                return TokenType::Boolean(false);
358                            }
359                            if self.peek_char() == Some('(') {
360                                return TokenType::Ident(name);
361                            }
362                            if self.mode == LexerMode::A1 {
363                                let parsed_reference = utils::parse_reference_a1(&name_upper);
364                                if parsed_reference.is_some()
365                                    || (utils::is_valid_column(name_upper.trim_start_matches('$'))
366                                        && next_char_is_colon)
367                                {
368                                    self.position = position - 1;
369                                    match self.consume_range_a1() {
370                                        Ok(ParsedRange { left, right }) => {
371                                            if let Some(right) = right {
372                                                return TokenType::Range {
373                                                    sheet: None,
374                                                    left,
375                                                    right,
376                                                };
377                                            } else {
378                                                return TokenType::Reference {
379                                                    sheet: None,
380                                                    column: left.column,
381                                                    row: left.row,
382                                                    absolute_row: left.absolute_row,
383                                                    absolute_column: left.absolute_column,
384                                                };
385                                            }
386                                        }
387                                        Err(error) => {
388                                            // This could be the range operator: ":"
389                                            if let Some(r) = parsed_reference {
390                                                if next_char_is_colon {
391                                                    self.position = position_indent;
392                                                    return TokenType::Reference {
393                                                        sheet: None,
394                                                        row: r.row,
395                                                        column: r.column,
396                                                        absolute_column: r.absolute_column,
397                                                        absolute_row: r.absolute_row,
398                                                    };
399                                                }
400                                            }
401                                            self.position = self.len;
402                                            return TokenType::Illegal(error);
403                                        }
404                                    }
405                                } else if utils::is_valid_a1_identifier(&name) {
406                                    if peek_char == Some('[') {
407                                        if let Ok(r) = self.consume_structured_reference(&name) {
408                                            return r;
409                                        }
410                                        return TokenType::Illegal(self.set_error(
411                                            "Invalid structured reference",
412                                            self.position,
413                                        ));
414                                    }
415                                    return TokenType::Ident(name);
416                                } else {
417                                    return TokenType::Illegal(
418                                        self.set_error("Invalid identifier (A1)", self.position),
419                                    );
420                                }
421                            } else {
422                                let pos = self.position;
423                                self.position = position - 1;
424                                match self.consume_range_r1c1() {
425                                    // it's a valid R1C1 range
426                                    // We need to check it's not something like R1C1P
427                                    Ok(ParsedRange { left, right }) => {
428                                        if pos > self.position {
429                                            self.position = pos;
430                                            if is_valid_r1c1_identifier(&name, self.peek_char()) {
431                                                return TokenType::Ident(name);
432                                            } else {
433                                                self.position = self.len;
434                                                return TokenType::Illegal(
435                                                    self.set_error(
436                                                        "Invalid identifier (R1C1)",
437                                                        pos,
438                                                    ),
439                                                );
440                                            }
441                                        }
442                                        if let Some(right) = right {
443                                            return TokenType::Range {
444                                                sheet: None,
445                                                left,
446                                                right,
447                                            };
448                                        } else {
449                                            return TokenType::Reference {
450                                                sheet: None,
451                                                column: left.column,
452                                                row: left.row,
453                                                absolute_row: left.absolute_row,
454                                                absolute_column: left.absolute_column,
455                                            };
456                                        }
457                                    }
458                                    Err(error) => {
459                                        self.position = position - 1;
460                                        if let Ok(r) = self.consume_reference_r1c1() {
461                                            if self.peek_char() == Some(':') {
462                                                return TokenType::Reference {
463                                                    sheet: None,
464                                                    row: r.row,
465                                                    column: r.column,
466                                                    absolute_column: r.absolute_column,
467                                                    absolute_row: r.absolute_row,
468                                                };
469                                            }
470                                        }
471                                        self.position = pos;
472
473                                        if is_valid_r1c1_identifier(&name, self.peek_char()) {
474                                            return TokenType::Ident(name);
475                                        } else {
476                                            return TokenType::Illegal(self.set_error(
477                                                &format!("Invalid identifier (R1C1): {name}"),
478                                                error.position,
479                                            ));
480                                        }
481                                    }
482                                }
483                            }
484                        }
485                        TokenType::Illegal(self.set_error("Unknown error", self.position))
486                    }
487                }
488            }
489            None => TokenType::EOF,
490        }
491    }
492
493    // Private methods
494
495    fn set_error(&mut self, message: &str, position: usize) -> LexerError {
496        self.position = self.len;
497        LexerError {
498            position,
499            message: message.to_string(),
500        }
501    }
502
503    fn peek_char(&mut self) -> Option<char> {
504        let position = self.position;
505        if position < self.len {
506            Some(self.chars[position])
507        } else {
508            None
509        }
510    }
511
512    fn expect_char(&mut self, ch_expected: char) -> Result<()> {
513        let position = self.position;
514        if position >= self.len {
515            return Err(self.set_error(
516                &format!("Error, expected {} found EOF", ch_expected),
517                self.position,
518            ));
519        } else {
520            let ch = self.chars[position];
521            if ch_expected != ch {
522                return Err(self.set_error(
523                    &format!("Error, expected {} found {}", ch_expected, ch),
524                    self.position,
525                ));
526            }
527            self.position += 1;
528        }
529        Ok(())
530    }
531
532    fn read_next_char(&mut self) -> Option<char> {
533        let position = self.position;
534        if position < self.len {
535            self.position = position + 1;
536            Some(self.chars[position])
537        } else {
538            None
539        }
540    }
541
542    // Consumes an integer from the input stream
543    fn consume_integer(&mut self, first: char) -> Result<i32> {
544        let mut position = self.position;
545        let len = self.len;
546        let mut chars = first.to_string();
547        while position < len {
548            let next_char = self.chars[position];
549            if next_char.is_ascii_digit() {
550                chars.push(next_char);
551            } else {
552                break;
553            }
554            position += 1;
555        }
556        self.position = position;
557        chars.parse::<i32>().map_err(|_| LexerError {
558            position,
559            message: format!("Failed to parse to int: {chars}"),
560        })
561    }
562
563    // Consumes a number in the current locale.
564    // It only takes into account the decimal separator
565    // Note that we do not parse the thousands separator
566    // Let's say ',' is the thousands separator. Then 1,234 would be an error.
567    // This is ok for most cases:
568    // =IF(A1=1,234, TRUE, FALSE) will not work
569    // If a user introduces a single number in the cell 1,234 we should be able to parse
570    // and format the cell appropriately
571    fn consume_number(&mut self, first: char) -> Result<f64> {
572        let mut position = self.position;
573        let len = self.len;
574        let mut chars = first.to_string();
575        // numbers before the decimal point
576        while position < len {
577            let x = self.chars[position];
578            if x.is_ascii_digit() {
579                chars.push(x);
580            } else {
581                break;
582            }
583            position += 1;
584        }
585        if position < len && self.chars[position].to_string() == self.locale.numbers.symbols.decimal
586        {
587            // numbers after the decimal point
588            chars.push('.');
589            position += 1;
590            while position < len {
591                let x = self.chars[position];
592                if x.is_ascii_digit() {
593                    chars.push(x);
594                } else {
595                    break;
596                }
597                position += 1;
598            }
599        }
600        if position + 1 < len && (self.chars[position] == 'e' || self.chars[position] == 'E') {
601            // exponential side
602            let x = self.chars[position + 1];
603            if x == '-' || x == '+' || x.is_ascii_digit() {
604                chars.push('e');
605                chars.push(x);
606                position += 2;
607                while position < len {
608                    let x = self.chars[position];
609                    if x.is_ascii_digit() {
610                        chars.push(x);
611                    } else {
612                        break;
613                    }
614                    position += 1;
615                }
616            }
617        }
618        self.position = position;
619        // A number too large to hold, like 1e400, is not a number: it would
620        // be infinity, which cannot be written back as a formula.
621        match parse_finite_number(&chars) {
622            None => Err(self.set_error(&format!("Failed to parse to double: {chars}"), position)),
623            Some(v) => Ok(v),
624        }
625    }
626
627    // Consumes an identifier from the input stream
628    fn consume_identifier(&mut self) -> String {
629        let mut position = self.position;
630        while position < self.len {
631            let next_char = self.chars[position];
632            if next_char.is_alphanumeric() || next_char == '_' || next_char == '.' {
633                position += 1;
634            } else {
635                break;
636            }
637        }
638        let chars = self.chars[self.position..position].iter().collect();
639        self.position = position;
640        chars
641    }
642
643    fn consume_string(&mut self) -> Result<String> {
644        let mut position = self.position;
645        let len = self.len;
646        let mut chars = "".to_string();
647        let mut terminated = false;
648        while position < len {
649            let x = self.chars[position];
650            position += 1;
651            if x != '"' {
652                chars.push(x);
653            } else if position < len && self.chars[position] == '"' {
654                chars.push(x);
655                chars.push(self.chars[position]);
656                position += 1;
657            } else {
658                terminated = true;
659                break;
660            }
661        }
662        self.position = position;
663
664        if !terminated {
665            return Err(self.set_error("Expected closing '\"' but found end of input", position));
666        }
667        Ok(chars)
668    }
669
670    // Consumes a quoted string from input
671    // 'This is a quoted string'
672    // ' Also is a ''quoted'' string'
673    // Returns an error if it does not find a closing quote
674    fn consume_single_quote_string(&mut self) -> Result<String> {
675        let mut position = self.position;
676        let len = self.len;
677        let mut success = false;
678        let mut needs_escape = false;
679        while position < len {
680            let next_char = self.chars[position];
681            position += 1;
682            if next_char == '\'' {
683                if position == len {
684                    success = true;
685                    break;
686                }
687                if self.chars[position] != '\'' {
688                    success = true;
689                    break;
690                } else {
691                    // In Excel we escape "'" with "''"
692                    needs_escape = true;
693                    position += 1;
694                }
695            }
696        }
697        if !success {
698            // We reached the end without the closing quote
699            return Err(self.set_error("Expected closing \"'\" but found end of input", position));
700        }
701        let chars: String = self.chars[self.position..position - 1].iter().collect();
702        self.position = position;
703        if needs_escape {
704            // In most cases we will not needs escaping so this would be an overkill
705            return Ok(chars.replace("''", "'"));
706        }
707
708        Ok(chars)
709    }
710
711    // Reads an error from the input stream
712    fn consume_error(&mut self) -> TokenType {
713        let errors = &self.language.errors;
714        let rest_of_formula: String = self.chars[self.position - 1..self.len].iter().collect();
715        if rest_of_formula.starts_with(&errors.r#ref) {
716            self.position += errors.r#ref.chars().count() - 1;
717            return TokenType::Error(Error::REF);
718        } else if rest_of_formula.starts_with(&errors.name) {
719            self.position += errors.name.chars().count() - 1;
720            return TokenType::Error(Error::NAME);
721        } else if rest_of_formula.starts_with(&errors.value) {
722            self.position += errors.value.chars().count() - 1;
723            return TokenType::Error(Error::VALUE);
724        } else if rest_of_formula.starts_with(&errors.div) {
725            self.position += errors.div.chars().count() - 1;
726            return TokenType::Error(Error::DIV);
727        } else if rest_of_formula.starts_with(&errors.na) {
728            self.position += errors.na.chars().count() - 1;
729            return TokenType::Error(Error::NA);
730        } else if rest_of_formula.starts_with(&errors.num) {
731            self.position += errors.num.chars().count() - 1;
732            return TokenType::Error(Error::NUM);
733        } else if rest_of_formula.starts_with(&errors.error) {
734            self.position += errors.error.chars().count() - 1;
735            return TokenType::Error(Error::ERROR);
736        } else if rest_of_formula.starts_with(&errors.nimpl) {
737            self.position += errors.nimpl.chars().count() - 1;
738            return TokenType::Error(Error::NIMPL);
739        } else if rest_of_formula.starts_with(&errors.spill) {
740            self.position += errors.spill.chars().count() - 1;
741            return TokenType::Error(Error::SPILL);
742        } else if rest_of_formula.starts_with(&errors.calc) {
743            self.position += errors.calc.chars().count() - 1;
744            return TokenType::Error(Error::CALC);
745        } else if rest_of_formula.starts_with(&errors.null) {
746            self.position += errors.null.chars().count() - 1;
747            return TokenType::Error(Error::NULL);
748        } else if rest_of_formula.starts_with(&errors.circ) {
749            self.position += errors.circ.chars().count() - 1;
750            return TokenType::Error(Error::CIRC);
751        }
752        // If it is not an error it _might_ be a spill operator.
753        TokenType::Spill
754    }
755
756    fn consume_whitespace(&mut self) {
757        let mut position = self.position;
758        let len = self.len;
759        while position < len {
760            let x = self.chars[position];
761            if !x.is_whitespace() {
762                break;
763            }
764            position += 1;
765        }
766        self.position = position;
767    }
768
769    fn consume_absolute_reference(&mut self) -> TokenType {
770        // This is an absolute reference.
771        // $A$4
772        if self.mode == LexerMode::R1C1 {
773            return TokenType::Illegal(
774                self.set_error("Cannot parse A1 reference in R1C1 mode", self.position),
775            );
776        }
777        self.position -= 1;
778        self.consume_range(None)
779    }
780
781    fn consume_quoted_sheet_reference(&mut self) -> TokenType {
782        // This is a reference:
783        // 'First Sheet'!A34
784        let sheet_name = match self.consume_single_quote_string() {
785            Ok(v) => v,
786            Err(error) => {
787                return TokenType::Illegal(error);
788            }
789        };
790        if self.next_token() != TokenType::Bang {
791            return TokenType::Illegal(self.set_error("Expected '!'", self.position));
792        }
793        self.consume_range(Some(sheet_name))
794    }
795
796    fn consume_range(&mut self, sheet: Option<String>) -> TokenType {
797        let m = if self.mode == LexerMode::A1 {
798            self.consume_range_a1()
799        } else {
800            self.consume_range_r1c1()
801        };
802        match m {
803            Ok(ParsedRange { left, right }) => {
804                if let Some(right) = right {
805                    TokenType::Range { sheet, left, right }
806                } else {
807                    TokenType::Reference {
808                        sheet,
809                        column: left.column,
810                        row: left.row,
811                        absolute_row: left.absolute_row,
812                        absolute_column: left.absolute_column,
813                    }
814                }
815            }
816            Err(error) => TokenType::Illegal(error),
817        }
818    }
819}