//! Definition of a lexer for the WebAssembly text format.

//!

//! This module provides a [`Lexer`][] type which is an iterate over the raw

//! tokens of a WebAssembly text file. A [`Lexer`][] accounts for every single

//! byte in a WebAssembly text field, returning tokens even for comments and

//! whitespace. Typically you'll ignore comments and whitespace, however.

//!

//! If you'd like to iterate over the tokens in a file you can do so via:

//!

//! ```

//! # fn foo() -> Result<(), wast::Error> {

//! use wast::lexer::Lexer;

//!

//! let wat = "(module (func $foo))";

//! for token in Lexer::new(wat).iter(0) {

//!     println!("{:?}", token?);

//! }

//! # Ok(())

//! # }

//! ```

//!

//! Note that you'll typically not use this module but will rather use

//! [`ParseBuffer`](crate::parser::ParseBuffer) instead.

//!

//! [`Lexer`]: crate::lexer::Lexer

use crate::token::Span;

use crate::Error;

use std::borrow::Cow;

use std::char;

use std::fmt;

use std::slice;

use std::str;

use std::str::Utf8Error;

/// A structure used to lex the s-expression syntax of WAT files.

///

/// This structure is used to generate [`Token`] items, which should account for

/// every single byte of the input as we iterate over it. A [`LexError`] is

/// returned for any non-lexable text.

#[derive(Clone)]

pub struct Lexer<'a> {

    input: &'a str,

    allow_confusing_unicode: bool,

}

/// A single token parsed from a `Lexer`.

#[derive(Copy, Clone, Debug, PartialEq)]

pub struct Token {

    /// The kind of token this represents, such as whether it's whitespace, a

    /// keyword, etc.

    pub kind: TokenKind,

    /// The byte offset within the original source for where this token came

    /// from.

    pub offset: usize,

    /// The byte length of this token as it resides in the original source.

    //

    // NB: this is `u32` to enable packing `Token` into two pointers of size.

    // This does limit a single token to being at most 4G large, but that seems

    // probably ok.

    pub len: u32,

}

#[test]

fn token_is_not_too_big() {

    assert!(std::mem::size_of::<Token>() <= std::mem::size_of::<u64>() * 2);

}

/// Classification of what was parsed from the input stream.

///

/// This enumeration contains all kinds of fragments, including comments and

/// whitespace.

#[derive(Copy, Clone, Debug, PartialEq)]

pub enum TokenKind {

    /// A line comment, preceded with `;;`

    LineComment,

    /// A block comment, surrounded by `(;` and `;)`. Note that these can be

    /// nested.

    BlockComment,

    /// A fragment of source that represents whitespace.

    Whitespace,

    /// A left-parenthesis, including the source text for where it comes from.

    LParen,

    /// A right-parenthesis, including the source text for where it comes from.

    RParen,

    /// A string literal, which is actually a list of bytes.

    String,

    /// An identifier (like `$foo`).

    ///

    /// All identifiers start with `$` and the payload here is the original

    /// source text.

    Id,

    /// A keyword, or something that starts with an alphabetic character.

    ///

    /// The payload here is the original source text.

    Keyword,

    /// An annotation (like `@foo`).

    ///

    /// All annotations start with `@` and the payload will be the name of the

    /// annotation.

    Annotation,

    /// A reserved series of `idchar` symbols. Unknown what this is meant to be

    /// used for, you'll probably generate an error about an unexpected token.

    Reserved,

    /// An integer.

    Integer(IntegerKind),

    /// A float.

    Float(FloatKind),

}

/// Description of the parsed integer from the source.

#[derive(Copy, Clone, Debug, PartialEq)]

pub struct IntegerKind {

    sign: Option<SignToken>,

    has_underscores: bool,

    hex: bool,

}

/// Description of a parsed float from the source.

#[allow(missing_docs)]

#[derive(Copy, Clone, Debug, PartialEq)]

pub enum FloatKind {

    #[doc(hidden)]

    Inf { negative: bool },

    #[doc(hidden)]

    Nan { negative: bool },

    #[doc(hidden)]

    NanVal {

        negative: bool,

        has_underscores: bool,

    },

    #[doc(hidden)]

    Normal { has_underscores: bool, hex: bool },

}

enum ReservedKind {

    /// "..."

    String,

    /// anything that's just a sequence of `idchars!()`

    Idchars,

    /// $"..."

    IdString,

    /// @"..."

    AnnotationString,

    /// everything else (a conglomeration of strings, idchars, etc)

    Reserved,

}

/// Errors that can be generated while lexing.

///

/// All lexing errors have line/colum/position information as well as a

/// `LexError` indicating what kind of error happened while lexing.

#[derive(Debug, Clone, PartialEq, Eq)]

#[non_exhaustive]

pub enum LexError {

    /// A dangling block comment was found with an unbalanced `(;` which was

    /// never terminated in the file.

    DanglingBlockComment,

    /// An unexpected character was encountered when generally parsing and

    /// looking for something else.

    Unexpected(char),

    /// An invalid `char` in a string literal was found.

    InvalidStringElement(char),

    /// An invalid string escape letter was found (the thing after the `\` in

    /// string literals)

    InvalidStringEscape(char),

    /// An invalid hexadecimal digit was found.

    InvalidHexDigit(char),

    /// An invalid base-10 digit was found.

    InvalidDigit(char),

    /// Parsing expected `wanted` but ended up finding `found` instead where the

    /// two characters aren't the same.

    Expected {

        /// The character that was expected to be found

        wanted: char,

        /// The character that was actually found

        found: char,

    },

    /// We needed to parse more but EOF (or end of the string) was encountered.

    UnexpectedEof,

    /// A number failed to parse because it was too big to fit within the target

    /// type.

    NumberTooBig,

    /// An invalid unicode value was found in a `\u{...}` escape in a string,

    /// only valid unicode scalars can be escaped that way.

    InvalidUnicodeValue(u32),

    /// A lone underscore was found when parsing a number, since underscores

    /// should always be preceded and succeeded with a digit of some form.

    LoneUnderscore,

    /// A "confusing" unicode character is present in a comment or a string

    /// literal, such as a character that changes the direction text is

    /// typically displayed in editors. This could cause the human-read

    /// version to behave differently than the compiler-visible version, so

    /// these are simply rejected for now.

    ConfusingUnicode(char),

    /// An invalid utf-8 sequence was found in a quoted identifier, such as

    /// `$"\ff"`.

    InvalidUtf8Id(Utf8Error),

    /// An empty identifier was found, or a lone `$`.

    EmptyId,

    /// An empty identifier was found, or a lone `@`.

    EmptyAnnotation,

}

/// A sign token for an integer.

#[derive(Clone, Copy, Debug, PartialEq, Eq)]

pub enum SignToken {

    /// Plus sign: "+",

    Plus,

    /// Minus sign: "-",

    Minus,

}

/// A fully parsed integer from a source string with a payload ready to parse

/// into an integral type.

#[derive(Debug, PartialEq)]

pub struct Integer<'a> {

    sign: Option<SignToken>,

    val: Cow<'a, str>,

    hex: bool,

}

/// Possible parsed float values

#[derive(Debug, PartialEq, Eq)]

pub enum Float<'a> {

    /// A float `NaN` representation

    Nan {

        /// The specific bits to encode for this float, optionally

        val: Option<Cow<'a, str>>,

        /// Whether or not this is a negative `NaN` or not.

        negative: bool,

    },

    /// An float infinite representation,

    Inf {

        #[allow(missing_docs)]

        negative: bool,

    },

    /// A parsed and separated floating point value

    Val {

        /// Whether or not the `integral` and `fractional` are specified in hex

        hex: bool,

        /// The float parts before the `.`

        integral: Cow<'a, str>,

        /// The float parts after the `.`

        fractional: Option<Cow<'a, str>>,

        /// The exponent to multiple this `integral.fractional` portion of the

        /// float by. If `hex` is true this is `2^exponent` and otherwise it's

        /// `10^exponent`

        exponent: Option<Cow<'a, str>>,

    },

}

// https://webassembly.github.io/spec/core/text/values.html#text-idchar

macro_rules! idchars {

    () => {

        b'0'..=b'9'

        | b'A'..=b'Z'

        | b'a'..=b'z'

        | b'!'

        | b'#'

        | b'$'

        | b'%'

        | b'&'

        | b'\''

        | b'*'

        | b'+'

        | b'-'

        | b'.'

        | b'/'

        | b':'

        | b'<'

        | b'='

        | b'>'

        | b'?'

        | b'@'

        | b'\\'

        | b'^'

        | b'_'

        | b'`'

        | b'|'

        | b'~'

    }

}

impl<'a> Lexer<'a> {

    /// Creates a new lexer which will lex the `input` source string.

    pub fn new(input: &str) -> Lexer<'_> {

        Lexer {

            input,

            allow_confusing_unicode: false,

        }

    }

    /// Returns the original source input that we're lexing.

    pub fn input(&self) -> &'a str {

        self.input

    }

    /// Configures whether "confusing" unicode characters are allowed while

    /// lexing.

    ///

    /// If allowed then no error will happen if these characters are found, but

    /// otherwise if disallowed a lex error will be produced when these

    /// characters are found. Confusing characters are denied by default.

    ///

    /// For now "confusing characters" are primarily related to the "trojan

    /// source" problem where it refers to characters which cause humans to read

    /// text differently than this lexer, such as characters that alter the

    /// left-to-right display of the source code.

    pub fn allow_confusing_unicode(&mut self, allow: bool) -> &mut Self {

        self.allow_confusing_unicode = allow;

        self

    }

    /// Lexes the next at the byte position `pos` in the input.

    ///

    /// Returns `Some` if a token is found or `None` if we're at EOF.

    ///

    /// The `pos` argument will be updated to point to the next token on a

    /// successful parse.

    ///

    /// # Errors

    ///

    /// Returns an error if the input is malformed.

    pub fn parse(&self, pos: &mut usize) -> Result<Option<Token>, Error> {

        let offset = *pos;

        Ok(match self.parse_kind(pos)
?0
 {

            Some(kind) => Some(Token {

                kind,

                offset,

                len: (*pos - offset).try_into().unwrap(),

            }),

            None => None,

        })

    }

    fn parse_kind(&self, pos: &mut usize) -> Result<Option<TokenKind>, Error> {

        let start = *pos;

        // This `match` generally parses the grammar specified at

        //

        // https://webassembly.github.io/spec/core/text/lexical.html#text-token

        let remaining = &self.input.as_bytes()[start..];

        let 
byte157k
 = match remaining.first() {

            Some(b) => b,

            None => return Ok(None),

        };

        match byte {

            // Open-parens check the next character to see if this is the start

            // of a block comment, otherwise it's just a bland left-paren

            // token.

            b'(' => match remaining.get(1) {

                Some(b';') => {

                    let mut level = 1;

                    // Note that we're doing a byte-level search here for the

                    // close-delimiter of `;)`. The actual source text is utf-8

                    // encode in `remaining` but due to how utf-8 works we

                    // can safely search for an ASCII byte since it'll never

                    // otherwise appear in the middle of a codepoint and if we

                    // find it then it's guaranteed to be the right byte.

                    //

                    // Mainly we're avoiding the overhead of decoding utf-8

                    // characters into a Rust `char` since it's otherwise

                    // unnecessary work.

                    let mut iter = remaining[2..].iter();

                    while let Some(ch) = iter.next() {

                        match ch {

                            b'(' => {

                                if let Some(b';') = iter.as_slice().first() {

                                    level += 1;

                                    iter.next();

                                }

                            }

                            b';' => {

                                if let Some(b')') = iter.as_slice().first() {

                                    level -= 1;

                                    iter.next();

                                    if level == 0 {

                                        let len = remaining.len() - iter.as_slice().len();

                                        let comment = &self.input[start..][..len];

                                        *pos += len;

                                        self.check_confusing_comment(*pos, comment)?;

                                        return Ok(Some(TokenKind::BlockComment));

                                    }

                                }

                            }

                            _ => {}

                        }

                    }

                    Err(self.error(start, LexError::DanglingBlockComment))

                }

                _ => {

                    *pos += 1;

                    Ok(Some(TokenKind::LParen))

                }

            },

            b')' => {

                *pos += 1;

                Ok(Some(TokenKind::RParen))

            }

            // https://webassembly.github.io/spec/core/text/lexical.html#white-space

            b' ' | b'\n' | b'\r' | b'\t' => {

                self.skip_ws(pos);

                Ok(Some(TokenKind::Whitespace))

            }

            c @ (
idchars!()44.4k
 | b'"') => {

                let (kind, src) = self.parse_reserved(pos)
?0
;

                match kind {

                    // If the reserved token was simply a single string then

                    // that is converted to a standalone string token

                    ReservedKind::String => return Ok(Some(TokenKind::String)),

                    // If only idchars were consumed then this could be a

                    // specific kind of standalone token we're interested in.

                    ReservedKind::Idchars => {

                        // https://webassembly.github.io/spec/core/text/values.html#integers

                        if let Some(
ret5.72k
) = self.classify_number(src) {

                            return Ok(Some(ret));

                        // https://webassembly.github.io/spec/core/text/values.html#text-id

                        } else if *c == b'$' {

                            return Ok(Some(TokenKind::Id));

                        // part of the WebAssembly/annotations proposal

                        // (no online url yet)

                        } else if *c == b'@' {

                            return Ok(Some(TokenKind::Annotation));

                        // https://webassembly.github.io/spec/core/text/lexical.html#text-keyword

                        } else if b'a' <= *c && *c <= b'z' {

                            return Ok(Some(TokenKind::Keyword));

                        }

                    }

                    ReservedKind::IdString => return Ok(Some(TokenKind::Id)),

                    ReservedKind::AnnotationString => return Ok(Some(TokenKind::Annotation)),

                    // ... otherwise this was a conglomeration of idchars,

                    // strings, or just idchars that don't match a prior rule,

                    // meaning this falls through to the fallback `Reserved`

                    // token.

                    ReservedKind::Reserved => {}

                }

                Ok(Some(TokenKind::Reserved))

            }

            // This could be a line comment, otherwise `;` is a reserved token.

            // The second byte is checked to see if it's a `;;` line comment

            //

            // Note that this character being considered as part of a

            // `reserved` token is part of the annotations proposal.

            b';' => match remaining.get(1) {

                Some(b';') => {

                    let remaining = &self.input[*pos..];

                    let byte_pos = memchr::memchr2(b'\n', b'\r', remaining.as_bytes())

                        .unwrap_or(remaining.len());

                    *pos += byte_pos;

                    let comment = &remaining[..byte_pos];

                    self.check_confusing_comment(*pos, comment)
?0
;

                    Ok(Some(TokenKind::LineComment))

                }

                _ => {

                    *pos += 1;

                    Ok(Some(TokenKind::Reserved))

                }

            },

            // Other known reserved tokens other than `;`

            //

            // Note that these characters being considered as part of a

            // `reserved` token is part of the annotations proposal.

            b',' | b'[' | b']' | b'{' | b'}' => {

                *pos += 1;

                Ok(Some(TokenKind::Reserved))

            }

            _ => {

                let ch = self.input[start..].chars().next().unwrap();

                Err(self.error(*pos, LexError::Unexpected(ch)))

            }

        }

    }

    fn skip_ws(&self, pos: &mut usize) {

        // This table is a byte lookup table to determine whether a byte is a

        // whitespace byte. There are only 4 whitespace bytes for the `*.wat`

        // format right now which are ' ', '\t', '\r', and '\n'. These 4 bytes

        // have a '1' in the table below.

        //

        // Due to how utf-8 works (our input is guaranteed to be utf-8) it is

        // known that if these bytes are found they're guaranteed to be the

        // whitespace byte, so they can be safely skipped and we don't have to

        // do full utf-8 decoding. This means that the goal of this function is

        // to find the first non-whitespace byte in `remaining`.

        //

        // For now this lookup table seems to be the fastest, but projects like

        // https://github.com/lemire/despacer show other simd algorithms which

        // can possibly accelerate this even more. Note that `*.wat` files often

        // have a lot of whitespace so this function is typically quite hot when

        // parsing inputs.

        #[rustfmt::skip]

        const WS: [u8; 256] = [

            //                                   \t \n       \r

            /* 0x00 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0,

            /* 0x10 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            //        ' '

            /* 0x20 */ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0x30 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0x40 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0x50 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0x60 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0x70 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0x80 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0x90 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0xa0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0xb0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0xc0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0xd0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0xe0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

            /* 0xf0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

        ];

        let remaining = &self.input[*pos..];

        let non_ws_pos = remaining

            .as_bytes()

            .iter()

            .position(|b| WS[*b as usize] != 1)

            .unwrap_or(remaining.len());

        *pos += non_ws_pos;

    }

    /// Splits off a "reserved" token which is then further processed later on

    /// to figure out which kind of token it is `depending on `ReservedKind`.

    ///

    /// For more information on this method see the clarification at

    /// <https://github.com/WebAssembly/spec/pull/1499> but the general gist is

    /// that this is parsing the grammar:

    ///

    /// ```text

    /// reserved := (idchar | string)+

    /// ```

    ///

    /// which means that it is eating any number of adjacent string/idchar

    /// tokens (e.g. `a"b"c`) and returning the classification of what was

    /// eaten. The classification assists in determining what the actual token

    /// here eaten looks like.

    fn parse_reserved(&self, pos: &mut usize) -> Result<(ReservedKind, &'a str), Error> {

        let mut idchars = 0u32;

        let mut strings = 0u32;

        let start = *pos;

        while let Some(byte) = self.input.as_bytes().get(*pos) {

            match byte {

                // Normal `idchars` production which appends to the reserved

                // token that's being produced.

                idchars!() => {

                    idchars += 1;

                    *pos += 1;

                }

                // https://webassembly.github.io/spec/core/text/values.html#text-string

                b'"' => {

                    strings += 1;

                    *pos += 1;

                    let mut it = self.input[*pos..].chars();

                    let result = Lexer::parse_str(&mut it, self.allow_confusing_unicode);

                    *pos = self.input.len() - it.as_str().len();

                    match result {

                        Ok(_) => {}

                        Err(e) => {

                            let err_pos = match &e {

                                LexError::UnexpectedEof => self.input.len(),

                                _ => self.input[..*pos].char_indices().next_back().unwrap().0,

                            };

                            return Err(self.error(err_pos, e));

                        }

                    }

                }

                // Nothing else is considered part of a reserved token

                _ => break,

            }

        }

        let ret = &self.input[start..*pos];

        Ok(match (idchars, strings) {

            (0, 0) => unreachable!(),

            (0, 1) => (ReservedKind::String, ret),

            (_, 0) => (ReservedKind::Idchars, ret),

            // Pattern match `@"..."` and `$"..."` for string-based

            // identifiers and annotations.

            (1, 1) if ret.starts_with("$") => (ReservedKind::IdString, ret),

            (1, 1) if ret.starts_with("@") => (ReservedKind::AnnotationString, ret),

            _ => (ReservedKind::Reserved, ret),

        })

    }

    fn classify_number(&self, src: &str) -> Option<TokenKind> {

        let (sign, num) = if let Some(
stripped0
) = src.strip_prefix('+') {

            (Some(SignToken::Plus), stripped)

        } else if let Some(
stripped3
) = src.strip_prefix('-') {

            (Some(SignToken::Minus), stripped)

        } else {

            (None, src)

        };

        let negative = sign == Some(SignToken::Minus);

        // Handle `inf` and `nan` which are special numbers here

        if num == "inf" {

            return Some(TokenKind::Float(FloatKind::Inf { negative }));

        } else if num == "nan" {

            return Some(TokenKind::Float(FloatKind::Nan { negative }));

        } else if let Some(
stripped0
) = num.strip_prefix("nan:0x") {

            let mut it = stripped.as_bytes().iter();

            let has_underscores = skip_underscores(&mut it, |x| char::from(x).is_ascii_hexdigit())?;

            if it.next().is_some() {

                return None;

            }

            return Some(TokenKind::Float(FloatKind::NanVal {

                negative,

                has_underscores,

            }));

        }

        // Figure out if we're a hex number or not

        let test_valid: fn(u8) -> bool;

        let (mut it, hex) = if let Some(
stripped15
) = num.strip_prefix("0x") {

            
test_valid = 15
|x: u8| char::from(x).is_ascii_hexdigit();

            (stripped.as_bytes().iter(), true)

        } else {

            
test_valid = 50.3k
|x: u8| char::from(x).is_ascii_digit();

            (num.as_bytes().iter(), false)

        };

        // Evaluate the first part, moving out all underscores

        let 
mut has_underscores5.72k
 = skip_underscores(&mut it, test_valid)
?44.6k
;

        match it.clone().next() {

            // If we're followed by something this may be a float so keep going.

            Some(_) => {}

            // Otherwise this is a valid integer literal!

            None => {

                return Some(TokenKind::Integer(IntegerKind {

                    has_underscores,

                    sign,

                    hex,

                }))

            }

        }

        // A number can optionally be after the dot so only actually try to

        // parse one if it's there.

        if it.clone().next() == Some(&b'.') {

            it.next();

            match it.clone().next() {

                Some(c) if test_valid(*c) => {

                    if skip_underscores(&mut it, test_valid)
?0
 {

                        has_underscores = true;

                    }

                }

                Some(_) | None => {}

            }

        };

        // Figure out if there's an exponential part here to make a float, and

        // if so parse it but defer its actual calculation until later.

        match (hex, it.next()) {

            (true, Some(b'p')) | (true, Some(b'P')) | (false, Some(b'e')) | (false, Some(b'E')) => {

                match it.clone().next() {

                    Some(b'-') => {

                        it.next();

                    }

                    Some(b'+') => {

                        it.next();

                    }

                    _ => {}

                }

                if skip_underscores(&mut it, |x| char::from(x).is_ascii_digit())? {

                    has_underscores = true;

                }

            }

            (_, None) => {}

            _ => return None,

        }

        // We should have eaten everything by now, if not then this is surely

        // not a float or integer literal.

        if it.next().is_some() {

            return None;

        }

        return Some(TokenKind::Float(FloatKind::Normal {

            has_underscores,

            hex,

        }));

        fn skip_underscores(it: &mut slice::Iter<'_, u8>, good: fn(u8) -> bool) -> Option<bool> {

            let mut last_underscore = false;

            let mut has_underscores = false;

            let first = *it.next()
?0
;

            if !good(first) {

                return None;

            }

            while let Some(
c2.66k
) = it.clone().next() {

                if *c == b'_' && 
!last_underscore9
 {

                    has_underscores = true;

                    it.next();

                    last_underscore = true;

                    continue;

                }

                if !good(*c) {

                    break;

                }

                last_underscore = false;

                it.next();

            }

            if last_underscore {

                return None;

            }

            Some(has_underscores)

        }

    }

    /// Verifies that `comment`, which is about to be returned, has a "confusing

    /// unicode character" in it and should instead be transformed into an

    /// error.

    fn check_confusing_comment(&self, end: usize, comment: &str) -> Result<(), Error> {

        if self.allow_confusing_unicode {

            return Ok(());

        }

        // In an effort to avoid utf-8 decoding the entire `comment` the search

        // here is a bit more optimized. This checks for the `0xe2` byte because

        // in the utf-8 encoding that's the leading encoding byte for all

        // "confusing characters". Each instance of 0xe2 is checked to see if it

        // starts a confusing character, and if so that's returned.

        //

        // Also note that 0xe2 will never be found in the middle of a codepoint,

        // it's always the start of a codepoint. This means that if our special

        // characters show up they're guaranteed to start with 0xe2 bytes.

        let bytes = comment.as_bytes();

        for 
pos0
 in memchr::Memchr::new(0xe2, bytes) {

            if let Some(c) = comment[pos..].chars().next() {

                if is_confusing_unicode(c) {

                    // Note that `self.cur()` accounts for already having

                    // parsed `comment`, so we move backwards to where

                    // `comment` started and then add the index within

                    // `comment`.

                    let pos = end - comment.len() + pos;

                    return Err(self.error(pos, LexError::ConfusingUnicode(c)));

                }

            }

        }

        Ok(())

    }

    fn parse_str(

        it: &mut str::Chars<'a>,

        allow_confusing_unicode: bool,

    ) -> Result<Cow<'a, [u8]>, LexError> {

        enum State {

            Start,

            String(Vec<u8>),

        }

        let orig = it.as_str();

        let mut state = State::Start;

        loop {

            match it.next().ok_or(LexError::UnexpectedEof)
?0
 {

                '"' => break,

                '\\' => {

                    match state {

                        State::String(_) => {}

                        State::Start => {

                            let pos = orig.len() - it.as_str().len() - 1;

                            state = State::String(orig[..pos].as_bytes().to_vec());

                        }

                    }

                    let buf = match &mut state {

                        State::String(b) => b,

                        State::Start => unreachable!(),

                    };

                    match it.next().ok_or(LexError::UnexpectedEof)
?0
 {

                        '"' => buf.push(b'"'),

                        '\'' => buf.push(b'\''),

                        't' => buf.push(b'\t'),

                        'n' => buf.push(b'\n'),

                        'r' => buf.push(b'\r'),

                        '\\' => buf.push(b'\\'),

                        'u' => {

                            Lexer::must_eat_char(it, '{')?;

                            let n = Lexer::hexnum(it)?;

                            let c = char::from_u32(n).ok_or(LexError::InvalidUnicodeValue(n))?;

                            buf.extend(c.encode_utf8(&mut [0; 4]).as_bytes());

                            Lexer::must_eat_char(it, '}')?;

                        }

                        c1 if c1.is_ascii_hexdigit() => {

                            let c2 = Lexer::hexdigit(it)
?0
;

                            buf.push(to_hex(c1) * 16 + c2);

                        }

                        c => return Err(LexError::InvalidStringEscape(c)),

                    }

                }

                
c0
 if (c as u32) < 0x20 || c as u32 == 0x7f => {

                    return Err(LexError::InvalidStringElement(c))

                }

                
c0
 if !allow_confusing_unicode && is_confusing_unicode(
c29.8k
) => {

                    return Err(LexError::ConfusingUnicode(c))

                }

                c => match &mut state {

                    State::Start => {}

                    State::String(v) => {

                        v.extend(c.encode_utf8(&mut [0; 4]).as_bytes());

                    }

                },

            }

        }

        match state {

            State::Start => Ok(orig[..orig.len() - it.as_str().len() - 1].as_bytes().into()),

            State::String(s) => Ok(s.into()),

        }

    }

    /// Parses an id-or-string-based name from `it`.

    ///

    /// Note that `it` should already have been lexed and this is just

    /// extracting the value. If the token lexed was `@a` then this should point

    /// to `a`.

    ///

    /// This will automatically detect quoted syntax such as `@"..."` and the

    /// byte string will be parsed and validated as utf-8.

    ///

    /// # Errors

    ///

    /// Returns an error if a quoted byte string is found and contains invalid

    /// utf-8.

    fn parse_name(it: &mut str::Chars<'a>) -> Result<Cow<'a, str>, LexError> {

        if it.clone().next() == Some('"') {

            it.next();

            match Lexer::parse_str(it, true)? {

                Cow::Borrowed(bytes) => match std::str::from_utf8(bytes) {

                    Ok(s) => Ok(Cow::Borrowed(s)),

                    Err(e) => Err(LexError::InvalidUtf8Id(e)),

                },

                Cow::Owned(bytes) => match String::from_utf8(bytes) {

                    Ok(s) => Ok(Cow::Owned(s)),

                    Err(e) => Err(LexError::InvalidUtf8Id(e.utf8_error())),

                },

            }

        } else {

            Ok(Cow::Borrowed(it.as_str()))

        }

    }

    fn hexnum(it: &mut str::Chars<'_>) -> Result<u32, LexError> {

        let n = Lexer::hexdigit(it)?;

        let mut last_underscore = false;

        let mut n = n as u32;

        while let Some(c) = it.clone().next() {

            if c == '_' {

                it.next();

                last_underscore = true;

                continue;

            }

            if !c.is_ascii_hexdigit() {

                break;

            }

            last_underscore = false;

            it.next();

            n = n

                .checked_mul(16)

                .and_then(|n| n.checked_add(to_hex(c) as u32))

                .ok_or(LexError::NumberTooBig)?;

        }

        if last_underscore {

            return Err(LexError::LoneUnderscore);

        }

        Ok(n)

    }

    /// Reads a hexidecimal digit from the input stream, returning where it's

    /// defined and the hex value. Returns an error on EOF or an invalid hex

    /// digit.

    fn hexdigit(it: &mut str::Chars<'_>) -> Result<u8, LexError> {

        let ch = Lexer::must_char(it)
?0
;

        if ch.is_ascii_hexdigit() {

            Ok(to_hex(ch))

        } else {

            Err(LexError::InvalidHexDigit(ch))

        }

    }

    /// Reads the next character from the input string and where it's located,

    /// returning an error if the input stream is empty.

    fn must_char(it: &mut str::Chars<'_>) -> Result<char, LexError> {

        it.next().ok_or(LexError::UnexpectedEof)

    }

    /// Expects that a specific character must be read next

    fn must_eat_char(it: &mut str::Chars<'_>, wanted: char) -> Result<(), LexError> {

        let found = Lexer::must_char(it)?;

        if wanted == found {

            Ok(())

        } else {

            Err(LexError::Expected { wanted, found })

        }

    }

    /// Creates an error at `pos` with the specified `kind`

    fn error(&self, pos: usize, kind: LexError) -> Error {

        Error::lex(Span { offset: pos }, self.input, kind)

    }

    /// Returns an iterator over all tokens in the original source string

    /// starting at the `pos` specified.

    pub fn iter(&self, mut pos: usize) -> impl Iterator<Item = Result<Token, Error>> + '_ {

        std::iter::from_fn(move || self.parse(&mut pos).transpose())

    }

    /// Returns whether an annotation is present at `pos`. If it is present then

    /// `Ok(Some(token))` is returned corresponding to the token, otherwise

    /// `Ok(None)` is returned. If the next token cannot be parsed then an error

    /// is returned.

    pub fn annotation(&self, mut pos: usize) -> Result<Option<Token>, Error> {

        let bytes = self.input.as_bytes();

        // Quickly reject anything that for sure isn't an annotation since this

        // method is used every time an lparen is parsed.

        if bytes.get(pos) != Some(&b'@') {

            return Ok(None);

        }

        match self.parse(&mut pos)? {

            Some(token) => match token.kind {

                TokenKind::Annotation => Ok(Some(token)),

                _ => Ok(None),

            },

            None => Ok(None),

        }

    }

}

impl Token {

    /// Returns the original source text for this token.

    pub fn src<'a>(&self, s: &'a str) -> &'a str {

        &s[self.offset..][..self.len.try_into().unwrap()]

    }

    /// Returns the identifier, without the leading `$` symbol, that this token

    /// represents.

    ///

    /// Note that this method returns the contents of the identifier. With a

    /// string-based identifier this means that escapes have been resolved to

    /// their string-based equivalent.

    ///

    /// Should only be used with `TokenKind::Id`.

    ///

    /// # Errors

    ///

    /// Returns an error if this is a string-based identifier (e.g. `$"..."`)

    /// which is invalid utf-8.

    pub fn id<'a>(&self, s: &'a str) -> Result<Cow<'a, str>, Error> {

        let mut ch = self.src(s).chars();

        let dollar = ch.next();

        debug_assert_eq!(dollar, Some('$'));

        let id = Lexer::parse_name(&mut ch).map_err(|e| 
self.error(s, e)0
)
?0
;

        if id.is_empty() {

            return Err(self.error(s, LexError::EmptyId));

        }

        Ok(id)

    }

    /// Returns the annotation, without the leading `@` symbol, that this token

    /// represents.

    ///

    /// Note that this method returns the contents of the identifier. With a

    /// string-based identifier this means that escapes have been resolved to

    /// their string-based equivalent.

    ///

    /// Should only be used with `TokenKind::Annotation`.

    ///

    /// # Errors

    ///

    /// Returns an error if this is a string-based identifier (e.g. `$"..."`)

    /// which is invalid utf-8.

    pub fn annotation<'a>(&self, s: &'a str) -> Result<Cow<'a, str>, Error> {

        let mut ch = self.src(s).chars();

        let at = ch.next();

        debug_assert_eq!(at, Some('@'));

        let id = Lexer::parse_name(&mut ch).map_err(|e| self.error(s, e))?;

        if id.is_empty() {

            return Err(self.error(s, LexError::EmptyAnnotation));

        }

        Ok(id)

    }

    /// Returns the keyword this token represents.

    ///

    /// Should only be used with [`TokenKind::Keyword`].

    pub fn keyword<'a>(&self, s: &'a str) -> &'a str {

        self.src(s)

    }

    /// Returns the reserved string this token represents.

    ///

    /// Should only be used with [`TokenKind::Reserved`].

    pub fn reserved<'a>(&self, s: &'a str) -> &'a str {

        self.src(s)

    }

    /// Returns the parsed string that this token represents.

    ///

    /// This returns either a raw byte slice into the source if that's possible

    /// or an owned representation to handle escaped characters and such.

    ///

    /// Should only be used with [`TokenKind::String`].

    pub fn string<'a>(&self, s: &'a str) -> Cow<'a, [u8]> {

        let mut ch = self.src(s).chars();