cl_token/
lib.rs

1//! # Token
2//!
3//! Stores a component of a file as a [Lexeme], [TKind], and [struct@Span]
4#![warn(clippy::all)]
5//! The Token defines an interface between lexer and parser
6
7use cl_structures::span::Span;
8
9/// A unit of lexical information produced by a Lexer
10#[derive(Clone, Debug)]
11pub struct Token {
12    pub lexeme: Lexeme,
13    pub kind: TKind,
14    pub span: Span,
15}
16
17impl Token {
18    /// Extracts the [`kind`](Token::kind) field of this Token
19    pub const fn kind(&self) -> TKind {
20        self.kind
21    }
22}
23
24/// The (possibly pre-processed) lexical information, in the form of a [String], [u128], or [char]
25#[derive(Clone, Debug)]
26pub enum Lexeme {
27    String(String),
28    Integer(u128, u32),
29    Char(char),
30}
31
32impl Lexeme {
33    pub fn string(self) -> Option<String> {
34        match self {
35            Self::String(s) => Some(s),
36            _ => None,
37        }
38    }
39    pub fn str(&self) -> Option<&str> {
40        match self {
41            Self::String(s) => Some(s),
42            _ => None,
43        }
44    }
45    pub const fn int(&self) -> Option<u128> {
46        match self {
47            Self::Integer(i, _) => Some(*i),
48            _ => None,
49        }
50    }
51    pub const fn char(&self) -> Option<char> {
52        match self {
53            Self::Char(c) => Some(*c),
54            _ => None,
55        }
56    }
57}
58
59impl std::fmt::Display for Lexeme {
60    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61        match self {
62            Self::String(v) => v.fmt(f),
63            Self::Integer(v, _) => v.fmt(f),
64            Self::Char(v) => v.fmt(f),
65        }
66    }
67}
68
69/// The lexical classification of a [Token].
70#[derive(Clone, Copy, Debug, PartialEq, Eq)]
71pub enum TKind {
72    /// Line or block comment
73    Comment,
74    /// Inner doc comment: //!.*
75    InDoc,
76    /// Outer doc comment ///.*
77    OutDoc,
78
79    As,
80    Break,
81    Catch,
82    Const,
83    Continue,
84    Defer,
85    Do,
86    Else,
87    Enum,
88    False,
89    Fn,
90    For,
91    If,
92    Impl,
93    In,
94    Let,
95    Loop,
96    Macro,
97    Match,
98    Mod,
99    Mut,
100    Pub,
101    Return,
102    Static,
103    Struct,
104    True,
105    Try,
106    Type,
107    Underscore,
108    Use,
109    While,
110
111    Identifier, // or Keyword
112    Character,
113    String,
114    /// `0(x[0-9A-Fa-f]* | d[0-9]* | o[0-7]* | b[0-1]*) | [1-9][0-9]*`
115    Integer,
116    /// {
117    LCurly,
118    /// }
119    RCurly,
120    /// [
121    LBrack,
122    /// ]
123    RBrack,
124    /// (
125    LParen,
126    /// )
127    RParen,
128    /// &
129    Amp,
130    /// &&
131    AmpAmp,
132    /// &=
133    AmpEq,
134    /// ->
135    Arrow,
136    /// @
137    At,
138    /// \
139    Backslash,
140    /// !
141    Bang,
142    /// !!
143    BangBang,
144    /// !=
145    BangEq,
146    /// |
147    Bar,
148    /// ||
149    BarBar,
150    /// |=
151    BarEq,
152    /// :
153    Colon,
154    /// ::
155    ColonColon,
156    /// ,
157    Comma,
158    /// $
159    Dollar,
160    /// .
161    Dot,
162    /// ..
163    DotDot,
164    /// ...
165    DotDotDot,
166    /// ..=
167    DotDotEq,
168    /// =
169    Eq,
170    /// ==
171    EqEq,
172    /// =>
173    FatArrow,
174    /// \`
175    Grave,
176    /// >
177    Gt,
178    /// >=
179    GtEq,
180    /// >>
181    GtGt,
182    /// >>=
183    GtGtEq,
184    /// #
185    Hash,
186    /// #!
187    HashBang,
188    /// <
189    Lt,
190    /// <=
191    LtEq,
192    /// <<
193    LtLt,
194    /// <<=
195    LtLtEq,
196    /// -
197    Minus,
198    /// -=
199    MinusEq,
200    /// +
201    Plus,
202    /// +=
203    PlusEq,
204    /// ?
205    Question,
206    /// %
207    Rem,
208    /// %=
209    RemEq,
210    /// ;
211    Semi,
212    /// /
213    Slash,
214    /// /=
215    SlashEq,
216    /// *
217    Star,
218    /// *=
219    StarEq,
220    /// ~
221    Tilde,
222    /// ^
223    Xor,
224    /// ^=
225    XorEq,
226    /// ^^
227    XorXor,
228}
229
230impl TKind {
231    pub const fn flip(self) -> Self {
232        match self {
233            Self::LCurly => Self::RCurly,
234            Self::RCurly => Self::LCurly,
235            Self::LBrack => Self::RBrack,
236            Self::RBrack => Self::LBrack,
237            Self::LParen => Self::RParen,
238            Self::RParen => Self::LParen,
239            Self::Gt => Self::Lt,
240            Self::GtGt => Self::LtLt,
241            Self::LtLt => Self::GtGt,
242            Self::Lt => Self::Gt,
243            _ => self,
244        }
245    }
246
247    /// Splits a single [TKind] into two, if possible, or returns the original.
248    pub const fn split(self) -> Result<(Self, Self), Self> {
249        Ok(match self {
250            Self::AmpAmp => (Self::Amp, Self::Amp),
251            Self::AmpEq => (Self::Amp, Self::Eq),
252            Self::Arrow => (Self::Minus, Self::Gt),
253            Self::BangBang => (Self::Bang, Self::Bang),
254            Self::BangEq => (Self::Bang, Self::Eq),
255            Self::BarBar => (Self::Bar, Self::Bar),
256            Self::BarEq => (Self::Bar, Self::Eq),
257            Self::ColonColon => (Self::Colon, Self::Colon),
258            Self::DotDot => (Self::Dot, Self::Dot),
259            Self::DotDotEq => (Self::DotDot, Self::Eq),
260            Self::EqEq => (Self::Eq, Self::Eq),
261            Self::FatArrow => (Self::Eq, Self::Gt),
262            Self::GtEq => (Self::Gt, Self::Eq),
263            Self::GtGt => (Self::Gt, Self::Gt),
264            Self::GtGtEq => (Self::Gt, Self::GtEq),
265            Self::HashBang => (Self::Hash, Self::Bang),
266            Self::LtEq => (Self::Lt, Self::Eq),
267            Self::LtLt => (Self::Lt, Self::Lt),
268            Self::LtLtEq => (Self::Lt, Self::LtEq),
269            Self::MinusEq => (Self::Minus, Self::Eq),
270            Self::PlusEq => (Self::Plus, Self::Eq),
271            Self::RemEq => (Self::Rem, Self::Eq),
272            Self::SlashEq => (Self::Slash, Self::Eq),
273            Self::StarEq => (Self::Star, Self::Eq),
274            Self::XorEq => (Self::Xor, Self::Eq),
275            Self::XorXor => (Self::Xor, Self::Xor),
276            _ => return Err(self),
277        })
278    }
279}
cl_token/lib.rs

cl_token/
lib.rs
