cl_token/
lib.rs

1//! # Token
2//!
3//! Stores a component of a file as a [Lexeme], [TKind], and [struct@Span]
4#![warn(clippy::all)]
5//! The Token defines an interface between lexer and parser
6
7use cl_structures::span::Span;
8
9/// A unit of lexical information produced by a Lexer
10#[derive(Clone, Debug)]
11pub struct Token {
12    pub lexeme: Lexeme,
13    pub kind: TKind,
14    pub span: Span,
15}
16
17impl Token {
18    /// Extracts the [`kind`](Token::kind) field of this Token
19    pub const fn kind(&self) -> TKind {
20        self.kind
21    }
22}
23
24/// The (possibly pre-processed) lexical information, in the form of a [String], [u128], or [char]
25#[derive(Clone, Debug)]
26pub enum Lexeme {
27    String(String),
28    Integer(u128, u32),
29    Char(char),
30}
31
32impl Lexeme {
33    pub fn string(self) -> Option<String> {
34        match self {
35            Self::String(s) => Some(s),
36            _ => None,
37        }
38    }
39    pub fn str(&self) -> Option<&str> {
40        match self {
41            Self::String(s) => Some(s),
42            _ => None,
43        }
44    }
45    pub const fn int(&self) -> Option<u128> {
46        match self {
47            Self::Integer(i, _) => Some(*i),
48            _ => None,
49        }
50    }
51    pub const fn char(&self) -> Option<char> {
52        match self {
53            Self::Char(c) => Some(*c),
54            _ => None,
55        }
56    }
57}
58
59impl std::fmt::Display for Lexeme {
60    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61        match self {
62            Self::String(v) => v.fmt(f),
63            Self::Integer(v, _) => v.fmt(f),
64            Self::Char(v) => v.fmt(f),
65        }
66    }
67}
68
69/// The lexical classification of a [Token].
70#[derive(Clone, Copy, Debug, PartialEq, Eq)]
71pub enum TKind {
72    /// Line or block comment
73    Comment,
74    /// Outer doc comment ///.*
75    OutDoc,
76    /// Inner doc comment: //!.*
77    InDoc,
78
79    As,
80    Break,
81    Const,
82    Continue,
83    Do,
84    Else,
85    Enum,
86    False,
87    Fn,
88    For,
89    If,
90    Impl,
91    In,
92    Let,
93    Loop,
94    Macro,
95    Match,
96    Mod,
97    Mut,
98    Pub,
99    Return,
100    Static,
101    Struct,
102    True,
103    Type,
104    Use,
105    While,
106
107    Identifier, // or Keyword
108    Character,
109    String,
110    /// `0(x[0-9A-Fa-f]* | d[0-9]* | o[0-7]* | b[0-1]*) | [1-9][0-9]*`
111    Integer,
112    /// {
113    LCurly,
114    /// }
115    RCurly,
116    /// [
117    LBrack,
118    /// ]
119    RBrack,
120    /// (
121    LParen,
122    /// )
123    RParen,
124    /// &
125    Amp,
126    /// &&
127    AmpAmp,
128    /// &=
129    AmpEq,
130    /// ->
131    Arrow,
132    /// @
133    At,
134    /// \
135    Backslash,
136    /// !
137    Bang,
138    /// !!
139    BangBang,
140    /// !=
141    BangEq,
142    /// |
143    Bar,
144    /// ||
145    BarBar,
146    /// |=
147    BarEq,
148    /// :
149    Colon,
150    /// ::
151    ColonColon,
152    /// ,
153    Comma,
154    /// $
155    Dollar,
156    /// .
157    Dot,
158    /// ..
159    DotDot,
160    /// ...
161    DotDotDot,
162    /// ..=
163    DotDotEq,
164    /// =
165    Eq,
166    /// ==
167    EqEq,
168    /// =>
169    FatArrow,
170    /// \`
171    Grave,
172    /// >
173    Gt,
174    /// >=
175    GtEq,
176    /// >>
177    GtGt,
178    /// >>=
179    GtGtEq,
180    /// #
181    Hash,
182    /// #!
183    HashBang,
184    /// <
185    Lt,
186    /// <=
187    LtEq,
188    /// <<
189    LtLt,
190    /// <<=
191    LtLtEq,
192    /// -
193    Minus,
194    /// -=
195    MinusEq,
196    /// +
197    Plus,
198    /// +=
199    PlusEq,
200    /// ?
201    Question,
202    /// %
203    Rem,
204    /// %=
205    RemEq,
206    /// ;
207    Semi,
208    /// /
209    Slash,
210    /// /=
211    SlashEq,
212    /// *
213    Star,
214    /// *=
215    StarEq,
216    /// ~
217    Tilde,
218    /// ^
219    Xor,
220    /// ^=
221    XorEq,
222    /// ^^
223    XorXor,
224}
225
226impl TKind {
227    pub const fn flip(self) -> Self {
228        match self {
229            Self::LCurly => Self::RCurly,
230            Self::RCurly => Self::LCurly,
231            Self::LBrack => Self::RBrack,
232            Self::RBrack => Self::LBrack,
233            Self::LParen => Self::RParen,
234            Self::RParen => Self::LParen,
235            Self::Gt => Self::Lt,
236            Self::GtGt => Self::LtLt,
237            Self::LtLt => Self::GtGt,
238            Self::Lt => Self::Gt,
239            _ => self,
240        }
241    }
242
243    /// Splits a single [TKind] into two, if possible, or returns the original.
244    pub const fn split(self) -> Result<(Self, Self), Self> {
245        Ok(match self {
246            Self::AmpAmp => (Self::Amp, Self::Amp),
247            Self::AmpEq => (Self::Amp, Self::Eq),
248            Self::Arrow => (Self::Minus, Self::Gt),
249            Self::BangBang => (Self::Bang, Self::Bang),
250            Self::BangEq => (Self::Bang, Self::Eq),
251            Self::BarBar => (Self::Bar, Self::Bar),
252            Self::BarEq => (Self::Bar, Self::Eq),
253            Self::ColonColon => (Self::Colon, Self::Colon),
254            Self::DotDot => (Self::Dot, Self::Dot),
255            Self::DotDotEq => (Self::DotDot, Self::Eq),
256            Self::EqEq => (Self::Eq, Self::Eq),
257            Self::FatArrow => (Self::Eq, Self::Gt),
258            Self::GtEq => (Self::Gt, Self::Eq),
259            Self::GtGt => (Self::Gt, Self::Gt),
260            Self::GtGtEq => (Self::Gt, Self::GtEq),
261            Self::HashBang => (Self::Hash, Self::Bang),
262            Self::LtEq => (Self::Lt, Self::Eq),
263            Self::LtLt => (Self::Lt, Self::Lt),
264            Self::LtLtEq => (Self::Lt, Self::LtEq),
265            Self::MinusEq => (Self::Minus, Self::Eq),
266            Self::PlusEq => (Self::Plus, Self::Eq),
267            Self::RemEq => (Self::Rem, Self::Eq),
268            Self::SlashEq => (Self::Slash, Self::Eq),
269            Self::StarEq => (Self::Star, Self::Eq),
270            Self::XorEq => (Self::Xor, Self::Eq),
271            Self::XorXor => (Self::Xor, Self::Xor),
272            _ => return Err(self),
273        })
274    }
275}
cl_token/lib.rs

cl_token/
lib.rs
