cl_token/
lib.rs

1//! # Token
2//!
3//! Stores a component of a file as a [Lexeme], [TKind], and [struct@Span]
4#![warn(clippy::all)]
5//! The Token defines an interface between lexer and parser
6
7use cl_structures::span::Span;
8
9/// A unit of lexical information produced by a Lexer
10#[derive(Clone, Debug)]
11pub struct Token {
12    pub lexeme: Lexeme,
13    pub kind: TKind,
14    pub span: Span,
15}
16
17impl Token {
18    /// Extracts the [`kind`](Token::kind) field of this Token
19    pub const fn kind(&self) -> TKind {
20        self.kind
21    }
22}
23
24/// The (possibly pre-processed) lexical information, in the form of a [String], [u128], or [char]
25#[derive(Clone, Debug)]
26pub enum Lexeme {
27    String(String),
28    Integer(u128, u32),
29    Char(char),
30}
31
32impl Lexeme {
33    pub fn string(self) -> Option<String> {
34        match self {
35            Self::String(s) => Some(s),
36            _ => None,
37        }
38    }
39    pub fn str(&self) -> Option<&str> {
40        match self {
41            Self::String(s) => Some(s),
42            _ => None,
43        }
44    }
45    pub const fn int(&self) -> Option<u128> {
46        match self {
47            Self::Integer(i, _) => Some(*i),
48            _ => None,
49        }
50    }
51    pub const fn char(&self) -> Option<char> {
52        match self {
53            Self::Char(c) => Some(*c),
54            _ => None,
55        }
56    }
57}
58
59impl std::fmt::Display for Lexeme {
60    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61        match self {
62            Self::String(v) => v.fmt(f),
63            Self::Integer(v, _) => v.fmt(f),
64            Self::Char(v) => v.fmt(f),
65        }
66    }
67}
68
69/// The lexical classification of a [Token].
70#[derive(Clone, Copy, Debug, PartialEq, Eq)]
71pub enum TKind {
72    /// Line or block comment
73    Comment,
74    /// Inner doc comment: //!.*
75    InDoc,
76    /// Outer doc comment ///.*
77    OutDoc,
78
79    As,
80    Break,
81    Catch,
82    Const,
83    Continue,
84    Defer,
85    Do,
86    Else,
87    Enum,
88    False,
89    Fn,
90    For,
91    If,
92    Impl,
93    In,
94    Let,
95    Loop,
96    Macro,
97    Match,
98    Mod,
99    Mut,
100    Pub,
101    Return,
102    Static,
103    Struct,
104    True,
105    Try,
106    Type,
107    Use,
108    While,
109
110    Identifier, // or Keyword
111    Character,
112    String,
113    /// `0(x[0-9A-Fa-f]* | d[0-9]* | o[0-7]* | b[0-1]*) | [1-9][0-9]*`
114    Integer,
115    /// {
116    LCurly,
117    /// }
118    RCurly,
119    /// [
120    LBrack,
121    /// ]
122    RBrack,
123    /// (
124    LParen,
125    /// )
126    RParen,
127    /// &
128    Amp,
129    /// &&
130    AmpAmp,
131    /// &=
132    AmpEq,
133    /// ->
134    Arrow,
135    /// @
136    At,
137    /// \
138    Backslash,
139    /// !
140    Bang,
141    /// !!
142    BangBang,
143    /// !=
144    BangEq,
145    /// |
146    Bar,
147    /// ||
148    BarBar,
149    /// |=
150    BarEq,
151    /// :
152    Colon,
153    /// ::
154    ColonColon,
155    /// ,
156    Comma,
157    /// $
158    Dollar,
159    /// .
160    Dot,
161    /// ..
162    DotDot,
163    /// ...
164    DotDotDot,
165    /// ..=
166    DotDotEq,
167    /// =
168    Eq,
169    /// ==
170    EqEq,
171    /// =>
172    FatArrow,
173    /// \`
174    Grave,
175    /// >
176    Gt,
177    /// >=
178    GtEq,
179    /// >>
180    GtGt,
181    /// >>=
182    GtGtEq,
183    /// #
184    Hash,
185    /// #!
186    HashBang,
187    /// <
188    Lt,
189    /// <=
190    LtEq,
191    /// <<
192    LtLt,
193    /// <<=
194    LtLtEq,
195    /// -
196    Minus,
197    /// -=
198    MinusEq,
199    /// +
200    Plus,
201    /// +=
202    PlusEq,
203    /// ?
204    Question,
205    /// %
206    Rem,
207    /// %=
208    RemEq,
209    /// ;
210    Semi,
211    /// /
212    Slash,
213    /// /=
214    SlashEq,
215    /// *
216    Star,
217    /// *=
218    StarEq,
219    /// ~
220    Tilde,
221    /// ^
222    Xor,
223    /// ^=
224    XorEq,
225    /// ^^
226    XorXor,
227}
228
229impl TKind {
230    pub const fn flip(self) -> Self {
231        match self {
232            Self::LCurly => Self::RCurly,
233            Self::RCurly => Self::LCurly,
234            Self::LBrack => Self::RBrack,
235            Self::RBrack => Self::LBrack,
236            Self::LParen => Self::RParen,
237            Self::RParen => Self::LParen,
238            Self::Gt => Self::Lt,
239            Self::GtGt => Self::LtLt,
240            Self::LtLt => Self::GtGt,
241            Self::Lt => Self::Gt,
242            _ => self,
243        }
244    }
245
246    /// Splits a single [TKind] into two, if possible, or returns the original.
247    pub const fn split(self) -> Result<(Self, Self), Self> {
248        Ok(match self {
249            Self::AmpAmp => (Self::Amp, Self::Amp),
250            Self::AmpEq => (Self::Amp, Self::Eq),
251            Self::Arrow => (Self::Minus, Self::Gt),
252            Self::BangBang => (Self::Bang, Self::Bang),
253            Self::BangEq => (Self::Bang, Self::Eq),
254            Self::BarBar => (Self::Bar, Self::Bar),
255            Self::BarEq => (Self::Bar, Self::Eq),
256            Self::ColonColon => (Self::Colon, Self::Colon),
257            Self::DotDot => (Self::Dot, Self::Dot),
258            Self::DotDotEq => (Self::DotDot, Self::Eq),
259            Self::EqEq => (Self::Eq, Self::Eq),
260            Self::FatArrow => (Self::Eq, Self::Gt),
261            Self::GtEq => (Self::Gt, Self::Eq),
262            Self::GtGt => (Self::Gt, Self::Gt),
263            Self::GtGtEq => (Self::Gt, Self::GtEq),
264            Self::HashBang => (Self::Hash, Self::Bang),
265            Self::LtEq => (Self::Lt, Self::Eq),
266            Self::LtLt => (Self::Lt, Self::Lt),
267            Self::LtLtEq => (Self::Lt, Self::LtEq),
268            Self::MinusEq => (Self::Minus, Self::Eq),
269            Self::PlusEq => (Self::Plus, Self::Eq),
270            Self::RemEq => (Self::Rem, Self::Eq),
271            Self::SlashEq => (Self::Slash, Self::Eq),
272            Self::StarEq => (Self::Star, Self::Eq),
273            Self::XorEq => (Self::Xor, Self::Eq),
274            Self::XorXor => (Self::Xor, Self::Xor),
275            _ => return Err(self),
276        })
277    }
278}
cl_token/lib.rs

cl_token/
lib.rs
