cl_lexer/
lib.rs

1//! A lobster
2//!
3//! The [Lexer] takes an input text and carves it up into [Tokens](cl_token::Token)
4//! which hold a [Lexeme](cl_token::Lexeme), [Token Kind](cl_token::TKind), and
5//! [Span](struct@cl_structures::span::Span).
6//!
7//! You can create a new [Lexer] with [Lexer::new], and pull a token with [Lexer::scan].
8//!
9//! # Examples:
10//!
11//! ```rust
12//! # use cl_lexer::LexError;
13//! # fn main() -> Result<(), LexError> {
14//! use cl_lexer::{Lexer, Symbol};
15//! use cl_token::{Token, Lexeme, TKind};
16//!
17//! let sample_text = r#"
18//!     "This is a string\n"
19//!     '\u{1f988}'
20//!     123456
21//!     098765
22//!     0x4141
23//!     0o7777
24//!     0b1100
25//!     x + 1
26//! "#;
27//!
28//! let mut lexer = Lexer::new(Symbol::from("lexer test"), sample_text);
29//!
30//! let a_string = lexer.scan()?;
31//! assert_eq!(a_string.lexeme.str(), Some("This is a string\n"));
32//! assert_eq!(a_string.kind, TKind::String);
33//!
34//! let a_char = lexer.scan()?;
35//! assert_eq!(a_char.lexeme.char(), Some('\u{1f988}'));
36//! assert_eq!(a_char.kind, TKind::Character);
37//!
38//! let num_123456 = lexer.scan()?;
39//! assert_eq!(num_123456.lexeme.int(), Some(123456));
40//! assert_eq!(num_123456.kind, TKind::Integer);
41//!
42//! let num_098765 = lexer.scan()?;
43//! assert_eq!(num_098765.lexeme.int(), Some(98765));
44//! assert_eq!(num_098765.kind, TKind::Integer);
45//!
46//! let num_0x4141 = lexer.scan()?;
47//! assert_eq!(num_0x4141.lexeme.int(), Some(16705));
48//! assert_eq!(num_0x4141.kind, TKind::Integer);
49//!
50//! let num_0o7777 = lexer.scan()?;
51//! assert_eq!(num_0o7777.lexeme.int(), Some(4095));
52//! assert_eq!(num_0o7777.kind, TKind::Integer);
53//!
54//! let num_0b1100 = lexer.scan()?;
55//! assert_eq!(num_0b1100.lexeme.int(), Some(12));
56//! assert_eq!(num_0b1100.kind, TKind::Integer);
57//!
58//! let id_x = lexer.scan()?;
59//! assert_eq!(id_x.lexeme.str(), Some("x"));
60//! assert_eq!(id_x.kind, TKind::Identifier);
61//!
62//! let op_plus = lexer.scan()?;
63//! assert_eq!(op_plus.lexeme.str(), Some("+"));
64//! assert_eq!(op_plus.kind, TKind::Plus);
65//!
66//! let num_1 = lexer.scan()?;
67//! assert_eq!(num_1.lexeme.int(), Some(1));
68//! assert_eq!(num_1.kind, TKind::Integer);
69//!
70//! assert!(lexer.scan().is_err());
71//!
72//! # Ok(())
73//! # }
74//! ```
75
76pub mod error;
77
78pub mod lexer;
79
80pub use crate::{
81    error::{EOF, LexError, LexFailure},
82    lexer::{Lexer, Symbol},
83};
cl_lexer/lib.rs

cl_lexer/
lib.rs
