Module
lexie/config
src/lexie/config.pookaEvery lexer begins as a wish list: these words, those symbols, numbers that look like this.
The specification: what the lexer should recognise, with its defaults, read from TOML.
Reading is lenient about missing sections, where defaults apply, and
strict about types and unknown keys. Every problem is collected as a
ConfigError, so all of them can be shown at once.
API reference
- classConfig
- classWhitespace
- classIdentifier
- classCharClass
- classRange
- classSet
- classLineComment
- classBlockComment
- classComments
- classStringStyle
- classChars
- classQuotedChars
- classPrefixedChars
- classNumbers
- classConfigError
- functiondefault_config
- functionparse
- functionknown_sections
- functionparse_class
- functionclass_chars
class Config(
/// `[module] name`: generated modules go in `src/<name>/`. Empty means
/// `src/` itself.
module_name: String,
/// `[output] dir`: the root folder for generated files.
output_dir: String,
/// `[highlighter] enabled`
highlighter: Bool,
whitespace: Whitespace,
keywords: List(String),
literals: List(String),
operators: List(String),
punctuation: List(String),
/// `[tokens.names]`: class names chosen by hand, keyed by source text.
names: List(#(String, String)),
identifier: Identifier,
comments: Comments,
/// `[strings.*]`, sorted by style name.
strings: List(StringStyle),
chars: Option(Chars),
numbers: Numbers,
/// `[errors] token`: the class for unexpected input.
error_token: String,
/// `location-info`: lex returns located lexemes when true, plain tokens otherwise.
location_info: Bool,
)
A whole specification.
module_name: String
Stringoutput_dir: String
Stringhighlighter: Bool
Boolwhitespace: Whitespace
lexie/config.Whitespacekeywords: List(String)
List(String)literals: List(String)
List(String)operators: List(String)
List(String)punctuation: List(String)
List(String)names: List(#(String, String))
List(#(String, String))identifier: Identifier
lexie/config.Identifiercomments: Comments
lexie/config.Commentsstrings: List(StringStyle)
List(lexie/config.StringStyle)chars: Option(Chars)
Option(lexie/config.Chars)numbers: Numbers
lexie/config.Numberserror_token: String
Stringlocation_info: Bool
Bool
fn(String, String, Bool, lexie/config.Whitespace, List(String), List(String), List(String), List(String), List(#(String, String)), lexie/config.Identifier, lexie/config.Comments, List(lexie/config.StringStyle), Option(lexie/config.Chars), lexie/config.Numbers, String, Bool) -> lexie/config.Configclass Whitespace(items: List(String), emit: Bool, token: String)
items: List(String)
List(String)emit: Bool
Booltoken: String
String
fn(List(String), Bool, String) -> lexie/config.Whitespaceclass Identifier(
enabled: Bool,
start: List(CharClass),
more: List(CharClass),
unicode: Bool,
token: String,
)
enabled: Bool
Boolstart: List(CharClass)
List(lexie/config.CharClass)more: List(CharClass)
List(lexie/config.CharClass)unicode: Bool
Booltoken: String
String
fn(Bool, List(lexie/config.CharClass), List(lexie/config.CharClass), Bool, String) -> lexie/config.Identifierclass CharClass
A character class entry, such as "a-z" or "_?!".
class Range(from: String, to: String) is CharClass
from: String
Stringto: String
String
fn(String, String) -> lexie/config.Rangeclass Set(characters: List(String)) is CharClass
characters: List(String)
List(String)
fn(List(String)) -> lexie/config.Setclass LineComment(start: String, emit: Bool, token: String)
start: String
Stringemit: Bool
Booltoken: String
String
fn(String, Bool, String) -> lexie/config.LineCommentclass BlockComment(
start: String,
finish: String,
nested: Bool,
emit: Bool,
token: String
)
start: String
Stringfinish: String
Stringnested: Bool
Boolemit: Bool
Booltoken: String
String
fn(String, String, Bool, Bool, String) -> lexie/config.BlockCommentclass Comments(
line: Option(LineComment),
doc: Option(LineComment),
module_doc: Option(LineComment),
block: Option(BlockComment),
)
line: Option(LineComment)
Option(lexie/config.LineComment)doc: Option(LineComment)
Option(lexie/config.LineComment)module_doc: Option(LineComment)
Option(lexie/config.LineComment)block: Option(BlockComment)
Option(lexie/config.BlockComment)
fn(Option(lexie/config.LineComment), Option(lexie/config.LineComment), Option(lexie/config.LineComment), Option(lexie/config.BlockComment)) -> lexie/config.Commentsclass StringStyle(
/// The `[strings.<name>]` key, used in the generated scanner's name.
name: String,
quote: String,
prefix: String,
escapes: Bool,
multiline: Bool,
token: String,
)
name: String
Stringquote: String
Stringprefix: String
Stringescapes: Bool
Boolmultiline: Bool
Booltoken: String
String
fn(String, String, String, Bool, Bool, String) -> lexie/config.StringStyleclass Chars
How characters are written.
class QuotedChars(quote: String, escapes: Bool, token: String) is Chars
'a': delimited on both sides.
quote: String
Stringescapes: Bool
Booltoken: String
String
fn(String, Bool, String) -> lexie/config.QuotedCharsclass PrefixedChars(prefix: String, escapes: Bool, token: String) is Chars
$a: a prefix, then one character.
prefix: String
Stringescapes: Bool
Booltoken: String
String
fn(String, Bool, String) -> lexie/config.PrefixedCharsclass Numbers(
int_enabled: Bool,
int_token: String,
float_enabled: Bool,
float_token: String,
/// Between digit groups, such as `_` in `1_000`.
separator: Option(String),
exponent_markers: List(String),
/// Let `1e3`, an exponent without a decimal point, be a float.
exponent_after_int: Bool,
hex_prefix: Option(String),
binary_prefix: Option(String),
octal_prefix: Option(String),
suffixes: List(String),
)
int_enabled: Bool
Boolint_token: String
Stringfloat_enabled: Bool
Boolfloat_token: String
Stringseparator: Option(String)
Option(String)exponent_markers: List(String)
List(String)exponent_after_int: Bool
Boolhex_prefix: Option(String)
Option(String)binary_prefix: Option(String)
Option(String)octal_prefix: Option(String)
Option(String)suffixes: List(String)
List(String)
fn(Bool, String, Bool, String, Option(String), List(String), Bool, Option(String), Option(String), Option(String), List(String)) -> lexie/config.Numbersclass ConfigError(location: String, message: String)
Something wrong with the specification: where, and what.
location: String
Stringmessage: String
String
fn(String, String) -> lexie/config.ConfigErrordefault_config() -> Config
fn() -> lexie/config.ConfigThe specification an empty file describes: identifiers, integers and floats, a few whitespace characters, and nothing else. Every section of a real specification is read on top of this.
import lexie/config
let defaults = config.default_config()
assert defaults.output_dir == "src"
assert defaults.identifier.token == "Ident"
assert defaults.numbers.int_token == "IntLiteral"
parse(doc: Toml) -> #(Config, List(ConfigError))
fn($package/tome/tome.Toml) -> #(lexie/config.Config, List(lexie/config.ConfigError))Build a specification from a TOML document, collecting every problem. Missing sections use defaults; invalid values remain in the problem list.
import lexie/config
import tome
let assert Ok(document) = tome.parse("[keywords]\nitems = [\"let\"]")
let #(specification, problems) = config.parse(document)
// specification.keywords is ["let"], and problems is empty.
known_sections() -> List(String)
fn() -> List(String)The top-level settings a specification may have.
parse_class(spec: String) -> CharClass
fn(String) -> lexie/config.CharClass"a-z" is a range; anything else is a set of characters.
import lexie/config
assert config.parse_class("a-z") == config.Range("a", "z")
assert config.parse_class("_?!") == config.Set(["_", "?", "!"])
class_chars(classes: List(CharClass)) -> List(String)
fn(List(lexie/config.CharClass)) -> List(String)Every member of the classes, in order, without repeats. Ranges are expanded only when both ends are ASCII.