Module

lexie/config

src/lexie/config.pooka

Every lexer begins as a wish list: these words, those symbols, numbers that look like this.

The specification: what the lexer should recognise, with its defaults, read from TOML.

Reading is lenient about missing sections, where defaults apply, and strict about types and unknown keys. Every problem is collected as a ConfigError, so all of them can be shown at once.

API reference

class

Config

src/lexie/config.pooka:21
class Config(
  /// `[module] name`: generated modules go in `src/<name>/`. Empty means
  /// `src/` itself.
  module_name: String,
  /// `[output] dir`: the root folder for generated files.
  output_dir: String,
  /// `[highlighter] enabled`
  highlighter: Bool,
  whitespace: Whitespace,
  keywords: List(String),
  literals: List(String),
  operators: List(String),
  punctuation: List(String),
  /// `[tokens.names]`: class names chosen by hand, keyed by source text.
  names: List(#(String, String)),
  identifier: Identifier,
  comments: Comments,
  /// `[strings.*]`, sorted by style name.
  strings: List(StringStyle),
  chars: Option(Chars),
  numbers: Numbers,
  /// `[errors] token`: the class for unexpected input.
  error_token: String,
  /// `location-info`: lex returns located lexemes when true, plain tokens otherwise.
  location_info: Bool,
)

A whole specification.

field

module_name

src/lexie/config.pooka:24
module_name: String
INFERREDString
field

output_dir

src/lexie/config.pooka:26
output_dir: String
INFERREDString
field

highlighter

src/lexie/config.pooka:28
highlighter: Bool
INFERREDBool
field

whitespace

src/lexie/config.pooka:29
whitespace: Whitespace
field

keywords

src/lexie/config.pooka:30
keywords: List(String)
INFERREDList(String)
field

literals

src/lexie/config.pooka:31
literals: List(String)
INFERREDList(String)
field

operators

src/lexie/config.pooka:32
operators: List(String)
INFERREDList(String)
field

punctuation

src/lexie/config.pooka:33
punctuation: List(String)
INFERREDList(String)
field

names

src/lexie/config.pooka:35
names: List(#(String, String))
INFERREDList(#(String, String))
field

identifier

src/lexie/config.pooka:36
identifier: Identifier
field

comments

src/lexie/config.pooka:37
comments: Comments
field

strings

src/lexie/config.pooka:39
strings: List(StringStyle)
field

chars

src/lexie/config.pooka:40
chars: Option(Chars)
INFERREDOption(lexie/config.Chars)
field

numbers

src/lexie/config.pooka:41
numbers: Numbers
field

error_token

src/lexie/config.pooka:43
error_token: String
INFERREDString
field

location_info

src/lexie/config.pooka:45
location_info: Bool
INFERREDBool
constructor

new

src/lexie/config.pooka:24
INFERREDfn(String, String, Bool, lexie/config.Whitespace, List(String), List(String), List(String), List(String), List(#(String, String)), lexie/config.Identifier, lexie/config.Comments, List(lexie/config.StringStyle), Option(lexie/config.Chars), lexie/config.Numbers, String, Bool) -> lexie/config.Config
class

Whitespace

src/lexie/config.pooka:48
class Whitespace(items: List(String), emit: Bool, token: String)
field

items

src/lexie/config.pooka:48
items: List(String)
INFERREDList(String)
field

emit

src/lexie/config.pooka:48
emit: Bool
INFERREDBool
field

token

src/lexie/config.pooka:48
token: String
INFERREDString
constructor

new

src/lexie/config.pooka:48
INFERREDfn(List(String), Bool, String) -> lexie/config.Whitespace
class

Identifier

src/lexie/config.pooka:50
class Identifier(
  enabled: Bool,
  start: List(CharClass),
  more: List(CharClass),
  unicode: Bool,
  token: String,
)
field

enabled

src/lexie/config.pooka:51
enabled: Bool
INFERREDBool
field

start

src/lexie/config.pooka:52
start: List(CharClass)
field

more

src/lexie/config.pooka:53
more: List(CharClass)
field

unicode

src/lexie/config.pooka:54
unicode: Bool
INFERREDBool
field

token

src/lexie/config.pooka:55
token: String
INFERREDString
constructor

new

src/lexie/config.pooka:51
INFERREDfn(Bool, List(lexie/config.CharClass), List(lexie/config.CharClass), Bool, String) -> lexie/config.Identifier
class

CharClass

src/lexie/config.pooka:59
class CharClass

A character class entry, such as "a-z" or "_?!".

class

Range

src/lexie/config.pooka:61
class Range(from: String, to: String) is CharClass
field

from

src/lexie/config.pooka:61
from: String
INFERREDString
field

to

src/lexie/config.pooka:61
to: String
INFERREDString
constructor

new

src/lexie/config.pooka:61
INFERREDfn(String, String) -> lexie/config.Range
class

Set

src/lexie/config.pooka:63
class Set(characters: List(String)) is CharClass
field

characters

src/lexie/config.pooka:63
characters: List(String)
INFERREDList(String)
constructor

new

src/lexie/config.pooka:63
INFERREDfn(List(String)) -> lexie/config.Set
class

LineComment

src/lexie/config.pooka:65
class LineComment(start: String, emit: Bool, token: String)
field

start

src/lexie/config.pooka:65
start: String
INFERREDString
field

emit

src/lexie/config.pooka:65
emit: Bool
INFERREDBool
field

token

src/lexie/config.pooka:65
token: String
INFERREDString
constructor

new

src/lexie/config.pooka:65
INFERREDfn(String, Bool, String) -> lexie/config.LineComment
class

BlockComment

src/lexie/config.pooka:67
class BlockComment(
  start: String,
  finish: String,
  nested: Bool,
  emit: Bool,
  token: String
)
field

start

src/lexie/config.pooka:68
start: String
INFERREDString
field

finish

src/lexie/config.pooka:69
finish: String
INFERREDString
field

nested

src/lexie/config.pooka:70
nested: Bool
INFERREDBool
field

emit

src/lexie/config.pooka:71
emit: Bool
INFERREDBool
field

token

src/lexie/config.pooka:72
token: String
INFERREDString
constructor

new

src/lexie/config.pooka:68
INFERREDfn(String, String, Bool, Bool, String) -> lexie/config.BlockComment
class

Comments

src/lexie/config.pooka:75
class Comments(
  line: Option(LineComment),
  doc: Option(LineComment),
  module_doc: Option(LineComment),
  block: Option(BlockComment),
)
field

line

src/lexie/config.pooka:76
line: Option(LineComment)
INFERREDOption(lexie/config.LineComment)
field

doc

src/lexie/config.pooka:77
doc: Option(LineComment)
INFERREDOption(lexie/config.LineComment)
field

module_doc

src/lexie/config.pooka:78
module_doc: Option(LineComment)
INFERREDOption(lexie/config.LineComment)
field

block

src/lexie/config.pooka:79
block: Option(BlockComment)
constructor

new

src/lexie/config.pooka:76
class

StringStyle

src/lexie/config.pooka:82
class StringStyle(
  /// The `[strings.<name>]` key, used in the generated scanner's name.
  name: String,
  quote: String,
  prefix: String,
  escapes: Bool,
  multiline: Bool,
  token: String,
)
field

name

src/lexie/config.pooka:84
name: String
INFERREDString
field

quote

src/lexie/config.pooka:85
quote: String
INFERREDString
field

prefix

src/lexie/config.pooka:86
prefix: String
INFERREDString
field

escapes

src/lexie/config.pooka:87
escapes: Bool
INFERREDBool
field

multiline

src/lexie/config.pooka:88
multiline: Bool
INFERREDBool
field

token

src/lexie/config.pooka:89
token: String
INFERREDString
constructor

new

src/lexie/config.pooka:84
INFERREDfn(String, String, String, Bool, Bool, String) -> lexie/config.StringStyle
class

Chars

src/lexie/config.pooka:93
class Chars

How characters are written.

class

QuotedChars

src/lexie/config.pooka:96
class QuotedChars(quote: String, escapes: Bool, token: String) is Chars

'a': delimited on both sides.

field

quote

src/lexie/config.pooka:96
quote: String
INFERREDString
field

escapes

src/lexie/config.pooka:96
escapes: Bool
INFERREDBool
field

token

src/lexie/config.pooka:96
token: String
INFERREDString
constructor

new

src/lexie/config.pooka:96
INFERREDfn(String, Bool, String) -> lexie/config.QuotedChars
class

PrefixedChars

src/lexie/config.pooka:99
class PrefixedChars(prefix: String, escapes: Bool, token: String) is Chars

$a: a prefix, then one character.

field

prefix

src/lexie/config.pooka:99
prefix: String
INFERREDString
field

escapes

src/lexie/config.pooka:99
escapes: Bool
INFERREDBool
field

token

src/lexie/config.pooka:99
token: String
INFERREDString
constructor

new

src/lexie/config.pooka:99
INFERREDfn(String, Bool, String) -> lexie/config.PrefixedChars
class

Numbers

src/lexie/config.pooka:101
class Numbers(
  int_enabled: Bool,
  int_token: String,
  float_enabled: Bool,
  float_token: String,
  /// Between digit groups, such as `_` in `1_000`.
  separator: Option(String),
  exponent_markers: List(String),
  /// Let `1e3`, an exponent without a decimal point, be a float.
  exponent_after_int: Bool,
  hex_prefix: Option(String),
  binary_prefix: Option(String),
  octal_prefix: Option(String),
  suffixes: List(String),
)
field

int_enabled

src/lexie/config.pooka:102
int_enabled: Bool
INFERREDBool
field

int_token

src/lexie/config.pooka:103
int_token: String
INFERREDString
field

float_enabled

src/lexie/config.pooka:104
float_enabled: Bool
INFERREDBool
field

float_token

src/lexie/config.pooka:105
float_token: String
INFERREDString
field

separator

src/lexie/config.pooka:107
separator: Option(String)
INFERREDOption(String)
field

exponent_markers

src/lexie/config.pooka:108
exponent_markers: List(String)
INFERREDList(String)
field

exponent_after_int

src/lexie/config.pooka:110
exponent_after_int: Bool
INFERREDBool
field

hex_prefix

src/lexie/config.pooka:111
hex_prefix: Option(String)
INFERREDOption(String)
field

binary_prefix

src/lexie/config.pooka:112
binary_prefix: Option(String)
INFERREDOption(String)
field

octal_prefix

src/lexie/config.pooka:113
octal_prefix: Option(String)
INFERREDOption(String)
field

suffixes

src/lexie/config.pooka:114
suffixes: List(String)
INFERREDList(String)
constructor

new

src/lexie/config.pooka:102
INFERREDfn(Bool, String, Bool, String, Option(String), List(String), Bool, Option(String), Option(String), Option(String), List(String)) -> lexie/config.Numbers
class

ConfigError

src/lexie/config.pooka:118
class ConfigError(location: String, message: String)

Something wrong with the specification: where, and what.

field

location

src/lexie/config.pooka:118
location: String
INFERREDString
field

message

src/lexie/config.pooka:118
message: String
INFERREDString
constructor

new

src/lexie/config.pooka:118
INFERREDfn(String, String) -> lexie/config.ConfigError
function

default_config

src/lexie/config.pooka:134
default_config() -> Config
INFERREDfn() -> lexie/config.Config

The specification an empty file describes: identifiers, integers and floats, a few whitespace characters, and nothing else. Every section of a real specification is read on top of this.

Pooka
import lexie/config

let defaults = config.default_config()
assert defaults.output_dir == "src"
assert defaults.identifier.token == "Ident"
assert defaults.numbers.int_token == "IntLiteral"
function

parse

src/lexie/config.pooka:198
parse(doc: Toml) -> #(Config, List(ConfigError))
INFERREDfn($package/tome/tome.Toml) -> #(lexie/config.Config, List(lexie/config.ConfigError))

Build a specification from a TOML document, collecting every problem. Missing sections use defaults; invalid values remain in the problem list.

Pooka
import lexie/config
import tome

let assert Ok(document) = tome.parse("[keywords]\nitems = [\"let\"]")
let #(specification, problems) = config.parse(document)
// specification.keywords is ["let"], and problems is empty.
function

known_sections

src/lexie/config.pooka:262
known_sections() -> List(String)
INFERREDfn() -> List(String)

The top-level settings a specification may have.

function

parse_class

src/lexie/config.pooka:746
parse_class(spec: String) -> CharClass
INFERREDfn(String) -> lexie/config.CharClass

"a-z" is a range; anything else is a set of characters.

Pooka
import lexie/config

assert config.parse_class("a-z") == config.Range("a", "z")
assert config.parse_class("_?!") == config.Set(["_", "?", "!"])
function

class_chars

src/lexie/config.pooka:755
class_chars(classes: List(CharClass)) -> List(String)
INFERREDfn(List(lexie/config.CharClass)) -> List(String)

Every member of the classes, in order, without repeats. Ranges are expanded only when both ends are ASCII.