move lexer tests to new module
This commit is contained in:
parent
3744490034
commit
117aa64c0a
GPG key ID:
801FE4AD456E922C
4 changed files with 185 additions and 186 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -4,4 +4,4 @@ version = 3
|
|||
|
||||
[[package]]
|
||||
name = "frostwalker"
|
||||
version = "0.0.1"
|
||||
version = "0.0.2"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "frostwalker"
|
||||
version = "0.0.1"
|
||||
version = "0.0.2"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
|
|
|||
182
src/lexer_tests.rs
Normal file
182
src/lexer_tests.rs
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
use super::lexer::{self, Token, Class};
|
||||
|
||||
#[test]
|
||||
fn single_key() {
|
||||
let tree = lexer::tokenize("key = \"value\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_array() {
|
||||
let tree = lexer::tokenize("key = [ \"\\\"value\", 6 ]");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) };
|
||||
let strn = Token { class: Class::STRING, value: Some("\"value".to_string()) };
|
||||
let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) };
|
||||
let strn2 = Token { class: Class::INTEGER, value: Some("6".to_string()) };
|
||||
let t3 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) };
|
||||
let manual_tree = vec![id, op, t1, strn, t2, strn2, t3];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_unknown_token() {
|
||||
let tree = lexer::tokenize("key = ^");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::UNKNOWN, value: Some("^".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_escaped_double_quote_in_string() {
|
||||
let tree = lexer::tokenize("key = \"\\\"value\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("\"value".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integers() {
|
||||
let tree = lexer::tokenize("key = 123567\r\nkey2 = -400");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::INTEGER, value: Some("123567".to_string()) };
|
||||
let nl = Token { class: Class::NEWLINE, value: None };
|
||||
let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) };
|
||||
let op2 = Token { class: Class::EQUALS, value: None };
|
||||
let strn2 = Token { class: Class::INTEGER, value: Some("-400".to_string()) };
|
||||
let manual_tree = vec![id, op, strn, nl, id2, op2, strn2];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn booleans() {
|
||||
let tree = lexer::tokenize("key = true\r\nkey2 = FALSE");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::TRUE, value: None };
|
||||
let nl = Token { class: Class::NEWLINE, value: None };
|
||||
let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) };
|
||||
let op2 = Token { class: Class::EQUALS, value: None };
|
||||
let strn2 = Token { class: Class::FALSE, value: None };
|
||||
let manual_tree = vec![id, op, strn, nl, id2, op2, strn2];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_space_in_string() {
|
||||
let tree = lexer::tokenize("key = \"a b c\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("a b c".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_double_space_in_string() {
|
||||
let tree = lexer::tokenize("key = \"a b c d\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("a b c d".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn triple_key() {
|
||||
let tree = lexer::tokenize("key = \"value\"\r\nkey2 = \"value2\"\r\nkey3 = \"value3\"");
|
||||
|
||||
let nl = Token { class: Class::NEWLINE, value: None };
|
||||
let nl2 = Token { class: Class::NEWLINE, value: None };
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) };
|
||||
let op2 = Token { class: Class::EQUALS, value: None };
|
||||
let strn2 = Token { class: Class::STRING, value: Some("value2".to_string()) };
|
||||
let id3 = Token { class: Class::IDENTIFIER, value: Some("key3".to_string()) };
|
||||
let op3 = Token { class: Class::EQUALS, value: None };
|
||||
let strn3 = Token { class: Class::STRING, value: Some("value3".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn, nl, id2, op2, strn2, nl2, id3, op3, strn3];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment() {
|
||||
let tree = lexer::tokenize("# This is a comment\r\nkey = \"value\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_comment() {
|
||||
let tree = lexer::tokenize("# This is a comment\r\nkey = \"value\" # I'm inline!");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_comment_no_spaces() {
|
||||
let tree = lexer::tokenize("#This is a comment\r\nkey = \"value\" #I'm inline!");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quoted_identifier() {
|
||||
let tree = lexer::tokenize("\"key\" = \"value\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn double_equals() {
|
||||
let tree = lexer::tokenize("\"key\" = =");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let op2 = Token { class: Class::EQUALS, value: None };
|
||||
|
||||
let manual_tree = vec![id, op, op2];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
185
src/lib.rs
185
src/lib.rs
|
|
@ -1,187 +1,4 @@
|
|||
pub mod lexer;
|
||||
|
||||
#[cfg(test)]
|
||||
mod lexer_tests {
|
||||
use super::lexer::{self, Token, Class};
|
||||
|
||||
#[test]
|
||||
fn single_key() {
|
||||
let tree = lexer::tokenize("key = \"value\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_array() {
|
||||
let tree = lexer::tokenize("key = [ \"\\\"value\", 6 ]");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) };
|
||||
let strn = Token { class: Class::STRING, value: Some("\"value".to_string()) };
|
||||
let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) };
|
||||
let strn2 = Token { class: Class::INTEGER, value: Some("6".to_string()) };
|
||||
let t3 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) };
|
||||
let manual_tree = vec![id, op, t1, strn, t2, strn2, t3];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_unknown_token() {
|
||||
let tree = lexer::tokenize("key = ^");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::UNKNOWN, value: Some("^".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_escaped_double_quote_in_string() {
|
||||
let tree = lexer::tokenize("key = \"\\\"value\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("\"value".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integers() {
|
||||
let tree = lexer::tokenize("key = 123567\r\nkey2 = -400");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::INTEGER, value: Some("123567".to_string()) };
|
||||
let nl = Token { class: Class::NEWLINE, value: None };
|
||||
let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) };
|
||||
let op2 = Token { class: Class::EQUALS, value: None };
|
||||
let strn2 = Token { class: Class::INTEGER, value: Some("-400".to_string()) };
|
||||
let manual_tree = vec![id, op, strn, nl, id2, op2, strn2];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn booleans() {
|
||||
let tree = lexer::tokenize("key = true\r\nkey2 = FALSE");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::TRUE, value: None };
|
||||
let nl = Token { class: Class::NEWLINE, value: None };
|
||||
let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) };
|
||||
let op2 = Token { class: Class::EQUALS, value: None };
|
||||
let strn2 = Token { class: Class::FALSE, value: None };
|
||||
let manual_tree = vec![id, op, strn, nl, id2, op2, strn2];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_space_in_string() {
|
||||
let tree = lexer::tokenize("key = \"a b c\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("a b c".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_key_double_space_in_string() {
|
||||
let tree = lexer::tokenize("key = \"a b c d\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("a b c d".to_string()) };
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn triple_key() {
|
||||
let tree = lexer::tokenize("key = \"value\"\r\nkey2 = \"value2\"\r\nkey3 = \"value3\"");
|
||||
|
||||
let nl = Token { class: Class::NEWLINE, value: None };
|
||||
let nl2 = Token { class: Class::NEWLINE, value: None };
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) };
|
||||
let op2 = Token { class: Class::EQUALS, value: None };
|
||||
let strn2 = Token { class: Class::STRING, value: Some("value2".to_string()) };
|
||||
let id3 = Token { class: Class::IDENTIFIER, value: Some("key3".to_string()) };
|
||||
let op3 = Token { class: Class::EQUALS, value: None };
|
||||
let strn3 = Token { class: Class::STRING, value: Some("value3".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn, nl, id2, op2, strn2, nl2, id3, op3, strn3];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment() {
|
||||
let tree = lexer::tokenize("# This is a comment\r\nkey = \"value\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_comment() {
|
||||
let tree = lexer::tokenize("# This is a comment\r\nkey = \"value\" # I'm inline!");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_comment_no_spaces() {
|
||||
let tree = lexer::tokenize("#This is a comment\r\nkey = \"value\" #I'm inline!");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quoted_identifier() {
|
||||
let tree = lexer::tokenize("\"key\" = \"value\"");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let strn = Token { class: Class::STRING, value: Some("value".to_string()) };
|
||||
|
||||
let manual_tree = vec![id, op, strn];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn double_equals() {
|
||||
let tree = lexer::tokenize("\"key\" = =");
|
||||
|
||||
let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
|
||||
let op = Token { class: Class::EQUALS, value: None };
|
||||
let op2 = Token { class: Class::EQUALS, value: None };
|
||||
|
||||
let manual_tree = vec![id, op, op2];
|
||||
assert_eq!(tree, manual_tree);
|
||||
}
|
||||
}
|
||||
|
||||
mod lexer_tests;
|
||||
|
|
|
|||
Loading…
Reference in a new issue