From 117aa64c0a707bf4cd4108299207660f152a36d6 Mon Sep 17 00:00:00 2001 From: threeoh6000 Date: Tue, 16 Jan 2024 20:31:41 +0000 Subject: [PATCH] move lexer tests to new module --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/lexer_tests.rs | 182 ++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 185 +-------------------------------------------- 4 files changed, 185 insertions(+), 186 deletions(-) create mode 100644 src/lexer_tests.rs diff --git a/Cargo.lock b/Cargo.lock index e5a8fa0..872d1e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,4 +4,4 @@ version = 3 [[package]] name = "frostwalker" -version = "0.0.1" +version = "0.0.2" diff --git a/Cargo.toml b/Cargo.toml index 84bf36e..9efd551 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frostwalker" -version = "0.0.1" +version = "0.0.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/lexer_tests.rs b/src/lexer_tests.rs new file mode 100644 index 0000000..e7ea07e --- /dev/null +++ b/src/lexer_tests.rs @@ -0,0 +1,182 @@ +use super::lexer::{self, Token, Class}; + +#[test] +fn single_key() { + let tree = lexer::tokenize("key = \"value\""); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; + let manual_tree = vec![id, op, strn]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn single_key_array() { + let tree = lexer::tokenize("key = [ \"\\\"value\", 6 ]"); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) }; + let strn = Token { class: Class::STRING, value: Some("\"value".to_string()) }; + let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) }; + let strn2 = Token { class: Class::INTEGER, value: Some("6".to_string()) }; + let t3 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) }; + let manual_tree = vec![id, op, t1, strn, t2, strn2, t3]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn single_key_unknown_token() { + let tree = lexer::tokenize("key = ^"); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::UNKNOWN, value: Some("^".to_string()) }; + let manual_tree = vec![id, op, strn]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn single_key_escaped_double_quote_in_string() { + let tree = lexer::tokenize("key = \"\\\"value\""); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::STRING, value: Some("\"value".to_string()) }; + let manual_tree = vec![id, op, strn]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn integers() { + let tree = lexer::tokenize("key = 123567\r\nkey2 = -400"); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::INTEGER, value: Some("123567".to_string()) }; + let nl = Token { class: Class::NEWLINE, value: None }; + let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) }; + let op2 = Token { class: Class::EQUALS, value: None }; + let strn2 = Token { class: Class::INTEGER, value: Some("-400".to_string()) }; + let manual_tree = vec![id, op, strn, nl, id2, op2, strn2]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn booleans() { + let tree = lexer::tokenize("key = true\r\nkey2 = FALSE"); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::TRUE, value: None }; + let nl = Token { class: Class::NEWLINE, value: None }; + let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) }; + let op2 = Token { class: Class::EQUALS, value: None }; + let strn2 = Token { class: Class::FALSE, value: None }; + let manual_tree = vec![id, op, strn, nl, id2, op2, strn2]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn single_key_space_in_string() { + let tree = lexer::tokenize("key = \"a b c\""); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::STRING, value: Some("a b c".to_string()) }; + let manual_tree = vec![id, op, strn]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn single_key_double_space_in_string() { + let tree = lexer::tokenize("key = \"a b c d\""); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::STRING, value: Some("a b c d".to_string()) }; + let manual_tree = vec![id, op, strn]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn triple_key() { + let tree = lexer::tokenize("key = \"value\"\r\nkey2 = \"value2\"\r\nkey3 = \"value3\""); + + let nl = Token { class: Class::NEWLINE, value: None }; + let nl2 = Token { class: Class::NEWLINE, value: None }; + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; + let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) }; + let op2 = Token { class: Class::EQUALS, value: None }; + let strn2 = Token { class: Class::STRING, value: Some("value2".to_string()) }; + let id3 = Token { class: Class::IDENTIFIER, value: Some("key3".to_string()) }; + let op3 = Token { class: Class::EQUALS, value: None }; + let strn3 = Token { class: Class::STRING, value: Some("value3".to_string()) }; + + let manual_tree = vec![id, op, strn, nl, id2, op2, strn2, nl2, id3, op3, strn3]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn comment() { + let tree = lexer::tokenize("# This is a comment\r\nkey = \"value\""); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; + + let manual_tree = vec![id, op, strn]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn inline_comment() { + let tree = lexer::tokenize("# This is a comment\r\nkey = \"value\" # I'm inline!"); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; + + let manual_tree = vec![id, op, strn]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn inline_comment_no_spaces() { + let tree = lexer::tokenize("#This is a comment\r\nkey = \"value\" #I'm inline!"); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; + + let manual_tree = vec![id, op, strn]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn quoted_identifier() { + let tree = lexer::tokenize("\"key\" = \"value\""); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; + + let manual_tree = vec![id, op, strn]; + assert_eq!(tree, manual_tree); +} + +#[test] +fn double_equals() { + let tree = lexer::tokenize("\"key\" = ="); + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let op2 = Token { class: Class::EQUALS, value: None }; + + let manual_tree = vec![id, op, op2]; + assert_eq!(tree, manual_tree); +} + diff --git a/src/lib.rs b/src/lib.rs index 220b882..73e423c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,187 +1,4 @@ pub mod lexer; #[cfg(test)] -mod lexer_tests { - use super::lexer::{self, Token, Class}; - - #[test] - fn single_key() { - let tree = lexer::tokenize("key = \"value\""); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; - let manual_tree = vec![id, op, strn]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn single_key_array() { - let tree = lexer::tokenize("key = [ \"\\\"value\", 6 ]"); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) }; - let strn = Token { class: Class::STRING, value: Some("\"value".to_string()) }; - let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) }; - let strn2 = Token { class: Class::INTEGER, value: Some("6".to_string()) }; - let t3 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) }; - let manual_tree = vec![id, op, t1, strn, t2, strn2, t3]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn single_key_unknown_token() { - let tree = lexer::tokenize("key = ^"); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::UNKNOWN, value: Some("^".to_string()) }; - let manual_tree = vec![id, op, strn]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn single_key_escaped_double_quote_in_string() { - let tree = lexer::tokenize("key = \"\\\"value\""); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::STRING, value: Some("\"value".to_string()) }; - let manual_tree = vec![id, op, strn]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn integers() { - let tree = lexer::tokenize("key = 123567\r\nkey2 = -400"); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::INTEGER, value: Some("123567".to_string()) }; - let nl = Token { class: Class::NEWLINE, value: None }; - let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) }; - let op2 = Token { class: Class::EQUALS, value: None }; - let strn2 = Token { class: Class::INTEGER, value: Some("-400".to_string()) }; - let manual_tree = vec![id, op, strn, nl, id2, op2, strn2]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn booleans() { - let tree = lexer::tokenize("key = true\r\nkey2 = FALSE"); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::TRUE, value: None }; - let nl = Token { class: Class::NEWLINE, value: None }; - let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) }; - let op2 = Token { class: Class::EQUALS, value: None }; - let strn2 = Token { class: Class::FALSE, value: None }; - let manual_tree = vec![id, op, strn, nl, id2, op2, strn2]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn single_key_space_in_string() { - let tree = lexer::tokenize("key = \"a b c\""); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::STRING, value: Some("a b c".to_string()) }; - let manual_tree = vec![id, op, strn]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn single_key_double_space_in_string() { - let tree = lexer::tokenize("key = \"a b c d\""); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::STRING, value: Some("a b c d".to_string()) }; - let manual_tree = vec![id, op, strn]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn triple_key() { - let tree = lexer::tokenize("key = \"value\"\r\nkey2 = \"value2\"\r\nkey3 = \"value3\""); - - let nl = Token { class: Class::NEWLINE, value: None }; - let nl2 = Token { class: Class::NEWLINE, value: None }; - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; - let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) }; - let op2 = Token { class: Class::EQUALS, value: None }; - let strn2 = Token { class: Class::STRING, value: Some("value2".to_string()) }; - let id3 = Token { class: Class::IDENTIFIER, value: Some("key3".to_string()) }; - let op3 = Token { class: Class::EQUALS, value: None }; - let strn3 = Token { class: Class::STRING, value: Some("value3".to_string()) }; - - let manual_tree = vec![id, op, strn, nl, id2, op2, strn2, nl2, id3, op3, strn3]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn comment() { - let tree = lexer::tokenize("# This is a comment\r\nkey = \"value\""); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; - - let manual_tree = vec![id, op, strn]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn inline_comment() { - let tree = lexer::tokenize("# This is a comment\r\nkey = \"value\" # I'm inline!"); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; - - let manual_tree = vec![id, op, strn]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn inline_comment_no_spaces() { - let tree = lexer::tokenize("#This is a comment\r\nkey = \"value\" #I'm inline!"); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; - - let manual_tree = vec![id, op, strn]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn quoted_identifier() { - let tree = lexer::tokenize("\"key\" = \"value\""); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let strn = Token { class: Class::STRING, value: Some("value".to_string()) }; - - let manual_tree = vec![id, op, strn]; - assert_eq!(tree, manual_tree); - } - - #[test] - fn double_equals() { - let tree = lexer::tokenize("\"key\" = ="); - - let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; - let op = Token { class: Class::EQUALS, value: None }; - let op2 = Token { class: Class::EQUALS, value: None }; - - let manual_tree = vec![id, op, op2]; - assert_eq!(tree, manual_tree); - } -} - +mod lexer_tests;