From ed882263a81a326d353576388c7c0a03d2728cf8 Mon Sep 17 00:00:00 2001 From: threeoh6000 Date: Fri, 19 Jan 2024 00:07:59 +0000 Subject: [PATCH] add formatter, move Token and Class types out to main library binary to make the validator and formatter less dependent on the lexer --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 4 +-- src/formatter.rs | 41 +++++++++++++++++++++++++++++ src/formatter_tests.rs | 60 ++++++++++++++++++++++++++++++++++++++++++ src/lexer.rs | 19 +------------ src/lexer_tests.rs | 3 ++- src/lib.rs | 24 +++++++++++++++++ src/validator.rs | 2 +- src/validator_tests.rs | 2 +- tests/integration.rs | 2 +- 11 files changed, 135 insertions(+), 26 deletions(-) create mode 100644 src/formatter.rs create mode 100644 src/formatter_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 909408e..195def2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,4 +4,4 @@ version = 3 [[package]] name = "frostwalker" -version = "0.0.5" +version = "0.0.6" diff --git a/Cargo.toml b/Cargo.toml index f548fc6..4e06241 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frostwalker" -version = "0.0.5" +version = "0.0.6" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README.md b/README.md index 6b818d2..a67c8ab 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Frostwalker -*Frostwalker is not stable nor ready for dependency development yet. Unless you plan to implement your own formatter, please wait for 0.1.0 to release. 0.0 versions are placeholders and will not be tagged.* +*Frostwalker is still in development and should be considered alpha level software. The parser stack is complete but evaluation is still ongoing.* -A TOML-like configuration language parser for Rust. At present, the crate has a lexer and an incomplete validator with unit tests written for common cases. A formatter will eventually be added. +A TOML-like configuration language parser for Rust. The crate features a parser stack made up of a lexer, validator, and formatter which all have unit tests and integration tests. ## Justification The Rust crate `toml` pulls in 8 other crates to do its job, including `serde`. While this may not be irksome to basically most Rust developers, I'm used to using severely underpowered Intel hardware so low compile times is a big focus for me so a crate made by me that requires only the standard library compared to one that requires other dependencies is preferrable. diff --git a/src/formatter.rs b/src/formatter.rs new file mode 100644 index 0000000..027d178 --- /dev/null +++ b/src/formatter.rs @@ -0,0 +1,41 @@ +use super::{Token, Class}; +use std::collections::HashMap; + +pub fn format(tree: Vec) -> HashMap { + let mut output = HashMap::new(); + let mut current_key = "".to_string(); + let mut current_index = 0; + let mut i = 0; + while i < tree.len() { + if tree[i].class == Class::IDENTIFIER { + current_key = tree[i].value.clone().unwrap_or("".to_string()); + i = i + 1; + continue; + } + + if tree[i].class == Class::LITERAL && current_key != "".to_string() { + output.insert(current_key, tree[i].value.clone().unwrap_or("".to_string())); + current_key = "".to_string(); + i = i + 1; + continue; + } + + if tree[i].class == Class::SEPARATOR && current_key != "".to_string() && tree[i].value.clone().unwrap_or("".to_string()) == "[".to_string() { + current_index = 0; + loop { + i = i + 1; + output.insert(format!("{}[{}]", current_key, current_index), tree[i].value.clone().unwrap_or("".to_string())); + current_index = current_index + 1; + i = i + 1; + if tree[i].class == Class::SEPARATOR && tree[i].value.clone().unwrap_or("".to_string()) == ",".to_string() { + continue; + } else { + break; + } + } + } + + i = i + 1; + } + return output; +} diff --git a/src/formatter_tests.rs b/src/formatter_tests.rs new file mode 100644 index 0000000..f15d530 --- /dev/null +++ b/src/formatter_tests.rs @@ -0,0 +1,60 @@ +use super::{Token, Class}; +use super::formatter; + +#[test] +fn single_key() { + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::LITERAL, value: Some("value".to_string()) }; + let manual_tree = vec![id, op, strn]; + let tree = formatter::format(manual_tree); + assert_eq!(format!("{:?}", tree), "{\"key\": \"value\"}"); +} + +#[test] +fn single_key_escaped_double_quote() { + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::LITERAL, value: Some("\"value".to_string()) }; + let manual_tree = vec![id, op, strn]; + let tree = formatter::format(manual_tree); + assert_eq!(format!("{:?}", tree), "{\"key\": \"\\\"value\"}"); +} + +#[test] +fn double_key() { + + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let strn = Token { class: Class::LITERAL, value: Some("value".to_string()) }; + let nl = Token { class: Class::NEWLINE, value: None }; + let id2 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) }; + let op2 = Token { class: Class::EQUALS, value: None }; + let strn2 = Token { class: Class::LITERAL, value: Some("value2".to_string()) }; + let manual_tree = vec![id, op, strn, nl, id2, op2, strn2]; + let tree = formatter::format(manual_tree); + if !(tree.get("key").unwrap() == "value") || !(tree.get("key2").unwrap() == "value2") { + println!("{:?}", tree); + panic!(); + } +} + +#[test] +fn array() { + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) }; + let strn = Token { class: Class::LITERAL, value: Some("value".to_string()) }; + let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) }; + let strn2 = Token { class: Class::LITERAL, value: Some("6".to_string()) }; + let t3 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) }; + + let manual_tree = vec![id, op, t1, strn, t2, strn2, t3]; + let tree = formatter::format(manual_tree); + if !(tree.get("key[0]").unwrap() == "value") || !(tree.get("key[1]").unwrap() == "6") { + println!("{:?}", tree); + panic!(); + } +} diff --git a/src/lexer.rs b/src/lexer.rs index 5030dbb..f5691d2 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,21 +1,4 @@ -#[derive(Debug)] -#[derive(PartialEq)] -pub struct Token { - pub class: Class, - pub value: Option, -} - -#[derive(Debug)] -#[derive(PartialEq)] -pub enum Class { - IDENTIFIER, - SEPARATOR, - EQUALS, - LITERAL, - NEWLINE, - BOOLEAN, - UNKNOWN, -} +use super::{Token, Class}; pub fn tokenise(source: &str) -> Vec { return tokenize(source); diff --git a/src/lexer_tests.rs b/src/lexer_tests.rs index 1c4a499..ef329c6 100644 --- a/src/lexer_tests.rs +++ b/src/lexer_tests.rs @@ -1,4 +1,5 @@ -use super::lexer::{self, Token, Class}; +use super::{Token, Class}; +use super::lexer; #[test] fn single_key() { diff --git a/src/lib.rs b/src/lib.rs index d3fbffd..238f65d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,32 @@ +#[derive(Debug)] +#[derive(PartialEq)] +pub struct Token { + pub class: Class, + pub value: Option, +} + +#[derive(Debug)] +#[derive(PartialEq)] +pub enum Class { + IDENTIFIER, + SEPARATOR, + EQUALS, + LITERAL, + NEWLINE, + BOOLEAN, + UNKNOWN, +} + pub mod lexer; pub mod validator; +pub mod formatter; #[cfg(test)] mod lexer_tests; #[cfg(test)] mod validator_tests; + +#[cfg(test)] +mod formatter_tests; + diff --git a/src/validator.rs b/src/validator.rs index b0ea801..b45089a 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -1,4 +1,4 @@ -use super::lexer::{Token, Class}; +use super::{Token, Class}; #[derive(Debug)] #[derive(PartialEq)] diff --git a/src/validator_tests.rs b/src/validator_tests.rs index 738422f..b460fbc 100644 --- a/src/validator_tests.rs +++ b/src/validator_tests.rs @@ -1,5 +1,5 @@ use super::validator; -use super::lexer::{Token, Class}; +use super::{Token, Class}; #[test] fn single_key() { diff --git a/tests/integration.rs b/tests/integration.rs index e45c1d7..912a594 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1,4 +1,4 @@ -use frostwalker::lexer::{self, Token, Class}; +use frostwalker::lexer; use frostwalker::validator; #[test]