From c83183e0e82de36e1d132198d25174271a027341 Mon Sep 17 00:00:00 2001 From: threeoh6000 Date: Tue, 16 Jan 2024 22:49:32 +0000 Subject: [PATCH] fix assorted issues relating to cutting keys off early --- Cargo.toml | 2 +- src/lexer.rs | 8 ++++ src/validator.rs | 52 ++++++++++++++++++++++--- src/validator_tests.rs | 88 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 143 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0febb4d..cc3aab7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frostwalker" -version = "0.0.3" +version = "0.0.4" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/lexer.rs b/src/lexer.rs index ff52821..b723abc 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -27,6 +27,7 @@ pub fn tokenize(source: &str) -> Vec { let mut tree: Vec = vec![]; for line in lines { + println!("{}", line); let mut added = false; let mut words: Vec<&str> = line.split(" ").collect(); let mut i = 0; @@ -138,6 +139,13 @@ pub fn tokenize(source: &str) -> Vec { break; } + if words.len() - i < 2 && tree[tree.len()-1].class == Class::NEWLINE { + tree.push(Token { class: Class::IDENTIFIER, value: Some(words[i].to_string()) }); + added = true; + i = i + 1; + continue; + } + tree.push(Token { class: Class::UNKNOWN, value: Some(words[i].to_string()) }); i = i + 1; } diff --git a/src/validator.rs b/src/validator.rs index eec7b92..b0ea801 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -7,6 +7,8 @@ enum ExpectedClass { EQUALS, NEWLINE, LITERALORSEPARATOR, + LITERAL, + SEPARATOR, } pub fn validate(tree: &Vec) -> Option { @@ -14,6 +16,7 @@ pub fn validate(tree: &Vec) -> Option { let mut i = 0; let mut line = 1; while i < tree.len() { + if tree[i].class == Class::NEWLINE && i == 0 { i = i + 1; line = line + 1; @@ -27,12 +30,11 @@ pub fn validate(tree: &Vec) -> Option { continue; } - if tree[i].class == Class::UNKNOWN { - return Some(format!("Invalid token {} at line {}.", tree[i].value.clone().unwrap_or("None".to_string()), line)); - } - if tree[i].class == Class::IDENTIFIER && expected_token == ExpectedClass::IDENTIFIER { i = i + 1; + if tree.len() - i == 0 { + return Some(format!("Key left undefined on line {}.", line)); + } expected_token = ExpectedClass::EQUALS; continue; } @@ -40,6 +42,9 @@ pub fn validate(tree: &Vec) -> Option { if tree[i].class == Class::EQUALS && expected_token == ExpectedClass::EQUALS { i = i + 1; expected_token = ExpectedClass::LITERALORSEPARATOR; + if tree.len() - i == 0 { + return Some(format!("Key left undefined on line {}.", line)); + } continue; } @@ -49,7 +54,44 @@ pub fn validate(tree: &Vec) -> Option { continue; } - return Some(format!("{:?} found where {:?} expected at line {}.", tree[i].class, expected_token, line)); + if (tree[i].class == Class::LITERAL || tree[i].class == Class::BOOLEAN) && expected_token == ExpectedClass::LITERAL { + i = i + 1; + expected_token = ExpectedClass::SEPARATOR; + continue; + } + + if tree[i].class == Class::SEPARATOR && expected_token == ExpectedClass::LITERALORSEPARATOR { + if tree[i].value.clone().unwrap_or("".to_string()) == "[".to_string() { + expected_token = ExpectedClass::LITERAL; + i = i + 1; + if tree.len() - i == 0 { + return Some(format!("Key left undefined on line {}.", line)); + } + continue; + } else { + return Some(format!("Invalid use of separator syntax on line {}.", line)); + } + } + + if tree[i].class == Class::SEPARATOR && expected_token == ExpectedClass::SEPARATOR { + if tree[i].value.clone().unwrap_or("".to_string()) == "]".to_string() { + expected_token = ExpectedClass::NEWLINE; + i = i + 1; + continue; + } else if tree[i].value.clone().unwrap_or("".to_string()) == ",".to_string() { + expected_token = ExpectedClass::LITERAL; + i = i + 1; + continue; + } else { + return Some(format!("Invalid use of separator syntax on line {}.", line)); + } + } + + if tree[i].value.is_none() { + return Some(format!("{:?} found where {:?} expected at line {}.", tree[i].class, expected_token, line)); + } else { + return Some(format!("{:?} {} found where {:?} expected at line {}.", tree[i].class, tree[i].value.clone().unwrap_or("None".to_string()), expected_token, line)); + } } return None; } diff --git a/src/validator_tests.rs b/src/validator_tests.rs index 5ba51a8..7b31460 100644 --- a/src/validator_tests.rs +++ b/src/validator_tests.rs @@ -11,6 +11,92 @@ fn single_key() { assert_eq!(result.is_none(), true); } +#[test] +fn single_key_array() { + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) }; + let strn = Token { class: Class::LITERAL, value: Some("value".to_string()) }; + let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) }; + let strn2 = Token { class: Class::LITERAL, value: Some("6".to_string()) }; + let t3 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) }; + let result = validator::validate(&vec![id, op, t1, strn, t2, strn2, t3]); + + assert_eq!(result.is_none(), true); +} + +#[test] +fn double_array() { + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) }; + let strn = Token { class: Class::LITERAL, value: Some("value".to_string()) }; + let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) }; + let t4 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) }; + let strn2 = Token { class: Class::LITERAL, value: Some("6".to_string()) }; + let t3 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) }; + let t5 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) }; + let result = validator::validate(&vec![id, op, t1, strn, t2, t4, strn2, t3, t5]); + + assert_eq!(result.unwrap_or("".to_string()), "SEPARATOR [ found where LITERAL expected at line 1."); +} + +#[test] +fn long_array() { + let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let op = Token { class: Class::EQUALS, value: None }; + let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) }; + let strn = Token { class: Class::LITERAL, value: Some("value".to_string()) }; + let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) }; + let strn2 = Token { class: Class::LITERAL, value: Some("6".to_string()) }; + let t3 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) }; + let strn3 = Token { class: Class::LITERAL, value: Some("7".to_string()) }; + let t4 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) }; + let strn4 = Token { class: Class::LITERAL, value: Some("8".to_string()) }; + let t5 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) }; + let strn5 = Token { class: Class::BOOLEAN, value: Some("true".to_string()) }; + let t6 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) }; + let result = validator::validate(&vec![id, op, t1, strn, t2, strn2, t3, strn3, t4, strn4, t5, strn5, t6]); + + assert_eq!(result.is_none(), true); +} + +#[test] +fn stray_element_with_no_value() { + let t1 = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let t2 = Token { class: Class::EQUALS, value: None }; + let t4 = Token { class: Class::EQUALS, value: None }; + let t3 = Token { class: Class::LITERAL, value: Some("10".to_string()) }; + let result = validator::validate(&vec![t1, t2, t4, t3]); + + assert_eq!(result.unwrap_or("".to_string()), "EQUALS found where LITERALORSEPARATOR expected at line 1.".to_string()); +} + +#[test] +fn unknown_element() { + let t1 = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let t4 = Token { class: Class::EQUALS, value: None }; + let t3 = Token { class: Class::UNKNOWN, value: Some("10".to_string()) }; + let result = validator::validate(&vec![t1, t4, t3]); + + assert_eq!(result.unwrap_or("".to_string()), "UNKNOWN 10 found where LITERALORSEPARATOR expected at line 1.".to_string()); +} + +#[test] +fn stray_element_with_no_value_on_newline() { + let at1 = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; + let at2 = Token { class: Class::EQUALS, value: None }; + let at3 = Token { class: Class::LITERAL, value: Some("10".to_string()) }; + let nl = Token { class: Class::NEWLINE, value: None }; + let t1 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) }; + let t2 = Token { class: Class::EQUALS, value: None }; + let t4 = Token { class: Class::EQUALS, value: None }; + let t3 = Token { class: Class::LITERAL, value: Some("11".to_string()) }; + let result = validator::validate(&vec![at1, at2, at3, nl, t1, t2, t4, t3]); + + assert_eq!(result.unwrap_or("".to_string()), "EQUALS found where LITERALORSEPARATOR expected at line 2.".to_string()); +} + #[test] fn triple_key() { let t1 = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) }; @@ -35,5 +121,5 @@ fn triple_key_no_newline() { let t6 = Token { class: Class::LITERAL, value: Some("13".to_string()) }; let result = validator::validate(&vec![t1, t2, t3, t4, t5, t6]); - assert_eq!(result.unwrap_or("".to_string()), "IDENTIFIER found where NEWLINE expected at line 1.".to_string()); + assert_eq!(result.unwrap_or("".to_string()), "IDENTIFIER key2 found where NEWLINE expected at line 1.".to_string()); }