fix assorted issues relating to cutting keys off early

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "frostwalker"
-version = "0.0.3"
+version = "0.0.4"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

src/lexer.rs

@@ -27,6 +27,7 @@ pub fn tokenize(source: &str) -> Vec<Token> {
 	let mut tree: Vec<Token> = vec![];
 
 	for line in lines {
+		println!("{}", line);
 		let mut added = false;
 		let mut words: Vec<&str> = line.split(" ").collect();
 		let mut i = 0;
@@ -138,6 +139,13 @@ pub fn tokenize(source: &str) -> Vec<Token> {
 				break;
 			}
 
+			if words.len() - i < 2 && tree[tree.len()-1].class == Class::NEWLINE {
+				tree.push(Token { class: Class::IDENTIFIER, value: Some(words[i].to_string()) });
+				added = true;
+				i = i + 1;
+				continue;
+			}
+
 			tree.push(Token { class: Class::UNKNOWN, value: Some(words[i].to_string()) });
 			i = i + 1;
 		}

src/validator.rs

@@ -7,6 +7,8 @@ enum ExpectedClass {
 	EQUALS,
 	NEWLINE,
 	LITERALORSEPARATOR,
+	LITERAL,
+	SEPARATOR,
 }
 
 pub fn validate(tree: &Vec<Token>) -> Option<String> {
@@ -14,6 +16,7 @@ pub fn validate(tree: &Vec<Token>) -> Option<String> {
 	let mut i = 0;
 	let mut line = 1;
 	while i < tree.len() {
+
 		if tree[i].class == Class::NEWLINE && i == 0 {
 			i = i + 1;
 			line = line + 1;
@@ -27,12 +30,11 @@ pub fn validate(tree: &Vec<Token>) -> Option<String> {
 			continue;
 		}
 
-		if tree[i].class == Class::UNKNOWN {
-			return Some(format!("Invalid token {} at line {}.", tree[i].value.clone().unwrap_or("None".to_string()), line));
-		}
-
 		if tree[i].class == Class::IDENTIFIER && expected_token == ExpectedClass::IDENTIFIER {
 			i = i + 1;
+			if tree.len() - i == 0 {
+				return Some(format!("Key left undefined on line {}.", line));
+			}
 			expected_token = ExpectedClass::EQUALS;
 			continue;
 		}
@@ -40,6 +42,9 @@ pub fn validate(tree: &Vec<Token>) -> Option<String> {
 		if tree[i].class == Class::EQUALS && expected_token == ExpectedClass::EQUALS {
 			i = i + 1;
 			expected_token = ExpectedClass::LITERALORSEPARATOR;
+			if tree.len() - i == 0 {
+				return Some(format!("Key left undefined on line {}.", line));
+			}
 			continue;
 		}
 
@@ -49,7 +54,44 @@ pub fn validate(tree: &Vec<Token>) -> Option<String> {
 			continue;
 		}
 
+		if (tree[i].class == Class::LITERAL || tree[i].class == Class::BOOLEAN) && expected_token == ExpectedClass::LITERAL {
+			i = i + 1;
+			expected_token = ExpectedClass::SEPARATOR;
+			continue;
+		}
+
+		if tree[i].class == Class::SEPARATOR && expected_token == ExpectedClass::LITERALORSEPARATOR {
+			if tree[i].value.clone().unwrap_or("".to_string()) == "[".to_string() {
+				expected_token = ExpectedClass::LITERAL;
+				i = i + 1;
+				if tree.len() - i == 0 {
+					return Some(format!("Key left undefined on line {}.", line));
+				}
+				continue;
+			} else {
+				return Some(format!("Invalid use of separator syntax on line {}.", line));
+			}
+		}
+
+		if tree[i].class == Class::SEPARATOR && expected_token == ExpectedClass::SEPARATOR {
+			if tree[i].value.clone().unwrap_or("".to_string()) == "]".to_string() {
+				expected_token = ExpectedClass::NEWLINE;
+				i = i + 1;
+				continue;
+			} else if tree[i].value.clone().unwrap_or("".to_string()) == ",".to_string() {
+				expected_token = ExpectedClass::LITERAL;
+				i = i + 1;
+				continue;
+			} else {
+				return Some(format!("Invalid use of separator syntax on line {}.", line));
+			}		
+		}
+
+		if tree[i].value.is_none() {
 			return Some(format!("{:?} found where {:?} expected at line {}.", tree[i].class, expected_token, line));
+		} else {
+			return Some(format!("{:?} {} found where {:?} expected at line {}.", tree[i].class, tree[i].value.clone().unwrap_or("None".to_string()), expected_token, line));
+		}
 	}
 	return None;
 }

src/validator_tests.rs

@@ -11,6 +11,92 @@ fn single_key() {
 	assert_eq!(result.is_none(), true);
 }
 
+#[test]
+fn single_key_array() {
+	let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
+	let op = Token { class: Class::EQUALS, value: None };
+	let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) };
+	let strn = Token { class: Class::LITERAL, value: Some("value".to_string()) };
+	let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) };
+	let strn2 = Token { class: Class::LITERAL, value: Some("6".to_string()) };
+	let t3 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) };
+	let result = validator::validate(&vec![id, op, t1, strn, t2, strn2, t3]);
+
+	assert_eq!(result.is_none(), true);
+}
+
+#[test]
+fn double_array() {
+	let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
+	let op = Token { class: Class::EQUALS, value: None };
+	let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) };
+	let strn = Token { class: Class::LITERAL, value: Some("value".to_string()) };
+	let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) };
+	let t4 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) };
+	let strn2 = Token { class: Class::LITERAL, value: Some("6".to_string()) };
+	let t3 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) };
+	let t5 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) };
+	let result = validator::validate(&vec![id, op, t1, strn, t2, t4, strn2, t3, t5]);
+
+	assert_eq!(result.unwrap_or("".to_string()), "SEPARATOR [ found where LITERAL expected at line 1.");
+}
+
+#[test]
+fn long_array() {
+	let id = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
+	let op = Token { class: Class::EQUALS, value: None };
+	let t1 = Token { class: Class::SEPARATOR, value: Some("[".to_string()) };
+	let strn = Token { class: Class::LITERAL, value: Some("value".to_string()) };
+	let t2 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) };
+	let strn2 = Token { class: Class::LITERAL, value: Some("6".to_string()) };
+	let t3 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) };
+	let strn3 = Token { class: Class::LITERAL, value: Some("7".to_string()) };
+	let t4 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) };
+	let strn4 = Token { class: Class::LITERAL, value: Some("8".to_string()) };
+	let t5 = Token { class: Class::SEPARATOR, value: Some(",".to_string()) };
+	let strn5 = Token { class: Class::BOOLEAN, value: Some("true".to_string()) };
+	let t6 = Token { class: Class::SEPARATOR, value: Some("]".to_string()) };
+	let result = validator::validate(&vec![id, op, t1, strn, t2, strn2, t3, strn3, t4, strn4, t5, strn5, t6]);
+
+	assert_eq!(result.is_none(), true);
+}
+
+#[test]
+fn stray_element_with_no_value() {
+	let t1 = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
+	let t2 = Token { class: Class::EQUALS, value: None };
+	let t4 = Token { class: Class::EQUALS, value: None };
+	let t3 = Token { class: Class::LITERAL, value: Some("10".to_string()) };
+	let result = validator::validate(&vec![t1, t2, t4, t3]);
+
+	assert_eq!(result.unwrap_or("".to_string()), "EQUALS found where LITERALORSEPARATOR expected at line 1.".to_string());
+}
+
+#[test]
+fn unknown_element() {
+	let t1 = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
+	let t4 = Token { class: Class::EQUALS, value: None };
+	let t3 = Token { class: Class::UNKNOWN, value: Some("10".to_string()) };
+	let result = validator::validate(&vec![t1, t4, t3]);
+
+	assert_eq!(result.unwrap_or("".to_string()), "UNKNOWN 10 found where LITERALORSEPARATOR expected at line 1.".to_string());
+}
+
+#[test]
+fn stray_element_with_no_value_on_newline() {
+	let at1 = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
+	let at2 = Token { class: Class::EQUALS, value: None };
+	let at3 = Token { class: Class::LITERAL, value: Some("10".to_string()) };
+	let nl = Token { class: Class::NEWLINE, value: None };
+	let t1 = Token { class: Class::IDENTIFIER, value: Some("key2".to_string()) };
+	let t2 = Token { class: Class::EQUALS, value: None };
+	let t4 = Token { class: Class::EQUALS, value: None };
+	let t3 = Token { class: Class::LITERAL, value: Some("11".to_string()) };
+	let result = validator::validate(&vec![at1, at2, at3, nl, t1, t2, t4, t3]);
+
+	assert_eq!(result.unwrap_or("".to_string()), "EQUALS found where LITERALORSEPARATOR expected at line 2.".to_string());
+}
+
 #[test]
 fn triple_key() {
 	let t1 = Token { class: Class::IDENTIFIER, value: Some("key".to_string()) };
@@ -35,5 +121,5 @@ fn triple_key_no_newline() {
 	let t6 = Token { class: Class::LITERAL, value: Some("13".to_string()) };
 	let result = validator::validate(&vec![t1, t2, t3, t4, t5, t6]);
 
-	assert_eq!(result.unwrap_or("".to_string()), "IDENTIFIER found where NEWLINE expected at line 1.".to_string());
+	assert_eq!(result.unwrap_or("".to_string()), "IDENTIFIER key2 found where NEWLINE expected at line 1.".to_string());
 }