diff --git a/src/parsing/checks/parser.rs b/src/parsing/checks/parser.rs index 3fd9df3b..3f3b9d91 100644 --- a/src/parsing/checks/parser.rs +++ b/src/parsing/checks/parser.rs @@ -485,7 +485,7 @@ fn character_delimited_blocks() { let mut input = Parser::new(); input.initialize("{ todo() }"); - let result = input.take_block_chars("inline code", '{', '}', true, |parser| { + let result = input.take_block_chars("inline code", '{', '}', |parser| { let text = parser.source; assert_eq!(text, " todo() "); Ok(true) @@ -497,7 +497,7 @@ fn character_delimited_blocks() { // we find ourselves parsing them, so subparser() won't work. input.initialize("XhelloX world"); - let result = input.take_block_chars("", 'X', 'X', false, |parser| { + let result = input.take_block_chars("", 'X', 'X', |parser| { let text = parser.source; assert_eq!(text, "hello"); Ok(true) @@ -511,7 +511,7 @@ fn skip_string_content_flag() { // Test skip_string_content: true - should ignore braces inside strings input.initialize(r#"{ "string with { brace" }"#); - let result = input.take_block_chars("code block", '{', '}', true, |parser| { + let result = input.take_block_chars("code block", '{', '}', |parser| { let text = parser.source; assert_eq!(text, r#" "string with { brace" "#); Ok(true) @@ -520,7 +520,7 @@ fn skip_string_content_flag() { // Test skip_string_content: false - should treat braces normally input.initialize(r#""string with } brace""#); - let result = input.take_block_chars("string content", '"', '"', false, |parser| { + let result = input.take_block_chars("string content", '"', '"', |parser| { let text = parser.source; assert_eq!(text, "string with } brace"); Ok(true) @@ -1212,6 +1212,46 @@ fn test_potential_procedure_declaration_is_superset() { assert!(!is_procedure_declaration("Ask these questions :")); assert!(!potential_procedure_declaration("Ask these questions :")); + // Nor prose, code, titles, or responses that merely contain a colon. A + // procedure name is a lowercase identifier, so none of these can be one. + // Lines taken from the example corpus + assert!(!is_procedure_declaration( + "Ask yourself: \"What can I do to influence the situation?\" Interpret" + )); + assert!(!is_procedure_declaration("Assuming you do, then:")); + assert!(!is_procedure_declaration("Warning: Important")); + assert!(!is_procedure_declaration("Ingredients: Leaves, Water")); + assert!(!is_procedure_declaration( + "bringing web1:80 and web2:80 into service" + )); + assert!(!is_procedure_declaration("# Choosing: Overview")); + assert!(!is_procedure_declaration("'Yes: proceed' | 'No'")); + assert!(!is_procedure_declaration("note: be careful here")); + assert!(!is_procedure_declaration( + "exec(\"curl http://127.0.0.1:48080/simple/\")" + )); + + // ... but a declaration whose signature is malformed is still one, the + // author's intent being plain. An empty or arrow-bearing remainder is + // enough to make the line an attempt at a declaration + assert!(is_procedure_declaration("broken_proc : A ->")); + assert!(is_procedure_declaration("f : B")); + assert!(potential_procedure_declaration("MyProcedure :")); + assert!(potential_procedure_declaration("my_proc(a, b :")); + assert!(potential_procedure_declaration("f( :")); + + // A declaration sets its colon apart from the name; prose punctuates the + // other way and so is never one, wherever it appears + assert!(is_procedure_declaration("foo : A -> B")); + assert!(is_procedure_declaration("foo : A -> B")); + assert!(is_procedure_declaration("foo\t: A -> B")); + assert!(is_procedure_declaration(" foo : A -> B")); + assert!(!is_procedure_declaration("foo: A -> B")); + assert!(!potential_procedure_declaration("foo: A -> B")); + + // ... except that a missing name has nothing to stand apart from + assert!(potential_procedure_declaration(": Ingredients -> Coffee")); + // Edge cases with whitespace assert!(!is_procedure_declaration(" :")); // No name assert!(!potential_procedure_declaration(" :")); @@ -2956,6 +2996,61 @@ https_proxy=http://10.0.0.1:8888/ curl -f https://www.example.com/ ); } +#[test] +fn colon_in_prose_or_code_is_not_a_declaration() { + let mut input = Parser::new(); + + // Same `http://` hazard as the fenced case above, but in a plain string. + // The colon is only a declaration when what follows it could be a + // signature. + let source = r#" +% technique v1 + +check_proxy : + +1. Make a request. + { + exec("curl -f http://1.2.3.4:8888/") + } +"# + .trim_ascii(); + + input.initialize(source); + let result = input.parse_collecting_errors(); + assert!( + result.is_ok(), + "string content must not be read as structure: {:?}", + result.err() + ); +} + +#[test] +fn literals_are_opaque_to_a_scan() { + // what a line scanner sees once the content of any literal is blanked out + fn mask(text: &str) -> String { + let mut literals = Literals::new(); + text.char_indices() + .map(|(i, c)| if literals.opaque(&text[i..]) { ' ' } else { c }) + .collect() + } + + assert_eq!(mask("plain : text"), "plain : text"); + assert_eq!(mask("exec(\"a : b\")"), "exec(\" \")"); + + // the delimiters survive, so a malformed response is still recognisable + assert_eq!(mask("\"Yes\" | \"No\""), "\" \" | \" \""); + + // a `"` string ends at the line ending, so an unbalanced quote cannot + // swallow everything that follows it + assert_eq!(mask("say \"oops\nfoo :"), "say \" \nfoo :"); + + // ... whereas a fence deliberately does span lines + assert_eq!(mask("```\nfoo :\n```"), " "); + + // a run of backticks longer than the delimiter is content past the third + assert_eq!(mask("````x```"), " "); +} + #[test] fn test_multiple_error_collection() { use std::path::Path; diff --git a/src/parsing/parser.rs b/src/parsing/parser.rs index 23a90bc0..5e056078 100644 --- a/src/parsing/parser.rs +++ b/src/parsing/parser.rs @@ -438,7 +438,6 @@ impl<'i> Parser<'i> { subject: &'static str, start_char: char, end_char: char, - skip_string_content: bool, function: F, ) -> Result where @@ -462,28 +461,28 @@ impl<'i> Parser<'i> { } } else { // Nesting case: different characters for start and end (like (...)) + let mut literals = Literals::new(); let mut depth = 0; - let mut in_string = false; for (i, c) in self .source .char_indices() { + if literals.opaque(&self.source[i..]) { + continue; + } + if !begun && c == start_char { begun = true; depth = 1; } else if begun { - if skip_string_content && c == '"' { - in_string = !in_string; - } else if !skip_string_content || !in_string { - if c == start_char { - depth += 1; - } else if c == end_char { - depth -= 1; - if depth == 0 { - l = i + 1; // add end character - break; - } + if c == start_char { + depth += 1; + } else if c == end_char { + depth -= 1; + if depth == 0 { + l = i + 1; // add end character + break; } } } @@ -577,8 +576,13 @@ impl<'i> Parser<'i> { F: Fn(&mut Parser<'i>) -> Result, { let content = self.source; + + // a delimiter inside a literal is that literal's text, not ours + let mut literals = Literals::new(); let end_pos = content - .find(pattern) + .char_indices() + .find(|&(i, c)| !literals.opaque(&content[i..]) && pattern.contains(&c)) + .map(|(i, _)| i) .unwrap_or(content.len()); let block = &content[..end_pos]; @@ -1449,7 +1453,7 @@ impl<'i> Parser<'i> { } fn read_code_block(&mut self) -> Result>, ParsingError> { - self.take_block_chars("a code block", '{', '}', true, |inner| { + self.take_block_chars("a code block", '{', '}', |inner| { let mut expressions = Vec::new(); loop { @@ -1553,16 +1557,14 @@ impl<'i> Parser<'i> { let span = self.span_since(start); Ok(Expression::Number(numeric, span)) } else if is_string_literal(content) { - let parts = self.take_block_chars("a string literal", '"', '"', false, |inner| { + let parts = self.take_block_chars("a string literal", '"', '"', |inner| { inner.parse_string_pieces(inner.source) })?; let span = self.span_since(start); Ok(Expression::String(parts, span)) } else if is_enum_response(content) { let value = - self.take_block_chars("a response literal", '\'', '\'', false, |inner| { - Ok(inner.source) - })?; + self.take_block_chars("a response literal", '\'', '\'', |inner| Ok(inner.source))?; let span = self.span_since(start); Ok(Expression::Response(value, span)) } else if is_invocation(content) { @@ -1658,7 +1660,7 @@ impl<'i> Parser<'i> { .starts_with('(') { // Parse parenthesized list: (id1, id2, ...) - self.take_block_chars("a list of identifiers", '(', ')', true, |outer| { + self.take_block_chars("a list of identifiers", '(', ')', |outer| { let mut identifiers = Vec::new(); loop { @@ -1755,7 +1757,7 @@ impl<'i> Parser<'i> { let start = self.offset; self.trim_whitespace(); self.advance(1); // consume '$' - let inner = self.take_block_chars("a cost", '(', ')', true, |outer| { + let inner = self.take_block_chars("a cost", '(', ')', |outer| { outer.trim_whitespace(); if outer .source @@ -1816,12 +1818,12 @@ impl<'i> Parser<'i> { return Ok(Expression::Tablet(vec![], self.span_since(start))); } - let elements = self.take_block_chars("a list", '[', ']', true, |outer| { + let elements = self.take_block_chars("a list", '[', ']', |outer| { outer.take_elements(true, |inner| { if is_pair(inner.source) { let pair_start = inner.offset; - let label = inner - .take_block_chars("a label", '"', '"', false, |label| Ok(label.source))?; + let label = + inner.take_block_chars("a label", '"', '"', |label| Ok(label.source))?; inner.trim_whitespace(); inner.advance(1); // consume '=' (is_pair guarantees it) inner.trim_whitespace(); @@ -1875,7 +1877,7 @@ impl<'i> Parser<'i> { /// `()` for an empty value. fn read_tuple_literal(&mut self) -> Result, ParsingError> { let start = self.offset; - let elements = self.take_block_chars("a tuple", '(', ')', true, |outer| { + let elements = self.take_block_chars("a tuple", '(', ')', |outer| { outer.take_elements(false, |inner| inner.read_expression()) })?; if elements.len() < 2 { @@ -2226,7 +2228,7 @@ impl<'i> Parser<'i> { /// so its presence marks the content as an external URI. fn read_target(&mut self) -> Result, ParsingError> { let start_offset = self.offset; - self.take_block_chars("an invocation", '<', '>', true, |inner| { + self.take_block_chars("an invocation", '<', '>', |inner| { let content = inner .source .trim(); @@ -2678,7 +2680,7 @@ impl<'i> Parser<'i> { /// comma-separated list of full expressions, e.g. `(a, b, other(c))`. /// Unlike a list, there's no newline form. fn read_parameters(&mut self) -> Result>, ParsingError> { - self.take_block_chars("parameters for a function", '(', ')', true, |outer| { + self.take_block_chars("parameters for a function", '(', ')', |outer| { outer.take_elements(false, |inner| inner.read_expression()) }) } @@ -3052,26 +3054,31 @@ where { let mut i = 0; let mut begun = false; - let mut in_fence = false; + let mut literals = Literals::new(); + let mut buffer = String::new(); for line in source.lines() { - let opaque = in_fence; - if line - .matches("```") - .count() - % 2 - == 1 - { - in_fence = !in_fence; - } - if opaque { - i += line.len() + 1; - continue; - } + // A predicate reads the shape of a line, so the content of any literal + // on it is blanked first. Most lines have none and are handed straight + // through; the rest are rewritten into a buffer we keep reusing. + let text = if !literals.in_literal() && !line.contains(['"', '`']) { + line + } else { + buffer.clear(); + for (j, c) in line.char_indices() { + if literals.opaque(&line[j..]) { + buffer.push(' '); + } else { + buffer.push(c); + } + } + &buffer + }; + literals.opaque("\n"); - if !begun && start(line) { + if !begun && start(text) { begun = true; - } else if begun && end(line) { + } else if begun && end(text) { // don't include this line break; } @@ -3114,13 +3121,20 @@ where /// example it must not match " a. And now: do something" or "b. Proceed /// with:". /// -/// This function, however, is permissive. It identifies lines that could be -/// intended as procedure declarations (including malformed ones) so that -/// proper validation and error messages can be provided during the actual -/// parsing phase. +/// The name must be an identifier and the colon must stand apart from it. The +/// signature is not validated here; `f : B` is a declaration whose signature is +/// wrong, which read_signature() will address once we commit to reading it. fn is_procedure_declaration(content: &str) -> bool { match content.split_once(':') { Some((before, _after)) => { + // a declaration is written `name : signature`, with the colon + // standing apart from the name. Prose punctuates the other way, as + // in `Warning: Important`, and so is never a declaration. A missing + // name has nothing to stand apart from + if !before.is_empty() && !before.ends_with([' ', '\t']) { + return false; + } + let before = before.trim_ascii(); // Check if the name part is valid @@ -3176,8 +3190,19 @@ fn begins_procedure_declaration(content: &str) -> bool { /// preventing us from attempting to parse it as a separate procedure and /// reporting what turns out to be a better error. fn potential_procedure_declaration(content: &str) -> bool { - match content.split_once(':') { + let line = content + .lines() + .next() + .unwrap_or(""); + + match line.split_once(':') { Some((before, after)) => { + // as in is_procedure_declaration(), a declaration sets its colon + // apart from the name; prose does not + if !before.is_empty() && !before.ends_with([' ', '\t']) { + return false; + } + let before = before.trim_ascii(); // Empty before colon -> only a declaration if there's something after @@ -3188,10 +3213,10 @@ fn potential_procedure_declaration(content: &str) -> bool { } // If it's a step patterns then it's not a procedure declaration! - if is_step_dependent(content) - || is_step_parallel(content) - || is_substep_dependent(content) - || is_substep_parallel(content) + if is_step_dependent(line) + || is_step_parallel(line) + || is_substep_dependent(line) + || is_substep_parallel(line) { return false; } @@ -3225,9 +3250,10 @@ fn potential_procedure_declaration(content: &str) -> bool { fn is_procedure_body(content: &str) -> bool { let line = content.trim_ascii(); - // Empty lines are not body content (continue reading declaration) + // A declaration ends at the blank line separating it from what follows, so + // by definition nothing after one is still part of it if line.is_empty() { - return false; + return true; } // Check for procedure body indicators. At the end, if it doesn't look like signature, it's body. @@ -3344,35 +3370,93 @@ fn is_function(content: &str) -> bool { re.is_match(content) } +// Tracks whether a scan has passed into a literal, where the text belongs to +// the string and not to the structure of the document. A `"` string ends at the +// line ending; a ``` fence spans lines. Callers drive it one character at a +// time, which keeps every scan working on borrowed slices. +enum Within { + Text, + String, + Fence, +} + +struct Literals { + within: Within, + delimiter: u8, +} + +impl Literals { + fn new() -> Literals { + Literals { + within: Within::Text, + delimiter: 0, + } + } + + fn in_literal(&self) -> bool { + match self.within { + Within::Text => false, + _ => true, + } + } + + // `rest` is the text from the current position onwards, so that the ``` + // delimiter is recognized whole rather than a backtick at a time + fn opaque(&mut self, rest: &str) -> bool { + if self.delimiter > 0 { + self.delimiter -= 1; + return true; + } + + match self.within { + Within::Text => { + if rest.starts_with("```") { + self.within = Within::Fence; + self.delimiter = 2; + true + } else if rest.starts_with('"') { + self.within = Within::String; + false + } else { + false + } + } + // a string is closed by the next quote, or by the line ending; a + // fence is how text is carried across lines + Within::String => { + if rest.starts_with('"') || rest.starts_with('\n') { + self.within = Within::Text; + false + } else { + true + } + } + Within::Fence => { + if rest.starts_with("```") { + self.within = Within::Text; + self.delimiter = 2; + } + true + } + } + } +} + // Iterate the `(offset, char)` pairs of `content` that sit at the top level — -// not nested inside `()`/`[]`, a `"..."` string, or a ``` multiline fence. +// not nested inside `()`/`[]` and not within a literal. // Shared by take_elements() and locate_statement_end() so the two don't drift. fn top_level_chars(content: &str) -> impl Iterator + '_ { + let mut literals = Literals::new(); let mut depth = 0i32; - let mut in_string = false; - let mut in_multiline = false; - let mut backticks = 0u8; content .char_indices() .filter_map(move |(i, c)| { - if c == '`' { - backticks += 1; - if backticks == 3 { - in_multiline = !in_multiline; - backticks = 0; - } + if literals.opaque(&content[i..]) { return None; } - backticks = 0; match c { - _ if in_multiline => None, - '"' => { - in_string = !in_string; - None - } - _ if in_string => None, '(' | '[' => { depth += 1; None diff --git a/tests/broken/parsing/DeclartionParentheses.tq b/tests/broken/parsing/DeclarationParentheses.tq. similarity index 100% rename from tests/broken/parsing/DeclartionParentheses.tq rename to tests/broken/parsing/DeclarationParentheses.tq. diff --git a/tests/samples/parsing/Choices.tq b/tests/samples/parsing/Choices.tq index 79e8e3d9..cc84b120 100644 --- a/tests/samples/parsing/Choices.tq +++ b/tests/samples/parsing/Choices.tq @@ -1,7 +1,13 @@ one_of_many : +# Choosing: Overview + Pick one! 1. From these 'One' | 'Two' | 'Three' + + 2. Or from these + + 'Yes: proceed' | 'No (stop): halt' diff --git a/tests/samples/parsing/LocalNetwork.tq b/tests/samples/parsing/LocalNetwork.tq index 354dc8f6..fb20bb9d 100644 --- a/tests/samples/parsing/LocalNetwork.tq +++ b/tests/samples/parsing/LocalNetwork.tq @@ -2,7 +2,8 @@ local_network : # Local Network Connectivity -Establish that the local network environment is functioning. +Establish that the local network environment is functioning (all of it: hosts, +routes, and names). 1. Check physical network interface { exec( ```bash @@ -13,3 +14,14 @@ Establish that the local network environment is functioning. 3. Check local DHCP is working 4. Check local DNS responding 5. Verify reachability of local network gateway + 6. Confirm the local package mirror is serving + { + exec("curl --silent http://127.0.0.1:48080/simple/") + } + 7. Report anything still down { exec( + ```bash + for i in $(ip -br link | awk '$2 != "UP" {print $1}') + do echo "still down: $i" + done + ``` + ) } diff --git a/tests/samples/parsing/Sequence.tq b/tests/samples/parsing/Sequence.tq index d0a9f8bb..5c0b1b70 100644 --- a/tests/samples/parsing/Sequence.tq +++ b/tests/samples/parsing/Sequence.tq @@ -14,3 +14,4 @@ instance we have to disable termination protection. select("Apply Immediately") click("Modify DB instance") } + 2. And now we list home { exec("ls ~/.local/bin") ~ listing }