Normalize tabs in multilines to four spaces when formatting

Author: istathar

src/domain/engine.rs

@@ -273,12 +273,8 @@ fn render_expression_parts(expr: &Expression) -> (String, Vec<String>) {
     if let Expression::Execution(func, _) = expr {
         let mut body = Vec::new();
         for param in &func.parameters {
-            if let Expression::Multiline(_, lines, _) = param {
-                body.extend(
-                    lines
-                        .iter()
-                        .map(|s| s.to_string()),
-                );
+            if let Expression::Multiline(multiline, _) = param {
+                body.push(multiline.content());
             }
         }
         if !body.is_empty() {
@@ -350,7 +346,7 @@ fn render_expression(expr: &Expression) -> String {
                 args.join(", ")
             )
         }
-        Expression::Multiline(_, lines, _) => lines.join("\n"),
+        Expression::Multiline(multiline, _) => multiline.content(),
         Expression::Variable(id, _) => id
             .value
             .to_string(),

src/formatting/formatter.rs

@@ -480,7 +480,7 @@ impl<'i> Formatter<'i> {
             return Vec::new();
         }
         match expr {
-            Expression::Multiline(_, _, _) => {
+            Expression::Multiline(_, _) => {
                 // These are not inline, caller should handle specially
                 Vec::new()
             }
@@ -913,7 +913,7 @@ impl<'i> Formatter<'i> {
                             line = self.builder();
                             line.add_word(Syntax::Structure, "}");
                         }
-                        Expression::Multiline(_, _, _) => {
+                        Expression::Multiline(_, _) => {
                             line.flush();
                             self.add_fragment_reference(Syntax::Structure, "{");
                             self.increase(4);
@@ -930,7 +930,7 @@ impl<'i> Formatter<'i> {
                                     .parameters
                                     .iter()
                                     .any(|p| {
-                                        if let Expression::Multiline(_, _, _) = p {
+                                        if let Expression::Multiline(_, _) = p {
                                             true
                                         } else {
                                             false
@@ -1284,28 +1284,21 @@ impl<'i> Formatter<'i> {
                 self.add_fragment_reference(Syntax::Quote, "'");
             }
             Expression::Number(numeric, _) => self.append_numeric(numeric),
-            Expression::Multiline(lang, lines, _) => {
+            Expression::Multiline(multiline, _) => {
                 self.append_char('\n');
 
                 self.indent();
                 self.add_fragment_reference(Syntax::Quote, "```");
-                if let Some(which) = lang {
+                if let Some(which) = multiline.language {
                     self.add_fragment_reference(Syntax::Language, which);
                 }
                 self.append_char('\n');
 
                 self.increase(4);
-                for line in lines {
-                    self.indent();
-                    // Break multiline content into words for wrapping
-                    for (i, word) in line
-                        .split_ascii_whitespace()
-                        .enumerate()
-                    {
-                        if i > 0 {
-                            self.add_fragment_reference(Syntax::Multiline, " ");
-                        }
-                        self.add_fragment_reference(Syntax::Multiline, word);
+                for line in multiline.lines() {
+                    if !line.is_empty() {
+                        self.indent();
+                        self.add_fragment(Syntax::Multiline, line);
                     }
                     self.append_char('\n');
                 }
@@ -1482,7 +1475,7 @@ impl<'i> Formatter<'i> {
 
         let mut has_multiline = false;
         for parameter in &function.parameters {
-            if let Expression::Multiline(_, _, _) = parameter {
+            if let Expression::Multiline(_, _) = parameter {
                 has_multiline = true;
                 break;
             }

src/language/mod.rs

@@ -1,10 +1,12 @@
 // Types representing the Technique language surface syntax
 
 mod error;
+mod multiline;
 mod quantity;
 mod types;
 
 // Re-export all public symbols
 pub use error::*;
+pub use multiline::*;
 pub use quantity::*;
 pub use types::*;

src/language/multiline.rs

@@ -0,0 +1,87 @@
+//! The content of a ``` fence
+
+use std::borrow::Cow;
+
+#[derive(Debug, Eq, PartialEq)]
+pub struct Multiline<'i> {
+    pub language: Option<&'i str>,
+    pub lines: Vec<&'i str>,
+}
+
+impl<'i> Multiline<'i> {
+    /// The lines as they were present in the input source but with the leading block
+    /// indentation removed.
+    pub fn lines(&self) -> impl Iterator<Item = Cow<'i, str>> + '_ {
+        let common = self
+            .lines
+            .iter()
+            .filter(|line| {
+                !line
+                    .trim_ascii()
+                    .is_empty()
+            })
+            .map(|line| indent(line))
+            .min()
+            .unwrap_or(0);
+
+        self.lines
+            .iter()
+            .map(move |line| strip(expand(line), common))
+    }
+
+    /// The lines, joined, suitable for use by builtin functions.
+    pub fn content(&self) -> String {
+        self.lines()
+            .collect::<Vec<Cow<'i, str>>>()
+            .join("\n")
+    }
+}
+
+fn indent(line: &str) -> usize {
+    let mut column = 0;
+
+    for c in line.chars() {
+        match c {
+            ' ' => column += 1,
+            '\t' => column = (column / 4 + 1) * 4,
+            _ => break,
+        }
+    }
+
+    column
+}
+
+/// Expand any tabs present into 4 spaces. If the author needs an actual tab
+/// character in a sting literal they can use the `\t` escape.
+fn expand(line: &str) -> Cow<'_, str> {
+    if !line.contains('\t') {
+        return Cow::Borrowed(line);
+    }
+
+    let mut result = String::with_capacity(line.len() + 8);
+    let mut column = 0;
+
+    for c in line.chars() {
+        if c == '\t' {
+            let stop = (column / 4 + 1) * 4;
+            result.push_str(&" ".repeat(stop - column));
+            column = stop;
+        } else {
+            result.push(c);
+            column += 1;
+        }
+    }
+
+    result.into()
+}
+
+/// Strip leading block indendation from a line.
+fn strip(line: Cow<'_, str>, common: usize) -> Cow<'_, str> {
+    match line {
+        Cow::Borrowed(text) => Cow::Borrowed(&text[common.min(text.len())..]),
+        Cow::Owned(mut text) => {
+            text.drain(..common.min(text.len()));
+            Cow::Owned(text)
+        }
+    }
+}

src/language/types.rs

@@ -1,5 +1,7 @@
 //! Types representing an Abstract Syntax Tree for the Technique language
 
+use crate::language::multiline::Multiline;
+use crate::language::quantity::Quantity;
 use crate::regex::*;
 
 /// Byte range within the original source. `length` excludes trailing whitespace.
@@ -523,7 +525,7 @@ pub enum Expression<'i> {
     String(Vec<Piece<'i>>, Span),
     Response(&'i str, Span),
     Number(Numeric<'i>, Span),
-    Multiline(Option<&'i str>, Vec<&'i str>, Span),
+    Multiline(Multiline<'i>, Span),
     Repeat(Box<Expression<'i>>, Span),
     Foreach(Vec<Identifier<'i>>, Box<Expression<'i>>, Span),
     Within(Box<Expression<'i>>, Span),
@@ -547,9 +549,7 @@ impl PartialEq for Expression<'_> {
             (Expression::String(a, _), Expression::String(b, _)) => a == b,
             (Expression::Response(a, _), Expression::Response(b, _)) => a == b,
             (Expression::Number(a, _), Expression::Number(b, _)) => a == b,
-            (Expression::Multiline(a1, a2, _), Expression::Multiline(b1, b2, _)) => {
-                a1 == b1 && a2 == b2
-            }
+            (Expression::Multiline(a, _), Expression::Multiline(b, _)) => a == b,
             (Expression::Repeat(a, _), Expression::Repeat(b, _)) => a == b,
             (Expression::Foreach(a1, a2, _), Expression::Foreach(b1, b2, _)) => {
                 a1 == b1 && a2 == b2
@@ -579,8 +579,6 @@ pub enum Numeric<'i> {
     Scientific(Quantity<'i>),
 }
 
-pub use crate::language::quantity::Quantity;
-
 // the validate functions all need to have start and end anchors, which seems
 // like it should be abstracted away.
 

src/linking/linker.rs

@@ -125,7 +125,7 @@ fn link_operation<'i>(
         Operation::Variable(_, _)
         | Operation::Number(_, _)
         | Operation::Response(_, _)
-        | Operation::Multiline(_, _, _)
+        | Operation::Verbatim(_, _)
         | Operation::Prose(_, _)
         | Operation::Hole(_)
         | Operation::Unit(_) => {}