A brush with an invalid shape keeps its handles (#66)

The editable outline only needs whole topology, not a valid shape: a brush
dragged into a concave or flat shape keeps its handles, drawn in the invalid
color, and can be dragged back. A brush's shape errors aren't logged any
more (they come with every drag while the shape is invalid); the inspector
shows them.

Author: LogicalError

Core/1.Input/Generators/Brush/ChiselBrushDefinition.cs

@@ -173,10 +173,10 @@ public bool Validate()
 				if (brushOutline.planes == null || brushOutline.planes.Length != brushOutline.polygons.Length)
 					brushOutline.CalculatePlanes();
 
-				// If the brush is concave, we set the generator to not be valid, so that when we commit, it will be reverted
+				// If the brush is concave, we set the generator to not be valid, so that when we commit, it will be reverted.
+				// Not logged: it happens while a shape is dragged, and the inspector shows why (GetMessages)
 				if (!brushOutline.ValidateShape(out errorMessage))
 				{
-					Debug.LogError(errorMessage);
 					validState = false;
 					return false;
 				}

Editor/ComponentEditors/Base/ChiselEditableOutline.cs

@@ -508,14 +508,20 @@ public void Rebuild()
             UpdateSelectionCenter();
         }
 
+        // Whether the outline can be edited: whole topology only, so a brush whose shape is invalid (concave, flat) keeps
+        // its handles and can be dragged back
         bool IsValidBrush()
         {
             if (brushMesh == null ||
                 brushMesh.vertices == null || brushMesh.vertices.Length == 0 ||
                 brushMesh.halfEdges == null || brushMesh.halfEdges.Length == 0 ||
                 brushMesh.polygons == null || brushMesh.polygons.Length == 0)
                 return false;
-			return brushMesh.Validate();
+            if (!brushMesh.ValidateData(out _))
+                return false;
+            if (brushMesh.planes == null || brushMesh.planes.Length != brushMesh.polygons.Length)
+                brushMesh.CalculatePlanes();
+            return true;
         }
 
 

Editor/Tests/ChiselEditableOutlineTests.cs

@@ -0,0 +1,52 @@
+using Chisel.Components;
+using Chisel.Core;
+using NUnit.Framework;
+using UnityEngine;
+
+namespace Chisel.Editors.Tests
+{
+    // A brush being edited keeps its handles when its shape is invalid (concave, flat), so it can be dragged back
+    public sealed class ChiselEditableOutlineTests
+    {
+        GameObject root;
+
+        [TearDown]
+        public void TearDown()
+        {
+            if (root)
+                Object.DestroyImmediate(root);
+        }
+
+        // A box with its top corner pushed in past its middle: concave
+        static BrushMesh ConcaveBox()
+        {
+            BrushMeshFactory.CreateBox(new Vector3(2, 2, 2), 0, out var box);
+            var corner = 0;
+            for (int v = 1; v < box.vertices.Length; v++)
+            {
+                if (box.vertices[v].x + box.vertices[v].y + box.vertices[v].z > box.vertices[corner].x + box.vertices[corner].y + box.vertices[corner].z)
+                    corner = v;
+            }
+            box.vertices[corner] = new Unity.Mathematics.float3(-0.25f, -0.25f, -0.25f);
+            box.CalculatePlanes();
+            return box;
+        }
+
+        [Test]
+        public void AConcaveBrush_KeepsItsHandles()
+        {
+            var outline = ConcaveBox();
+            Assert.That(outline.ValidateShape(out _), Is.False, "the brush isn't concave");
+
+            root = new GameObject("Brush");
+            var brush = root.AddComponent<ChiselBrushComponent>();
+            brush.definition.BrushOutline = outline;
+            brush.OnValidate();
+            Assert.That(brush.definition.IsValid, Is.False, "an invalid brush is drawn as it is");
+
+            var editable = new ChiselEditableOutline(brush);
+            Assert.That(editable.polygonCenters, Is.Not.Null, "no handles");
+            Assert.That(editable.polygonCenters.Length, Is.EqualTo(outline.polygons.Length));
+        }
+    }
+}