Harden atomic double addition

Author: HadenSmith

Numerics/Utilities/Tools.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic; 
 using System.IO;
 using System.IO.Compression;
+using System.Runtime.CompilerServices;
 using System.Threading;
 
 namespace Numerics
@@ -666,20 +667,55 @@ public static double Max(IList<double> values, IList<int> indicators, bool useCo
         /// </summary>
         /// <param name="valueToAddTo">The accumulator to update atomically.</param>
         /// <param name="valueToAdd">The value to add.</param>
-        /// <returns>The value committed by the successful compare-and-swap operation.</returns>
+        /// <returns>The accumulator value produced by the atomic addition.</returns>
         /// <remarks>
-        /// Concurrent updates are not lost, but their order depends on thread scheduling. Floating-point
-        /// reductions using this method are therefore not guaranteed to be bit-reproducible.
+        /// All concurrent writes to <paramref name="valueToAddTo"/> must use this method or another
+        /// interlocked operation. The compare-and-swap operation can retry under contention. Concurrent
+        /// updates are not lost, but their order depends on thread scheduling, so floating-point reductions
+        /// using this method are not guaranteed to be bit-reproducible.
         /// </remarks>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static double ParallelAdd(ref double valueToAddTo, double valueToAdd)
         {
-            double newCurrentValue = valueToAddTo;
+            double currentValue = valueToAddTo;
+            double newValue = currentValue + valueToAdd;
+            double observedValue = Interlocked.CompareExchange(ref valueToAddTo, newValue, currentValue);
+            if (observedValue == currentValue &&
+                (currentValue != 0d ||
+                 BitConverter.DoubleToInt64Bits(observedValue) == BitConverter.DoubleToInt64Bits(currentValue)))
+            {
+                return newValue;
+            }
+            if (double.IsNaN(observedValue) && double.IsNaN(currentValue)) return newValue;
+            return ParallelAddRetry(ref valueToAddTo, valueToAdd, observedValue);
+        }
+
+        /// <summary>
+        /// Retries an atomic double addition after a failed compare-and-swap operation.
+        /// </summary>
+        /// <param name="valueToAddTo">The accumulator to update atomically.</param>
+        /// <param name="valueToAdd">The value to add.</param>
+        /// <param name="observedValue">The accumulator value observed by the failed operation.</param>
+        /// <returns>The accumulator value produced by the atomic addition.</returns>
+        /// <remarks>
+        /// The observed value seeds the lock-free retry loop so that another non-volatile read is unnecessary.
+        /// Signed zeros require a representation comparison because numerical equality does not distinguish
+        /// positive zero from negative zero, while compare-and-swap does.
+        /// </remarks>
+        private static double ParallelAddRetry(ref double valueToAddTo, double valueToAdd, double observedValue)
+        {
+            double newCurrentValue = observedValue;
             while (true)
             {
                 double currentValue = newCurrentValue;
                 double newValue = currentValue + valueToAdd;
                 newCurrentValue = Interlocked.CompareExchange(ref valueToAddTo, newValue, currentValue);
-                if (newCurrentValue == currentValue) return newValue;
+                if (newCurrentValue == currentValue &&
+                    (currentValue != 0d ||
+                     BitConverter.DoubleToInt64Bits(newCurrentValue) == BitConverter.DoubleToInt64Bits(currentValue)))
+                {
+                    return newValue;
+                }
                 if (double.IsNaN(newCurrentValue) && double.IsNaN(currentValue)) return newValue;
             }
         }

Test_Numerics/Utilities/Test_Tools.cs

@@ -2,6 +2,7 @@
 using Numerics;
 using System;
 using System.Collections.Generic;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace Utilities
@@ -374,6 +375,118 @@ public void Test_ParallelAdd_NaNAccumulatorTerminates()
             Assert.IsTrue(double.IsNaN(result));
         }
 
+        /// <summary>
+        /// Verifies signed zero, NaN, and infinity follow double-precision addition semantics.
+        /// </summary>
+        [TestMethod]
+        public void Test_ParallelAdd_HandlesSpecialValues()
+        {
+            double negativeZero = BitConverter.Int64BitsToDouble(long.MinValue);
+
+            double total = negativeZero;
+            double result = Tools.ParallelAdd(ref total, negativeZero);
+
+            Assert.AreEqual(long.MinValue, BitConverter.DoubleToInt64Bits(total));
+            Assert.AreEqual(long.MinValue, BitConverter.DoubleToInt64Bits(result));
+
+            total = negativeZero;
+            result = Tools.ParallelAdd(ref total, 0d);
+
+            Assert.AreEqual(0L, BitConverter.DoubleToInt64Bits(total));
+            Assert.AreEqual(0L, BitConverter.DoubleToInt64Bits(result));
+
+            total = 1d;
+            result = Tools.ParallelAdd(ref total, double.NaN);
+
+            Assert.IsTrue(double.IsNaN(total));
+            Assert.IsTrue(double.IsNaN(result));
+
+            total = double.PositiveInfinity;
+            result = Tools.ParallelAdd(ref total, 1d);
+
+            Assert.AreEqual(double.PositiveInfinity, total);
+            Assert.AreEqual(double.PositiveInfinity, result);
+
+            total = double.NegativeInfinity;
+            result = Tools.ParallelAdd(ref total, -1d);
+
+            Assert.AreEqual(double.NegativeInfinity, total);
+            Assert.AreEqual(double.NegativeInfinity, result);
+
+            total = double.PositiveInfinity;
+            result = Tools.ParallelAdd(ref total, double.NegativeInfinity);
+
+            Assert.IsTrue(double.IsNaN(total));
+            Assert.IsTrue(double.IsNaN(result));
+        }
+
+        /// <summary>
+        /// Verifies concurrent mixed-sign additions are all committed when the accumulator revisits values.
+        /// </summary>
+        [TestMethod]
+        public void Test_ParallelAdd_CommitsMixedSignUpdates()
+        {
+            const int updateCount = 25000;
+            double total = 0d;
+
+            Parallel.For(
+                0,
+                updateCount,
+                index => Tools.ParallelAdd(ref total, index % 2 == 0 ? 2d : -1d));
+
+            Assert.AreEqual(updateCount / 2d, total);
+        }
+
+        /// <summary>
+        /// Verifies a negative-to-positive zero race cannot cause finite concurrent updates to be lost.
+        /// </summary>
+        [TestMethod]
+        public void Test_ParallelAdd_SignedZeroContentionDoesNotLoseUpdates()
+        {
+            const int rounds = 256;
+            double total = 0d;
+            bool everyUpdateCommitted = true;
+            double negativeZero = BitConverter.Int64BitsToDouble(long.MinValue);
+
+            using (var startBarrier = new Barrier(5))
+            using (var finishBarrier = new Barrier(5))
+            {
+                Func<double, Task> createWorker = valueToAdd => Task.Factory.StartNew(
+                    () =>
+                    {
+                        for (int round = 0; round < rounds; round++)
+                        {
+                            startBarrier.SignalAndWait();
+                            Tools.ParallelAdd(ref total, valueToAdd);
+                            finishBarrier.SignalAndWait();
+                        }
+                    },
+                    CancellationToken.None,
+                    TaskCreationOptions.LongRunning,
+                    TaskScheduler.Default);
+
+                Task[] workers =
+                {
+                    createWorker(0d),
+                    createWorker(0d),
+                    createWorker(1d),
+                    createWorker(1d)
+                };
+
+                for (int round = 0; round < rounds; round++)
+                {
+                    total = negativeZero;
+                    startBarrier.SignalAndWait();
+                    finishBarrier.SignalAndWait();
+                    if (total != 2d) everyUpdateCommitted = false;
+                }
+
+                Task.WaitAll(workers);
+            }
+
+            Assert.IsTrue(everyUpdateCommitted, "A finite update was lost during signed-zero contention.");
+        }
+
         /// <summary>
         /// Testing the Log-sum-exponential function.
         /// <para>