|
| 1 | +using System; |
| 2 | +using System.IO; |
| 3 | +using System.Linq; |
| 4 | +using System.Text; |
| 5 | +using BenchmarkDotNet.Attributes; |
| 6 | +using BenchmarkDotNet.Columns; |
| 7 | +using BenchmarkDotNet.Configs; |
| 8 | +using BenchmarkDotNet.Reports; |
| 9 | +using BenchmarkDotNet.Running; |
| 10 | +using SimdUnicode; |
| 11 | + |
| 12 | +namespace SimdUnicodeBenchmarks |
| 13 | +{ |
| 14 | + // Speed in GB/s of UTF-16 input (2 bytes per code unit). |
| 15 | + public class Utf16Speed : IColumn |
| 16 | + { |
| 17 | + public string GetValue(Summary summary, BenchmarkCase benchmarkCase) |
| 18 | + { |
| 19 | + if (summary is null || benchmarkCase is null || benchmarkCase.Parameters is null) |
| 20 | + { |
| 21 | + return "N/A"; |
| 22 | + } |
| 23 | + var ourReport = summary.Reports.First(x => x.BenchmarkCase.Equals(benchmarkCase)); |
| 24 | + if (ourReport is null || ourReport.ResultStatistics is null) |
| 25 | + { |
| 26 | + return "N/A"; |
| 27 | + } |
| 28 | + var fileName = (string)benchmarkCase.Parameters["FileName"]; |
| 29 | + var errors = (int)benchmarkCase.Parameters["ErrorsPerMillion"]; |
| 30 | + long bytes = 2L * UTF16WellFormedBenchmark.Load(fileName, errors).Length; |
| 31 | + return $"{(bytes / ourReport.ResultStatistics.Mean):#####.00}"; |
| 32 | + } |
| 33 | + |
| 34 | + public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase); |
| 35 | + public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; |
| 36 | + public bool IsAvailable(Summary summary) => true; |
| 37 | + |
| 38 | + public string Id { get; } = nameof(Utf16Speed); |
| 39 | + public string ColumnName { get; } = "Speed (GB/s)"; |
| 40 | + public bool AlwaysShow { get; } = true; |
| 41 | + public ColumnCategory Category { get; } = ColumnCategory.Custom; |
| 42 | + public int PriorityInCategory { get; } |
| 43 | + public bool IsNumeric { get; } |
| 44 | + public UnitType UnitType { get; } = UnitType.Dimensionless; |
| 45 | + public string Legend { get; } = "The speed in gigabytes per second (UTF-16 input)"; |
| 46 | + } |
| 47 | + |
| 48 | + // Replacing lone surrogates by U+FFFD (JavaScript's toWellFormed). |
| 49 | + // Run with: dotnet run -c Release --filter "*UTF16WellFormed*" |
| 50 | + [SimpleJob(launchCount: 1, warmupCount: 3, iterationCount: 5)] |
| 51 | + [Config(typeof(Config))] |
| 52 | + public class UTF16WellFormedBenchmark |
| 53 | + { |
| 54 | +#pragma warning disable CA1812 |
| 55 | + private sealed class Config : ManualConfig |
| 56 | + { |
| 57 | + public Config() |
| 58 | + { |
| 59 | + AddColumn(new Utf16Speed()); |
| 60 | + } |
| 61 | + } |
| 62 | + |
| 63 | + [Params(@"data/twitter.json", |
| 64 | + @"data/Arabic-Lipsum.utf8.txt", |
| 65 | + @"data/Chinese-Lipsum.utf8.txt", |
| 66 | + @"data/Emoji-Lipsum.utf8.txt", |
| 67 | + @"data/Hebrew-Lipsum.utf8.txt", |
| 68 | + @"data/Hindi-Lipsum.utf8.txt", |
| 69 | + @"data/Japanese-Lipsum.utf8.txt", |
| 70 | + @"data/Korean-Lipsum.utf8.txt", |
| 71 | + @"data/Latin-Lipsum.utf8.txt", |
| 72 | + @"data/Russian-Lipsum.utf8.txt")] |
| 73 | +#pragma warning disable CA1051 |
| 74 | + public string FileName = ""; |
| 75 | + |
| 76 | + // 0: well-formed input (the common case). Otherwise, about this many code units |
| 77 | + // per million are overwritten with lone surrogates. |
| 78 | + [Params(0, 100)] |
| 79 | + public int ErrorsPerMillion; |
| 80 | + |
| 81 | + private string input = ""; |
| 82 | + private char[] output = Array.Empty<char>(); |
| 83 | + // The same content, cut into short strings of 1 to 64 code units. |
| 84 | + private string[] shortStrings = Array.Empty<string>(); |
| 85 | + |
| 86 | + public static string Load(string fileName, int errorsPerMillion) |
| 87 | + { |
| 88 | + char[] chars = Encoding.UTF8.GetString(File.ReadAllBytes(fileName)).ToCharArray(); |
| 89 | + if (errorsPerMillion > 0) |
| 90 | + { |
| 91 | + var rand = new Random(1234); |
| 92 | + long count = (long)chars.Length * errorsPerMillion / 1_000_000 + 1; |
| 93 | + for (long k = 0; k < count; k++) |
| 94 | + { |
| 95 | + int pos = rand.Next(chars.Length); |
| 96 | + chars[pos] = (char)(rand.Next(2) == 0 ? 0xD800 + rand.Next(0x400) : 0xDC00 + rand.Next(0x400)); |
| 97 | + } |
| 98 | + } |
| 99 | + return new string(chars); |
| 100 | + } |
| 101 | + |
| 102 | + [GlobalSetup] |
| 103 | + public void Setup() |
| 104 | + { |
| 105 | + input = Load(FileName, ErrorsPerMillion); |
| 106 | + output = new char[input.Length]; |
| 107 | + var rand = new Random(4321); |
| 108 | + var pieces = new System.Collections.Generic.List<string>(); |
| 109 | + for (int i = 0; i < input.Length;) |
| 110 | + { |
| 111 | + int len = Math.Min(rand.Next(1, 65), input.Length - i); |
| 112 | + pieces.Add(input.Substring(i, len)); |
| 113 | + i += len; |
| 114 | + } |
| 115 | + shortStrings = pieces.ToArray(); |
| 116 | + string expected = RunesToWellFormed(input); |
| 117 | + if (UTF16.ToWellFormed(input) != expected || IndexOfAnyToWellFormed(input) != expected) |
| 118 | + { |
| 119 | + throw new InvalidOperationException("Mismatch between implementations."); |
| 120 | + } |
| 121 | + } |
| 122 | + |
| 123 | + // The idiomatic version: decode rune by rune. |
| 124 | + private static string RunesToWellFormed(string s) => string.Concat(s.EnumerateRunes()); |
| 125 | + |
| 126 | + // The best version we know that uses only public .NET APIs: return the input |
| 127 | + // when possible and skip non-surrogates with the vectorized IndexOfAnyInRange. |
| 128 | + private static string IndexOfAnyToWellFormed(string s) |
| 129 | + { |
| 130 | + int first = NextError(s, 0); |
| 131 | + if (first < 0) |
| 132 | + { |
| 133 | + return s; |
| 134 | + } |
| 135 | + return string.Create(s.Length, (s, first), static (dst, state) => |
| 136 | + { |
| 137 | + var (src, i) = state; |
| 138 | + src.AsSpan().CopyTo(dst); |
| 139 | + do |
| 140 | + { |
| 141 | + dst[i] = '\uFFFD'; |
| 142 | + i = NextError(src, i + 1); |
| 143 | + } while (i >= 0); |
| 144 | + }); |
| 145 | + } |
| 146 | + |
| 147 | + private static int NextError(ReadOnlySpan<char> s, int start) |
| 148 | + { |
| 149 | + int i = start; |
| 150 | + while (true) |
| 151 | + { |
| 152 | + int k = s.Slice(i).IndexOfAnyInRange('\uD800', '\uDFFF'); |
| 153 | + if (k < 0) |
| 154 | + { |
| 155 | + return -1; |
| 156 | + } |
| 157 | + i += k; |
| 158 | + if (char.IsHighSurrogate(s[i]) && i + 1 < s.Length && char.IsLowSurrogate(s[i + 1])) |
| 159 | + { |
| 160 | + i += 2; |
| 161 | + } |
| 162 | + else |
| 163 | + { |
| 164 | + return i; |
| 165 | + } |
| 166 | + } |
| 167 | + } |
| 168 | + |
| 169 | + [Benchmark] |
| 170 | + [BenchmarkCategory("string")] |
| 171 | + public string StringRunes() => RunesToWellFormed(input); |
| 172 | + |
| 173 | + [Benchmark] |
| 174 | + [BenchmarkCategory("string")] |
| 175 | + public string StringIndexOfAnyInRange() => IndexOfAnyToWellFormed(input); |
| 176 | + |
| 177 | + // Encoding.Unicode replaces lone surrogates by U+FFFD when encoding. |
| 178 | + [Benchmark] |
| 179 | + [BenchmarkCategory("string")] |
| 180 | + public string StringEncodingRoundTrip() => Encoding.Unicode.GetString(Encoding.Unicode.GetBytes(input)); |
| 181 | + |
| 182 | + [Benchmark] |
| 183 | + [BenchmarkCategory("string")] |
| 184 | + public string StringSimdUnicode() => UTF16.ToWellFormed(input); |
| 185 | + |
| 186 | + // Short strings (1 to 64 code units), one call per string. |
| 187 | + [Benchmark] |
| 188 | + [BenchmarkCategory("short")] |
| 189 | + public int ShortIndexOfAnyInRange() |
| 190 | + { |
| 191 | + int count = 0; |
| 192 | + foreach (string s in shortStrings) |
| 193 | + { |
| 194 | + count += IndexOfAnyToWellFormed(s).Length; |
| 195 | + } |
| 196 | + return count; |
| 197 | + } |
| 198 | + |
| 199 | + [Benchmark] |
| 200 | + [BenchmarkCategory("short")] |
| 201 | + public int ShortSimdUnicode() |
| 202 | + { |
| 203 | + int count = 0; |
| 204 | + foreach (string s in shortStrings) |
| 205 | + { |
| 206 | + count += UTF16.ToWellFormed(s).Length; |
| 207 | + } |
| 208 | + return count; |
| 209 | + } |
| 210 | + |
| 211 | + // Buffer to buffer: always writes the whole output. |
| 212 | + // A plain copy, as a reference for the best we can hope for. |
| 213 | + [Benchmark] |
| 214 | + [BenchmarkCategory("buffer")] |
| 215 | + public void BufferCopyOnly() => input.AsSpan().CopyTo(output); |
| 216 | + |
| 217 | + [Benchmark] |
| 218 | + [BenchmarkCategory("buffer")] |
| 219 | + public unsafe void BufferScalar() |
| 220 | + { |
| 221 | + fixed (char* pIn = input) |
| 222 | + fixed (char* pOut = output) |
| 223 | + { |
| 224 | + UTF16.ToWellFormedScalar(pIn, input.Length, pOut); |
| 225 | + } |
| 226 | + } |
| 227 | + |
| 228 | + [Benchmark] |
| 229 | + [BenchmarkCategory("buffer")] |
| 230 | + public void BufferCopyThenIndexOfAnyInRange() |
| 231 | + { |
| 232 | + input.AsSpan().CopyTo(output); |
| 233 | + Span<char> dst = output; |
| 234 | + int i = NextError(dst, 0); |
| 235 | + while (i >= 0) |
| 236 | + { |
| 237 | + dst[i] = '\uFFFD'; |
| 238 | + i = NextError(dst, i + 1); |
| 239 | + } |
| 240 | + } |
| 241 | + |
| 242 | + [Benchmark] |
| 243 | + [BenchmarkCategory("buffer")] |
| 244 | + public unsafe void BufferSimdUnicode() |
| 245 | + { |
| 246 | + fixed (char* pIn = input) |
| 247 | + fixed (char* pOut = output) |
| 248 | + { |
| 249 | + UTF16.ToWellFormed(pIn, input.Length, pOut); |
| 250 | + } |
| 251 | + } |
| 252 | + |
| 253 | + // Validation only (isWellFormed). |
| 254 | + [Benchmark] |
| 255 | + [BenchmarkCategory("validate")] |
| 256 | + public bool ValidateIndexOfAnyInRange() => NextError(input, 0) < 0; |
| 257 | + |
| 258 | + [Benchmark] |
| 259 | + [BenchmarkCategory("validate")] |
| 260 | + public bool ValidateSimdUnicode() => UTF16.IsWellFormed(input); |
| 261 | + } |
| 262 | +} |
0 commit comments