Skip to content

Commit 59bc661

Browse files
authored
Merge pull request #1 from GrAxOS/feat/foundation-perf-phase1-2
2 parents ea688c8 + 90e5a42 commit 59bc661

15 files changed

Lines changed: 1260 additions & 68 deletions

File tree

.github/workflows/ci.yml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: ci
2+
3+
on:
4+
push:
5+
branches: ["**"]
6+
pull_request:
7+
workflow_dispatch:
8+
9+
permissions:
10+
contents: read
11+
12+
concurrency:
13+
group: ci-${{ github.ref }}
14+
cancel-in-progress: true
15+
16+
jobs:
17+
build-test:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- name: Environment
21+
run: |
22+
set -eux
23+
dotnet --info
24+
25+
- name: Clone source
26+
run: |
27+
set -eux
28+
git clone --no-checkout "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" repo
29+
cd repo
30+
git checkout --detach "${{ github.sha }}"
31+
ls -la
32+
33+
- name: Restore
34+
working-directory: repo
35+
run: dotnet restore HydraDB.sln
36+
37+
- name: Build
38+
working-directory: repo
39+
run: dotnet build HydraDB.sln -c Release --no-restore
40+
41+
- name: Test
42+
working-directory: repo
43+
run: dotnet test HydraDB.sln -c Release --no-build --logger "console;verbosity=minimal"
44+
45+
- name: CLI smoke (.stats)
46+
working-directory: repo
47+
run: |
48+
set -eux
49+
printf 'create table t (id int primary key, v int)\ninsert into t values (1,10), (2,20), (3,30)\n.checkpoint\n.stats\n' | dotnet run -c Release --project src/HydraDB.Cli -- ./ci-data

.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,8 @@ bin/
22
obj/
33
hydradata/
44
*.user
5+
*.log
6+
TestResults/
7+
.vs/
8+
.idea/
9+
*.trx

HydraDB.sln

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
Microsoft Visual Studio Solution File, Format Version 12.00
2+
# Visual Studio Version 17
3+
VisualStudioVersion = 17.0.31903.59
4+
MinimumVisualStudioVersion = 10.0.40219.1
5+
Project("{9A19103F-16F7-32B7-A4EE-C7B4B7F0E0E7}") = "HydraDB.Core", "src\HydraDB.Core\HydraDB.Core.csproj", "{C0A5D001-0001-4000-9000-000000000001}"
6+
EndProject
7+
Project("{9A19103F-16F7-32B7-A4EE-C7B4B7F0E0E7}") = "HydraDB.Cli", "src\HydraDB.Cli\HydraDB.Cli.csproj", "{C0A5D001-0001-4000-9000-000000000002}"
8+
EndProject
9+
Project("{9A19103F-16F7-32B7-A4EE-C7B4B7F0E0E7}") = "HydraDB.Tests", "tests\HydraDB.Tests\HydraDB.Tests.csproj", "{C0A5D001-0001-4000-9000-000000000003}"
10+
EndProject
11+
Global
12+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
13+
Debug|Any CPU = Debug|Any CPU
14+
Release|Any CPU = Release|Any CPU
15+
EndGlobalSection
16+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
17+
{C0A5D001-0001-4000-9000-000000000001}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
18+
{C0A5D001-0001-4000-9000-000000000001}.Debug|Any CPU.Build.0 = Debug|Any CPU
19+
{C0A5D001-0001-4000-9000-000000000001}.Release|Any CPU.ActiveCfg = Release|Any CPU
20+
{C0A5D001-0001-4000-9000-000000000001}.Release|Any CPU.Build.0 = Release|Any CPU
21+
{C0A5D001-0001-4000-9000-000000000002}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
22+
{C0A5D001-0001-4000-9000-000000000002}.Debug|Any CPU.Build.0 = Debug|Any CPU
23+
{C0A5D001-0001-4000-9000-000000000002}.Release|Any CPU.ActiveCfg = Release|Any CPU
24+
{C0A5D001-0001-4000-9000-000000000002}.Release|Any CPU.Build.0 = Release|Any CPU
25+
{C0A5D001-0001-4000-9000-000000000003}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
26+
{C0A5D001-0001-4000-9000-000000000003}.Debug|Any CPU.Build.0 = Debug|Any CPU
27+
{C0A5D001-0001-4000-9000-000000000003}.Release|Any CPU.ActiveCfg = Release|Any CPU
28+
{C0A5D001-0001-4000-9000-000000000003}.Release|Any CPU.Build.0 = Release|Any CPU
29+
EndGlobalSection
30+
GlobalSection(SolutionProperties) = preSolution
31+
HideSolutionNode = FALSE
32+
EndGlobalSection
33+
EndGlobal

src/HydraDB.Cli/Program.cs

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using HydraDB.Core;
22
using HydraDB.Core.Exec;
33
using HydraDB.Core.Graph;
4+
using HydraDB.Core.Index;
45

56
string directory = args.Length > 0
67
? args[0]
@@ -43,12 +44,24 @@
4344
}
4445

4546
case ".stats":
47+
{
4648
Console.WriteLine(
4749
$"tables={engine.Store.Tables.Count} " +
4850
$"commits={engine.Commits} " +
4951
$"commitSeq={engine.Store.CommitSeq} " +
50-
$"wal={engine.WalBytes}B");
52+
$"wal={engine.WalBytes}B " +
53+
$"lastPlan={engine.LastPlan}");
54+
55+
foreach (var entry in engine.GetAllStats().OrderBy(e => e.Key, StringComparer.OrdinalIgnoreCase))
56+
{
57+
IndexStats s = entry.Value;
58+
Console.WriteLine(
59+
$" {s.Table}: order={s.Order} height={s.Height} keys={s.KeyCount} " +
60+
$"pages={s.PageCount} fill={s.FillFactor:F1}% log={s.LogBytes}B " +
61+
$"crc={(s.CrcOk ? "ok" : "BAD")} secondary={s.SecondaryCount}");
62+
}
5163
break;
64+
}
5265

5366
case ".checkpoint":
5467
engine.Checkpoint();

src/HydraDB.Core/Engine.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@ public Engine(string directory)
5454

5555
public long WalBytes => _wal.SizeBytes;
5656

57+
/// <summary>Access path used by the most recent scan. Diagnostic only.</summary>
58+
public ScanPlan LastPlan => _executor.LastPlan;
59+
60+
/// <summary>Shape of every registered primary index. Runs a full validation per index.</summary>
61+
public Dictionary<string, IndexStats> GetAllStats() => _indexes.GetAllStats();
62+
5763
public Transaction Begin() => _store.BeginTxn();
5864

5965
public QueryResult Execute(string sql, Transaction? txn = null)

src/HydraDB.Core/Exec/DataChunk.cs

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
namespace HydraDB.Core.Exec;
2+
3+
/// <summary>
4+
/// Positions inside a chunk that survived a filter. Filters never move rows: they only
5+
/// narrow this vector, so a chunk can be reused without copying row data.
6+
/// </summary>
7+
public sealed class SelectionVector
8+
{
9+
public SelectionVector(int capacity) => Indices = new int[capacity];
10+
11+
public int[] Indices { get; }
12+
13+
public int Count { get; private set; }
14+
15+
public void Add(int index) => Indices[Count++] = index;
16+
17+
public void Clear() => Count = 0;
18+
19+
/// <summary>Selects the first <paramref name="count"/> rows in order.</summary>
20+
public void SelectAll(int count)
21+
{
22+
for (int i = 0; i < count; i++) Indices[i] = i;
23+
Count = count;
24+
}
25+
}
26+
27+
/// <summary>
28+
/// A batch of rows in columnar-friendly form. Reference type on purpose: one instance is
29+
/// filled, filtered, and yielded repeatedly, so the scan allocates once per query rather
30+
/// than once per row. <see cref="Keys"/> holds the filter column already unboxed so
31+
/// <see cref="VectorFilter"/> can load it straight into a Vector&lt;long&gt;.
32+
/// </summary>
33+
public sealed class DataChunk
34+
{
35+
public const int Capacity = 1024;
36+
37+
public DataChunk(int columnCount)
38+
{
39+
ColumnCount = columnCount;
40+
Rows = new object?[Capacity][];
41+
RowIds = new long[Capacity];
42+
Keys = new long[Capacity];
43+
KeyIsNull = new bool[Capacity];
44+
Selection = new SelectionVector(Capacity);
45+
}
46+
47+
public int ColumnCount { get; }
48+
49+
/// <summary>References to the live row arrays. Never copies of them.</summary>
50+
public object?[][] Rows { get; }
51+
52+
public long[] RowIds { get; }
53+
54+
/// <summary>Unboxed filter column, laid out contiguously for SIMD loads.</summary>
55+
public long[] Keys { get; }
56+
57+
public bool[] KeyIsNull { get; }
58+
59+
public SelectionVector Selection { get; }
60+
61+
public int Count { get; private set; }
62+
63+
public bool IsFull => Count == Capacity;
64+
65+
public void Reset()
66+
{
67+
Count = 0;
68+
Selection.Clear();
69+
}
70+
71+
public void Append(long rowId, object?[] values, int keyColumn)
72+
{
73+
int slot = Count++;
74+
Rows[slot] = values;
75+
RowIds[slot] = rowId;
76+
77+
object? key = keyColumn >= 0 && keyColumn < values.Length ? values[keyColumn] : null;
78+
79+
if (key is long unboxed)
80+
{
81+
Keys[slot] = unboxed;
82+
KeyIsNull[slot] = false;
83+
}
84+
else
85+
{
86+
// NULL never matches a comparison, so the payload value is irrelevant.
87+
Keys[slot] = 0;
88+
KeyIsNull[slot] = true;
89+
}
90+
}
91+
}
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
using System.Numerics;
2+
3+
namespace HydraDB.Core.Exec;
4+
5+
/// <summary>Comparison operators the vectorized filter can evaluate.</summary>
6+
public enum CompareOp
7+
{
8+
Equal,
9+
NotEqual,
10+
Less,
11+
LessOrEqual,
12+
Greater,
13+
GreaterOrEqual
14+
}
15+
16+
/// <summary>
17+
/// SIMD predicate evaluation over a chunk's unboxed key column. Allocation-free: the only
18+
/// values created are structs, and the surviving positions are written into the chunk's
19+
/// existing selection vector. The tail that does not fill a full vector is done scalar.
20+
/// </summary>
21+
public static class VectorFilter
22+
{
23+
public static void Apply(DataChunk chunk, CompareOp op, long threshold)
24+
{
25+
chunk.Selection.Clear();
26+
27+
long[] keys = chunk.Keys;
28+
bool[] isNull = chunk.KeyIsNull;
29+
int count = chunk.Count;
30+
int width = Vector<long>.Count;
31+
var probe = new Vector<long>(threshold);
32+
int i = 0;
33+
34+
for (; i + width <= count; i += width)
35+
{
36+
var block = new Vector<long>(keys, i);
37+
38+
Vector<long> mask = op switch
39+
{
40+
CompareOp.Equal => Vector.Equals(block, probe),
41+
CompareOp.NotEqual => Vector.OnesComplement(Vector.Equals(block, probe)),
42+
CompareOp.Less => Vector.LessThan(block, probe),
43+
CompareOp.LessOrEqual => Vector.LessThanOrEqual(block, probe),
44+
CompareOp.Greater => Vector.GreaterThan(block, probe),
45+
_ => Vector.GreaterThanOrEqual(block, probe)
46+
};
47+
48+
for (int lane = 0; lane < width; lane++)
49+
{
50+
int position = i + lane;
51+
if (mask[lane] != 0 && !isNull[position]) chunk.Selection.Add(position);
52+
}
53+
}
54+
55+
for (; i < count; i++)
56+
if (!isNull[i] && Matches(keys[i], op, threshold)) chunk.Selection.Add(i);
57+
}
58+
59+
public static bool Matches(long value, CompareOp op, long threshold) => op switch
60+
{
61+
CompareOp.Equal => value == threshold,
62+
CompareOp.NotEqual => value != threshold,
63+
CompareOp.Less => value < threshold,
64+
CompareOp.LessOrEqual => value <= threshold,
65+
CompareOp.Greater => value > threshold,
66+
_ => value >= threshold
67+
};
68+
}

0 commit comments

Comments
 (0)