|
| 1 | +using System.Security.Cryptography; |
| 2 | +using System.Text; |
| 3 | +using Npgsql; |
| 4 | +using SourceFlow.Query; |
| 5 | + |
| 6 | +namespace SourceFlow.Storage.PostgreSQL.Query; |
| 7 | + |
| 8 | +/// <summary>One managed index (doc 14 §7.2).</summary> |
| 9 | +/// <param name="Name">The index name: <c>sfq_</c> + a hash of the definition, so a changed definition is a new index.</param> |
| 10 | +/// <param name="Table">The table (<c>events</c> or <c>documents</c>).</param> |
| 11 | +/// <param name="Source">What it serves, for example <c>bank.account.opened.customerId</c>.</param> |
| 12 | +/// <param name="Sql">The <c>CREATE INDEX</c> statement.</param> |
| 13 | +public sealed record QueryIndex(string Name, string Table, string Source, string Sql); |
| 14 | + |
| 15 | +/// <summary>What an index sync did or would do.</summary> |
| 16 | +/// <param name="Create">Indexes to create (missing).</param> |
| 17 | +/// <param name="Drop">Managed indexes no longer in the catalog.</param> |
| 18 | +/// <param name="Unchanged">Indexes already present.</param> |
| 19 | +public sealed record QueryIndexPlan(IReadOnlyList<QueryIndex> Create, IReadOnlyList<string> Drop, IReadOnlyList<string> Unchanged); |
| 20 | + |
| 21 | +/// <summary> |
| 22 | +/// Index sync for SFQL (doc 14 §7.2; <c>dotnet sourceflow db index sync</c>): one expression index per <c>[Queryable]</c> |
| 23 | +/// field, a partial index per event type over <c>events</c> and per read-model generation over <c>documents</c>. The |
| 24 | +/// expression is exactly the one the SFQL compiler emits, so PostgreSQL matches it; indexes follow the schema and are never |
| 25 | +/// written by hand. Timestamp fields are skipped: the text-to-timestamptz cast isn't immutable, so it can't be indexed. |
| 26 | +/// </summary> |
| 27 | +public static class PostgreSqlIndexSync |
| 28 | +{ |
| 29 | + /// <summary>The prefix of managed index names; sync drops managed indexes the catalog no longer needs.</summary> |
| 30 | + public const string Prefix = "sfq_"; |
| 31 | + |
| 32 | + /// <summary>The indexes the catalog needs.</summary> |
| 33 | + public static IReadOnlyList<QueryIndex> Indexes(QueryCatalog catalog, string schema) |
| 34 | + { |
| 35 | + ArgumentNullException.ThrowIfNull(catalog); |
| 36 | + ArgumentException.ThrowIfNullOrWhiteSpace(schema); |
| 37 | + var indexes = new List<QueryIndex>(); |
| 38 | + foreach (QueryEventType type in catalog.EventTypes) |
| 39 | + { |
| 40 | + foreach ((IReadOnlyList<object> path, QueryField field) in Queryable(type.Latest.Fields, [])) |
| 41 | + { |
| 42 | + string expression = PostgreSqlQueryCompiler.Extract("events", path, field.Type).Replace("events.data", "data", StringComparison.Ordinal); |
| 43 | + indexes.Add(Create(schema, "events", $"{type.Name}.{string.Join('.', path)}", $"tenant_id, ({expression})", $"event_type = {Literal(type.Name)}")); |
| 44 | + } |
| 45 | + } |
| 46 | + |
| 47 | + foreach (QueryReadModel model in catalog.ReadModels) |
| 48 | + { |
| 49 | + foreach ((IReadOnlyList<object> path, QueryField field) in Queryable(model.Fields, [])) |
| 50 | + { |
| 51 | + string expression = PostgreSqlQueryCompiler.Extract("documents", path, field.Type).Replace("documents.data", "data", StringComparison.Ordinal); |
| 52 | + indexes.Add(Create(schema, "documents", $"{model.Collection}.{string.Join('.', path)}", $"tenant_id, ({expression})", $"collection = {Literal(model.Collection)}")); |
| 53 | + } |
| 54 | + } |
| 55 | + |
| 56 | + return [.. indexes.OrderBy(i => i.Source, StringComparer.Ordinal)]; |
| 57 | + } |
| 58 | + |
| 59 | + /// <summary>Compares the catalog's indexes with the database's managed indexes.</summary> |
| 60 | + public static async ValueTask<QueryIndexPlan> PlanAsync(NpgsqlDataSource dataSource, QueryCatalog catalog, string schema, CancellationToken cancellationToken) |
| 61 | + { |
| 62 | + ArgumentNullException.ThrowIfNull(dataSource); |
| 63 | + IReadOnlyList<QueryIndex> wanted = Indexes(catalog, schema); |
| 64 | + HashSet<string> existing = await ExistingAsync(dataSource, schema, cancellationToken).ConfigureAwait(false); |
| 65 | + return new QueryIndexPlan( |
| 66 | + [.. wanted.Where(i => !existing.Contains(i.Name))], |
| 67 | + [.. existing.Where(name => wanted.All(i => i.Name != name)).Order(StringComparer.Ordinal)], |
| 68 | + [.. wanted.Where(i => existing.Contains(i.Name)).Select(i => i.Name)]); |
| 69 | + } |
| 70 | + |
| 71 | + /// <summary> |
| 72 | + /// Creates missing indexes and drops stale managed ones. <paramref name="concurrently"/> (the default) builds without |
| 73 | + /// blocking writes; each statement then runs on its own, outside a transaction. |
| 74 | + /// </summary> |
| 75 | + public static async ValueTask<QueryIndexPlan> ApplyAsync(NpgsqlDataSource dataSource, QueryCatalog catalog, string schema, bool concurrently = true, CancellationToken cancellationToken = default) |
| 76 | + { |
| 77 | + QueryIndexPlan plan = await PlanAsync(dataSource, catalog, schema, cancellationToken).ConfigureAwait(false); |
| 78 | + await using NpgsqlConnection connection = await dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); |
| 79 | + foreach (string name in plan.Drop) |
| 80 | + { |
| 81 | + await using var drop = new NpgsqlCommand($"DROP INDEX {(concurrently ? "CONCURRENTLY " : string.Empty)}IF EXISTS {schema}.{name}", connection); |
| 82 | + await drop.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); |
| 83 | + } |
| 84 | + |
| 85 | + foreach (QueryIndex index in plan.Create) |
| 86 | + { |
| 87 | + string sql = concurrently ? index.Sql.Replace("CREATE INDEX ", "CREATE INDEX CONCURRENTLY ", StringComparison.Ordinal) : index.Sql; |
| 88 | + await using var create = new NpgsqlCommand(sql, connection); |
| 89 | + await create.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); |
| 90 | + } |
| 91 | + |
| 92 | + return plan; |
| 93 | + } |
| 94 | + |
| 95 | + private static QueryIndex Create(string schema, string table, string source, string columns, string predicate) |
| 96 | + { |
| 97 | + string definition = $"{table}|{columns}|{predicate}"; |
| 98 | + string name = Prefix + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(definition)))[..20]; |
| 99 | + return new QueryIndex(name, table, source, $"CREATE INDEX IF NOT EXISTS {name} ON {schema}.{table} ({columns}) WHERE {predicate}"); |
| 100 | + } |
| 101 | + |
| 102 | + private static IEnumerable<(IReadOnlyList<object> Path, QueryField Field)> Queryable(IReadOnlyList<QueryField> fields, IReadOnlyList<object> prefix) |
| 103 | + { |
| 104 | + foreach (QueryField field in fields) |
| 105 | + { |
| 106 | + IReadOnlyList<object> path = [.. prefix, field.Name]; |
| 107 | + if (field.Queryable && !field.PersonalData && field.Type is not (QueryType.Timestamp or QueryType.Object or QueryType.Array or QueryType.Any)) |
| 108 | + { |
| 109 | + yield return (path, field); |
| 110 | + } |
| 111 | + |
| 112 | + foreach ((IReadOnlyList<object> Path, QueryField Field) nested in Queryable(field.Fields, path)) |
| 113 | + { |
| 114 | + yield return nested; |
| 115 | + } |
| 116 | + } |
| 117 | + } |
| 118 | + |
| 119 | + private static async ValueTask<HashSet<string>> ExistingAsync(NpgsqlDataSource dataSource, string schema, CancellationToken cancellationToken) |
| 120 | + { |
| 121 | + await using NpgsqlCommand command = dataSource.CreateCommand("SELECT indexname FROM pg_indexes WHERE schemaname = @schema AND starts_with(indexname, 'sfq_')"); |
| 122 | + command.Parameters.Add(new NpgsqlParameter<string>("schema", schema)); |
| 123 | + var names = new HashSet<string>(StringComparer.Ordinal); |
| 124 | + await using NpgsqlDataReader reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); |
| 125 | + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) |
| 126 | + { |
| 127 | + names.Add(reader.GetString(0)); |
| 128 | + } |
| 129 | + |
| 130 | + return names; |
| 131 | + } |
| 132 | + |
| 133 | + // Catalog names (types, collections) in DDL: quoted SQL string constants; DDL takes no parameters. |
| 134 | + private static string Literal(string value) |
| 135 | + { |
| 136 | + return "'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; |
| 137 | + } |
| 138 | +} |
0 commit comments