Skip to content

Commit 03977d8

Browse files
ramonsmitsramonsmits
committed
Approve RavenDB index definitions to catch changes that trigger a rebuild
1 parent dd100b9 commit 03977d8

6 files changed

Lines changed: 487 additions & 0 deletions

File tree

‎docs/testing.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ On Windows, [PackagedRavenDBTests](../src/ServiceControlInstaller.Engine.UnitTes
2727
Persistence tests check assumptions at the persistence seam level by exercising each persister.
2828
For local setup details, see [Local testing of persistence providers](testing-persistence.md).
2929

30+
The RavenDB persistence test projects contain an `IndexDefinitionApprovals` test that approves the definition of every index the persister deploys. RavenDB rebuilds an index side by side when its definition changes, and the rebuild of a large database can take days. When the test fails, record the changed indexes in the upgrade guide of the next release before you approve the new definition. An operator who locked an index must unlock it to receive the change.
31+
3032
## Transport tests
3133

3234
Transport tests run the transport test suite for each transport.

‎src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/IndexDefinitionApprovals.Verify_index_definitions_are_unchanged.approved.txt‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
FailedAuditImportIndex
2+
Maps:
3+
docs.FailedAuditImports.Select(cc => new {
4+
Id = Id(cc)
5+
})
6+
7+
MessagesViewIndex
8+
Maps:
9+
docs.ProcessedMessages.Select(message => new {
10+
MessageId = ((string) message.MessageMetadata["MessageId"]),
11+
MessageType = ((string) message.MessageMetadata["MessageType"]),
12+
IsSystemMessage = ((bool) message.MessageMetadata["IsSystemMessage"]),
13+
Status = ((bool) message.MessageMetadata["IsRetried"]) ? "ResolvedSuccessfully" : "Successful",
14+
TimeSent = ((DateTime) message.MessageMetadata["TimeSent"]),
15+
ProcessedAt = message.ProcessedAt,
16+
ReceivingEndpointName = ((message.MessageMetadata["ReceivingEndpoint"])).Name,
17+
SendingEndpointName = ((message.MessageMetadata["SendingEndpoint"])).Name,
18+
CriticalTime = ((TimeSpan ? ) message.MessageMetadata["CriticalTime"]),
19+
ProcessingTime = ((TimeSpan ? ) message.MessageMetadata["ProcessingTime"]),
20+
DeliveryTime = ((TimeSpan ? ) message.MessageMetadata["DeliveryTime"]),
21+
Query = Enumerable.ToArray(DynamicEnumerable.Union(message.MessageMetadata.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(_ => _.Value.ToString()), new string[] {
22+
String.Join(" ", message.Headers.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(x => x.Value))
23+
})),
24+
ConversationId = ((string) message.MessageMetadata["ConversationId"])
25+
})
26+
Fields:
27+
Query: Indexing=Search, Analyzer=Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net, Version=3.0.3.0, Culture=neutral, PublicKeyToken=85089178b9ac3181
28+
29+
MessagesViewIndexWithFullTextSearch
30+
Maps:
31+
docs.ProcessedMessages.Select(message => new {
32+
MessageId = ((string) message.MessageMetadata["MessageId"]),
33+
MessageType = ((string) message.MessageMetadata["MessageType"]),
34+
IsSystemMessage = ((bool) message.MessageMetadata["IsSystemMessage"]),
35+
Status = ((bool) message.MessageMetadata["IsRetried"]) ? "ResolvedSuccessfully" : "Successful",
36+
TimeSent = ((DateTime) message.MessageMetadata["TimeSent"]),
37+
ProcessedAt = message.ProcessedAt,
38+
ReceivingEndpointName = ((message.MessageMetadata["ReceivingEndpoint"])).Name,
39+
SendingEndpointName = ((message.MessageMetadata["SendingEndpoint"])).Name,
40+
CriticalTime = ((TimeSpan ? ) message.MessageMetadata["CriticalTime"]),
41+
ProcessingTime = ((TimeSpan ? ) message.MessageMetadata["ProcessingTime"]),
42+
DeliveryTime = ((TimeSpan ? ) message.MessageMetadata["DeliveryTime"]),
43+
Query = Enumerable.ToArray(DynamicEnumerable.Union(message.MessageMetadata.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(_ => _.Value.ToString()), new string[] {
44+
String.Join(" ", message.Headers.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(x => x.Value)),
45+
this.LoadAttachment(message, "body").GetContentAsString()
46+
})),
47+
ConversationId = ((string) message.MessageMetadata["ConversationId"])
48+
})
49+
Fields:
50+
Query: Indexing=Search, Analyzer=Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net, Version=3.0.3.0, Culture=neutral, PublicKeyToken=85089178b9ac3181
51+
52+
SagaDetailsIndex
53+
Maps:
54+
docs.SagaHistories.Select(doc => new {
55+
SagaId = doc.SagaId,
56+
Id = doc.SagaId,
57+
SagaType = doc.SagaType,
58+
Changes = doc.Changes
59+
})
60+
docs.SagaSnapshots.Select(doc => new {
61+
SagaId = doc.SagaId,
62+
Id = doc.SagaId,
63+
SagaType = doc.SagaType,
64+
Changes = new[] {
65+
new {
66+
Endpoint = doc.Endpoint,
67+
FinishTime = doc.FinishTime,
68+
InitiatingMessage = doc.InitiatingMessage,
69+
OutgoingMessages = doc.OutgoingMessages,
70+
StartTime = doc.StartTime,
71+
StateAfterChange = doc.StateAfterChange,
72+
Status = doc.Status
73+
}
74+
}
75+
})
76+
Reduce:
77+
results.GroupBy(result => result.SagaId).Select(g => new {
78+
g = g,
79+
first = DynamicEnumerable.FirstOrDefault(g)
80+
}).Select(this0 => new {
81+
Id = this0.first.SagaId,
82+
SagaId = this0.first.SagaId,
83+
SagaType = this0.first.SagaType,
84+
Changes = Enumerable.ToList(DynamicEnumerable.OrderByDescending(this0.g.SelectMany(x => x.Changes), x0 => x0.FinishTime).Take(50000))
85+
})
86+
Fields:
87+
Changes: Indexing=No
88+
SagaId: Indexing=Exact
89+
SagaType: Indexing=No
90+

‎src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexDefinitionApprovals.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
namespace ServiceControl.Audit.Persistence.Tests;
2+
3+
using NUnit.Framework;
4+
using Particular.Approvals;
5+
using Persistence.RavenDB;
6+
using TestHelper;
7+
8+
[TestFixture]
9+
class IndexDefinitionApprovals
10+
{
11+
// A changed definition makes RavenDB rebuild the index side by side at the next start-up, which can take days.
12+
// Record the change in the upgrade guide before approving it, see docs/testing.md.
13+
[Test]
14+
public void Verify_index_definitions_are_unchanged() =>
15+
Approver.Verify(IndexDefinitionText.Render(typeof(DatabaseSetup).Assembly));
16+
}

‎src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/IndexDefinitionApprovals.Verify_index_definitions_are_unchanged.approved.txt‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
ArchivedGroupsViewIndex
2+
Maps:
3+
docs.FailedMessages.Where(doc => doc.Status == "Archived").Select(doc => new {
4+
doc = doc,
5+
failureTimes = doc.ProcessingAttempts.Select(x => x.FailureDetails.TimeOfFailure)
6+
}).SelectMany(this0 => this0.doc.FailureGroups, (this0, failureGroup) => new {
7+
Id = failureGroup.Id,
8+
Title = failureGroup.Title,
9+
Count = 1,
10+
First = DynamicEnumerable.Min(this0.failureTimes),
11+
Last = DynamicEnumerable.Max(this0.failureTimes),
12+
Type = failureGroup.Type
13+
})
14+
Reduce:
15+
results.GroupBy(result => new {
16+
Id = result.Id,
17+
Title = result.Title,
18+
Type = result.Type
19+
}).Select(g => new {
20+
Id = g.Key.Id,
21+
Title = g.Key.Title,
22+
Count = Enumerable.Sum(g, x => ((int) x.Count)),
23+
First = DynamicEnumerable.Min(g, x0 => ((DateTime) x0.First)),
24+
Last = DynamicEnumerable.Max(g, x1 => ((DateTime) x1.Last)),
25+
Type = g.Key.Type
26+
})
27+
28+
CustomChecksIndex
29+
Maps:
30+
docs.CustomChecks.Select(cc => new {
31+
Status = cc.Status,
32+
ReportedAt = cc.ReportedAt,
33+
Category = cc.Category,
34+
CustomCheckId = cc.CustomCheckId
35+
})
36+
37+
EndpointIndex
38+
Maps:
39+
docs.EndpointDocuments.Select(message => new {
40+
ThroughputSource = message.EndpointId.ThroughputSource,
41+
Name = message.EndpointId.Name
42+
})
43+
44+
FailedErrorImportIndex
45+
Maps:
46+
docs.FailedErrorImports.Select(cc => new {
47+
Id = Id(cc)
48+
})
49+
50+
FailedMessageFacetsIndex
51+
Maps:
52+
docs.FailedMessages.Where(failure => failure.Status == "Unresolved").Select(failure => new {
53+
failure = failure,
54+
t = ((DynamicEnumerable.LastOrDefault(failure.ProcessingAttempts)).MessageMetadata["ReceivingEndpoint"])
55+
}).Select(this0 => new {
56+
Name = this0.t.Name,
57+
Host = this0.t.Host,
58+
MessageType = (DynamicEnumerable.LastOrDefault(this0.failure.ProcessingAttempts)).MessageMetadata["MessageType"]
59+
})
60+
Fields:
61+
Host: Indexing=Exact
62+
MessageType: Indexing=Exact
63+
Name: Indexing=Exact
64+
65+
FailedMessageRetries/ByBatch
66+
Maps:
67+
docs.FailedMessageRetries.Select(doc => new {
68+
RetryBatchId = doc.RetryBatchId
69+
})
70+
71+
FailedMessageViewIndex
72+
Maps:
73+
docs.FailedMessages.Select(message => new {
74+
message = message,
75+
processingAttemptsLast = DynamicEnumerable.LastOrDefault(message.ProcessingAttempts)
76+
}).Select(this0 => new {
77+
MessageId = this0.processingAttemptsLast.MessageMetadata["MessageId"],
78+
MessageType = this0.processingAttemptsLast.MessageMetadata["MessageType"],
79+
Status = this0.message.Status,
80+
TimeSent = ((DateTime) this0.processingAttemptsLast.MessageMetadata["TimeSent"]),
81+
ReceivingEndpointName = ((this0.processingAttemptsLast.MessageMetadata["ReceivingEndpoint"])).Name,
82+
QueueAddress = this0.processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint,
83+
TimeOfFailure = this0.processingAttemptsLast.FailureDetails.TimeOfFailure,
84+
LastModified = (this.MetadataFor(this0.message).Value < DateTime > ("@last-modified")).Ticks
85+
})
86+
87+
FailedMessages/ByGroup
88+
Maps:
89+
docs.FailedMessages.Select(doc => new {
90+
doc = doc,
91+
processingAttemptsLast = DynamicEnumerable.LastOrDefault(doc.ProcessingAttempts)
92+
}).SelectMany(this0 => this0.doc.FailureGroups, (this0, failureGroup) => new {
93+
Id = Id(this0.doc),
94+
MessageId = this0.doc.UniqueMessageId,
95+
FailureGroupId = failureGroup.Id,
96+
FailureGroupName = failureGroup.Title,
97+
Status = this0.doc.Status,
98+
MessageType = ((string) this0.processingAttemptsLast.MessageMetadata["MessageType"]),
99+
TimeSent = ((DateTime) this0.processingAttemptsLast.MessageMetadata["TimeSent"]),
100+
TimeOfFailure = this0.processingAttemptsLast.FailureDetails.TimeOfFailure,
101+
LastModified = (this.MetadataFor(this0.doc).Value < DateTime > ("@last-modified")).Ticks
102+
})
103+
Fields:
104+
__all_fields: Storage=Yes
105+
106+
FailureGroupsViewIndex
107+
Maps:
108+
docs.FailedMessages.Where(doc => doc.Status == "Unresolved").Select(doc => new {
109+
doc = doc,
110+
failureTimes = doc.ProcessingAttempts.Select(x => x.FailureDetails.TimeOfFailure)
111+
}).SelectMany(this0 => this0.doc.FailureGroups, (this0, failureGroup) => new {
112+
Id = failureGroup.Id,
113+
Title = failureGroup.Title,
114+
Count = 1,
115+
First = DynamicEnumerable.Min(this0.failureTimes),
116+
Last = DynamicEnumerable.Max(this0.failureTimes),
117+
Type = failureGroup.Type
118+
})
119+
Reduce:
120+
results.GroupBy(result => new {
121+
Id = result.Id,
122+
Title = result.Title,
123+
Type = result.Type
124+
}).Select(g => new {
125+
Id = g.Key.Id,
126+
Title = g.Key.Title,
127+
Count = Enumerable.Sum(g, x => ((int) x.Count)),
128+
First = DynamicEnumerable.Min(g, x0 => ((DateTime) x0.First)),
129+
Last = DynamicEnumerable.Max(g, x1 => ((DateTime) x1.Last)),
130+
Type = g.Key.Type
131+
})
132+
133+
GroupCommentIndex
134+
Maps:
135+
docs.GroupComments.Select(gc => new {
136+
Id = Id(gc),
137+
Comment = gc.Comment
138+
})
139+
140+
KnownEndpointIndex
141+
Maps:
142+
docs.KnownEndpoints.Select(message => new {
143+
EndpointDetails_Name = message.EndpointDetails.Name,
144+
EndpointDetails_Host = message.EndpointDetails.Host,
145+
HostDisplayName = message.HostDisplayName,
146+
Monitored = message.Monitored
147+
})
148+
149+
MessagesViewIndex
150+
Maps:
151+
docs.FailedMessages.Select(message => new {
152+
message = message,
153+
last = DynamicEnumerable.LastOrDefault(message.ProcessingAttempts)
154+
}).Select(this0 => new {
155+
MessageId = this0.last.MessageId,
156+
MessageType = ((string) this0.last.MessageMetadata["MessageType"]),
157+
IsSystemMessage = ((bool) this0.last.MessageMetadata["IsSystemMessage"]),
158+
Status = this0.message.Status == "Archived" ? "ArchivedFailure" : (this0.message.Status == "Resolved" ? "ResolvedSuccessfully" : (this0.message.ProcessingAttempts.Count == 1 ? "Failed" : "RepeatedFailure")),
159+
TimeSent = ((DateTime) this0.last.MessageMetadata["TimeSent"]),
160+
ProcessedAt = this0.last.AttemptedAt,
161+
ReceivingEndpointName = ((this0.last.MessageMetadata["ReceivingEndpoint"])).Name,
162+
CriticalTime = ((TimeSpan ? ) this0.last.MessageMetadata["CriticalTime"]),
163+
ProcessingTime = ((TimeSpan ? ) this0.last.MessageMetadata["ProcessingTime"]),
164+
DeliveryTime = ((TimeSpan ? ) this0.last.MessageMetadata["DeliveryTime"]),
165+
Query = Enumerable.ToArray(DynamicEnumerable.Union(this0.last.MessageMetadata.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(_ => _.Value.ToString()), new string[] {
166+
String.Join(" ", this0.last.Headers.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(x => x.Value))
167+
})),
168+
ConversationId = ((string) this0.last.MessageMetadata["ConversationId"])
169+
})
170+
Fields:
171+
Query: Indexing=Search
172+
173+
QueueAddressIndex
174+
Maps:
175+
docs.FailedMessages.Select(message => new {
176+
message = message,
177+
processingAttemptsLast = DynamicEnumerable.LastOrDefault(message.ProcessingAttempts)
178+
}).Select(this0 => new {
179+
PhysicalAddress = this0.processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint,
180+
FailedMessageCount = 1
181+
})
182+
Reduce:
183+
results.GroupBy(result => result.PhysicalAddress).Select(g => new {
184+
PhysicalAddress = g.Key,
185+
FailedMessageCount = Enumerable.Sum(g, m => ((int) m.FailedMessageCount))
186+
})
187+
188+
RetryBatches/ByStatus/ReduceInitialBatchSize
189+
Maps:
190+
docs.RetryBatches.Select(doc => new {
191+
RequestId = doc.RequestId,
192+
RetryType = doc.RetryType,
193+
HasStagingBatches = doc.Status == "Staging",
194+
HasForwardingBatches = doc.Status == "Forwarding",
195+
InitialBatchSize = doc.InitialBatchSize,
196+
Originator = doc.Originator,
197+
Classifier = doc.Classifier,
198+
StartTime = doc.StartTime,
199+
Last = doc.Last
200+
})
201+
Reduce:
202+
results.GroupBy(result => new {
203+
RequestId = result.RequestId,
204+
RetryType = result.RetryType
205+
}).Select(g => new {
206+
RequestId = g.Key.RequestId,
207+
RetryType = g.Key.RetryType,
208+
Originator = (DynamicEnumerable.FirstOrDefault(g)).Originator,
209+
HasStagingBatches = Enumerable.Any(g, x => x.HasStagingBatches),
210+
HasForwardingBatches = Enumerable.Any(g, x0 => x0.HasForwardingBatches),
211+
InitialBatchSize = Enumerable.Sum(g, x1 => ((int) x1.InitialBatchSize)),
212+
StartTime = (DynamicEnumerable.FirstOrDefault(g)).StartTime,
213+
Last = DynamicEnumerable.Max(g, x2 => ((DateTime) x2.Last)),
214+
Classifier = (DynamicEnumerable.FirstOrDefault(g)).Classifier
215+
})
216+
217+
RetryBatches/ByStatusAndSession
218+
Maps:
219+
docs.RetryBatches.Select(doc => new {
220+
RetrySessionId = doc.RetrySessionId,
221+
Status = doc.Status
222+
})
223+

‎src/ServiceControl.Persistence.Tests.RavenDB/IndexDefinitionApprovals.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
namespace ServiceControl.Persistence.Tests.RavenDB
2+
{
3+
using NUnit.Framework;
4+
using Particular.Approvals;
5+
using ServiceControl.Persistence.RavenDB;
6+
using TestHelper;
7+
8+
[TestFixture]
9+
class IndexDefinitionApprovals
10+
{
11+
// A changed definition makes RavenDB rebuild the index side by side at the next start-up, which can take days.
12+
// Record the change in the upgrade guide before approving it, see docs/testing.md.
13+
[Test]
14+
public void Verify_index_definitions_are_unchanged() =>
15+
Approver.Verify(IndexDefinitionText.Render(typeof(DatabaseSetup).Assembly));
16+
}
17+
}

0 commit comments

Comments
 (0)