Skip to content

Commit c772bc8

Browse files
Merge pull request #2123 from abays/fix/custom-image-validation-race
Fix OpenStackVersion custom image validation races
2 parents 6b9156e + 1ac5830 commit c772bc8

5 files changed

Lines changed: 124 additions & 106 deletions

File tree

‎api/core/v1beta1/openstackversion_webhook.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,6 @@ func (r *OpenStackVersion) ValidateUpdate(ctx context.Context, old runtime.Objec
143143
if !ok {
144144
return nil, apierrors.NewInternalError(fmt.Errorf("failed to convert old object to OpenStackVersion"))
145145
}
146-
147146
// Validate that the target stage annotation is not from earlier stage while a minor update is in progress
148147
if err := validateMinorUpdateTargetStageAnnotationProgress(oldVersion, r); err != nil {
149148
return nil, err
@@ -163,9 +162,10 @@ func (r *OpenStackVersion) ValidateUpdate(ctx context.Context, old runtime.Objec
163162
// unless skip annotation is present
164163
if !skipValidation && hasAnyCustomImage(r.Spec.CustomContainerImages) {
165164

166-
// Get the tracked custom images for the previous version
167-
if r.Status.TrackedCustomImages != nil {
168-
if trackedImages, exists := r.Status.TrackedCustomImages[oldVersion.Spec.TargetVersion]; exists {
165+
// Compare against the tracked custom images persisted for the old version.
166+
// The incoming object's status can be stale while the controller is updating it.
167+
if oldVersion.Status.TrackedCustomImages != nil {
168+
if trackedImages, exists := oldVersion.Status.TrackedCustomImages[oldVersion.Spec.TargetVersion]; exists {
169169
// Compare current CustomContainerImages with tracked ones
170170
if !customContainerImagesAllModified(r.Spec.CustomContainerImages, trackedImages) {
171171
return nil, apierrors.NewForbidden(

‎api/core/v1beta1/openstackversion_webhook_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -161,14 +161,15 @@ var _ = Describe("OpenStackVersion Webhook", func() {
161161
Expect(err).ToNot(HaveOccurred())
162162
})
163163

164-
It("should handle edge case where TrackedCustomImages is nil", func() {
164+
It("should use the old version's tracked images when the incoming status is nil", func() {
165165
newVersion.Status.TrackedCustomImages = nil
166166
_, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil)
167-
Expect(err).ToNot(HaveOccurred())
167+
Expect(err).To(HaveOccurred())
168+
Expect(err.Error()).To(ContainSubstring("CustomContainerImages must be updated when changing targetVersion"))
168169
})
169170

170171
It("should handle edge case where previous version not found in TrackedCustomImages", func() {
171-
newVersion.Status.TrackedCustomImages = map[string]CustomContainerImages{
172+
oldVersion.Status.TrackedCustomImages = map[string]CustomContainerImages{
172173
"0.9.0": {}, // Different version than oldVersion.Spec.TargetVersion
173174
}
174175
_, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil)

‎internal/openstack/version.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package openstack
33
import (
44
"context"
55
"fmt"
6+
"maps"
67
"reflect"
78
"regexp"
89
"strings"
@@ -151,8 +152,8 @@ func getOpenstackNetworkExporterImg(customImages corev1beta1.CustomContainerImag
151152
func GetContainerImages(defaults *corev1beta1.ContainerDefaults, instance corev1beta1.OpenStackVersion) corev1beta1.ContainerImages {
152153

153154
containerImages := corev1beta1.ContainerImages{
154-
CinderVolumeImages: instance.Spec.CustomContainerImages.CinderVolumeImages,
155-
ManilaShareImages: instance.Spec.CustomContainerImages.ManilaShareImages,
155+
CinderVolumeImages: maps.Clone(instance.Spec.CustomContainerImages.CinderVolumeImages),
156+
ManilaShareImages: maps.Clone(instance.Spec.CustomContainerImages.ManilaShareImages),
156157
CeilometerProxyImage: getImg(instance.Spec.CustomContainerImages.ApacheImage, defaults.ApacheImage),
157158
OctaviaApacheImage: getImg(instance.Spec.CustomContainerImages.ApacheImage, defaults.ApacheImage),
158159
ContainerTemplate: corev1beta1.ContainerTemplate{

‎internal/openstack/version_test.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
Copyright 2026.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package openstack
18+
19+
import (
20+
"testing"
21+
22+
. "github.com/onsi/gomega" //revive:disable:dot-imports
23+
24+
corev1 "github.com/openstack-k8s-operators/openstack-operator/api/core/v1beta1"
25+
)
26+
27+
func TestGetContainerImagesDoesNotMutateCustomImageSpec(t *testing.T) {
28+
g := NewWithT(t)
29+
cinderImage := "custom.registry/cinder-volume:custom-tag"
30+
manilaImage := "custom.registry/manila-share:custom-tag"
31+
cinderDefault := "default.registry/cinder-volume:tag"
32+
manilaDefault := "default.registry/manila-share:tag"
33+
34+
instance := corev1.OpenStackVersion{
35+
Spec: corev1.OpenStackVersionSpec{
36+
CustomContainerImages: corev1.CustomContainerImages{
37+
CinderVolumeImages: map[string]*string{"backend1": &cinderImage},
38+
ManilaShareImages: map[string]*string{"share-backend1": &manilaImage},
39+
},
40+
},
41+
}
42+
43+
images := GetContainerImages(&corev1.ContainerDefaults{
44+
CinderVolumeImage: &cinderDefault,
45+
ManilaShareImage: &manilaDefault,
46+
}, instance)
47+
48+
g.Expect(instance.Spec.CustomContainerImages.CinderVolumeImages).To(HaveLen(1))
49+
g.Expect(instance.Spec.CustomContainerImages.CinderVolumeImages).NotTo(HaveKey("default"))
50+
g.Expect(instance.Spec.CustomContainerImages.ManilaShareImages).To(HaveLen(1))
51+
g.Expect(instance.Spec.CustomContainerImages.ManilaShareImages).NotTo(HaveKey("default"))
52+
g.Expect(images.CinderVolumeImages).To(HaveKeyWithValue("default", &cinderDefault))
53+
g.Expect(images.ManilaShareImages).To(HaveKeyWithValue("default", &manilaDefault))
54+
}

‎test/functional/ctlplane/openstackversion_controller_test.go‎

Lines changed: 59 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package functional_test
1919
import (
2020
"errors"
2121
"os"
22-
"strings"
2322

2423
. "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports
2524
. "github.com/onsi/gomega" //revive:disable:dot-imports
@@ -36,6 +35,26 @@ import (
3635
"k8s.io/apimachinery/pkg/types"
3736
)
3837

38+
func markOpenStackVersionDeployedWithTrackedImages(version *corev1.OpenStackVersion) {
39+
// The target version may include a build suffix, so retain its full value as the key.
40+
targetVersion := version.Spec.TargetVersion
41+
version.Status.DeployedVersion = &targetVersion
42+
if version.Status.TrackedCustomImages == nil {
43+
version.Status.TrackedCustomImages = make(map[string]corev1.CustomContainerImages)
44+
}
45+
version.Status.TrackedCustomImages[targetVersion] = version.Spec.CustomContainerImages
46+
}
47+
48+
func expectCurrentTargetCustomImagesTracked(name types.NamespacedName) {
49+
Eventually(func(g Gomega) {
50+
version := GetOpenStackVersion(name)
51+
g.Expect(version.Status.DeployedVersion).Should(Not(BeNil()))
52+
g.Expect(*version.Status.DeployedVersion).Should(Equal(version.Spec.TargetVersion))
53+
g.Expect(version.Status.TrackedCustomImages).Should(HaveKey(version.Spec.TargetVersion))
54+
g.Expect(version.Status.TrackedCustomImages[version.Spec.TargetVersion]).Should(Equal(version.Spec.CustomContainerImages))
55+
}, timeout, interval).Should(Succeed())
56+
}
57+
3958
var _ = Describe("OpenStackVersion controller", func() {
4059
BeforeEach(func() {
4160
// lib-common uses OPERATOR_TEMPLATES env var to locate the "templates"
@@ -1145,20 +1164,10 @@ var _ = Describe("OpenStackVersion controller", func() {
11451164
// Simulate deployment by setting DeployedVersion
11461165
Eventually(func(g Gomega) {
11471166
version := GetOpenStackVersion(names.OpenStackVersionName)
1148-
version.Status.DeployedVersion = &initialVersion
1149-
1150-
// Track the custom images for the initial version
1151-
if version.Status.TrackedCustomImages == nil {
1152-
version.Status.TrackedCustomImages = make(map[string]corev1.CustomContainerImages)
1153-
}
1154-
version.Status.TrackedCustomImages[initialVersion] = corev1.CustomContainerImages{
1155-
ContainerTemplate: corev1.ContainerTemplate{
1156-
KeystoneAPIImage: &customKeystoneImage,
1157-
},
1158-
}
1159-
1167+
markOpenStackVersionDeployedWithTrackedImages(version)
11601168
g.Expect(th.K8sClient.Status().Update(th.Ctx, version)).To(Succeed())
11611169
}, timeout, interval).Should(Succeed())
1170+
expectCurrentTargetCustomImagesTracked(names.OpenStackVersionName)
11621171

11631172
// Remove finalizer as needed for cleanup
11641173
DeferCleanup(
@@ -1169,25 +1178,21 @@ var _ = Describe("OpenStackVersion controller", func() {
11691178
})
11701179

11711180
It("should prevent targetVersion modification when CustomContainerImages are unchanged", func() {
1172-
// Attempt to update targetVersion without changing CustomContainerImages
1173-
// This should be rejected by the webhook validation
1174-
Eventually(func(g Gomega) {
1181+
var updateErr error
1182+
Eventually(func() bool {
11751183
version := GetOpenStackVersion(names.OpenStackVersionName)
1176-
1177-
// Try to update to the new version without changing custom images
11781184
version.Spec.TargetVersion = updatedVersion
1179-
// Keep the same custom container images (this should trigger validation error)
11801185
version.Spec.CustomContainerImages = corev1.CustomContainerImages{
11811186
ContainerTemplate: corev1.ContainerTemplate{
11821187
KeystoneAPIImage: &customKeystoneImage,
11831188
},
11841189
}
1185-
1186-
err := k8sClient.Update(ctx, version)
1187-
g.Expect(err).Should(HaveOccurred())
1188-
g.Expect(err.Error()).Should(ContainSubstring("CustomContainerImages must be updated when changing targetVersion"))
1189-
g.Expect(err.Error()).Should(ContainSubstring("prevents proper version tracking and validation"))
1190-
}, timeout, interval).Should(Succeed())
1190+
updateErr = k8sClient.Update(ctx, version)
1191+
return !k8s_errors.IsConflict(updateErr)
1192+
}, timeout, interval).Should(BeTrue())
1193+
Expect(updateErr).Should(HaveOccurred())
1194+
Expect(updateErr.Error()).Should(ContainSubstring("CustomContainerImages must be updated when changing targetVersion"))
1195+
Expect(updateErr.Error()).Should(ContainSubstring("prevents proper version tracking and validation"))
11911196
})
11921197

11931198
It("should allow targetVersion modification when CustomContainerImages are updated", func() {
@@ -1285,20 +1290,10 @@ var _ = Describe("OpenStackVersion controller", func() {
12851290
// Simulate deployment by setting DeployedVersion
12861291
Eventually(func(g Gomega) {
12871292
version := GetOpenStackVersion(names.OpenStackVersionName)
1288-
version.Status.DeployedVersion = &initialVersion
1289-
1290-
// Track the custom images for the initial version
1291-
if version.Status.TrackedCustomImages == nil {
1292-
version.Status.TrackedCustomImages = make(map[string]corev1.CustomContainerImages)
1293-
}
1294-
version.Status.TrackedCustomImages[initialVersion] = corev1.CustomContainerImages{
1295-
CinderVolumeImages: map[string]*string{
1296-
"backend1": &customCinderVolumeImage,
1297-
},
1298-
}
1299-
1293+
markOpenStackVersionDeployedWithTrackedImages(version)
13001294
g.Expect(th.K8sClient.Status().Update(th.Ctx, version)).To(Succeed())
13011295
}, timeout, interval).Should(Succeed())
1296+
expectCurrentTargetCustomImagesTracked(names.OpenStackVersionName)
13021297

13031298
// Remove finalizer as needed for cleanup
13041299
DeferCleanup(
@@ -1309,25 +1304,20 @@ var _ = Describe("OpenStackVersion controller", func() {
13091304
})
13101305

13111306
It("should prevent targetVersion modification when CinderVolumeImages are unchanged", func() {
1312-
// Attempt to update targetVersion without changing CinderVolumeImages
1313-
Eventually(func(g Gomega) {
1307+
var updateErr error
1308+
Eventually(func() bool {
13141309
version := GetOpenStackVersion(names.OpenStackVersionName)
13151310
version.Spec.TargetVersion = updatedVersion
1316-
// Keep the same CinderVolumeImages (this should trigger validation error)
13171311
version.Spec.CustomContainerImages = corev1.CustomContainerImages{
13181312
CinderVolumeImages: map[string]*string{
13191313
"backend1": &customCinderVolumeImage,
13201314
},
13211315
}
1322-
1323-
err := k8sClient.Update(ctx, version)
1324-
if err != nil && strings.Contains(err.Error(), "the object has been modified") {
1325-
// Retry on conflict errors
1326-
return
1327-
}
1328-
g.Expect(err).Should(HaveOccurred())
1329-
g.Expect(err.Error()).Should(ContainSubstring("CustomContainerImages must be updated when changing targetVersion"))
1330-
}, timeout, interval).Should(Succeed())
1316+
updateErr = k8sClient.Update(ctx, version)
1317+
return !k8s_errors.IsConflict(updateErr)
1318+
}, timeout, interval).Should(BeTrue())
1319+
Expect(updateErr).Should(HaveOccurred())
1320+
Expect(updateErr.Error()).Should(ContainSubstring("CustomContainerImages must be updated when changing targetVersion"))
13311321
})
13321322

13331323
It("should allow targetVersion modification when CinderVolumeImages are updated", func() {
@@ -1401,20 +1391,10 @@ var _ = Describe("OpenStackVersion controller", func() {
14011391
// Simulate deployment by setting DeployedVersion
14021392
Eventually(func(g Gomega) {
14031393
version := GetOpenStackVersion(names.OpenStackVersionName)
1404-
version.Status.DeployedVersion = &initialVersion
1405-
1406-
// Track the custom images for the initial version
1407-
if version.Status.TrackedCustomImages == nil {
1408-
version.Status.TrackedCustomImages = make(map[string]corev1.CustomContainerImages)
1409-
}
1410-
version.Status.TrackedCustomImages[initialVersion] = corev1.CustomContainerImages{
1411-
ManilaShareImages: map[string]*string{
1412-
"share-backend1": &customManilaShareImage,
1413-
},
1414-
}
1415-
1394+
markOpenStackVersionDeployedWithTrackedImages(version)
14161395
g.Expect(th.K8sClient.Status().Update(th.Ctx, version)).To(Succeed())
14171396
}, timeout, interval).Should(Succeed())
1397+
expectCurrentTargetCustomImagesTracked(names.OpenStackVersionName)
14181398

14191399
// Remove finalizer as needed for cleanup
14201400
DeferCleanup(
@@ -1425,26 +1405,21 @@ var _ = Describe("OpenStackVersion controller", func() {
14251405
})
14261406

14271407
It("should prevent targetVersion modification when ManilaShareImages are unchanged", func() {
1428-
// Attempt to update targetVersion without changing ManilaShareImages
1429-
Eventually(func(g Gomega) {
1408+
var updateErr error
1409+
Eventually(func() bool {
14301410
version := GetOpenStackVersion(names.OpenStackVersionName)
14311411
version.Spec.TargetVersion = updatedVersion
1432-
// Keep the same ManilaShareImages (this should trigger validation error)
14331412
version.Spec.CustomContainerImages = corev1.CustomContainerImages{
14341413
ManilaShareImages: map[string]*string{
14351414
"share-backend1": &customManilaShareImage,
14361415
},
14371416
}
1438-
1439-
err := k8sClient.Update(ctx, version)
1440-
if err != nil && strings.Contains(err.Error(), "the object has been modified") {
1441-
// Retry on conflict errors
1442-
return
1443-
}
1444-
g.Expect(err).Should(HaveOccurred())
1445-
g.Expect(err.Error()).Should(ContainSubstring("CustomContainerImages must be updated when changing targetVersion"))
1446-
g.Expect(err.Error()).Should(ContainSubstring("prevents proper version tracking and validation"))
1447-
}, timeout, interval).Should(Succeed())
1417+
updateErr = k8sClient.Update(ctx, version)
1418+
return !k8s_errors.IsConflict(updateErr)
1419+
}, timeout, interval).Should(BeTrue())
1420+
Expect(updateErr).Should(HaveOccurred())
1421+
Expect(updateErr.Error()).Should(ContainSubstring("CustomContainerImages must be updated when changing targetVersion"))
1422+
Expect(updateErr.Error()).Should(ContainSubstring("prevents proper version tracking and validation"))
14481423
})
14491424

14501425
It("should allow targetVersion modification when ManilaShareImages are updated", func() {
@@ -1520,20 +1495,10 @@ var _ = Describe("OpenStackVersion controller", func() {
15201495
// Simulate deployment by setting DeployedVersion
15211496
Eventually(func(g Gomega) {
15221497
version := GetOpenStackVersion(names.OpenStackVersionName)
1523-
version.Status.DeployedVersion = &initialVersion
1524-
1525-
// Track the custom images for the initial version
1526-
if version.Status.TrackedCustomImages == nil {
1527-
version.Status.TrackedCustomImages = make(map[string]corev1.CustomContainerImages)
1528-
}
1529-
version.Status.TrackedCustomImages[initialVersion] = corev1.CustomContainerImages{
1530-
ContainerTemplate: corev1.ContainerTemplate{
1531-
KeystoneAPIImage: &customKeystoneImage,
1532-
},
1533-
}
1534-
1498+
markOpenStackVersionDeployedWithTrackedImages(version)
15351499
g.Expect(th.K8sClient.Status().Update(th.Ctx, version)).To(Succeed())
15361500
}, timeout, interval).Should(Succeed())
1501+
expectCurrentTargetCustomImagesTracked(names.OpenStackVersionName)
15371502

15381503
// Remove finalizer as needed for cleanup
15391504
DeferCleanup(
@@ -1606,8 +1571,8 @@ var _ = Describe("OpenStackVersion controller", func() {
16061571
})
16071572

16081573
It("should prevent targetVersion modification without skip annotation when CustomContainerImages are unchanged", func() {
1609-
// Attempt to update targetVersion without skip annotation and unchanged custom images
1610-
Eventually(func(g Gomega) {
1574+
var updateErr error
1575+
Eventually(func() bool {
16111576
version := GetOpenStackVersion(names.OpenStackVersionName)
16121577

16131578
// Ensure no skip annotation
@@ -1623,15 +1588,12 @@ var _ = Describe("OpenStackVersion controller", func() {
16231588
},
16241589
}
16251590

1626-
err := k8sClient.Update(ctx, version)
1627-
if err != nil && strings.Contains(err.Error(), "the object has been modified") {
1628-
// Retry on conflict errors
1629-
return
1630-
}
1631-
g.Expect(err).Should(HaveOccurred())
1632-
g.Expect(err.Error()).Should(ContainSubstring("CustomContainerImages must be updated when changing targetVersion"))
1633-
g.Expect(err.Error()).Should(ContainSubstring("prevents proper version tracking and validation"))
1634-
}, timeout, interval).Should(Succeed())
1591+
updateErr = k8sClient.Update(ctx, version)
1592+
return !k8s_errors.IsConflict(updateErr)
1593+
}, timeout, interval).Should(BeTrue())
1594+
Expect(updateErr).Should(HaveOccurred())
1595+
Expect(updateErr.Error()).Should(ContainSubstring("CustomContainerImages must be updated when changing targetVersion"))
1596+
Expect(updateErr.Error()).Should(ContainSubstring("prevents proper version tracking and validation"))
16351597
})
16361598
})
16371599
})

0 commit comments

Comments
 (0)