Skip to content

Commit 2d00f1c

Browse files
grooverdangrooverdan
committed
MDBF-1185 msan/ubsan/asan with clang 22 (experiments)
With the new clang 22 msan, ubasan builders there is the oportunity to test new suites/options. As new version allow a little time to experiment with different options, added an experimental option to helpers. The spider suite is enabled in Debug but not RelWithDeb info Debug is proved stable, so check if the non-debug is also. This proved quick and stable locally. On testing cursor and view protocols on msan and ubasan in both debug and non-debug, the ubasan non-debug was showing the same few trivial tests to correct as elsewhere and ran significantly faster. msan view showed a test failure that occurred without MSAN. This was fast enough in a non-debug mode.
1 parent ff553bf commit 2d00f1c

3 files changed

Lines changed: 138 additions & 15 deletions

File tree

‎configuration/builders/sequences/helpers.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,102 @@ def get_mtr_normal_steps(
6565
return steps
6666

6767

68+
def get_mtr_cursor_steps(
69+
jobs,
70+
path_to_test_runner: PurePath,
71+
halt_on_failure: bool = True,
72+
step_wrapping_fn=lambda step: step,
73+
additional_mtr_options: list[MTROption] = [],
74+
env_vars: list[tuple] = [],
75+
):
76+
steps = []
77+
steps.append(
78+
step_wrapping_fn(
79+
ShellStep(
80+
MTRTest(
81+
name="cursor",
82+
save_logs_path=MTR_PATH_TO_SAVE_LOGS / "cursor",
83+
workdir=path_to_test_runner,
84+
testcase=MTRGenerator(
85+
flags=[
86+
MTROption(MTR.VERBOSE_RESTART, True),
87+
MTROption(MTR.FORCE, True),
88+
MTROption(MTR.CURSOR_PROTOCOL, True),
89+
MTROption(MTR.MAX_SAVE_CORE, 2),
90+
MTROption(MTR.MAX_SAVE_DATADIR, 1),
91+
MTROption(MTR.MAX_TEST_FAIL, 20),
92+
MTROption(MTR.PARALLEL, jobs * 2),
93+
MTROption(MTR.VARDIR, "/dev/shm/cursor"),
94+
MTROption(
95+
MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "cursor.xml"
96+
),
97+
]
98+
+ additional_mtr_options,
99+
suite_collection=TestSuiteCollection(
100+
[
101+
SUITE.MAIN,
102+
]
103+
),
104+
),
105+
),
106+
options=StepOptions(
107+
haltOnFailure=halt_on_failure, descriptionDone="MTR cursor"
108+
),
109+
env_vars=env_vars,
110+
)
111+
)
112+
)
113+
return steps
114+
115+
116+
def get_mtr_view_steps(
117+
jobs,
118+
path_to_test_runner: PurePath,
119+
halt_on_failure: bool = True,
120+
step_wrapping_fn=lambda step: step,
121+
additional_mtr_options: list[MTROption] = [],
122+
env_vars: list[tuple] = [],
123+
):
124+
steps = []
125+
steps.append(
126+
step_wrapping_fn(
127+
ShellStep(
128+
MTRTest(
129+
name="view",
130+
save_logs_path=MTR_PATH_TO_SAVE_LOGS / "view",
131+
workdir=path_to_test_runner,
132+
testcase=MTRGenerator(
133+
flags=[
134+
MTROption(MTR.VERBOSE_RESTART, True),
135+
MTROption(MTR.FORCE, True),
136+
MTROption(MTR.VIEW_PROTOCOL, True),
137+
MTROption(MTR.MAX_SAVE_CORE, 2),
138+
MTROption(MTR.MAX_SAVE_DATADIR, 1),
139+
MTROption(MTR.MAX_TEST_FAIL, 20),
140+
MTROption(MTR.PARALLEL, jobs * 2),
141+
MTROption(MTR.VARDIR, "/dev/shm/view"),
142+
MTROption(
143+
MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "view.xml"
144+
),
145+
]
146+
+ additional_mtr_options,
147+
suite_collection=TestSuiteCollection(
148+
[
149+
SUITE.MAIN,
150+
]
151+
),
152+
),
153+
),
154+
options=StepOptions(
155+
haltOnFailure=halt_on_failure, descriptionDone="MTR view"
156+
),
157+
env_vars=env_vars,
158+
)
159+
)
160+
)
161+
return steps
162+
163+
68164
def get_mtr_rocksdb_steps(
69165
jobs,
70166
path_to_test_runner: PurePath,

‎configuration/builders/sequences/sanitizers.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
InContainer,
77
)
88
from configuration.builders.sequences.helpers import (
9+
get_mtr_cursor_steps,
10+
get_mtr_view_steps,
911
get_mtr_normal_steps,
1012
get_mtr_s3_steps,
1113
get_mtr_spider_steps,
@@ -34,6 +36,7 @@ def asan_ubsan(
3436
config: DockerConfig,
3537
jobs: int,
3638
isDebugBuildType: bool,
39+
isExperimental: bool,
3740
):
3841
sequence = BuildSequence()
3942

@@ -179,13 +182,34 @@ def asan_ubsan(
179182
):
180183
sequence.add_step(step)
181184

185+
if isExperimental and not isDebugBuildType:
186+
for step in get_mtr_cursor_steps(
187+
jobs=jobs,
188+
env_vars=env_vars,
189+
halt_on_failure=False,
190+
path_to_test_runner=PurePath("bld", "mysql-test"),
191+
step_wrapping_fn=lambda step: InContainer(
192+
docker_environment=config, step=step
193+
),
194+
) + get_mtr_view_steps(
195+
jobs=jobs,
196+
env_vars=env_vars,
197+
halt_on_failure=False,
198+
path_to_test_runner=PurePath("bld", "mysql-test"),
199+
step_wrapping_fn=lambda step: InContainer(
200+
docker_environment=config, step=step
201+
),
202+
):
203+
sequence.add_step(step)
204+
182205
return sequence
183206

184207

185208
def msan(
186209
config: DockerConfig,
187210
jobs: int,
188211
isDebugBuildType: bool,
212+
isExperimental: bool,
189213
):
190214
sequence = BuildSequence()
191215

@@ -214,7 +238,7 @@ def msan(
214238
CMakeOption(WITH.ZLIB, "bundled"),
215239
CMakeOption(WITH.SYSTEMD, "no"),
216240
CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False),
217-
CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType),
241+
CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType or isExperimental),
218242
CMakeOption(PLUGIN.ROCKSDB_STORAGE_ENGINE, False),
219243
CMakeOption(PLUGIN.OQGRAPH_STORAGE_ENGINE, False),
220244
]
@@ -279,7 +303,7 @@ def msan(
279303
path_to_test_runner=PurePath("bld", "mysql-test"),
280304
step_wrapping_fn=lambda step: InContainer(docker_environment=config, step=step),
281305
)
282-
if isDebugBuildType:
306+
if isDebugBuildType or isExperimental:
283307
steps += get_mtr_spider_steps(
284308
jobs=jobs,
285309
env_vars=env_vars,
@@ -290,6 +314,7 @@ def msan(
290314
docker_environment=config, step=step
291315
),
292316
)
317+
# Save results
293318
steps += [
294319
save_mtr_logs(
295320
step_wrapping_fn=lambda step: InContainer(

‎master-migration/master.cfg‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ c["builders"].append(
160160

161161

162162
def ubasan_builder(
163-
name: str, image: str, debug: bool, workers: Iterable[WorkerBase]
163+
name: str, image: str, debug: bool, experimental: bool, workers: Iterable[WorkerBase]
164164
) -> GenericBuilder:
165165
tags_ubasan = ("Debian", "clang", "asan", "ubsan", "big")
166166
jobs = 12
@@ -174,6 +174,7 @@ def ubasan_builder(
174174
jobs=jobs,
175175
config=docker_config(image=image, shm_size="24g"),
176176
isDebugBuildType=debug,
177+
isExperimental=experimental,
177178
)
178179
],
179180
).get_config(
@@ -185,25 +186,25 @@ def ubasan_builder(
185186

186187
sg_bbw1 = WORKER_POOL.get_workers_for_arch(arch="amd64", names=["sg-bbw1"])
187188

188-
for builder, image, workers in [
189-
("amd64-ubasan-clang-20", "debian12-msan-clang-20", DEFAULT_AMD64_WORKER_POOL),
189+
for builder, image, experimental, workers in [
190+
("amd64-ubasan-clang-20", "debian12-msan-clang-20", False, DEFAULT_AMD64_WORKER_POOL),
190191
(
191192
"amd64-ubasan-clang-20-debug",
192-
"debian12-msan-clang-20",
193+
"debian12-msan-clang-20", False,
193194
DEFAULT_AMD64_WORKER_POOL,
194195
),
195-
("amd64-ubasan-clang-22", "debian13-msan-clang-22", sg_bbw1),
196-
("amd64-ubasan-clang-22-debug", "debian13-msan-clang-22", sg_bbw1),
196+
("amd64-ubasan-clang-22", "debian13-msan-clang-22", True, sg_bbw1),
197+
("amd64-ubasan-clang-22-debug", "debian13-msan-clang-22", True, sg_bbw1),
197198
]:
198199
c["builders"].append(
199200
ubasan_builder(
200-
name=builder, image=image, debug=builder.endswith("debug"), workers=workers
201+
name=builder, image=image, debug=builder.endswith("debug"), experimental, workers=workers
201202
)
202203
)
203204

204205

205206
def msan_builder(
206-
name: str, image: str, debug: bool, workers: Iterable[WorkerBase]
207+
name: str, image: str, debug: bool, experimental: bool, workers: Iterable[WorkerBase]
207208
) -> GenericBuilder:
208209
tags_msan = ("Debian", "clang", "msan", "big")
209210
jobs = 12
@@ -218,6 +219,7 @@ def msan_builder(
218219
jobs=jobs,
219220
config=docker_config(image=image, shm_size="24g"),
220221
isDebugBuildType=debug,
222+
isExperimental=experimental,
221223
)
222224
],
223225
).get_config(
@@ -227,14 +229,14 @@ def msan_builder(
227229
)
228230

229231

230-
for builder, image, workers in [
231-
("amd64-msan-clang-20-debug", "debian12-msan-clang-20", DEFAULT_AMD64_WORKER_POOL),
232-
("amd64-msan-clang-22", "debian13-msan-clang-22", sg_bbw1),
233-
("amd64-msan-clang-22-debug", "debian13-msan-clang-22", sg_bbw1),
232+
for builder, image, experimental, workers in [
233+
("amd64-msan-clang-20-debug", "debian12-msan-clang-20", False, DEFAULT_AMD64_WORKER_POOL),
234+
("amd64-msan-clang-22", "debian13-msan-clang-22", True, sg_bbw1),
235+
("amd64-msan-clang-22-debug", "debian13-msan-clang-22", True, sg_bbw1),
234236
]:
235237
c["builders"].append(
236238
msan_builder(
237-
name=builder, image=image, debug=builder.endswith("debug"), workers=workers
239+
name=builder, image=image, debug=builder.endswith("debug"), experimental, workers=workers
238240
)
239241
)
240242

0 commit comments

Comments
 (0)