Skip to content

Commit 9aa06fc

Browse files
RazvanLiviuVarzaruRazvanLiviuVarzaru
committed
Build TidesDB
1 parent 15c342d commit 9aa06fc

7 files changed

Lines changed: 316 additions & 31 deletions

File tree

‎configuration/builders/definitions/foundry/builders.py‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import yaml
44

55
from configuration.builders.base import GenericBuilder
6+
from configuration.builders.common import docker_config
67
from configuration.builders.sequences.foundry import autobake, dispatcher
78

89
with open(Path(__file__).parent / "foundry.yaml") as f:
@@ -12,42 +13,72 @@
1213
"rpm": autobake.rpm,
1314
"deb": autobake.deb,
1415
}
16+
_REPO_FILE_BY_PACKAGE_TYPE = {
17+
"rpm": "MariaDB.repo",
18+
"deb": "mariadb.sources",
19+
}
20+
# TEMPORARY: pull the MariaDB-devel/libmariadb-dev repo from production CI
21+
# rather than $ARTIFACTS_URL (dev), to build against real tarballs while this
22+
# pipeline is still dev-only. Revert to os.environ["ARTIFACTS_URL"] once done.
23+
_DEVEL_REPO_ARTIFACTS_URL = "https://ci.mariadb.org"
1524

1625

1726
def foundry_scheduler_name(mariadb_version):
1827
return f"foundry_{mariadb_version.replace('.', '_')}_scheduler"
1928

2029

30+
# Selectable via foundry_force_scheduler's "plugin" parameter.
31+
FOUNDRY_PLUGINS = _FOUNDRY_CONFIG["plugins"]
32+
33+
2134
FOUNDRY_BUILDERS_BY_ARCH = {}
2235
# Builders grouped by (ops, os version), regardless of arch.
2336
FOUNDRY_BUILDERS_BY_PACKAGE = {}
2437
for package_config in _FOUNDRY_CONFIG["packages"]:
2538
ops = package_config["ops"]
2639
version = package_config["version"]
2740
package = f"{ops}-{version}"
28-
sequence_fn = _SEQUENCE_BY_PACKAGE_TYPE[package_config["type"]]
41+
package_type = package_config["type"]
42+
sequence_fn = _SEQUENCE_BY_PACKAGE_TYPE[package_type]
43+
repo_file = _REPO_FILE_BY_PACKAGE_TYPE[package_type]
2944
for arch in package_config["arch"]:
45+
# Server autobake builder that publishes MariaDB-devel/libmariadb-dev
46+
# for this platform, e.g. "amd64-debian-12-deb-autobake" -- see
47+
# BUILDERS_AUTOBAKE in constants.py.
48+
autobake_builder = f"{arch}-{package_config['os_info_key']}-{package_type}-autobake"
49+
repo_file_url = (
50+
f"{_DEVEL_REPO_ARTIFACTS_URL}/%(prop:tarbuildnum)s/{autobake_builder}/{repo_file}"
51+
)
3052
builder = GenericBuilder(
3153
name=f"foundry-{arch}-{ops}-{version}",
32-
sequences=[sequence_fn()],
54+
sequences=[
55+
sequence_fn(docker_config(image=f"{ops}{version}"), repo_file_url)
56+
],
3357
)
3458
FOUNDRY_BUILDERS_BY_ARCH.setdefault(arch, []).append(builder)
3559
FOUNDRY_BUILDERS_BY_PACKAGE.setdefault(package, []).append(builder)
3660

37-
# Supported MariaDB versions, and which packages get built for each one.
61+
# Supported MariaDB versions: which packages get built for each, and the
62+
# tarbuildnum of the server CI build to install devel packages from.
3863
# Configured in configuration/builders/definitions/foundry/foundry.yaml.
3964
FOUNDRY_MARIADB_VERSIONS = _FOUNDRY_CONFIG["mariadb_versions"]
40-
for packages in FOUNDRY_MARIADB_VERSIONS.values():
41-
for package in packages:
65+
for version_config in FOUNDRY_MARIADB_VERSIONS.values():
66+
for package in version_config["packages"]:
4267
assert package in FOUNDRY_BUILDERS_BY_PACKAGE, f"Unknown foundry package: {package}"
4368

4469
DISPATCHER_BUILDER = GenericBuilder(
4570
name="foundry-trigger-builders",
4671
sequences=[
4772
dispatcher.trigger_foundry(
4873
[
49-
(foundry_scheduler_name(version), {"mariadb_version": version})
50-
for version in FOUNDRY_MARIADB_VERSIONS
74+
(
75+
foundry_scheduler_name(version),
76+
{
77+
"mariadb_version": version,
78+
"tarbuildnum": version_config["tarbuildnum"],
79+
},
80+
)
81+
for version, version_config in FOUNDRY_MARIADB_VERSIONS.items()
5182
]
5283
)
5384
],

‎configuration/builders/definitions/foundry/foundry.yaml‎

Lines changed: 115 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,127 @@
11
---
2+
# Plugins registered in https://github.com/vuvova/foundry (one top-level
3+
# directory each). Selectable via foundry_force_scheduler's "plugin" param
4+
# and passed straight to "cmake -P run.cmake <plugin>" in the child builders.
5+
plugins:
6+
- disabled-functions
7+
- levenshtein
8+
- tidesql
9+
210
# Foundry package builders: one builder per (package, arch) combination.
11+
# "version" + "ops" concatenate to the docker image tag (see os_info.yaml),
12+
# e.g. ops: ubuntu, version: "22.04" -> image "ubuntu22.04".
13+
# "os_info_key" is the matching top-level key in os_info.yaml -- it's used to
14+
# derive the real MariaDB server autobake builder name
15+
# ("{arch}-{os_info_key}-{type}-autobake", see constants.py) that publishes
16+
# the MariaDB.repo/mariadb.sources file we install -devel packages from. It
17+
# doesn't always match "{ops}-{version}" (e.g. centos-stream9, ubuntu-2204).
318
packages:
419
- ops: centosstream
520
version: "9"
621
type: rpm
7-
arch:
8-
- amd64
9-
- aarch64
10-
22+
os_info_key: centos-stream9
23+
arch: [amd64, aarch64]
24+
- ops: centosstream
25+
version: "10"
26+
type: rpm
27+
os_info_key: centos-stream10
28+
arch: [amd64, aarch64]
29+
- ops: rhel
30+
version: "8"
31+
type: rpm
32+
os_info_key: rhel-8
33+
arch: [amd64, aarch64]
34+
- ops: rhel
35+
version: "9"
36+
type: rpm
37+
os_info_key: rhel-9
38+
arch: [amd64, aarch64]
39+
- ops: rhel
40+
version: "10"
41+
type: rpm
42+
os_info_key: rhel-10
43+
arch: [amd64, aarch64]
44+
- ops: debian
45+
version: "11"
46+
type: deb
47+
os_info_key: debian-11
48+
arch: [amd64, aarch64]
1149
- ops: debian
1250
version: "12"
1351
type: deb
14-
arch:
15-
- amd64
16-
- aarch64
52+
os_info_key: debian-12
53+
arch: [amd64, aarch64]
54+
- ops: debian
55+
version: "13"
56+
type: deb
57+
os_info_key: debian-13
58+
arch: [amd64, aarch64]
59+
- ops: ubuntu
60+
version: "22.04"
61+
type: deb
62+
os_info_key: ubuntu-2204
63+
arch: [amd64, aarch64]
64+
- ops: ubuntu
65+
version: "24.04"
66+
type: deb
67+
os_info_key: ubuntu-2404
68+
arch: [amd64, aarch64]
69+
- ops: ubuntu
70+
version: "26.04"
71+
type: deb
72+
os_info_key: ubuntu-2604
73+
arch: [amd64, aarch64]
74+
# sles/opensuse images only exist for amd64 -- see os_info.yaml.
75+
- ops: sles
76+
version: "1507"
77+
type: rpm
78+
os_info_key: sles-1507
79+
arch: [amd64]
80+
- ops: sles
81+
version: "1600"
82+
type: rpm
83+
os_info_key: sles-1600
84+
arch: [amd64]
85+
- ops: opensuse
86+
version: "1600"
87+
type: rpm
88+
os_info_key: opensuse-1600
89+
arch: [amd64]
1790

18-
# Supported MariaDB versions, and which packages (by "{ops}-{version}") get
19-
# built for each one. Add or remove a version here to change what
20-
# "foundry-trigger-builders" dispatches.
91+
# Supported MariaDB versions: which packages get built for each, and the
92+
# tarbuildnum of the server CI build to install MariaDB-devel/libmariadb-dev
93+
# from (i.e. https://ci.mariadb.org/<tarbuildnum>/). Add or remove a version
94+
# here to change what "foundry-trigger-builders" dispatches.
95+
#
96+
# TODO: these tarbuildnum values are placeholders -- replace with the actual
97+
# last-known-good tarbuildnum for each branch.
2198
mariadb_versions:
2299
"11.4":
23-
- centosstream-9
24-
- debian-12
25-
"11.8":
26-
- debian-12
100+
tarbuildnum: 72543
101+
packages:
102+
- centosstream-9
103+
- centosstream-10
104+
- rhel-8
105+
- rhel-9
106+
- rhel-10
107+
- debian-11
108+
- debian-12
109+
- ubuntu-22.04
110+
- ubuntu-24.04
111+
# "11.8":
112+
# tarbuildnum: 0
113+
# packages:
114+
# - centosstream-9
115+
# - centosstream-10
116+
# - rhel-8
117+
# - rhel-9
118+
# - rhel-10
119+
# - debian-11
120+
# - debian-12
121+
# - debian-13
122+
# - ubuntu-22.04
123+
# - ubuntu-24.04
124+
# - ubuntu-26.04
125+
# - sles-1507
126+
# - sles-1600
127+
# - opensuse-1600

‎configuration/builders/sequences/foundry/autobake.py‎

Lines changed: 63 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,80 @@
33
DockerConfig,
44
InContainer,
55
)
6-
from configuration.steps.remote import PropFromShellStep, ShellStep
7-
from configuration.steps.commands.util import PrintEnvironmentDetails
6+
from configuration.steps.commands.download import GitInitFromCommit
7+
from configuration.steps.commands.foundry import BuildPlugin
8+
from configuration.steps.commands.packages import (
9+
InstallDEBPackages,
10+
InstallRPMPackages,
11+
SetupDEBRepoFromURL,
12+
SetupRPMRepoFromURL,
13+
)
14+
from configuration.steps.remote import ShellStep
815

916
_MARIADB_VERSION_ENV = [("MARIADB_VERSION", "%(prop:mariadb_version)s")]
1017

1118

12-
def deb():
19+
def _clone_foundry_step(config: DockerConfig):
20+
return InContainer(
21+
ShellStep(
22+
# revision is always empty on foundry_force_scheduler, so fetch
23+
# the branch tip instead of a pinned commit.
24+
command=GitInitFromCommit(
25+
repo_url="%(prop:repository)s",
26+
commit="%(prop:branch)s",
27+
),
28+
),
29+
docker_environment=config,
30+
)
31+
32+
33+
def deb(config: DockerConfig, repo_file_url: str):
1334
sequence = BuildSequence()
35+
sequence.add_step(_clone_foundry_step(config))
36+
sequence.add_step(
37+
InContainer(
38+
ShellStep(command=SetupDEBRepoFromURL(repo_file_url)),
39+
docker_environment=config,
40+
container_commit=True,
41+
)
42+
)
1443
sequence.add_step(
15-
ShellStep(command=PrintEnvironmentDetails(), env_vars=_MARIADB_VERSION_ENV)
44+
InContainer(
45+
ShellStep(command=InstallDEBPackages(packages=["libmariadb-dev"])),
46+
docker_environment=config,
47+
container_commit=True,
48+
)
49+
)
50+
sequence.add_step(
51+
InContainer(
52+
ShellStep(command=BuildPlugin("DEB"), env_vars=_MARIADB_VERSION_ENV),
53+
docker_environment=config,
54+
)
1655
)
1756
return sequence
1857

1958

20-
def rpm():
59+
def rpm(config: DockerConfig, repo_file_url: str):
2160
sequence = BuildSequence()
61+
sequence.add_step(_clone_foundry_step(config))
62+
sequence.add_step(
63+
InContainer(
64+
ShellStep(command=SetupRPMRepoFromURL(repo_file_url)),
65+
docker_environment=config,
66+
container_commit=True,
67+
)
68+
)
69+
sequence.add_step(
70+
InContainer(
71+
ShellStep(command=InstallRPMPackages(packages=["MariaDB-devel"])),
72+
docker_environment=config,
73+
container_commit=True,
74+
)
75+
)
2276
sequence.add_step(
23-
ShellStep(command=PrintEnvironmentDetails(), env_vars=_MARIADB_VERSION_ENV)
77+
InContainer(
78+
ShellStep(command=BuildPlugin("RPM"), env_vars=_MARIADB_VERSION_ENV),
79+
docker_environment=config,
80+
)
2481
)
2582
return sequence

‎configuration/schedulers/foundry.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,14 @@
99
schedulers.ForceScheduler(
1010
name="foundry_force_scheduler",
1111
builderNames=[foundry_builders.DISPATCHER_BUILDER.name],
12+
properties=[
13+
util.ChoiceStringParameter(
14+
name="plugin",
15+
label="Plugin to build",
16+
choices=foundry_builders.FOUNDRY_PLUGINS,
17+
default=foundry_builders.FOUNDRY_PLUGINS[0],
18+
),
19+
],
1220
codebases=[
1321
util.CodebaseParameter(
1422
codebase="",
@@ -35,9 +43,9 @@
3543
name=foundry_builders.foundry_scheduler_name(version),
3644
builderNames=[
3745
builder.name
38-
for package in packages
46+
for package in version_config["packages"]
3947
for builder in foundry_builders.FOUNDRY_BUILDERS_BY_PACKAGE[package]
4048
],
4149
)
42-
for version, packages in foundry_builders.FOUNDRY_MARIADB_VERSIONS.items()
50+
for version, version_config in foundry_builders.FOUNDRY_MARIADB_VERSIONS.items()
4351
]

‎configuration/steps/commands/foundry.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
from pathlib import PurePath
2+
3+
from buildbot.plugins import util
4+
from configuration.steps.commands.base import Command
5+
6+
7+
class BuildPlugin(Command):
8+
# package_type: "RPM" or "DEB" -- the -D flag run.cmake expects.
9+
def __init__(self, package_type: str, workdir: PurePath = PurePath(".")):
10+
self.package_type = package_type
11+
super().__init__(name=f"Build plugin ({package_type})", workdir=workdir)
12+
13+
def as_cmd_arg(self) -> list[str]:
14+
return [
15+
"bash",
16+
"-exc",
17+
util.Interpolate(f"cmake -D{self.package_type}=1 -P run.cmake %(prop:plugin)s"),
18+
]

0 commit comments

Comments
 (0)