master.cfg

# -*- python -*-
# ex: set filetype=python:

# git branch filter using fnmatch
import fnmatch
import os

from collections import defaultdict

from buildbot.plugins import schedulers, steps, util, worker
from constants import ALL_PLATFORMS, BUILDERS_GALERA, OS_INFO
from master_common import base_master_config
from utils import createWorker, savePackageIfBranchMatch, waitIfStaging, nextBuild


cfg_dir = os.path.abspath(os.path.dirname(__file__))

# Non autogen master. For now the directory structure is:
# <srcdir>
# └── <master-name>
#     ├── buildbot.tac
#     └── master.cfg
#
# Non autogen masters load from <srcdir> for now.
base_dir = os.path.abspath(f"{cfg_dir}/../")

# Load the slave, database passwords and 3rd-party tokens from an external private file, so
# that the rest of the configuration can be public.
config = {"private": {}}
with open(os.path.join(base_dir, "master-private.cfg"), "r") as file:
    exec(file.read(), config, {})

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = base_master_config(config)

FQDN = os.environ["BUILDMASTER_WG_IP"]


####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.

BRANCHES_MAIN = ["mariadb-3.x", "mariadb-4.x", "bb-*"]
SAVED_PACKAGE_BRANCHES_GALERA = ["mariadb-3.x", "mariadb-4.x", "bb-*"]


def upstream_branch_fn(branch):
    return (
        branch in BRANCHES_MAIN
        or fnmatch.fnmatch(branch, "mariadb-3.x")
        or fnmatch.fnmatch(branch, "mariadb-4.x")
        or fnmatch.fnmatch(branch, "bb-*")
        or fnmatch.fnmatch(branch, "refs/pull/*")
    )


# Override schedulers.
# TODO(cvicentiu): Move this to base_master_config maybe?
c["schedulers"] = []


schedulerTrigger = schedulers.AnyBranchScheduler(
    name="s_upstream_galera",
    change_filter=util.ChangeFilter(
        repository="https://github.com/MariaDB/galera", branch_fn=upstream_branch_fn
    ),
    treeStableTimer=60,
    builderNames=["trigger-galera-builds"],
)
schedulerGaleraBuilders = schedulers.Triggerable(
    name="s_galera_builders", builderNames=BUILDERS_GALERA
)

c["schedulers"].append(schedulerTrigger)
c["schedulers"].append(schedulerGaleraBuilders)

if os.environ["ENVIRON"] == "DEV":
    schedulerTrigger = schedulers.AnyBranchScheduler(
        name="s_upstream_galera_vlad",
        change_filter=util.ChangeFilter(
            repository="https://github.com/RazvanLiviuVarzaru/galera",
            branch_fn=upstream_branch_fn,
        ),
        treeStableTimer=60,
        builderNames=["trigger-galera-builds"],
    )
    c["schedulers"].append(schedulerTrigger)

####### WORKERS

# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.
c["workers"] = []

# Docker workers
GALERA_PACKAGES = os.environ["GALERA_PACKAGES_DIR"]

c["workers"].append(
    worker.DockerLatentWorker(
        "hz-bbw1-docker-galera-trigger",
        None,
        docker_host=config["private"]["docker_workers"]["hz-bbw1-docker"],
        image=os.environ["CONTAINER_REGISTRY_URL"] + "debian12",
        followStartupLogs=False,
        autopull=True,
        alwaysPull=True,
        masterFQDN=FQDN,
        hostconfig={"shm_size": "1G"},
        max_builds=1,
        build_wait_timeout=0,
        volumes=[GALERA_PACKAGES + "/:/packages"],
    )
)

c["workers"].append(
    worker.DockerLatentWorker(
        "hz-bbw4-docker-galera-trigger",
        None,
        docker_host=config["private"]["docker_workers"]["hz-bbw4-docker"],
        image=os.environ["CONTAINER_REGISTRY_URL"] + "debian12",
        followStartupLogs=False,
        autopull=True,
        alwaysPull=True,
        masterFQDN=FQDN,
        hostconfig={"shm_size": "1G"},
        max_builds=1,
        build_wait_timeout=0,
        volumes=[GALERA_PACKAGES + "/:/packages"],
    )
)

workers = defaultdict(list)


def addWorker(
    worker_name_prefix,
    worker_id,
    worker_type,
    dockerfile,
    jobs=5,
    save_packages=False,
    shm_size="15G",
):
    base_name, name, instance = createWorker(
        worker_name_prefix,
        worker_id,
        worker_type,
        dockerfile,
        jobs,
        save_packages,
        shm_size,
        worker_name_suffix="-galera",
        volumes=[
            "/srv/buildbot/ccache:/mnt/ccache",
            "/srv/buildbot/packages:/mnt/packages",
            GALERA_PACKAGES + "/:/packages",
        ],
    )

    workers[base_name].append(name)
    c["workers"].append(instance)


for platform in ALL_PLATFORMS:
    jobs = None
    if platform == "amd64":
        machines = ["hz-bbw"]
        worker_ids = [1, 2, 4, 5]
        jobs = 7
    elif platform == "aarch64":
        machines = ["aarch64-bbw"]
        # Only use bbw[5-7] for aarch64 since the others don't use wireguard
        worker_ids = range(5, 8)
        jobs = 4
    elif platform == "ppc64le":
        machines = ["ppc64le-osuosl-bbw"]
        worker_ids = [1]
        jobs = 12
    elif platform == "s390x":
        machines = ["s390x-bbw"]
        worker_ids = range(1, 3)
        jobs = 8
    elif platform == "x86":
        machines = ["hz-bbw"]
        worker_ids = [2]
        jobs = 7

    assert jobs is not None

    for w_name in machines:
        for i in worker_ids:
            for os_str in OS_INFO:
                if (
                    "install_only" in OS_INFO[os_str]
                    and OS_INFO[os_str]["install_only"]
                ):
                    continue
                if platform in OS_INFO[os_str]["arch"]:
                    quay_name = os.environ["CONTAINER_REGISTRY_URL"] + "".join(
                        os_str.split("-")
                    )
                    os_name = os_str
                    if "ubuntu" in quay_name:
                        quay_name = quay_name[:-2] + "." + quay_name[-2:]
                    if platform == "x86":
                        quay_name += "-386"
                        os_name += "-i386"
                    addWorker(
                        w_name,
                        i,
                        os_name,
                        quay_name,
                        jobs=jobs,
                        save_packages=True,
                    )


def dpkgDeb():
    return steps.ShellCommand(
        name="apt-ftparchive",
        haltOnFailure=True,
        command=[
            "sh",
            "-xc",
            util.Interpolate(
                """set -e
    mkdir -p debs
    find .. -maxdepth 1 -type f -exec cp {} debs/ \;
    cd debs
    apt-ftparchive packages . >Packages
    apt-ftparchive sources . >Sources
    apt-ftparchive release . >Release
    cd ..
    find debs -type f -exec sha256sum {} \; | sort > sha256sums.txt
"""
            ),
        ],
        doStepIf=(
            lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA)
        ),
    )


def isPushToSavedBranch(step):
    return step.getProperty("event") == "push" and any(
        fnmatch.fnmatch(step.getProperty("branch"), pattern)
        for pattern in SAVED_PACKAGE_BRANCHES_GALERA
    )


def rpmSave():
    return steps.ShellCommand(
        name="move rpm files",
        haltOnFailure=True,
        command=[
            "sh",
            "-xc",
            util.Interpolate(
                """set -e
    mkdir -p rpms srpms
    cp `find *.rpm -maxdepth 1 -type f` rpms
    find rpms -type f -exec sha256sum {} \\; | sort > sha256sums.txt
"""
            ),
        ],
        doStepIf=(
            lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA)
        ),
    )


####### FACTORY CODE

f_trigger_builds = util.BuildFactory()
f_trigger_builds.addStep(
    steps.GitHub(
        repourl=util.Property("repository"),
        mode="full",
        method="clobber",
        workdir="build",
        submodules=True,
    )
)
f_trigger_builds.addStep(
    steps.ShellCommand(
        name="build source archive",
        haltOnFailure=True,
        workdir="build",
        command=[
            "bash",
            "-xc",
            """set -e
./scripts/build.sh -a
galera_version=$(basename galera-*.tar.gz .tar.gz)
dest=$PWD/$galera_version
# build.sh's gen_source_archive uses `git checkout-index`, which is
# submodule-unaware and leaves submodule dirs empty (the same gap GitHub
# Releases tarballs have). Copy each submodule's checked-out content in
# before repacking, so the archive is actually self-contained.
git submodule foreach 'cp -rvi * "'"$dest"'/${path}/"'
tar czf ${galera_version}.tar.gz ${galera_version}/
""",
        ],
    )
)
f_trigger_builds.addStep(
    steps.ShellCommand(
        name="save_source_archive",
        timeout=7200,
        haltOnFailure=True,
        command=util.Interpolate(
            """set -e
mkdir -p /packages/%(prop:branch)s/%(prop:revision)s
cp -v galera-*.tar.gz /packages/%(prop:branch)s/%(prop:revision)s/
"""
        ),
        doStepIf=(lambda step: isPushToSavedBranch(step)),
    )
)
f_trigger_builds.addStep(
    steps.Trigger(
        schedulerNames=["s_galera_builders"],
        waitForFinish=False,
        updateSourceStamp=False,
        doStepIf=waitIfStaging,
    )
)

## f_deb_build - create source tarball
f_deb_build = util.BuildFactory()
f_deb_build.addStep(
    steps.ShellCommand(command=["echo", " revision: ", util.Property("revision")])
)
f_deb_build.addStep(
    steps.GitHub(
        repourl=util.Property("repository"),
        mode="full",
        method="clobber",
        workdir="build",
        submodules=True,
    )
)
f_deb_build.addStep(
    steps.ShellCommand(
        name="build packages",
        command=[
            "bash",
            "-xc",
            util.Interpolate(
                """set -e
./scripts/build.sh -p"""
            ),
        ],
        workdir="build",
        env={"DEBIAN": "1", "JOBS": util.Interpolate("%(prop:jobs)s")},
    )
)
f_deb_build.addStep(dpkgDeb())
f_deb_build.addStep(
    steps.ShellCommand(
        name="save_packages",
        timeout=7200,
        haltOnFailure=True,
        command=util.Interpolate(
            """
        . /etc/os-release; \
        mkdir -p /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s \
        && cp -r debs/ sha256sums.txt /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/ \
        && cat << EOF > /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.sources
X-Repolib-Name: Galera
Types: deb
URIs: %(kw:url)s/galera/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/debs
Suites: ./
Trusted: yes
EOF
        ln -sf %(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.sources /packages/%(prop:branch)s-latest-%(prop:buildername)s.sources \
        && sync /packages/%(prop:branch)s/%(prop:revision)s
""",
            url=os.environ["ARTIFACTS_URL"],
        ),
        doStepIf=(
            lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA)
        ),
    )
)
f_deb_build.addStep(
    steps.ShellCommand(
        name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True
    )
)

## f_rpm_build - create source tarball
f_rpm_build = util.BuildFactory()
f_rpm_build.addStep(
    steps.ShellCommand(command=["echo", " revision: ", util.Property("revision")])
)
f_rpm_build.addStep(
    steps.GitHub(
        repourl=util.Property("repository"),
        mode="full",
        method="clobber",
        workdir="build",
        submodules=True,
    )
)
f_rpm_build.addStep(
    steps.ShellCommand(
        name="build packages",
        command=["bash", "-xc", "./scripts/build.sh -p"],
        env={"JOBS": util.Interpolate("%(prop:jobs)s")},
        workdir="build",
    )
)
f_rpm_build.addStep(rpmSave())
f_rpm_build.addStep(
    steps.ShellCommand(
        name="save_packages",
        timeout=7200,
        haltOnFailure=True,
        command=util.Interpolate(
            """
        mkdir -p /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s \
        && cat << EOF > /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.repo
[Galera-%(prop:branch)s]
name=Galera %(prop:branch)s repo (build %(prop:tarbuildnum)s)
baseurl=%(kw:url)s/galera/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/rpms
gpgcheck=0
EOF
        case "%(prop:buildername)s" in
        *rhel-8|*stream8)
            echo "module_hotfixes = 1" >> /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.repo
            ;;
        esac \
        && createrepo rpms/ \
        && cp -r rpms srpms sha256sums.txt /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/ \
        && ln -sf %(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.repo /packages/%(prop:branch)s-latest-%(prop:buildername)s.repo \
        && sync /packages/%(prop:branch)s/%(prop:revision)s
""",
            url=os.environ["ARTIFACTS_URL"],
        ),
        doStepIf=(
            lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA)
        ),
    )
)
f_rpm_build.addStep(
    steps.ShellCommand(
        name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True
    )
)


####### BUILDERS LIST
c["builders"] = []

c["builders"].append(
    util.BuilderConfig(
        name="trigger-galera-builds",
        workernames=[
            "hz-bbw1-docker-galera-trigger",
            "hz-bbw4-docker-galera-trigger",
        ],
        tags=["galera"],
        collapseRequests=True,
        nextBuild=nextBuild,
        factory=f_trigger_builds,
    )
)


for os_i in OS_INFO:
    if "install_only" in OS_INFO[os_i] and OS_INFO[os_i]["install_only"]:
        continue
    for arch in OS_INFO[os_i]["arch"]:
        builder_name = "gal-" + arch + "-" + os_i

        assert builder_name in BUILDERS_GALERA

        worker_name = arch + "-bbw-docker-" + os_i
        if arch == "amd64":
            worker_name = "x64-bbw-docker-" + os_i
        if arch == "x86":
            worker_name = "x64-bbw-docker-" + os_i + "-i386"

        if OS_INFO[os_i]["type"] == "rpm":
            factory = f_rpm_build
        else:
            factory = f_deb_build

        env = {}

        c["builders"].append(
            util.BuilderConfig(
                name=builder_name,
                workernames=workers[worker_name],
                tags=[os_i, "galera", "gcc"],
                collapseRequests=True,
                nextBuild=nextBuild,
                env=env,
                factory=factory,
            )
        )