Skip to content

Commit a401737

Browse files
RazvanLiviuVarzaruRazvanLiviuVarzaru
committed
Load scripts into bash -c . Make LDD checks command more general
For the base command: - add support for loading a script and replace `VARIABLE=` from **kwargs - add scripts dir under commands. One must be able to run a script alone hence the chosen method of replacing variables LDD check: - make it more general, accept a dict of any number of binaries and linked libs - generate a shell script
1 parent 0943abe commit a401737

7 files changed

Lines changed: 166 additions & 115 deletions

File tree

‎configuration/builders/sequences/debug.py‎

Lines changed: 8 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
from configuration.steps.commands.download import FetchTarball
99
from configuration.steps.commands.mtr import MTRTest
1010
from configuration.steps.commands.util import (
11-
AnyCommand,
11+
GetSSLTests,
12+
LDDCheck,
1213
PrintEnvironmentDetails,
1314
UBIEnableFIPS,
1415
)
@@ -106,11 +107,11 @@ def openssl_fips(
106107
InContainer(
107108
docker_environment=config,
108109
step=ShellStep(
109-
command=AnyCommand(
110-
name="Check if libcrypto is dynamically linked",
111-
command="""
112-
ldd ./client/mariadb | grep libcrypto
113-
ldd ./sql/mariadbd | grep libcrypto""",
110+
command=LDDCheck(
111+
binary_checks={
112+
"./client/mariadb": ["libcrypto"],
113+
"./sql/mariadbd": ["libcrypto"],
114+
},
114115
),
115116
options=StepOptions(
116117
descriptionDone="Check if libcrypto is dynamically linked"
@@ -123,58 +124,7 @@ def openssl_fips(
123124
InContainer(
124125
docker_environment=config,
125126
step=ShellStep(
126-
command=AnyCommand(
127-
name="Extract tests to run",
128-
command="""
129-
set +x
130-
131-
extract_test() {
132-
local filepath="$1"
133-
134-
awk -F'/' -v path="$filepath" '
135-
BEGIN {
136-
n = split(path, parts, "/")
137-
test = parts[n]
138-
sub(/\.test$/, "", test)
139-
dir1 = parts[n-1]
140-
if (dir1 == "t") {
141-
suite = parts[n-2]
142-
} else {
143-
suite = dir1
144-
}
145-
print suite "." test
146-
}
147-
'
148-
}
149-
150-
tests_to_run="mysql-test/tests_to_run.txt"
151-
152-
# Extract all encryption tests
153-
find mysql-test/suite/encryption -type f -name "*.test" | while read -r file; do
154-
extract_test "$file" >> $tests_to_run
155-
done
156-
157-
# Extract all tests having SSL in their name
158-
find mysql-test -name "*ssl*.test" | while read -r file; do
159-
extract_test "$file" >> $tests_to_run
160-
done
161-
162-
# Extract all plugin tests
163-
find plugin/**/* -name "*.test" | while read -r file; do
164-
extract_test "$file" >> $tests_to_run
165-
done
166-
167-
# Extract all tests related to encoding, encryption, and hashing
168-
grep -rliE --include="*.test" 'encode|des_encrypt|aes_encrypt|md5|sha[12]' mysql-test | while read -r file; do
169-
extract_test "$file" >> $tests_to_run
170-
done
171-
172-
# Sort and remove duplicates
173-
sort -u "$tests_to_run" -o "$tests_to_run"
174-
175-
cat $tests_to_run
176-
""",
177-
),
127+
command=GetSSLTests(output_file="mysql-test/tests_to_run.txt"),
178128
options=StepOptions(
179129
descriptionDone="Extract tests to run",
180130
),

‎configuration/steps/commands/base.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,22 @@
11
from abc import ABC, abstractmethod
22
from dataclasses import dataclass
3-
from pathlib import PurePath
3+
from pathlib import Path, PurePath
44

55
from twisted.internet import defer
66

77
from buildbot.plugins import steps, util
88
from buildbot.process.properties import Interpolate
99

10+
# Use if you need to load script files to commands
11+
COMMAND_SCRIPT_BASE_DIR = Path(__file__).parent / "scripts"
12+
13+
14+
def load_script(script_name) -> str:
15+
script_path = COMMAND_SCRIPT_BASE_DIR / script_name
16+
with open(script_path, "r") as f:
17+
script = f.read()
18+
return script
19+
1020

1121
class Command(ABC):
1222
"""
@@ -28,6 +38,25 @@ def as_cmd_arg(self) -> list[str]:
2838
pass
2939

3040

41+
class BashScriptCommand(Command):
42+
def __init__(
43+
self, script_name: str, args: list[str] = None, user: str = "buildbot"
44+
):
45+
name = f"Run {script_name}"
46+
super().__init__(name=name, workdir=PurePath("."), user=user)
47+
self.script_name = script_name
48+
self.args = args if args is not None else []
49+
50+
def as_cmd_arg(self) -> list[str]:
51+
return [
52+
"bash",
53+
"-exc",
54+
util.Interpolate(load_script(script_name=self.script_name)),
55+
"--",
56+
*self.args,
57+
]
58+
59+
3160
@dataclass
3261
class URL:
3362
url: str

‎configuration/steps/commands/mtr.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ class MTRTest(Command):
1717
save_logs_path (PurePath): The path where logs will be saved.
1818
log_path (PurePath): The path where MTR logs are stored.
1919
archive_name (str): The name of the archive file to create.
20-
tests_from_file (str): Optional path to a file containing tests to run. Do not specify suites in this case.
20+
tests_from_file (str): Optional path to a file containing tests to run.
2121
"""
2222

2323
def __init__(

‎configuration/steps/commands/scripts/get_fips_mtr_tests.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
3+
4+
output_file=${1:-"fips_mtr_tests.txt"}
5+
6+
7+
set +x
8+
9+
extract_test() {
10+
local filepath="$1"
11+
12+
awk -F'/' -v path="$filepath" '
13+
BEGIN {
14+
n = split(path, parts, "/")
15+
test = parts[n]
16+
sub(/\.test$/, "", test)
17+
dir1 = parts[n-1]
18+
if (dir1 == "t") {
19+
suite = parts[n-2]
20+
} else {
21+
suite = dir1
22+
}
23+
print suite "." test
24+
}
25+
'
26+
}
27+
28+
29+
# Extract all encryption tests
30+
find mysql-test/suite/encryption -type f -name "*.test" | while read -r file; do
31+
extract_test "$file" >> "$output_file"
32+
done
33+
34+
# Extract all tests having SSL in their name
35+
find mysql-test -name "*ssl*.test" | while read -r file; do
36+
extract_test "$file" >> "$output_file"
37+
done
38+
39+
# Extract all plugin tests
40+
find plugin/**/* -name "*.test" | while read -r file; do
41+
extract_test "$file" >> "$output_file"
42+
done
43+
44+
# Extract all tests related to encoding, encryption, and hashing
45+
grep -rliE --include="*.test" 'encode|des_encrypt|aes_encrypt|md5|sha[12]' mysql-test | while read -r file; do
46+
extract_test "$file" >> "$output_file"
47+
done
48+
49+
# Sort and remove duplicates
50+
sort -u "$output_file" -o "$output_file"
51+
52+
cat "$output_file"

‎configuration/steps/commands/scripts/ldd_check.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
echo "Checking dynamic library dependencies..."
5+
6+
# bash ldd_check.sh "/usr/bin/curl:libssl.so.1.1,libcrypto.so.1.1" "./myapp:libstdc++.so.6,libm.so.6"
7+
for entry in "$@"; do
8+
binary="${entry%%:*}"
9+
libs="${entry#*:}"
10+
11+
echo "Checking $binary..."
12+
[[ -f "$binary" ]] || { echo "Missing binary: $binary"; exit 1; }
13+
14+
IFS=',' read -ra lib_array <<< "$libs"
15+
for lib in "${lib_array[@]}"; do
16+
if ldd "$binary" | grep -q "$lib"; then
17+
echo "$lib found in $binary"
18+
else
19+
echo "$lib NOT found in $binary"
20+
exit 1
21+
fi
22+
done
23+
done
24+
25+
echo "All checks passed."

‎configuration/steps/commands/scripts/ubi_enable_fips.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
3+
# Check if OpenSSL is installed
4+
if ! command -v openssl &> /dev/null; then
5+
echo "OpenSSL is not installed."
6+
exit 1
7+
fi
8+
9+
# Enable FIPS mode in RedHat OpenSSL
10+
sed -i -e '/\[ evp_properties \]/a default_properties = fips=yes' \
11+
-e '/opensslcnf.config/a .include = /etc/crypto-policies/back-ends/openssl_fips.config' \
12+
-e '/\[provider_sect\]/a fips = fips_sect' \
13+
/etc/pki/tls/openssl.cnf
14+
15+
16+
# List providers. If FIPS is not listed, it may not be enabled.
17+
if ! openssl list -providers | grep -q 'fips'; then
18+
echo "FIPS provider is not enabled."
19+
exit 1
20+
fi
21+
22+
# If FIPS is enabled then generating a hash with MD5 should fail
23+
if openssl dgst -md5 /dev/null &> /dev/null; then
24+
echo "FIPS mode is not enabled."
25+
exit 1
26+
else
27+
echo "FIPS mode is enabled."
28+
exit 0
29+
fi

‎configuration/steps/commands/util.py‎

Lines changed: 21 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from pathlib import PurePath
22

33
from buildbot.plugins import util
4-
from configuration.steps.commands.base import Command
4+
from configuration.steps.commands.base import BashScriptCommand, Command
55

66

77
class CreateS3Bucket(Command):
@@ -143,67 +143,33 @@ def as_cmd_arg(self) -> list[str]:
143143
)
144144

145145

146-
class UBIEnableFIPS(Command):
146+
class UBIEnableFIPS(BashScriptCommand):
147147
"""
148-
A command to enable FIPS mode in Red Hat-based systems.
149-
Attributes:
150-
name (str): The name of the command.
151-
workdir (PurePath): The working directory for the command.
148+
A command to enable FIPS mode on UBI containers.
152149
"""
153150

154151
def __init__(self):
155-
name = "Enable FIPS in UBI container"
156-
super().__init__(name=name, workdir=PurePath("."), user="root")
157-
158-
def as_cmd_arg(self) -> list[str]:
159-
return [
160-
"bash",
161-
"-exc",
162-
"""
163-
# Check if OpenSSL is installed
164-
if ! command -v openssl &> /dev/null; then
165-
echo "OpenSSL is not installed."
166-
exit 1
167-
fi
168-
169-
# Enable FIPS mode in RedHat OpenSSL
170-
sed -i -e '/\[ evp_properties \]/a default_properties = fips=yes' \
171-
-e '/opensslcnf.config/a .include = /etc/crypto-policies/back-ends/openssl_fips.config' \
172-
-e '/\[provider_sect\]/a fips = fips_sect' \
173-
/etc/pki/tls/openssl.cnf
174-
175-
176-
# List providers. If FIPS is not listed, it may not be enabled.
177-
if ! openssl list -providers | grep -q 'fips'; then
178-
echo "FIPS provider is not enabled."
179-
exit 1
180-
fi
181-
182-
# If FIPS is enabled then generating a hash with MD5 should fail
183-
if openssl dgst -md5 /dev/null &> /dev/null; then
184-
echo "FIPS mode is not enabled."
185-
exit 1
186-
else
187-
echo "FIPS mode is enabled."
188-
exit 0
189-
fi
190-
""",
191-
]
152+
super().__init__(script_name="ubi_enable_fips.sh", user="root")
192153

193154

194-
class AnyCommand(Command):
155+
class GetSSLTests(BashScriptCommand):
195156
"""
196-
A command that executes any arbitrary shell command.
197-
This command is useful for running custom shell commands that do not fit into predefined categories.
198-
Attributes:
199-
name (str): The name of the command.
200-
workdir (PurePath): The working directory for the command.
201-
command (str): The shell command to execute. Support for interpolation is provided.
157+
A command to extract the list of SSL tests to run from the MariaDB test suite.
202158
"""
203159

204-
def __init__(self, name: str, command: str, workdir: PurePath = PurePath(".")):
205-
super().__init__(name=name, workdir=workdir)
206-
self.command = command
160+
def __init__(self, output_file: str):
161+
args = [output_file]
162+
super().__init__(script_name="get_fips_mtr_tests.sh", args=args)
207163

208-
def as_cmd_arg(self) -> list[str]:
209-
return ["bash", "-exc", util.Interpolate(self.command)]
164+
165+
class LDDCheck(BashScriptCommand):
166+
"""
167+
A command to check dynamic library dependencies of specified binaries.
168+
"""
169+
170+
def __init__(
171+
self,
172+
binary_checks: dict[str, list[str]],
173+
):
174+
args = [f"{binary}:{','.join(libs)}" for binary, libs in binary_checks.items()]
175+
super().__init__(script_name="ldd_check.sh", args=args)

0 commit comments

Comments
 (0)