Skip to content

Commit 76bd7e7

Browse files
committed
fix end-to-end test
Signed-off-by: ziadhany <ziadhany2016@gmail.com>
1 parent a3e8fe3 commit 76bd7e7

2 files changed

Lines changed: 183 additions & 98 deletions

File tree

‎tests/end_to_end/base.py‎

Lines changed: 136 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,12 @@
1717
# along with this program. If not, see <http://www.gnu.org/licenses/>.
1818
#
1919

20+
import html
2021
import logging
2122
import os
23+
import re
2224
import signal
25+
import socket
2326
import subprocess
2427
import sys
2528
import tempfile
@@ -39,23 +42,41 @@
3942
from grimoirelab_metrics.cli import grimoirelab_metrics
4043

4144
GRIMOIRELAB_URL = "http://localhost:8000"
45+
GRIMOIRELAB_PORT = 8000
4246
GRIMOIRELAB_USER = "admin"
4347
GRIMOIRELAB_PASSWORD = "admin"
4448
GRIMOIRELAB_ECOSYSTEM = "npm-training-set"
4549
GRIMOIRELAB_PROJECT = "npm-popular-components"
4650

47-
# The default OpenSearch image used by testcontainers (1.3.x) is too old
48-
# for GrimoireLab: it rejects the 'copy_alias' field of the ISM rollover
49-
# action, which makes the archivists crash on startup. A recent image is
50-
# required (docker-compose uses 'opensearchproject/opensearch:3').
51-
OPENSEARCH_IMAGE = "opensearchproject/opensearch:2.19.2"
51+
OPENSEARCH_IMAGE = "opensearchproject/opensearch:3"
5252
OPENSEARCH_USER = "admin"
5353
OPENSEARCH_PASSWORD = "admin"
5454
OPENSEARCH_INDEX = "events"
5555

5656
GRIMOIRELAB_SERVER_TIMEOUT = 120
5757
GRIMOIRELAB_WORKERS_TIMEOUT = 120
5858

59+
TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
60+
ARCHIVED_REPOS_FILE = os.path.join(TESTS_DIR, "data", "archived_repos.spdx.xml")
61+
62+
PORT_IN_USE_HELP = (
63+
"Port 8000, where the tests run the GrimoireLab server, is already in use.\n"
64+
"Most likely there are processes or containers left over from a previous "
65+
"run (they can survive when a run is interrupted). Clean them up with:\n"
66+
"\n"
67+
' pkill -f "grimoirelab run"\n'
68+
' docker ps --filter "label=org.testcontainers=true" -q | xargs -r docker rm -f\n'
69+
"\n"
70+
"Also stop the docker-compose stack if it is running, as its nginx "
71+
"container publishes port 8000 too:\n"
72+
"\n"
73+
" docker compose down\n"
74+
"\n"
75+
"Verify the port is free afterwards with:\n"
76+
"\n"
77+
" ss -ltnp | grep :8000"
78+
)
79+
5980

6081
class EndToEndTestCase(TestCase):
6182
"""Base class to build end to end tests.
@@ -82,28 +103,22 @@ def setUpClass(cls):
82103
cls._grimoirelab_logs = {}
83104

84105
try:
106+
cls._check_grimoirelab_port_is_free()
85107
cls._start_redis_container()
86108
cls._start_database_container()
87109
cls._start_opensearch_container()
88110
cls._start_grimoirelab()
89111
cls._create_grimoirelab_resources()
90112
cls._preload_repositories()
91113
except Exception:
92-
# unittest does not call tearDownClass when setUpClass fails,
93-
# so clean up here to avoid leaking containers and processes.
94114
cls._cleanup()
95115
raise
96116

97117
@classmethod
98118
def tearDownClass(cls):
99-
# Give the workers some extra time to finish their pending jobs.
100119
time.sleep(20)
101120
cls._cleanup()
102121

103-
##
104-
# Command execution helpers
105-
##
106-
107122
@staticmethod
108123
def _reset_logging():
109124
"""Remove the logging handlers configured by previous invocations.
@@ -139,9 +154,25 @@ def _result_details(result):
139154

140155
return details
141156

142-
##
143-
# Containers
144-
##
157+
@staticmethod
158+
def _port_is_free(port):
159+
"""Check whether nothing is listening on '127.0.0.1:<port>'."""
160+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
161+
sock.settimeout(1)
162+
return sock.connect_ex(("127.0.0.1", port)) != 0
163+
164+
@classmethod
165+
def _check_grimoirelab_port_is_free(cls):
166+
"""Ensure the port used by the GrimoireLab server is available.
167+
168+
When an orphaned 'grimoirelab run server' (or the nginx container
169+
of the docker-compose stack) is holding the port, the server of
170+
the tests dies with a bind error and the requests are answered
171+
by that orphan, whose database containers no longer exist. The
172+
symptom is 500 errors like "Can't connect to MySQL server".
173+
"""
174+
if not cls._port_is_free(GRIMOIRELAB_PORT):
175+
raise RuntimeError(PORT_IN_USE_HELP)
145176

146177
@classmethod
147178
def _start_database_container(cls):
@@ -156,7 +187,7 @@ def _start_redis_container(cls):
156187
@classmethod
157188
def _start_opensearch_container(cls):
158189
# Keep the security plugin disabled so the server is reachable
159-
# through plain HTTP without authentication, like before.
190+
# through plain HTTP without authentication.
160191
cls.opensearch_container = (
161192
OpenSearchContainer(image=OPENSEARCH_IMAGE)
162193
.with_exposed_ports(9200)
@@ -172,18 +203,11 @@ def _start_opensearch_container(cls):
172203
cls.opensearch_user = OPENSEARCH_USER
173204
cls.opensearch_password = OPENSEARCH_PASSWORD
174205

175-
##
176-
# GrimoireLab
177-
##
178-
179206
@classmethod
180207
def _start_grimoirelab_process(cls, name, args):
181208
"""Start a GrimoireLab process, storing its output in a log file."""
182209
log_file = tempfile.NamedTemporaryFile(delete=False, prefix=f"grimoirelab_{name}_", suffix=".log")
183210
cls._grimoirelab_logs[name] = log_file
184-
185-
# Unbuffered output, so logs reach the file while the process
186-
# is still running (needed by '_wait_for_worker_logs').
187211
env = dict(os.environ)
188212
env["PYTHONUNBUFFERED"] = "1"
189213

@@ -308,17 +332,42 @@ def _wait_for_grimoirelab_server(cls, timeout=GRIMOIRELAB_SERVER_TIMEOUT):
308332
raise RuntimeError("GrimoireLab server exited unexpectedly.\n" + cls._dump_grimoirelab_logs())
309333
try:
310334
urllib.request.urlopen(GRIMOIRELAB_URL, timeout=5)
335+
cls._confirm_server_is_ours()
311336
return
312337
except urllib.error.HTTPError:
313-
# Any HTTP response means the server is already running.
338+
cls._confirm_server_is_ours()
314339
return
340+
except RuntimeError:
341+
raise
315342
except Exception:
316343
time.sleep(1)
317344

318345
raise RuntimeError(
319346
f"GrimoireLab server was not ready after {timeout} seconds.\n" + cls._dump_grimoirelab_logs()
320347
)
321348

349+
@classmethod
350+
def _confirm_server_is_ours(cls):
351+
"""Confirm the server answering the requests is the one the tests started.
352+
353+
If port 8000 is taken by another process (an orphaned
354+
'grimoirelab run server' from a previous run, or the nginx
355+
container of the docker-compose stack), the server of the tests
356+
dies with a bind error while that other process answers the
357+
requests using the ports of containers that no longer exist.
358+
"""
359+
time.sleep(2)
360+
if cls.grimoirelab_server.poll() is not None:
361+
raise RuntimeError(
362+
"The GrimoireLab server process of the tests died right after "
363+
"starting, but SOMETHING is still answering on port 8000. "
364+
"Another process took the port and is answering the requests "
365+
"with the configuration of a previous run.\n\n"
366+
+ PORT_IN_USE_HELP
367+
+ "\n\n--- server output ---\n"
368+
+ cls._dump_grimoirelab_logs()
369+
)
370+
322371
@classmethod
323372
def _wait_for_worker_logs(cls, name, proc, timeout=GRIMOIRELAB_WORKERS_TIMEOUT):
324373
"""Wait until a worker process reports it is listening for jobs."""
@@ -344,19 +393,14 @@ def _wait_for_worker_logs(cls, name, proc, timeout=GRIMOIRELAB_WORKERS_TIMEOUT):
344393
timeout,
345394
)
346395

347-
##
348-
# Ecosystem and project creation
349-
##
350-
351396
@classmethod
352397
def _create_grimoirelab_resources(cls):
353398
"""Create the ecosystem and the project used by the tests.
354399
355400
The metrics CLI takes 'grimoirelab-ecosystem' and
356401
'grimoirelab-project' as parameters and queries their repos
357402
endpoint, but it does not create them; they must exist
358-
beforehand, or every call becomes a 404 on
359-
'ecosystems/None/projects/None'.
403+
beforehand, or every call becomes a 404.
360404
"""
361405
api_url = f"{GRIMOIRELAB_URL}/api/v1"
362406
headers = cls._api_headers(api_url)
@@ -379,14 +423,23 @@ def _create_grimoirelab_resources(cls):
379423
@classmethod
380424
def _api_headers(cls, api_url):
381425
"""Return the headers to authenticate against the GrimoireLab API."""
382-
response = requests.post(
383-
f"{GRIMOIRELAB_URL}/token/",
384-
json={"username": GRIMOIRELAB_USER, "password": GRIMOIRELAB_PASSWORD},
385-
timeout=30,
386-
)
426+
try:
427+
response = requests.post(
428+
f"{GRIMOIRELAB_URL}/token/",
429+
json={"username": GRIMOIRELAB_USER, "password": GRIMOIRELAB_PASSWORD},
430+
timeout=30,
431+
)
432+
except requests.RequestException as exc:
433+
raise RuntimeError(
434+
f"Error connecting to the GrimoireLab server: {exc!r}\n\n" + cls._server_diagnostics()
435+
)
436+
387437
if response.status_code != 200:
388438
raise RuntimeError(
389-
f"Error getting the API token. Status: {response.status_code}. Body: {response.text}"
439+
"Error getting the API token. "
440+
f"Status: {response.status_code}. "
441+
f"Detail: {cls._html_error_detail(response.text)}\n\n"
442+
+ cls._server_diagnostics()
390443
)
391444

392445
data = response.json()
@@ -404,34 +457,72 @@ def _api_headers(cls, api_url):
404457
raise RuntimeError("None of the authentication schemes was accepted by the GrimoireLab API.")
405458

406459
@staticmethod
407-
def _api_get_or_create(headers, get_url, post_url, body, kind):
460+
def _html_error_detail(body):
461+
"""Extract the exception message from a Django debug page."""
462+
match = re.search(r'<pre class="exception_value">(.*?)</pre>', body, re.DOTALL)
463+
if match:
464+
return html.unescape(match.group(1)).strip()
465+
# Not a Django debug page; return a truncated copy.
466+
return body[:500]
467+
468+
@classmethod
469+
def _server_diagnostics(cls):
470+
"""Diagnostics to append when a request to the server fails."""
471+
lines = []
472+
473+
server = getattr(cls, "grimoirelab_server", None)
474+
if server is not None:
475+
if server.poll() is None:
476+
lines.append("* Server process started by the tests: running.")
477+
else:
478+
lines.append(
479+
f"* Server process started by the tests: DEAD (exit code {server.returncode}). "
480+
"The requests were answered by ANOTHER process using port 8000."
481+
)
482+
483+
db_port = os.environ.get("GRIMOIRELAB_DB_PORT")
484+
if db_port:
485+
if cls._port_is_free(int(db_port)):
486+
lines.append(
487+
f"* MariaDB container on port {db_port}: NOT reachable. Either the "
488+
"container died, or the requests were answered by an orphaned server "
489+
"configured with the ports of a previous run."
490+
)
491+
else:
492+
lines.append(f"* MariaDB container on port {db_port}: reachable.")
493+
494+
lines.append(cls._dump_grimoirelab_logs())
495+
return "\n".join(lines)
496+
497+
@classmethod
498+
def _api_get_or_create(cls, headers, get_url, post_url, body, kind):
408499
"""Create a resource through the API if it does not exist yet."""
409500
response = requests.get(get_url, headers=headers, timeout=30)
410501
if response.status_code == 200:
411502
return
412503

413504
if response.status_code != 404:
414505
raise RuntimeError(
415-
f"Error checking the {kind}. Status: {response.status_code}. Body: {response.text}"
506+
f"Error checking the {kind}. Status: {response.status_code}. "
507+
f"Detail: {cls._html_error_detail(response.text)}\n\n"
508+
+ cls._server_diagnostics()
416509
)
417510

418511
response = requests.post(post_url, headers=headers, json=body, timeout=30)
419512
if response.status_code not in (200, 201):
420513
raise RuntimeError(
421-
f"Error creating the {kind}. Status: {response.status_code}. Body: {response.text}"
514+
f"Error creating the {kind}. Status: {response.status_code}. "
515+
f"Detail: {cls._html_error_detail(response.text)}\n\n"
516+
+ cls._server_diagnostics()
422517
)
423518

424-
##
425-
# Preload
426-
##
427-
428519
@classmethod
429520
def _preload_repositories(cls):
430521
cls._reset_logging()
431522
result = cls.runner.invoke(
432523
grimoirelab_metrics,
433524
[
434-
"./data/archived_repos.spdx.xml",
525+
ARCHIVED_REPOS_FILE,
435526
"--grimoirelab-url", GRIMOIRELAB_URL,
436527
"--grimoirelab-user", GRIMOIRELAB_USER,
437528
"--grimoirelab-password", GRIMOIRELAB_PASSWORD,
@@ -454,10 +545,6 @@ def _preload_repositories(cls):
454545
)
455546
time.sleep(20)
456547

457-
##
458-
# Teardown
459-
##
460-
461548
@classmethod
462549
def _cleanup(cls):
463550
"""Stop GrimoireLab processes and containers, and remove temp files.
@@ -516,4 +603,4 @@ def _stop_process_group(proc):
516603
try:
517604
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
518605
except (ProcessLookupError, PermissionError, OSError):
519-
proc.kill()
606+
proc.kill()

0 commit comments

Comments
 (0)