1717# along with this program. If not, see <http://www.gnu.org/licenses/>.
1818#
1919
20+ import html
2021import logging
2122import os
23+ import re
2224import signal
25+ import socket
2326import subprocess
2427import sys
2528import tempfile
3942from grimoirelab_metrics .cli import grimoirelab_metrics
4043
4144GRIMOIRELAB_URL = "http://localhost:8000"
45+ GRIMOIRELAB_PORT = 8000
4246GRIMOIRELAB_USER = "admin"
4347GRIMOIRELAB_PASSWORD = "admin"
4448GRIMOIRELAB_ECOSYSTEM = "npm-training-set"
4549GRIMOIRELAB_PROJECT = "npm-popular-components"
4650
47- # The default OpenSearch image used by testcontainers (1.3.x) is too old
48- # for GrimoireLab: it rejects the 'copy_alias' field of the ISM rollover
49- # action, which makes the archivists crash on startup. A recent image is
50- # required (docker-compose uses 'opensearchproject/opensearch:3').
51- OPENSEARCH_IMAGE = "opensearchproject/opensearch:2.19.2"
51+ OPENSEARCH_IMAGE = "opensearchproject/opensearch:3"
5252OPENSEARCH_USER = "admin"
5353OPENSEARCH_PASSWORD = "admin"
5454OPENSEARCH_INDEX = "events"
5555
5656GRIMOIRELAB_SERVER_TIMEOUT = 120
5757GRIMOIRELAB_WORKERS_TIMEOUT = 120
5858
59+ TESTS_DIR = os .path .dirname (os .path .abspath (__file__ ))
60+ ARCHIVED_REPOS_FILE = os .path .join (TESTS_DIR , "data" , "archived_repos.spdx.xml" )
61+
62+ PORT_IN_USE_HELP = (
63+ "Port 8000, where the tests run the GrimoireLab server, is already in use.\n "
64+ "Most likely there are processes or containers left over from a previous "
65+ "run (they can survive when a run is interrupted). Clean them up with:\n "
66+ "\n "
67+ ' pkill -f "grimoirelab run"\n '
68+ ' docker ps --filter "label=org.testcontainers=true" -q | xargs -r docker rm -f\n '
69+ "\n "
70+ "Also stop the docker-compose stack if it is running, as its nginx "
71+ "container publishes port 8000 too:\n "
72+ "\n "
73+ " docker compose down\n "
74+ "\n "
75+ "Verify the port is free afterwards with:\n "
76+ "\n "
77+ " ss -ltnp | grep :8000"
78+ )
79+
5980
6081class EndToEndTestCase (TestCase ):
6182 """Base class to build end to end tests.
@@ -82,28 +103,22 @@ def setUpClass(cls):
82103 cls ._grimoirelab_logs = {}
83104
84105 try :
106+ cls ._check_grimoirelab_port_is_free ()
85107 cls ._start_redis_container ()
86108 cls ._start_database_container ()
87109 cls ._start_opensearch_container ()
88110 cls ._start_grimoirelab ()
89111 cls ._create_grimoirelab_resources ()
90112 cls ._preload_repositories ()
91113 except Exception :
92- # unittest does not call tearDownClass when setUpClass fails,
93- # so clean up here to avoid leaking containers and processes.
94114 cls ._cleanup ()
95115 raise
96116
97117 @classmethod
98118 def tearDownClass (cls ):
99- # Give the workers some extra time to finish their pending jobs.
100119 time .sleep (20 )
101120 cls ._cleanup ()
102121
103- ##
104- # Command execution helpers
105- ##
106-
107122 @staticmethod
108123 def _reset_logging ():
109124 """Remove the logging handlers configured by previous invocations.
@@ -139,9 +154,25 @@ def _result_details(result):
139154
140155 return details
141156
142- ##
143- # Containers
144- ##
157+ @staticmethod
158+ def _port_is_free (port ):
159+ """Check whether nothing is listening on '127.0.0.1:<port>'."""
160+ with socket .socket (socket .AF_INET , socket .SOCK_STREAM ) as sock :
161+ sock .settimeout (1 )
162+ return sock .connect_ex (("127.0.0.1" , port )) != 0
163+
164+ @classmethod
165+ def _check_grimoirelab_port_is_free (cls ):
166+ """Ensure the port used by the GrimoireLab server is available.
167+
168+ When an orphaned 'grimoirelab run server' (or the nginx container
169+ of the docker-compose stack) is holding the port, the server of
170+ the tests dies with a bind error and the requests are answered
171+ by that orphan, whose database containers no longer exist. The
172+ symptom is 500 errors like "Can't connect to MySQL server".
173+ """
174+ if not cls ._port_is_free (GRIMOIRELAB_PORT ):
175+ raise RuntimeError (PORT_IN_USE_HELP )
145176
146177 @classmethod
147178 def _start_database_container (cls ):
@@ -156,7 +187,7 @@ def _start_redis_container(cls):
156187 @classmethod
157188 def _start_opensearch_container (cls ):
158189 # Keep the security plugin disabled so the server is reachable
159- # through plain HTTP without authentication, like before .
190+ # through plain HTTP without authentication.
160191 cls .opensearch_container = (
161192 OpenSearchContainer (image = OPENSEARCH_IMAGE )
162193 .with_exposed_ports (9200 )
@@ -172,18 +203,11 @@ def _start_opensearch_container(cls):
172203 cls .opensearch_user = OPENSEARCH_USER
173204 cls .opensearch_password = OPENSEARCH_PASSWORD
174205
175- ##
176- # GrimoireLab
177- ##
178-
179206 @classmethod
180207 def _start_grimoirelab_process (cls , name , args ):
181208 """Start a GrimoireLab process, storing its output in a log file."""
182209 log_file = tempfile .NamedTemporaryFile (delete = False , prefix = f"grimoirelab_{ name } _" , suffix = ".log" )
183210 cls ._grimoirelab_logs [name ] = log_file
184-
185- # Unbuffered output, so logs reach the file while the process
186- # is still running (needed by '_wait_for_worker_logs').
187211 env = dict (os .environ )
188212 env ["PYTHONUNBUFFERED" ] = "1"
189213
@@ -308,17 +332,42 @@ def _wait_for_grimoirelab_server(cls, timeout=GRIMOIRELAB_SERVER_TIMEOUT):
308332 raise RuntimeError ("GrimoireLab server exited unexpectedly.\n " + cls ._dump_grimoirelab_logs ())
309333 try :
310334 urllib .request .urlopen (GRIMOIRELAB_URL , timeout = 5 )
335+ cls ._confirm_server_is_ours ()
311336 return
312337 except urllib .error .HTTPError :
313- # Any HTTP response means the server is already running.
338+ cls . _confirm_server_is_ours ()
314339 return
340+ except RuntimeError :
341+ raise
315342 except Exception :
316343 time .sleep (1 )
317344
318345 raise RuntimeError (
319346 f"GrimoireLab server was not ready after { timeout } seconds.\n " + cls ._dump_grimoirelab_logs ()
320347 )
321348
349+ @classmethod
350+ def _confirm_server_is_ours (cls ):
351+ """Confirm the server answering the requests is the one the tests started.
352+
353+ If port 8000 is taken by another process (an orphaned
354+ 'grimoirelab run server' from a previous run, or the nginx
355+ container of the docker-compose stack), the server of the tests
356+ dies with a bind error while that other process answers the
357+ requests using the ports of containers that no longer exist.
358+ """
359+ time .sleep (2 )
360+ if cls .grimoirelab_server .poll () is not None :
361+ raise RuntimeError (
362+ "The GrimoireLab server process of the tests died right after "
363+ "starting, but SOMETHING is still answering on port 8000. "
364+ "Another process took the port and is answering the requests "
365+ "with the configuration of a previous run.\n \n "
366+ + PORT_IN_USE_HELP
367+ + "\n \n --- server output ---\n "
368+ + cls ._dump_grimoirelab_logs ()
369+ )
370+
322371 @classmethod
323372 def _wait_for_worker_logs (cls , name , proc , timeout = GRIMOIRELAB_WORKERS_TIMEOUT ):
324373 """Wait until a worker process reports it is listening for jobs."""
@@ -344,19 +393,14 @@ def _wait_for_worker_logs(cls, name, proc, timeout=GRIMOIRELAB_WORKERS_TIMEOUT):
344393 timeout ,
345394 )
346395
347- ##
348- # Ecosystem and project creation
349- ##
350-
351396 @classmethod
352397 def _create_grimoirelab_resources (cls ):
353398 """Create the ecosystem and the project used by the tests.
354399
355400 The metrics CLI takes 'grimoirelab-ecosystem' and
356401 'grimoirelab-project' as parameters and queries their repos
357402 endpoint, but it does not create them; they must exist
358- beforehand, or every call becomes a 404 on
359- 'ecosystems/None/projects/None'.
403+ beforehand, or every call becomes a 404.
360404 """
361405 api_url = f"{ GRIMOIRELAB_URL } /api/v1"
362406 headers = cls ._api_headers (api_url )
@@ -379,14 +423,23 @@ def _create_grimoirelab_resources(cls):
379423 @classmethod
380424 def _api_headers (cls , api_url ):
381425 """Return the headers to authenticate against the GrimoireLab API."""
382- response = requests .post (
383- f"{ GRIMOIRELAB_URL } /token/" ,
384- json = {"username" : GRIMOIRELAB_USER , "password" : GRIMOIRELAB_PASSWORD },
385- timeout = 30 ,
386- )
426+ try :
427+ response = requests .post (
428+ f"{ GRIMOIRELAB_URL } /token/" ,
429+ json = {"username" : GRIMOIRELAB_USER , "password" : GRIMOIRELAB_PASSWORD },
430+ timeout = 30 ,
431+ )
432+ except requests .RequestException as exc :
433+ raise RuntimeError (
434+ f"Error connecting to the GrimoireLab server: { exc !r} \n \n " + cls ._server_diagnostics ()
435+ )
436+
387437 if response .status_code != 200 :
388438 raise RuntimeError (
389- f"Error getting the API token. Status: { response .status_code } . Body: { response .text } "
439+ "Error getting the API token. "
440+ f"Status: { response .status_code } . "
441+ f"Detail: { cls ._html_error_detail (response .text )} \n \n "
442+ + cls ._server_diagnostics ()
390443 )
391444
392445 data = response .json ()
@@ -404,34 +457,72 @@ def _api_headers(cls, api_url):
404457 raise RuntimeError ("None of the authentication schemes was accepted by the GrimoireLab API." )
405458
406459 @staticmethod
407- def _api_get_or_create (headers , get_url , post_url , body , kind ):
460+ def _html_error_detail (body ):
461+ """Extract the exception message from a Django debug page."""
462+ match = re .search (r'<pre class="exception_value">(.*?)</pre>' , body , re .DOTALL )
463+ if match :
464+ return html .unescape (match .group (1 )).strip ()
465+ # Not a Django debug page; return a truncated copy.
466+ return body [:500 ]
467+
468+ @classmethod
469+ def _server_diagnostics (cls ):
470+ """Diagnostics to append when a request to the server fails."""
471+ lines = []
472+
473+ server = getattr (cls , "grimoirelab_server" , None )
474+ if server is not None :
475+ if server .poll () is None :
476+ lines .append ("* Server process started by the tests: running." )
477+ else :
478+ lines .append (
479+ f"* Server process started by the tests: DEAD (exit code { server .returncode } ). "
480+ "The requests were answered by ANOTHER process using port 8000."
481+ )
482+
483+ db_port = os .environ .get ("GRIMOIRELAB_DB_PORT" )
484+ if db_port :
485+ if cls ._port_is_free (int (db_port )):
486+ lines .append (
487+ f"* MariaDB container on port { db_port } : NOT reachable. Either the "
488+ "container died, or the requests were answered by an orphaned server "
489+ "configured with the ports of a previous run."
490+ )
491+ else :
492+ lines .append (f"* MariaDB container on port { db_port } : reachable." )
493+
494+ lines .append (cls ._dump_grimoirelab_logs ())
495+ return "\n " .join (lines )
496+
497+ @classmethod
498+ def _api_get_or_create (cls , headers , get_url , post_url , body , kind ):
408499 """Create a resource through the API if it does not exist yet."""
409500 response = requests .get (get_url , headers = headers , timeout = 30 )
410501 if response .status_code == 200 :
411502 return
412503
413504 if response .status_code != 404 :
414505 raise RuntimeError (
415- f"Error checking the { kind } . Status: { response .status_code } . Body: { response .text } "
506+ f"Error checking the { kind } . Status: { response .status_code } . "
507+ f"Detail: { cls ._html_error_detail (response .text )} \n \n "
508+ + cls ._server_diagnostics ()
416509 )
417510
418511 response = requests .post (post_url , headers = headers , json = body , timeout = 30 )
419512 if response .status_code not in (200 , 201 ):
420513 raise RuntimeError (
421- f"Error creating the { kind } . Status: { response .status_code } . Body: { response .text } "
514+ f"Error creating the { kind } . Status: { response .status_code } . "
515+ f"Detail: { cls ._html_error_detail (response .text )} \n \n "
516+ + cls ._server_diagnostics ()
422517 )
423518
424- ##
425- # Preload
426- ##
427-
428519 @classmethod
429520 def _preload_repositories (cls ):
430521 cls ._reset_logging ()
431522 result = cls .runner .invoke (
432523 grimoirelab_metrics ,
433524 [
434- "./data/archived_repos.spdx.xml" ,
525+ ARCHIVED_REPOS_FILE ,
435526 "--grimoirelab-url" , GRIMOIRELAB_URL ,
436527 "--grimoirelab-user" , GRIMOIRELAB_USER ,
437528 "--grimoirelab-password" , GRIMOIRELAB_PASSWORD ,
@@ -454,10 +545,6 @@ def _preload_repositories(cls):
454545 )
455546 time .sleep (20 )
456547
457- ##
458- # Teardown
459- ##
460-
461548 @classmethod
462549 def _cleanup (cls ):
463550 """Stop GrimoireLab processes and containers, and remove temp files.
@@ -516,4 +603,4 @@ def _stop_process_group(proc):
516603 try :
517604 os .killpg (os .getpgid (proc .pid ), signal .SIGKILL )
518605 except (ProcessLookupError , PermissionError , OSError ):
519- proc .kill ()
606+ proc .kill ()
0 commit comments