Skip to content

Commit 93442e3

Browse files
authored
Merge pull request #5 from aboutcode-org/add_eco_pro_parameters_2_cli
[cli] Add two new parameters to the grimoirelab-metrics cli
2 parents 28b3f93 + 0145336 commit 93442e3

1 file changed

Lines changed: 58 additions & 27 deletions

File tree

‎src/grimoirelab_metrics/cli.py‎

Lines changed: 58 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@
6161
)
6262
@click.option("--grimoirelab-user", help="GrimoireLab API user")
6363
@click.option("--grimoirelab-password", help="GrimoireLab API password")
64+
@click.option("--grimoirelab-ecosystem", help="GrimoireLab will classify the data under this ecosystem name")
65+
@click.option(
66+
"--grimoirelab-project",
67+
help="GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem"
68+
)
6469
@click.option(
6570
"--opensearch-url",
6671
help="OpenSearch URL server",
@@ -110,6 +115,8 @@ def grimoirelab_metrics(
110115
grimoirelab_url: str,
111116
grimoirelab_user: str,
112117
grimoirelab_password: str,
118+
grimoirelab_ecosystem: str,
119+
grimoirelab_project: str,
113120
opensearch_url: str,
114121
opensearch_index: str,
115122
opensearch_user: str | None = None,
@@ -166,10 +173,12 @@ def grimoirelab_metrics(
166173
logging.info("Could not find any git repositories to analyze")
167174
sys.exit(0)
168175

169-
schedule_repositories(git_urls, grimoirelab_client)
176+
schedule_repositories(git_urls, grimoirelab_client, grimoirelab_ecosystem, grimoirelab_project)
170177

171178
metrics = generate_metrics_when_ready(
172179
grimoirelab_client=grimoirelab_client,
180+
grimoirelab_ecosystem=grimoirelab_ecosystem,
181+
grimoirelab_project=grimoirelab_project,
173182
repositories=git_urls,
174183
opensearch_url=opensearch_url,
175184
opensearch_index=opensearch_index,
@@ -262,24 +271,34 @@ def get_sbom_packages(file: str) -> dict[str, str]:
262271
return packages
263272

264273

265-
def schedule_repositories(repositories: list[str], grimoirelab_client: GrimoireLabClient) -> None:
274+
def schedule_repositories(
275+
repositories: list[str],
276+
grimoirelab_client: GrimoireLabClient,
277+
grimoirelab_ecosystem: str,
278+
grimoirelab_project: str
279+
) -> None:
266280
"""Schedule tasks to collect data from a list of repositories.
267281
268282
:param repositories: List of git repositories.
269283
:param grimoirelab_client: GrimoireLab API client.
284+
:param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name.
285+
:param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem.
286+
270287
"""
271288
logging.info("Scheduling data collection tasks with GrimoireLab")
272289
for package_url in repositories:
273290
logging.debug(f"Scheduling task to fetch commits from {package_url}")
274291
try:
275-
schedule_repository(grimoirelab_client=grimoirelab_client, uri=package_url, datasource="git", category="commit")
292+
schedule_repository(grimoirelab_client, grimoirelab_ecosystem, grimoirelab_project, package_url, "git", "commit")
276293
except (requests.HTTPError, requests.ConnectionError) as e:
277294
logging.error(f"Error scheduling task: {e}")
278295
raise e
279296

280297

281298
def generate_metrics_when_ready(
282299
grimoirelab_client: GrimoireLabClient,
300+
grimoirelab_ecosystem: str,
301+
grimoirelab_project: str,
283302
repositories: list[str],
284303
opensearch_url: str,
285304
opensearch_index: str,
@@ -300,6 +319,8 @@ def generate_metrics_when_ready(
300319
"""Generate metrics once the repositories have finished the collection.
301320
302321
:param grimoirelab_client: GrimoireLab API client.
322+
:param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name
323+
:param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem
303324
:param repositories: List of repositories.
304325
:param opensearch_url: OpenSearch URL.
305326
:param opensearch_index: OpenSearch index.
@@ -327,7 +348,7 @@ def generate_metrics_when_ready(
327348
while pending_repositories:
328349
processed = set()
329350
for repository in pending_repositories:
330-
if repository_ready(grimoirelab_client, repository, after_date):
351+
if repository_ready(grimoirelab_client, grimoirelab_ecosystem, grimoirelab_project, repository, after_date):
331352
metrics["repositories"][repository] = get_repository_metrics(
332353
repository=repository,
333354
opensearch_url=opensearch_url,
@@ -363,19 +384,23 @@ def generate_metrics_when_ready(
363384
return metrics
364385

365386

366-
def repository_ready(grimoirelab_client: GrimoireLabClient, repository: str, after_date: datetime.datetime) -> bool:
387+
def repository_ready(
388+
grimoirelab_client: GrimoireLabClient,
389+
grimoirelab_ecosystem: str,
390+
grimoirelab_project: str,
391+
repository: str,
392+
after_date: datetime.datetime
393+
) -> bool:
367394
"""
368395
Check if the task related to the repository has finished.
369396
370397
:param grimoirelab_client: GrimoireLab API client.
398+
:param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name
399+
:param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem
371400
:param repository: Repository URI
372401
:param after_date: Date to check if the task has finished
373402
"""
374-
#FIXME hardcoded parameters
375-
ecosystem ="npm-training-set"
376-
project = "npm-popular-components"
377-
378-
endpoint = f"api/v1/ecosystems/{ecosystem}/projects/{project}/repos/"
403+
endpoint = f"api/v1/ecosystems/{grimoirelab_ecosystem}/projects/{grimoirelab_project}/repos/"
379404
try:
380405
r = grimoirelab_client.get(endpoint, params={"uri": repository})
381406
except requests.HTTPError as e:
@@ -409,19 +434,26 @@ def is_valid(repository: str) -> bool:
409434
return repository and not isinstance(repository, SpdxNone) and not isinstance(repository, SpdxNoAssertion)
410435

411436

412-
def schedule_repository(grimoirelab_client: GrimoireLabClient, uri: str, datasource: str, category: str) -> Any:
437+
def schedule_repository(
438+
grimoirelab_client: GrimoireLabClient,
439+
grimoirelab_ecosystem: str,
440+
grimoirelab_project: str,
441+
uri: str,
442+
datasource: str,
443+
category: str
444+
) -> Any:
413445
"""Schedule a task to fetch a Git repository.
414446
415447
:param grimoirelab_client: GrimoireLab API client.
448+
:param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name.
449+
:param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem.
416450
:param uri: Repository URI.
417451
:param datasource: Data source type.
418452
:param category: Data source category.
419-
:param ecosystem: Ecosystem name.
420-
:param project: Project name.
421453
422454
:return: Scheduled task.
423455
"""
424-
#FIXME clarify parameters
456+
425457
data = {
426458
"uri": uri,
427459
"datasource_type": datasource,
@@ -433,14 +465,9 @@ def schedule_repository(grimoirelab_client: GrimoireLabClient, uri: str, datasou
433465
}
434466
}
435467

436-
#FIXME hardcoded parameters
437-
ecosystem ="npm-training-set"
438-
project = "npm-popular-components"
439-
468+
if is_added(grimoirelab_client, grimoirelab_ecosystem, grimoirelab_project, uri): return True
440469

441-
if is_added(grimoirelab_client, uri): return True
442-
443-
endpoint = f"api/v1/ecosystems/{ecosystem}/projects/{project}/repos/"
470+
endpoint = f"api/v1/ecosystems/{grimoirelab_ecosystem}/projects/{grimoirelab_project}/repos/"
444471
try:
445472
res = grimoirelab_client.post(endpoint, json=data)
446473
res.raise_for_status()
@@ -454,14 +481,18 @@ def schedule_repository(grimoirelab_client: GrimoireLabClient, uri: str, datasou
454481
# If it's a different HTTP error (500, 404, 403), re-raise it
455482
raise e
456483

457-
def is_added(grimoirelab_client: GrimoireLabClient, uri: str) -> bool:
458-
"""Check if the repository is already scheduled"""
484+
def is_added(grimoirelab_client: GrimoireLabClient, grimoirelab_ecosystem: str, grimoirelab_project: str, uri: str) -> bool:
485+
"""Check if the repository is already scheduled
459486
460-
#FIXME hardcoded parameters
461-
ecosystem ="npm-training-set"
462-
project = "npm-popular-components"
487+
:param grimoirelab_client: GrimoireLab API client.
488+
:param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name.
489+
:param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem.
490+
:param uri: Repository URI.
491+
492+
:return: Boolean.
493+
"""
463494

464-
endpoint = f"api/v1/ecosystems/{ecosystem}/projects/{project}/repos/"
495+
endpoint = f"api/v1/ecosystems/{grimoirelab_ecosystem}/projects/{grimoirelab_project}/repos/"
465496
params = {"uri": uri}
466497

467498
try:

0 commit comments

Comments
 (0)