6161)
6262@click .option ("--grimoirelab-user" , help = "GrimoireLab API user" )
6363@click .option ("--grimoirelab-password" , help = "GrimoireLab API password" )
64+ @click .option ("--grimoirelab-ecosystem" , help = "GrimoireLab will classify the data under this ecosystem name" )
65+ @click .option (
66+ "--grimoirelab-project" ,
67+ help = "GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem"
68+ )
6469@click .option (
6570 "--opensearch-url" ,
6671 help = "OpenSearch URL server" ,
@@ -110,6 +115,8 @@ def grimoirelab_metrics(
110115 grimoirelab_url : str ,
111116 grimoirelab_user : str ,
112117 grimoirelab_password : str ,
118+ grimoirelab_ecosystem : str ,
119+ grimoirelab_project : str ,
113120 opensearch_url : str ,
114121 opensearch_index : str ,
115122 opensearch_user : str | None = None ,
@@ -166,10 +173,12 @@ def grimoirelab_metrics(
166173 logging .info ("Could not find any git repositories to analyze" )
167174 sys .exit (0 )
168175
169- schedule_repositories (git_urls , grimoirelab_client )
176+ schedule_repositories (git_urls , grimoirelab_client , grimoirelab_ecosystem , grimoirelab_project )
170177
171178 metrics = generate_metrics_when_ready (
172179 grimoirelab_client = grimoirelab_client ,
180+ grimoirelab_ecosystem = grimoirelab_ecosystem ,
181+ grimoirelab_project = grimoirelab_project ,
173182 repositories = git_urls ,
174183 opensearch_url = opensearch_url ,
175184 opensearch_index = opensearch_index ,
@@ -262,24 +271,34 @@ def get_sbom_packages(file: str) -> dict[str, str]:
262271 return packages
263272
264273
265- def schedule_repositories (repositories : list [str ], grimoirelab_client : GrimoireLabClient ) -> None :
274+ def schedule_repositories (
275+ repositories : list [str ],
276+ grimoirelab_client : GrimoireLabClient ,
277+ grimoirelab_ecosystem : str ,
278+ grimoirelab_project : str
279+ ) -> None :
266280 """Schedule tasks to collect data from a list of repositories.
267281
268282 :param repositories: List of git repositories.
269283 :param grimoirelab_client: GrimoireLab API client.
284+ :param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name.
285+ :param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem.
286+
270287 """
271288 logging .info ("Scheduling data collection tasks with GrimoireLab" )
272289 for package_url in repositories :
273290 logging .debug (f"Scheduling task to fetch commits from { package_url } " )
274291 try :
275- schedule_repository (grimoirelab_client = grimoirelab_client , uri = package_url , datasource = "git" , category = "commit" )
292+ schedule_repository (grimoirelab_client , grimoirelab_ecosystem , grimoirelab_project , package_url , "git" , "commit" )
276293 except (requests .HTTPError , requests .ConnectionError ) as e :
277294 logging .error (f"Error scheduling task: { e } " )
278295 raise e
279296
280297
281298def generate_metrics_when_ready (
282299 grimoirelab_client : GrimoireLabClient ,
300+ grimoirelab_ecosystem : str ,
301+ grimoirelab_project : str ,
283302 repositories : list [str ],
284303 opensearch_url : str ,
285304 opensearch_index : str ,
@@ -300,6 +319,8 @@ def generate_metrics_when_ready(
300319 """Generate metrics once the repositories have finished the collection.
301320
302321 :param grimoirelab_client: GrimoireLab API client.
322+ :param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name
323+ :param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem
303324 :param repositories: List of repositories.
304325 :param opensearch_url: OpenSearch URL.
305326 :param opensearch_index: OpenSearch index.
@@ -327,7 +348,7 @@ def generate_metrics_when_ready(
327348 while pending_repositories :
328349 processed = set ()
329350 for repository in pending_repositories :
330- if repository_ready (grimoirelab_client , repository , after_date ):
351+ if repository_ready (grimoirelab_client , grimoirelab_ecosystem , grimoirelab_project , repository , after_date ):
331352 metrics ["repositories" ][repository ] = get_repository_metrics (
332353 repository = repository ,
333354 opensearch_url = opensearch_url ,
@@ -363,19 +384,23 @@ def generate_metrics_when_ready(
363384 return metrics
364385
365386
366- def repository_ready (grimoirelab_client : GrimoireLabClient , repository : str , after_date : datetime .datetime ) -> bool :
387+ def repository_ready (
388+ grimoirelab_client : GrimoireLabClient ,
389+ grimoirelab_ecosystem : str ,
390+ grimoirelab_project : str ,
391+ repository : str ,
392+ after_date : datetime .datetime
393+ ) -> bool :
367394 """
368395 Check if the task related to the repository has finished.
369396
370397 :param grimoirelab_client: GrimoireLab API client.
398+ :param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name
399+ :param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem
371400 :param repository: Repository URI
372401 :param after_date: Date to check if the task has finished
373402 """
374- #FIXME hardcoded parameters
375- ecosystem = "npm-training-set"
376- project = "npm-popular-components"
377-
378- endpoint = f"api/v1/ecosystems/{ ecosystem } /projects/{ project } /repos/"
403+ endpoint = f"api/v1/ecosystems/{ grimoirelab_ecosystem } /projects/{ grimoirelab_project } /repos/"
379404 try :
380405 r = grimoirelab_client .get (endpoint , params = {"uri" : repository })
381406 except requests .HTTPError as e :
@@ -409,19 +434,26 @@ def is_valid(repository: str) -> bool:
409434 return repository and not isinstance (repository , SpdxNone ) and not isinstance (repository , SpdxNoAssertion )
410435
411436
412- def schedule_repository (grimoirelab_client : GrimoireLabClient , uri : str , datasource : str , category : str ) -> Any :
437+ def schedule_repository (
438+ grimoirelab_client : GrimoireLabClient ,
439+ grimoirelab_ecosystem : str ,
440+ grimoirelab_project : str ,
441+ uri : str ,
442+ datasource : str ,
443+ category : str
444+ ) -> Any :
413445 """Schedule a task to fetch a Git repository.
414446
415447 :param grimoirelab_client: GrimoireLab API client.
448+ :param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name.
449+ :param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem.
416450 :param uri: Repository URI.
417451 :param datasource: Data source type.
418452 :param category: Data source category.
419- :param ecosystem: Ecosystem name.
420- :param project: Project name.
421453
422454 :return: Scheduled task.
423455 """
424- #FIXME clarify parameters
456+
425457 data = {
426458 "uri" : uri ,
427459 "datasource_type" : datasource ,
@@ -433,14 +465,9 @@ def schedule_repository(grimoirelab_client: GrimoireLabClient, uri: str, datasou
433465 }
434466 }
435467
436- #FIXME hardcoded parameters
437- ecosystem = "npm-training-set"
438- project = "npm-popular-components"
439-
468+ if is_added (grimoirelab_client , grimoirelab_ecosystem , grimoirelab_project , uri ): return True
440469
441- if is_added (grimoirelab_client , uri ): return True
442-
443- endpoint = f"api/v1/ecosystems/{ ecosystem } /projects/{ project } /repos/"
470+ endpoint = f"api/v1/ecosystems/{ grimoirelab_ecosystem } /projects/{ grimoirelab_project } /repos/"
444471 try :
445472 res = grimoirelab_client .post (endpoint , json = data )
446473 res .raise_for_status ()
@@ -454,14 +481,18 @@ def schedule_repository(grimoirelab_client: GrimoireLabClient, uri: str, datasou
454481 # If it's a different HTTP error (500, 404, 403), re-raise it
455482 raise e
456483
457- def is_added (grimoirelab_client : GrimoireLabClient , uri : str ) -> bool :
458- """Check if the repository is already scheduled"""
484+ def is_added (grimoirelab_client : GrimoireLabClient , grimoirelab_ecosystem : str , grimoirelab_project : str , uri : str ) -> bool :
485+ """Check if the repository is already scheduled
459486
460- #FIXME hardcoded parameters
461- ecosystem = "npm-training-set"
462- project = "npm-popular-components"
487+ :param grimoirelab_client: GrimoireLab API client.
488+ :param grimoirelab_ecosystem: GrimoireLab will classify the data under this ecosystem name.
489+ :param grimoirelab_project: GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem.
490+ :param uri: Repository URI.
491+
492+ :return: Boolean.
493+ """
463494
464- endpoint = f"api/v1/ecosystems/{ ecosystem } /projects/{ project } /repos/"
495+ endpoint = f"api/v1/ecosystems/{ grimoirelab_ecosystem } /projects/{ grimoirelab_project } /repos/"
465496 params = {"uri" : uri }
466497
467498 try :
0 commit comments