Skip to content

Commit a608b80

Browse files
matentznmatentzn
authored
Merge pull request #24 from mapping-commons/gz-handling
2 parents af4623e + 45c73a7 commit a608b80

2 files changed

Lines changed: 78 additions & 49 deletions

File tree

‎data/mapping-specifications.json‎

Lines changed: 60 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -343,19 +343,18 @@
343343
"metadata_completeness_score": 0.2727
344344
},
345345
{
346-
"id": "https://w3id.org/sssom/mappings/67df215d-fffd-4d37-b42f-268fa6cea80c",
347-
"license": "https://w3id.org/sssom/license/unspecified",
348-
"type": "sssom",
346+
"id": "http://w3id.org/sssom/commons/monarch/mappings/ncit-hp.sssom.tsv",
349347
"content_url": "http://w3id.org/sssom/commons/monarch/mappings/ncit-hp.sssom.tsv",
348+
"type": "sssom",
349+
"status": "fetch_error",
350350
"registries": [
351351
{
352352
"id": "https://w3id.org/sssom/commons/monarch",
353353
"name": "Monarch Mapping Commons",
354354
"url": "https://github.com/monarch-initiative/monarch-mapping-commons"
355355
}
356356
],
357-
"status": "ok",
358-
"metadata_completeness_score": 0.2727
357+
"metadata_completeness_score": 0.0
359358
},
360359
{
361360
"id": "https://w3id.org/sssom/mappings/467f578f-3319-4fb4-92fb-bddaf41b0c1f",
@@ -557,20 +556,18 @@
557556
"metadata_completeness_score": 0.3636
558557
},
559558
{
560-
"id": "https://doi.org/10.5281/zenodo.6949696",
561-
"version": "V1.1",
562-
"license": "https://creativecommons.org/publicdomain/zero/1.0/",
563-
"type": "sssom",
559+
"id": "https://w3id.org/cpont/mappings/omop2obo_drug_exposure.sssom.tsv",
564560
"content_url": "https://w3id.org/cpont/mappings/omop2obo_drug_exposure.sssom.tsv",
561+
"type": "sssom",
562+
"status": "fetch_error",
565563
"registries": [
566564
{
567565
"id": "https://w3id.org/cpont/mappings",
568566
"name": "C-Path Mapping Commons",
569567
"url": "https://gitlab.c-path.org/c-pathontology/mapping-commons"
570568
}
571569
],
572-
"status": "ok",
573-
"metadata_completeness_score": 0.3636
570+
"metadata_completeness_score": 0.0
574571
},
575572
{
576573
"id": "https://doi.org/10.5281/zenodo.6949858",
@@ -618,7 +615,8 @@
618615
"registries": [
619616
{
620617
"id": "https://github.com/biopragmatics/mapping-registry",
621-
"name": "Biopragmatics Mapping Registry"
618+
"name": "Biopragmatics Mapping Registry",
619+
"url": "https://github.com/biopragmatics"
622620
}
623621
],
624622
"status": "ok",
@@ -633,7 +631,8 @@
633631
"registries": [
634632
{
635633
"id": "https://github.com/biopragmatics/mapping-registry",
636-
"name": "Biopragmatics Mapping Registry"
634+
"name": "Biopragmatics Mapping Registry",
635+
"url": "https://github.com/biopragmatics"
637636
}
638637
],
639638
"status": "ok",
@@ -643,11 +642,12 @@
643642
"id": "https://zenodo.org/records/15504009/files/mappings.sssom.tsv.gz",
644643
"content_url": "https://zenodo.org/records/15504009/files/mappings.sssom.tsv.gz",
645644
"type": "sssom",
646-
"status": "nonstandard_format",
645+
"status": "no_metadata",
647646
"registries": [
648647
{
649648
"id": "https://github.com/biopragmatics/mapping-registry",
650-
"name": "Biopragmatics Mapping Registry"
649+
"name": "Biopragmatics Mapping Registry",
650+
"url": "https://github.com/biopragmatics"
651651
}
652652
],
653653
"metadata_completeness_score": 0.0
@@ -656,11 +656,12 @@
656656
"id": "https://zenodo.org/records/15826794/files/priority.sssom.tsv.gz",
657657
"content_url": "https://zenodo.org/records/15826794/files/priority.sssom.tsv.gz",
658658
"type": "sssom",
659-
"status": "nonstandard_format",
659+
"status": "no_metadata",
660660
"registries": [
661661
{
662662
"id": "https://github.com/biopragmatics/mapping-registry",
663-
"name": "Biopragmatics Mapping Registry"
663+
"name": "Biopragmatics Mapping Registry",
664+
"url": "https://github.com/biopragmatics"
664665
}
665666
],
666667
"metadata_completeness_score": 0.0
@@ -669,11 +670,12 @@
669670
"id": "https://zenodo.org/records/15826794/files/processed.sssom.tsv.gz",
670671
"content_url": "https://zenodo.org/records/15826794/files/processed.sssom.tsv.gz",
671672
"type": "sssom",
672-
"status": "nonstandard_format",
673+
"status": "no_metadata",
673674
"registries": [
674675
{
675676
"id": "https://github.com/biopragmatics/mapping-registry",
676-
"name": "Biopragmatics Mapping Registry"
677+
"name": "Biopragmatics Mapping Registry",
678+
"url": "https://github.com/biopragmatics"
677679
}
678680
],
679681
"metadata_completeness_score": 0.0
@@ -682,11 +684,12 @@
682684
"id": "https://zenodo.org/records/15826794/files/raw.sssom.tsv.gz",
683685
"content_url": "https://zenodo.org/records/15826794/files/raw.sssom.tsv.gz",
684686
"type": "sssom",
685-
"status": "nonstandard_format",
687+
"status": "no_metadata",
686688
"registries": [
687689
{
688690
"id": "https://github.com/biopragmatics/mapping-registry",
689-
"name": "Biopragmatics Mapping Registry"
691+
"name": "Biopragmatics Mapping Registry",
692+
"url": "https://github.com/biopragmatics"
690693
}
691694
],
692695
"metadata_completeness_score": 0.0
@@ -695,11 +698,12 @@
695698
"id": "https://zenodo.org/records/15826779/files/priority.sssom.tsv.gz",
696699
"content_url": "https://zenodo.org/records/15826779/files/priority.sssom.tsv.gz",
697700
"type": "sssom",
698-
"status": "nonstandard_format",
701+
"status": "no_metadata",
699702
"registries": [
700703
{
701704
"id": "https://github.com/biopragmatics/mapping-registry",
702-
"name": "Biopragmatics Mapping Registry"
705+
"name": "Biopragmatics Mapping Registry",
706+
"url": "https://github.com/biopragmatics"
703707
}
704708
],
705709
"metadata_completeness_score": 0.0
@@ -708,11 +712,12 @@
708712
"id": "https://zenodo.org/records/15826779/files/processed.sssom.tsv.gz",
709713
"content_url": "https://zenodo.org/records/15826779/files/processed.sssom.tsv.gz",
710714
"type": "sssom",
711-
"status": "nonstandard_format",
715+
"status": "no_metadata",
712716
"registries": [
713717
{
714718
"id": "https://github.com/biopragmatics/mapping-registry",
715-
"name": "Biopragmatics Mapping Registry"
719+
"name": "Biopragmatics Mapping Registry",
720+
"url": "https://github.com/biopragmatics"
716721
}
717722
],
718723
"metadata_completeness_score": 0.0
@@ -721,11 +726,12 @@
721726
"id": "https://zenodo.org/records/15826779/files/raw.sssom.tsv.gz",
722727
"content_url": "https://zenodo.org/records/15826779/files/raw.sssom.tsv.gz",
723728
"type": "sssom",
724-
"status": "nonstandard_format",
729+
"status": "no_metadata",
725730
"registries": [
726731
{
727732
"id": "https://github.com/biopragmatics/mapping-registry",
728-
"name": "Biopragmatics Mapping Registry"
733+
"name": "Biopragmatics Mapping Registry",
734+
"url": "https://github.com/biopragmatics"
729735
}
730736
],
731737
"metadata_completeness_score": 0.0
@@ -734,11 +740,12 @@
734740
"id": "https://zenodo.org/records/15826768/files/priority.sssom.tsv.gz",
735741
"content_url": "https://zenodo.org/records/15826768/files/priority.sssom.tsv.gz",
736742
"type": "sssom",
737-
"status": "nonstandard_format",
743+
"status": "no_metadata",
738744
"registries": [
739745
{
740746
"id": "https://github.com/biopragmatics/mapping-registry",
741-
"name": "Biopragmatics Mapping Registry"
747+
"name": "Biopragmatics Mapping Registry",
748+
"url": "https://github.com/biopragmatics"
742749
}
743750
],
744751
"metadata_completeness_score": 0.0
@@ -747,11 +754,12 @@
747754
"id": "https://zenodo.org/records/15826768/files/processed.sssom.tsv.gz",
748755
"content_url": "https://zenodo.org/records/15826768/files/processed.sssom.tsv.gz",
749756
"type": "sssom",
750-
"status": "nonstandard_format",
757+
"status": "no_metadata",
751758
"registries": [
752759
{
753760
"id": "https://github.com/biopragmatics/mapping-registry",
754-
"name": "Biopragmatics Mapping Registry"
761+
"name": "Biopragmatics Mapping Registry",
762+
"url": "https://github.com/biopragmatics"
755763
}
756764
],
757765
"metadata_completeness_score": 0.0
@@ -760,11 +768,12 @@
760768
"id": "https://zenodo.org/records/15826768/files/raw.sssom.tsv.gz",
761769
"content_url": "https://zenodo.org/records/15826768/files/raw.sssom.tsv.gz",
762770
"type": "sssom",
763-
"status": "nonstandard_format",
771+
"status": "no_metadata",
764772
"registries": [
765773
{
766774
"id": "https://github.com/biopragmatics/mapping-registry",
767-
"name": "Biopragmatics Mapping Registry"
775+
"name": "Biopragmatics Mapping Registry",
776+
"url": "https://github.com/biopragmatics"
768777
}
769778
],
770779
"metadata_completeness_score": 0.0
@@ -773,11 +782,12 @@
773782
"id": "https://zenodo.org/records/15826754/files/priority.sssom.tsv.gz",
774783
"content_url": "https://zenodo.org/records/15826754/files/priority.sssom.tsv.gz",
775784
"type": "sssom",
776-
"status": "nonstandard_format",
785+
"status": "no_metadata",
777786
"registries": [
778787
{
779788
"id": "https://github.com/biopragmatics/mapping-registry",
780-
"name": "Biopragmatics Mapping Registry"
789+
"name": "Biopragmatics Mapping Registry",
790+
"url": "https://github.com/biopragmatics"
781791
}
782792
],
783793
"metadata_completeness_score": 0.0
@@ -786,11 +796,12 @@
786796
"id": "https://zenodo.org/records/15826754/files/processed.sssom.tsv.gz",
787797
"content_url": "https://zenodo.org/records/15826754/files/processed.sssom.tsv.gz",
788798
"type": "sssom",
789-
"status": "nonstandard_format",
799+
"status": "no_metadata",
790800
"registries": [
791801
{
792802
"id": "https://github.com/biopragmatics/mapping-registry",
793-
"name": "Biopragmatics Mapping Registry"
803+
"name": "Biopragmatics Mapping Registry",
804+
"url": "https://github.com/biopragmatics"
794805
}
795806
],
796807
"metadata_completeness_score": 0.0
@@ -799,11 +810,12 @@
799810
"id": "https://zenodo.org/records/15826754/files/raw.sssom.tsv.gz",
800811
"content_url": "https://zenodo.org/records/15826754/files/raw.sssom.tsv.gz",
801812
"type": "sssom",
802-
"status": "nonstandard_format",
813+
"status": "no_metadata",
803814
"registries": [
804815
{
805816
"id": "https://github.com/biopragmatics/mapping-registry",
806-
"name": "Biopragmatics Mapping Registry"
817+
"name": "Biopragmatics Mapping Registry",
818+
"url": "https://github.com/biopragmatics"
807819
}
808820
],
809821
"metadata_completeness_score": 0.0
@@ -812,11 +824,12 @@
812824
"id": "https://zenodo.org/records/15826693/files/priority.sssom.tsv.gz",
813825
"content_url": "https://zenodo.org/records/15826693/files/priority.sssom.tsv.gz",
814826
"type": "sssom",
815-
"status": "nonstandard_format",
827+
"status": "no_metadata",
816828
"registries": [
817829
{
818830
"id": "https://github.com/biopragmatics/mapping-registry",
819-
"name": "Biopragmatics Mapping Registry"
831+
"name": "Biopragmatics Mapping Registry",
832+
"url": "https://github.com/biopragmatics"
820833
}
821834
],
822835
"metadata_completeness_score": 0.0
@@ -825,11 +838,12 @@
825838
"id": "https://zenodo.org/records/15826693/files/processed.sssom.tsv.gz",
826839
"content_url": "https://zenodo.org/records/15826693/files/processed.sssom.tsv.gz",
827840
"type": "sssom",
828-
"status": "nonstandard_format",
841+
"status": "no_metadata",
829842
"registries": [
830843
{
831844
"id": "https://github.com/biopragmatics/mapping-registry",
832-
"name": "Biopragmatics Mapping Registry"
845+
"name": "Biopragmatics Mapping Registry",
846+
"url": "https://github.com/biopragmatics"
833847
}
834848
],
835849
"metadata_completeness_score": 0.0
@@ -838,11 +852,12 @@
838852
"id": "https://zenodo.org/records/15826693/files/raw.sssom.tsv.gz",
839853
"content_url": "https://zenodo.org/records/15826693/files/raw.sssom.tsv.gz",
840854
"type": "sssom",
841-
"status": "nonstandard_format",
855+
"status": "no_metadata",
842856
"registries": [
843857
{
844858
"id": "https://github.com/biopragmatics/mapping-registry",
845-
"name": "Biopragmatics Mapping Registry"
859+
"name": "Biopragmatics Mapping Registry",
860+
"url": "https://github.com/biopragmatics"
846861
}
847862
],
848863
"metadata_completeness_score": 0.0

‎scripts/cli.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
import yaml
33
import requests
44
import json
5+
import gzip
6+
import io
57
import os
68
import logging
79
import traceback
@@ -33,10 +35,18 @@ def _registry_info(registry_data: dict) -> dict:
3335
}
3436

3537

36-
def _classify_error(mapping_set_uri: str, error: Exception) -> str:
38+
def _iter_gz_lines(response: requests.Response):
39+
"""Yield decompressed text lines from a gzipped HTTP response, streaming."""
40+
decompressor = gzip.GzipFile(fileobj=response.raw)
41+
reader = io.TextIOWrapper(decompressor, encoding="utf-8")
42+
for line in reader:
43+
yield line.rstrip("\n")
44+
45+
46+
def _classify_error(error: Exception) -> str:
3747
"""Classify a mapping set processing error into a status string."""
3848
error_msg = str(error).lower()
39-
if "bytes-like object" in error_msg or mapping_set_uri.endswith(".gz"):
49+
if "bytes-like object" in error_msg:
4050
return "nonstandard_format"
4151
if "no #-commented header" in error_msg:
4252
return "no_metadata"
@@ -67,7 +77,11 @@ def _process_sssom_mapping_set(mapping_set_id: str, mapping_set_uri: str) -> dic
6777
raise RuntimeError(f"HTTP {response.status_code} fetching {mapping_set_uri}")
6878

6979
try:
70-
meta = parse_sssom_tsv(response.iter_lines(decode_unicode=True))
80+
if mapping_set_uri.endswith(".gz"):
81+
lines = _iter_gz_lines(response)
82+
else:
83+
lines = response.iter_lines(decode_unicode=True)
84+
meta = parse_sssom_tsv(lines)
7185
finally:
7286
response.close()
7387

@@ -184,7 +198,7 @@ def prepare_mapping_registry(registry_file, output_file, log_file):
184198
mapping_set_id, registry_id, registry_title,
185199
)
186200
except Exception as e:
187-
status = _classify_error(mapping_set_uri, e)
201+
status = _classify_error(e)
188202
stub = _stub_spec(mapping_set_id, status)
189203
stub["registries"] = [info]
190204
specifications.append(stub)

0 commit comments

Comments
 (0)