Merge pull request #39 from TranslatorSRI/claude/focused-brahmagupta-e1iltd

Migrate novelty/scoring TRAPI access to 1.5+ binding format

Author: maximusunc

.github/workflows/dev.yml

@@ -28,16 +28,6 @@ jobs:
       - name: Exit if there are any test failures
         run: '[[ $TEST_OUTPUT != *FAILED* ]]'
 
-  verify-lock:
-    name: Verify lockfiles are up to date
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout the repository
-        uses: actions/checkout@v2
-
-      - name: Verify lockfiles
-        run: python manage.py verify_locked
-
   check-format:
     name: Check that code matches Black formatter
     runs-on: ubuntu-latest

app/clinical_evidence/compute_clinical_evidence.py

@@ -18,10 +18,10 @@ def compute_clinical_evidence(
     found_edges = []
     # loop over all analyses in the given result and append any clinical kp edges to found_edges
     for analysis in result.get("analyses") or []:
-        for edge_bindings in analysis.get("edge_bindings", {}).values():
-            for edge_binding in edge_bindings:
+        for edge_binding in analysis.get("edge_bindings", {}).values():
+            for edge_id in edge_binding["ids"]:
                 try:
-                    kg_edge = message["knowledge_graph"]["edges"][edge_binding["id"]]
+                    kg_edge = message["knowledge_graph"]["edges"][edge_id]
                 except KeyError:
                     # this is malformed TRAPI
                     logger.error("malformed TRAPI")

app/clinical_evidence/merge.py

@@ -1,13 +1,15 @@
 """Script for combining multiple KGX node and edge files."""
 
+from datetime import datetime
 import glob
 import json
 import jsonlines
-
+from tqdm import tqdm
 
 if __name__ == "__main__":
-    node_files = glob.glob("./kgx/*_nodes.jsonl")
-    edge_files = glob.glob("./kgx/*_edges.jsonl")
+    time = datetime.now().strftime("%Y_%m_%d")
+    node_files = glob.glob("./kgx/full/*_nodes.jsonl")
+    edge_files = glob.glob("./kgx/full/*_edges.jsonl")
 
     node_ids = set()
     nodes = {}
@@ -19,7 +21,7 @@
                 if node["id"] not in node_ids:
                     nodes[node["id"]] = node
 
-    with open("nodes_merged.json", "w") as f:
+    with open(f"kgx/nodes_merged_{time}.json", "w") as f:
         json.dump(nodes, f, indent=2)
 
     supported_categories = ["biolink:Drug", "biolink:ChemicalEntity", "biolink:Disease"]
@@ -29,7 +31,8 @@
     for edge_file in edge_files:
         print(f"merging {edge_file}")
         with jsonlines.open(edge_file) as reader:
-            for edge in reader:
+            print("loaded edge file")
+            for edge in tqdm(reader):
                 # get subject node
                 if type(edge["subject"]) == dict:
                     # from ehr
@@ -133,6 +136,23 @@
                                 "log_odds_analysis_result"
                             ].get("total_sample_size", 0)
 
+                        elif (
+                            edge.get("primary_knowledge_source")
+                            == "infores:isb-EHRMLA-data"
+                        ):
+                            # from ehr may treat
+                            save_edge["supporting_data_source"] = (
+                                "infores:isb-EHRMLA-data"
+                            )
+                            save_edge["predicate"] = edge["predicate"]
+                            save_edge["log_odds_ratio"] = edge.get("log_odds_ratio", 0)
+                            save_edge["log_odds_ratio_95_ci"] = edge.get(
+                                "log_odds_ratio_95_ci", [0, 0]
+                            )
+                            save_edge["total_sample_size"] = edge.get(
+                                "total_sample_size", 0
+                            )
+
                         if save_edge["log_odds_ratio"] > 10:
                             save_edge["log_odds_ratio"] = 10
                             save_edge["log_odds_ratio_95_ci"] = [10, 10]
@@ -148,6 +168,6 @@
                             ]
 
     print("Writing output edges...")
-    with open("edges_merged.json", "w") as f:
+    with open(f"kgx/edges_merged_{time}.json", "w") as f:
         json.dump(edges, f, indent=2)
     print("Merge Complete!")

app/novelty/compute_novelty.py

@@ -10,6 +10,20 @@
 from .gene_nmf_adapter import get_gene_nmf_novelty_for_gene_list
 from ..config import settings
 
+
+def result_node_id(result, query_id_node):
+    """Return the KG id of the non-query (result) node for a TRAPI result.
+
+    Picks the first node-bound Knowledge Graph identifier that is not the
+    queried node.
+    """
+    for binding in result["node_bindings"].values():
+        for kg_id in binding["ids"]:
+            if kg_id != query_id_node:
+                return kg_id
+    return None
+
+
 """
 This script computes the novelty score for a list of results obtained for a 1-H response using publications from 5 ARAs.
 The steps for the ideal workflow are as follows:
@@ -114,27 +128,11 @@ async def molecular_sim(known, unknown, message, query_id):
     known_ids = []
     if len(unknown) > 0:
         for drug in unknown:
-            s = list(message["results"][drug]["node_bindings"].keys())
-            if message["results"][drug]["node_bindings"][s[0]][0]["id"] == query_id:
-                unknown_ids.append(
-                    message["results"][drug]["node_bindings"][s[1]][0]["id"]
-                )
-            else:
-                unknown_ids.append(
-                    message["results"][drug]["node_bindings"][s[0]][0]["id"]
-                )
+            unknown_ids.append(result_node_id(message["results"][drug], query_id))
 
     if len(known) > 0:
         for drug in known:
-            s = list(message["results"][drug]["node_bindings"].keys())
-            if message["results"][drug]["node_bindings"][s[0]][0]["id"] == query_id:
-                known_ids.append(
-                    message["results"][drug]["node_bindings"][s[1]][0]["id"]
-                )
-            else:
-                known_ids.append(
-                    message["results"][drug]["node_bindings"][s[0]][0]["id"]
-                )
+            known_ids.append(result_node_id(message["results"][drug], query_id))
 
     smile_unkown = await mol_to_smile_molpro(unknown_ids)
     smile_known = await mol_to_smile_molpro(known_ids)
@@ -225,10 +223,10 @@ def extracting_publications(message, result):
     publications = []
     for idi, i in enumerate(result["analyses"]):
         edge_keys = list(i["edge_bindings"].keys())
-        for idj, j in enumerate(i["edge_bindings"][edge_keys[0]]):
+        for edge_id in i["edge_bindings"][edge_keys[0]]["ids"]:
             aux_graph, edges = [], []
             for idl, l in enumerate(
-                message["knowledge_graph"]["edges"][j["id"]]["attributes"]
+                message["knowledge_graph"]["edges"][edge_id]["attributes"]
             ):
                 if l["attribute_type_id"] == "biolink:publications":
                     publications.extend(l["value"])
@@ -271,19 +269,17 @@ def extract_results(message, unknown, known):
         if idi in unknown:
             results.append([])
             for idj, j in enumerate(i["analyses"]):
-                for idk, k in enumerate(
-                    j["edge_bindings"][list(j["edge_bindings"].keys())[0]]
-                ):
-                    results[ukid].append(k["id"])
+                edge_key = list(j["edge_bindings"].keys())[0]
+                for edge_id in j["edge_bindings"][edge_key]["ids"]:
+                    results[ukid].append(edge_id)
             ukid += 1
 
         elif idi in known:
             results_known.append([])
             for idj, j in enumerate(i["analyses"]):
-                for idk, k in enumerate(
-                    j["edge_bindings"][list(j["edge_bindings"].keys())[0]]
-                ):
-                    results_known[kid].append(k["id"])
+                edge_key = list(j["edge_bindings"].keys())[0]
+                for edge_id in j["edge_bindings"][edge_key]["ids"]:
+                    results_known[kid].append(edge_id)
             kid += 1
     return results, results_known
 
@@ -335,11 +331,7 @@ async def compute_novelty(
         novelty_score_rec_tdl, novelty_score_rec_clin = [], []
         for idi, i in enumerate(message["results"]):
             curated = 0
-            node_binding_keys = list(i["node_bindings"].keys())
-            if i["node_bindings"][node_binding_keys[0]][0]["id"] == query_id_node:
-                result_id_node = i["node_bindings"][node_binding_keys[1]][0]["id"]
-            else:
-                result_id_node = i["node_bindings"][node_binding_keys[0]][0]["id"]
+            result_id_node = result_node_id(i, query_id_node)
             df_numpy.append([query_id_node, result_id_node])
             result_node_cat = message["knowledge_graph"]["nodes"][result_id_node][
                 "categories"
@@ -363,39 +355,17 @@ async def compute_novelty(
                 correct_results.append(idi)
                 for idj, j in enumerate(i["analyses"]):
                     edge_keys = list(j["edge_bindings"].keys())
-                    for idk, k in enumerate(j["edge_bindings"][edge_keys[0]]):
+                    for edge_id in j["edge_bindings"][edge_keys[0]]["ids"]:
                         knowledge_graph_edge = message["knowledge_graph"]["edges"][
-                            k["id"]
+                            edge_id
                         ]
-                        epc_found = 0
-                        for idl, l in enumerate(knowledge_graph_edge["attributes"]):
-                            if l["attribute_type_id"] == "biolink:knowledge_level":
-                                epc_found = 1
-                                if l["value"] != "prediction":
-                                    curated = 1
-                                    df_numpy[idi].extend(
-                                        [l["attribute_type_id"], l["value"]]
-                                    )
-                                    break
-                        if curated == 1 and epc_found == 1:
+                        knowledge_level = knowledge_graph_edge["knowledge_level"]
+                        if knowledge_level != "prediction":
+                            curated = 1
+                            df_numpy[idi].extend(
+                                ["biolink:knowledge_level", knowledge_level]
+                            )
                             break
-                        elif curated == 0 and epc_found == 0:
-                            for idl, l in enumerate(knowledge_graph_edge["sources"]):
-                                if l["resource_role"] == "primary_knowledge_source":
-                                    if l["resource_id"] not in [
-                                        "infores:arax",
-                                        "infores:aragorn",
-                                        "infores:biothings-explorer",
-                                        "infores:unsecret-agent",
-                                        "infores:improving-agent",
-                                        "infores:cqs",
-                                    ]:
-                                        curated = 1
-                                        df_numpy[idi].extend(
-                                            [l["resource_role"], l["resource_id"]]
-                                        )
-                                        break
-
                     if curated == 1:
                         break
                 if curated == 1:
@@ -470,14 +440,7 @@ async def compute_novelty(
             map_result_keys = list(map_result["gene_results"].keys())
             for idi, i in enumerate(message["results"]):
                 if idi in unknown_list:
-                    node_binding_keys = list(i["node_bindings"].keys())
-                    if (
-                        i["node_bindings"][node_binding_keys[0]][0]["id"]
-                        == query_id_node
-                    ):
-                        res = i["node_bindings"][node_binding_keys[1]][0]["id"]
-                    else:
-                        res = i["node_bindings"][node_binding_keys[0]][0]["id"]
+                    res = result_node_id(i, query_id_node)
                     if res in map_result_keys:
                         gene_distinct = (
                             1 - map_result["gene_results"][res]["novelty_score"]
@@ -589,14 +552,7 @@ async def compute_novelty(
             similarity_map_keys = list(similarity_map.keys())
             for idi, i in enumerate(message["results"]):
                 if idi in unknown_list:
-                    node_binding_keys = list(i["node_bindings"].keys())
-                    if (
-                        i["node_bindings"][node_binding_keys[0]][0]["id"]
-                        == query_id_node
-                    ):
-                        res = i["node_bindings"][node_binding_keys[1]][0]["id"]
-                    else:
-                        res = i["node_bindings"][node_binding_keys[0]][0]["id"]
+                    res = result_node_id(i, query_id_node)
 
                     if res in similarity_map_keys and similarity_map[res] != []:
                         similarity = similarity_map[res][0][1]

app/novelty/dcc/compute_utils.py

@@ -386,7 +386,7 @@ def compute_beta_tildes(
 
     # MPD - change so only return pvalues
     # return finalize_regression(beta_tildes=beta_tildes, ses=ses, se_inflation_factors=se_inflation_factors)
-    (cal_beta_tildes, cal_ses, cal_z_scores, cal_p_values, cal_se_inflation_factors) = (
+    cal_beta_tildes, cal_ses, cal_z_scores, cal_p_values, cal_se_inflation_factors = (
         finalize_regression(
             beta_tildes=beta_tildes, ses=ses, se_inflation_factors=se_inflation_factors
         )
@@ -845,7 +845,7 @@ def _calc_X_shift_scale(X, y_corr_cholesky=None):
         scale_factors = np.array([])
         mean_shifts = np.array([])
         for X_b, begin, end, batch in _get_X_blocks_internal(X, y_corr_cholesky):
-            (cur_mean_shifts, cur_scale_factors) = _calc_shift_scale(X_b)
+            cur_mean_shifts, cur_scale_factors = _calc_shift_scale(X_b)
             mean_shifts = np.append(mean_shifts, cur_mean_shifts)
             scale_factors = np.append(scale_factors, cur_scale_factors)
 

app/novelty/dcc/file_utils.py

@@ -35,7 +35,6 @@
 
 from .dcc_utils import get_logger
 
-
 # constants
 logger = get_logger(__name__)
 

app/novelty/dcc/gui_utils.py

@@ -33,7 +33,6 @@
 from .matrix_utils import sum_of_gene_row
 from .dcc_utils import get_logger
 
-
 # constants
 logger = get_logger(__name__)
 

app/novelty/dcc/sql_utils.py

@@ -34,7 +34,6 @@
 
 from .dcc_utils import get_logger
 
-
 # constants
 logger = get_logger(__name__)
 SQL_SELECT_ALL_GENES = (

app/novelty/gene_nmf_adapter.py

@@ -42,7 +42,6 @@
 from .dcc import sql_utils as sql_utils
 from .dcc import gui_utils as gutils
 
-
 # constants
 P_VALUE_CUTOFF = 0.3
 MAX_NUMBER_GENE_SETS_FOR_COMPUTATION = 100
@@ -75,7 +74,7 @@
 )
 
 # get the other 2 cached matrices
-(mean_shifts, scale_factors) = cutils._calc_X_shift_scale(X=matrix_gene_sets)
+mean_shifts, scale_factors = cutils._calc_X_shift_scale(X=matrix_gene_sets)
 
 print("================ Bayes NMF data structures LOADED! ===========================")
 

app/ordering_components.py

@@ -8,7 +8,6 @@
 from .clinical_evidence.compute_clinical_evidence import compute_clinical_evidence
 from .novelty.compute_novelty import compute_novelty
 
-
 redis_pool = redis.ConnectionPool(
     host=settings.redis_host,
     port=settings.redis_port,
@@ -73,8 +72,7 @@ async def get_ordering_components(message, logger):
             "clinical_evidence": clinical_evidence_score,
             "novelty": 0.0,
         }
-        for node_bindings in result.get("node_bindings", {}).values():
-            for node_binding in node_bindings:
-                result["ordering_components"]["novelty"] = novelty_scores.get(
-                    node_binding["id"], 0.0
-                )
+        for binding in result.get("node_bindings", {}).values():
+            for kg_id in binding["ids"]:
+                if kg_id in novelty_scores:
+                    result["ordering_components"]["novelty"] = novelty_scores[kg_id]