Merge pull request #9 from ga4gh/dc-cache-and-clean

Removed dictionary cache

Author: dashrath-chauhan

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.12-slim
 
 WORKDIR /app
 

app/callbacks/epmc_callbacks.py

@@ -271,6 +271,10 @@ def show_epmc_details(selected_rows, search_value, year_filter, affiliation_filt
         )
         affiliation_rows = get_affiliations_by_article(pm_id) if pm_id else []
         affiliation_rows = [r for r in affiliation_rows if isinstance(r, dict)]
+
+        def _row_display_affiliation_order(row):
+            return row.get("display_affiliation_order") or row.get("affiliation_order")
+
         affiliation_rows = sorted(
             affiliation_rows,
             key=lambda r: (
@@ -303,8 +307,8 @@ def show_epmc_details(selected_rows, search_value, year_filter, affiliation_filt
         for row in sorted(
             affiliation_rows,
             key=lambda r: (
-                r.get("affiliation_order") is None,
-                r.get("affiliation_order") or 0,
+                _row_display_affiliation_order(r) is None,
+                _row_display_affiliation_order(r) or 0,
                 r.get("author_order") is None,
                 r.get("author_order") or 0,
             ),

app/constants/api.py

@@ -2,9 +2,11 @@
 All external API endpoints live here
 """
 
+from app.config import settings
+
 #BASE API endpoint
-BASE_API = "http://AnalyticsDashboardAlbBalancer-1386294349.us-east-2.elb.amazonaws.com:8000"
-#BASE_API = "http://0.0.0.0:8000" # local test url
+#BASE_API = "http://AnalyticsDashboardAlbBalancer-1386294349.us-east-2.elb.amazonaws.com:8000"
+BASE_API = settings.API_BASE_URL.rstrip("/")
 
 # PyPI API endpoint
 ALL_PACKAGES_API = BASE_API + "/pypi/all-packages"
@@ -32,4 +34,4 @@
 EPMC_ALL_ARTICLES = BASE_API + "/epmc/all-articles"
 EPMC_GET_AUTHORS_BY_ARTICLE = BASE_API + "/epmc/get-authors-by-article-id/"
 EPMC_CITATION_OVER_YEARS = BASE_API + "/epmc/citations-over-years"
-EPMC_AFFILIATION_BY_ARTICLE = BASE_API + "/epmc/get-affiliations-by-article-id/"
+EPMC_AFFILIATION_BY_ARTICLE = BASE_API + "/epmc/get-affiliations-by-article-id/"

app/layouts/datatables_layout.py

@@ -39,9 +39,10 @@ def get_datatables_layout(
 
     if not epmc_entries_df.empty:
         if "pub_year" in epmc_entries_df.columns:
+            years = pd.to_numeric(epmc_entries_df["pub_year"], errors="coerce").dropna().astype(int).unique()
             epmc_year_options = [
                 {"label": str(y), "value": str(y)}
-                for y in sorted(epmc_entries_df["pub_year"].dropna().unique(), reverse=True)
+                for y in sorted(years, reverse=True)
             ]
 
         if "raw_json" in epmc_entries_df.columns:

app/services/epmc_client.py

@@ -1,14 +1,17 @@
+import logging
 import requests
 import pandas as pd
 import json
 import app.constants.api as api_constants
 
+logger = logging.getLogger(__name__)
+
 
 def get_json(endpoint):
     """
     Generic GET → JSON helper (same as pypi_client.get_json).
     """
-    print(f"Calling API: {endpoint}")
+    logger.debug("Calling API: %s", endpoint)
     resp = requests.get(endpoint, timeout=30)
     resp.raise_for_status()
     return resp.json()
@@ -25,7 +28,7 @@ def get_all_paginated(endpoint, limit=1000):
 
     while True:
         params = {"limit": limit, "skip": skip}
-        print(f"Calling API: {endpoint} params={params}")
+        logger.debug("Calling API: %s params=%s", endpoint, params)
         resp = requests.get(endpoint, params=params, timeout=30)
         resp.raise_for_status()
         data = resp.json()
@@ -187,13 +190,6 @@ def get_affiliations_by_article(pm_id):
 
 
 
-# ---------------------------------------------------------------------------
-# Convenience: prepare a DataFrame ready for the layout / callbacks
-# ---------------------------------------------------------------------------
-
-_epmc_cache = {}
-
-
 def _normalize_pub_year(value):
     """Return a 4-digit publication year as int, or None when invalid."""
     if value is None:
@@ -210,20 +206,16 @@ def prepare_epmc_data():
     """
     Fetch and process all EPMC data in a single pass to avoid redundant API calls.
     Returns all data needed for the dashboard: DataFrames, counts, and metadata.
-    Results are cached after the first call.
 
     Returns:
         tuple: (entries_df, countries_df, authors_df, total_entries, citations,
                 unique_authors_count, top_authors_data)
     """
-    if "result" in _epmc_cache:
-        return _epmc_cache["result"]
     # Fetch all API data upfront (no redundancy)
     raw_entries = get_all_articles(limit=1000)
     total_entries = len(raw_entries)
 
     raw_countries = get_affiliation_countries_count()
-    raw_authors = get_all_pmc_authors()
 
     unique_authors_resp = get_json(api_constants.EPMC_UNIQUE_AUTHOR_COUNT)
     unique_authors_count = unique_authors_resp.get("unique_authors", 0) if isinstance(unique_authors_resp, dict) else 0
@@ -257,9 +249,8 @@ def prepare_epmc_data():
     else:
         countries_df = pd.DataFrame()
 
-    # Build authors DataFrame
-    authors_df = pd.DataFrame.from_records(raw_authors) if raw_authors and isinstance(raw_authors, list) else pd.DataFrame()
+    # The current UI uses summary author endpoints and article-specific author lookups,
+    # so avoid fetching every author row during app startup.
+    authors_df = pd.DataFrame()
 
-    result = (entries_df, countries_df, authors_df, total_entries, citations, unique_authors_count, top_authors_data)
-    _epmc_cache["result"] = result
-    return result
+    return entries_df, countries_df, authors_df, total_entries, citations, unique_authors_count, top_authors_data

app/services/github_client.py

@@ -1,3 +1,4 @@
+import logging
 import requests
 import pandas as pd
 import numpy as np
@@ -6,24 +7,21 @@
 
 import app.constants.api as api_constants
 
+logger = logging.getLogger(__name__)
+
 
 def get_json(endpoint: str, token: Optional[str] = None):
     headers = {}
     if token:
         headers["Authorization"] = f"token {token}"
 
-    print(f"Calling API: {endpoint}")
+    logger.debug("Calling API: %s", endpoint)
     resp = requests.get(endpoint, headers=headers, timeout=30)
     resp.raise_for_status()
     return resp.json()
 
 
-_github_cache = {}
-
 def prepare_github_data(fetch_date="2025-10-01"):
-    if "result" in _github_cache:
-        return _github_cache["result"]
-
     GA4GH_json = get_json(api_constants.GITHUB_REPOS_API)
 
     gh_df = pd.DataFrame.from_records(GA4GH_json)
@@ -74,5 +72,4 @@ def prepare_github_data(fetch_date="2025-10-01"):
     workstreams = gh_df["workstream"].dropna().unique().tolist()
 
     result = (gh_df, gh_activity_df, gh_activity_counts, gh_interest_df, total_repositories, workstreams)
-    _github_cache["result"] = result
     return result

app/services/pypi_client.py

@@ -1,7 +1,11 @@
+import logging
 import requests
 import app.constants.api as api_constants
 import pandas as pd
 
+logger = logging.getLogger(__name__)
+
+
 def get_json(endpoint):
     """
     Function to get JSON response from API for get_json
@@ -10,13 +14,11 @@ def get_json(endpoint):
     Returns:
         dict: JSON response from the API
     """
-    print(f"Calling API: {endpoint}")
+    logger.debug("Calling API: %s", endpoint)
     resp = requests.get(endpoint, timeout=30)
     resp.raise_for_status()
     return resp.json()
 
-_pypi_cache = {}
-
 def get_all_packages():
     """
     Returns the full list of all PyPI package records stored in DB.
@@ -32,19 +34,14 @@ def get_total_packages():
     Returns:
         int: total number of packages
     """
-    if "total" in _pypi_cache:
-        return _pypi_cache["total"]
     total_packages = get_json(api_constants.TOTAL_PACKAGES_API)
     total_packages = int(total_packages.get("total_packages", 0))
-    _pypi_cache["total"] = total_packages
     return total_packages
 
 def get_pypi_details():
     """
     Returns detailed metadata for each PyPI project.
     """
-    if "details" in _pypi_cache:
-        return _pypi_cache["details"]
     pypi_details = get_json(api_constants.PYPI_DETAILS_API)
     # Build DataFrame
     rows = []
@@ -61,15 +58,11 @@ def get_pypi_details():
         })
 
     df = pd.DataFrame(rows)
-    _pypi_cache["details"] = df
     return df
 
 def get_first_releases():
     """
     Returns the first release date for each PyPI project.
     """
-    if "first_releases" in _pypi_cache:
-        return _pypi_cache["first_releases"]
     first_releases = get_json(api_constants.FIRST_RELEASES_API)
-    _pypi_cache["first_releases"] = first_releases
-    return first_releases
+    return first_releases

app/services/service_map_client.py

@@ -1,7 +1,10 @@
+import logging
 import requests
 import pandas as pd
 from typing import Optional
 
+logger = logging.getLogger(__name__)
+
 BASE_URL = "https://implementation-registry.ga4gh.org/api"
 STANDARDS_ENDPOINT = f"{BASE_URL}/standards"
 SERVICES_ENDPOINT = f"{BASE_URL}/services"
@@ -12,25 +15,18 @@ def get_json(endpoint: str, token: Optional[str] = None):
     if token:
         headers["Authorization"] = f"token {token}"
 
-    print(f"Calling API: {endpoint}")
+    logger.debug("Calling API: %s", endpoint)
     resp = requests.get(endpoint, headers=headers, timeout=30)
     resp.raise_for_status()
     return resp.json()
 
-_service_map_cache = {}
-
 def prepare_service_map_data(fetch_date="2025-10-01"):
-    if "result" in _service_map_cache:
-        return _service_map_cache["result"]
-    
     standards_json = get_json(STANDARDS_ENDPOINT)
     services_json = get_json(SERVICES_ENDPOINT)
     deployments_json = get_json(DEPLOYMENTS_ENDPOINT)
 
     standards_df = pd.DataFrame.from_records(standards_json)
     services_df = pd.DataFrame.from_records(services_json)
     deployments_df = pd.DataFrame.from_records(deployments_json)
-    
-    _service_map_cache["result"] = (standards_df, services_df, deployments_df)
 
-    return _service_map_cache["result"]
+    return standards_df, services_df, deployments_df

run.py

@@ -3,4 +3,4 @@
 server = app.server
 
 if __name__ == "__main__":
-    app.run(debug=True, host="0.0.0.0", port=8050)
+    app.run(debug=True, host="0.0.0.0", port=8050, use_reloader=False)