33"""
44
55import json
6- import os
6+ from pathlib import Path
77import sys
88
99import pandas as pd
10- import yfinance as yf
1110
12- DATA_DIR = "data"
11+ try :
12+ import yfinance as yf
13+ except ImportError :
14+ yf = None
15+
16+ ROOT_DIR = Path (__file__ ).resolve ().parent .parent
17+ DATA_DIR = ROOT_DIR / "data"
18+ PUBLIC_DIR = ROOT_DIR / "public"
1319BASELINE_YEAR = 2000
1420LATEST_YEAR = 2023
1521
4046 "USA" : ("hist" , "SPY" ),
4147}
4248
49+ # Region metadata used by the final narrative site.
50+ REGION_BY_COUNTRY : dict [str , str ] = {
51+ "AUS" : "Asia-Pacific" ,
52+ "CAN" : "North America" ,
53+ "SWE" : "Europe" ,
54+ "DEU" : "Europe" ,
55+ "HKG" : "Asia-Pacific" ,
56+ "ITA" : "Europe" ,
57+ "JPN" : "Asia-Pacific" ,
58+ "BEL" : "Europe" ,
59+ "CHE" : "Europe" ,
60+ "MYS" : "Asia-Pacific" ,
61+ "NLD" : "Europe" ,
62+ "AUT" : "Europe" ,
63+ "ESP" : "Europe" ,
64+ "FRA" : "Europe" ,
65+ "SGP" : "Asia-Pacific" ,
66+ "GBR" : "Europe" ,
67+ "MEX" : "North America" ,
68+ "KOR" : "Asia-Pacific" ,
69+ "BRA" : "Latin America" ,
70+ "USA" : "North America" ,
71+ }
72+
73+ METRIC_METADATA : dict [str , dict [str , str | bool ]] = {
74+ "GDP" : {
75+ "unit" : "current US$" ,
76+ "absoluteLabel" : "Economic weight" ,
77+ "aggregate" : "sum" ,
78+ "absoluteComparable" : True ,
79+ },
80+ "GDP per Capita" : {
81+ "unit" : "current US$ per person" ,
82+ "absoluteLabel" : "Prosperity" ,
83+ "aggregate" : "mean" ,
84+ "absoluteComparable" : True ,
85+ },
86+ "ETF Price" : {
87+ "unit" : "ETF share price, adjusted close" ,
88+ "absoluteLabel" : "ETF price" ,
89+ "aggregate" : "mean" ,
90+ "absoluteComparable" : False ,
91+ },
92+ "Market Cap" : {
93+ "unit" : "current US$" ,
94+ "absoluteLabel" : "Listed company market value" ,
95+ "aggregate" : "sum" ,
96+ "absoluteComparable" : True ,
97+ },
98+ }
99+
43100WB_SKIPROWS = 4
44101_yf_cache : dict [str , pd .DataFrame ] = {}
45102
46103
47104# ── Data helpers ──────────────────────────────────────────────────────────────
48105
49106def load_wb (filename : str ) -> pd .DataFrame :
50- path = os . path . join ( DATA_DIR , "worldbank" , filename )
107+ path = DATA_DIR / "worldbank" / filename
51108 df = pd .read_csv (path , skiprows = WB_SKIPROWS , index_col = "Country Code" )
52109 for col in df .columns :
53110 if str (col ).strip ().isdigit ():
@@ -59,6 +116,10 @@ def _get_yf_supplement(symbol: str) -> pd.DataFrame:
59116 """Download 2020-2024 data via yfinance (cached). Returns df with 'close' column."""
60117 if symbol in _yf_cache :
61118 return _yf_cache [symbol ]
119+ if yf is None :
120+ print (f" yfinance unavailable for { symbol } ; using local ETF history only." , file = sys .stderr )
121+ _yf_cache [symbol ] = pd .DataFrame ()
122+ return _yf_cache [symbol ]
62123 try :
63124 ticker = yf .Ticker (symbol )
64125 hist = ticker .history (start = "2019-12-01" , end = "2024-06-30" , auto_adjust = True )
@@ -80,11 +141,11 @@ def load_etf_data(source: str, symbol: str) -> pd.DataFrame:
80141 """Load ETF data from local file, supplement with yfinance for post-2020 years.
81142 Returns a DataFrame with a 'close' column, indexed by date (ascending)."""
82143 if source == "etf" :
83- path = os . path . join ( DATA_DIR , "nasdaq" , "etf" , f"{ symbol } .csv" )
144+ path = DATA_DIR / "nasdaq" / "etf" / f"{ symbol } .csv"
84145 df = pd .read_csv (path , index_col = "Date" , parse_dates = True )
85146 df = df .rename (columns = {"Close" : "close" })
86147 elif source == "hist" :
87- path = os . path . join ( DATA_DIR , "stock" , "history" , f"{ symbol } .csv" )
148+ path = DATA_DIR / "stock" / "history" / f"{ symbol } .csv"
88149 df = pd .read_csv (path , index_col = "date" , parse_dates = True )
89150 df = df .rename (columns = {"close" : "close" })
90151 df = df .sort_index ()
@@ -166,12 +227,29 @@ def normalize_series(raw: dict, years: list[int], base: float) -> list:
166227 ]
167228
168229
230+ def absolute_series (raw : dict , years : list [int ]) -> list :
231+ return [
232+ round (raw [y ], 4 ) if raw .get (y ) is not None else None
233+ for y in years
234+ ]
235+
236+
237+ def make_metric_series (raw : dict , years : list [int ], base : float , metric : str ) -> dict :
238+ metadata = METRIC_METADATA [metric ]
239+ return {
240+ "normalized" : normalize_series (raw , years , base ),
241+ "absolute" : absolute_series (raw , years ),
242+ "unit" : metadata ["unit" ],
243+ "absoluteComparable" : metadata ["absoluteComparable" ],
244+ }
245+
246+
169247# ── Main ──────────────────────────────────────────────────────────────────────
170248
171249def main () -> None :
172250 print ("Loading World Bank data..." )
173251 iso_codes = pd .read_csv (
174- os . path . join ( DATA_DIR , "iso" , "countries.csv" ) , index_col = "alpha-3"
252+ DATA_DIR / "iso" / "countries.csv" , index_col = "alpha-3"
175253 )
176254 gdp = load_wb ("gdp-current-usd-2026.csv" )
177255 gdp_pc = load_wb ("gdp-capita-current-usd-2026.csv" )
@@ -212,9 +290,11 @@ def main() -> None:
212290 etf_raw = {y : get_annual_etf_price (etf_df , y ) for y in years }
213291
214292 timeseries : dict = {
215- "GDP" : normalize_series (gdp_series , years , gdp_base ),
216- "GDP per Capita" : normalize_series (gdp_pc_series , years , gdp_pc_base ),
217- "ETF Price" : normalize_series (etf_raw , years , etf_base ),
293+ "GDP" : make_metric_series (gdp_series , years , gdp_base , "GDP" ),
294+ "GDP per Capita" : make_metric_series (
295+ gdp_pc_series , years , gdp_pc_base , "GDP per Capita"
296+ ),
297+ "ETF Price" : make_metric_series (etf_raw , years , etf_base , "ETF Price" ),
218298 }
219299
220300 # Market cap (optional; excluded for CHN, IND, SWE, RUS)
@@ -223,7 +303,9 @@ def main() -> None:
223303 mc_base = mc_series .get (BASELINE_YEAR ) if mc_series else None
224304 if mc_series and mc_base and mc_base > 0 :
225305 mc_filled , n_filled = fill_mc_gaps (mc_series , etf_df , years )
226- timeseries ["Market Cap" ] = normalize_series (mc_filled , years , mc_base )
306+ timeseries ["Market Cap" ] = make_metric_series (
307+ mc_filled , years , mc_base , "Market Cap"
308+ )
227309 suffix = f"MC ok ({ n_filled } years ETF-proxy filled)"
228310 else :
229311 suffix = "MC skipped: no 2000 baseline"
@@ -235,6 +317,7 @@ def main() -> None:
235317 )
236318 countries_out [iso3 ] = {
237319 "name" : str (country_name ),
320+ "region" : REGION_BY_COUNTRY [iso3 ],
238321 "etf" : symbol ,
239322 "timeseries" : timeseries ,
240323 }
@@ -250,14 +333,17 @@ def main() -> None:
250333 "latestYear" : LATEST_YEAR ,
251334 "years" : years ,
252335 "axes" : universal_axes ,
336+ "metricMetadata" : METRIC_METADATA ,
337+ "regions" : sorted (set (REGION_BY_COUNTRY .values ())),
253338 "mcExcluded" : sorted (MC_EXCLUDE ),
254339 "countries" : countries_out ,
255340 }
256341
257- os . makedirs ( "../public" , exist_ok = True )
258- output_path = "../public/ spider_data.json"
342+ PUBLIC_DIR . mkdir ( exist_ok = True )
343+ output_path = PUBLIC_DIR / " spider_data.json"
259344 with open (output_path , "w" ) as f :
260- json .dump (output , f , separators = ("," , ":" ))
345+ json .dump (output , f , indent = 2 )
346+ f .write ("\n " )
261347
262348 # summary
263349 print (f"\n Wrote { len (countries_out )} countries to { output_path } " )
0 commit comments