Skip to content

Commit 447e41b

Browse files
soodokuclaude
soodoku
and
claude
committed
Store EXIF dumps gzipped and read them compressed
data/*/exif/*_exif.txt -> *_exif.txt.gz (1,014 files, 6.5 GB -> 422 MB, each verified byte-identical after round-trip). 00_process_videos.py writes new dumps gzipped and still reads a legacy .txt; 01_extract_face_frames.py and rebuild_csvs_from_exif.py read through gzip. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012f4vYASsYbLKVaHVz3XNoN
0 parents  commit 447e41b

1,435 files changed

Lines changed: 616453 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: CI
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
9+
permissions:
10+
contents: read
11+
12+
jobs:
13+
test:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
17+
- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
18+
with:
19+
version: "0.12.0"
20+
enable-cache: true
21+
- run: uv python install 3.13
22+
- run: uv sync --locked --dev
23+
- run: make ci

‎.gitignore‎

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
._*
6+
7+
# C extensions
8+
*.so
9+
10+
# Distribution / packaging
11+
.Python
12+
build/
13+
develop-eggs/
14+
dist/
15+
downloads/
16+
eggs/
17+
.eggs/
18+
lib/
19+
lib64/
20+
parts/
21+
sdist/
22+
var/
23+
wheels/
24+
share/python-wheels/
25+
*.egg-info/
26+
.installed.cfg
27+
*.egg
28+
MANIFEST
29+
30+
# PyInstaller
31+
# Usually these files are written by a python script from a template
32+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
33+
*.manifest
34+
*.spec
35+
36+
# Installer logs
37+
pip-log.txt
38+
pip-delete-this-directory.txt
39+
40+
# Unit test / coverage reports
41+
htmlcov/
42+
.tox/
43+
.nox/
44+
.coverage
45+
.coverage.*
46+
.cache
47+
nosetests.xml
48+
coverage.xml
49+
*.cover
50+
*.py,cover
51+
.hypothesis/
52+
.pytest_cache/
53+
cover/
54+
55+
# Translations
56+
*.mo
57+
*.pot
58+
59+
# Django stuff:
60+
*.log
61+
local_settings.py
62+
db.sqlite3
63+
db.sqlite3-journal
64+
65+
# Flask stuff:
66+
instance/
67+
.webassets-cache
68+
69+
# Scrapy stuff:
70+
.scrapy
71+
72+
# Sphinx documentation
73+
docs/_build/
74+
75+
# PyBuilder
76+
.pybuilder/
77+
target/
78+
79+
# Jupyter Notebook
80+
.ipynb_checkpoints
81+
82+
# IPython
83+
profile_default/
84+
ipython_config.py
85+
86+
# pyenv
87+
# For a library or package, you might want to ignore these files since the code is
88+
# intended to run in multiple environments; otherwise, check them in:
89+
# .python-version
90+
91+
# pipenv
92+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
93+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
94+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
95+
# install all needed dependencies.
96+
#Pipfile.lock
97+
98+
# UV
99+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
100+
# This is especially recommended for binary packages to ensure reproducibility, and is more
101+
# commonly ignored for libraries.
102+
#uv.lock
103+
104+
# poetry
105+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
106+
# This is especially recommended for binary packages to ensure reproducibility, and is more
107+
# commonly ignored for libraries.
108+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
109+
#poetry.lock
110+
111+
# pdm
112+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
113+
#pdm.lock
114+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
115+
# in version control.
116+
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
117+
.pdm.toml
118+
.pdm-python
119+
.pdm-build/
120+
121+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
122+
__pypackages__/
123+
124+
# Celery stuff
125+
celerybeat-schedule
126+
celerybeat.pid
127+
128+
# SageMath parsed files
129+
*.sage.py
130+
131+
# Environments
132+
.env
133+
.venv
134+
env/
135+
venv/
136+
ENV/
137+
env.bak/
138+
venv.bak/
139+
140+
# Spyder project settings
141+
.spyderproject
142+
.spyproject
143+
144+
# Rope project settings
145+
.ropeproject
146+
147+
# mkdocs documentation
148+
/site
149+
150+
# mypy
151+
.mypy_cache/
152+
.dmypy.json
153+
dmypy.json
154+
155+
# Pyre type checker
156+
.pyre/
157+
158+
# pytype static type analyzer
159+
.pytype/
160+
161+
# Cython debug symbols
162+
cython_debug/
163+
164+
# PyCharm
165+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
166+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
167+
# and can be added to the global gitignore or merged into this file. For a more nuclear
168+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
169+
#.idea/
170+
171+
# PyPI configuration file
172+
.pypirc
173+
.DS_Store
174+
data/._.DS_Store
175+
scripts/._.DS_Store
176+
177+
# Large data directories (gitignored)
178+
/data/frames
179+
/data/frames_new
180+
/data/annotation_frames
181+
/data/annotation_frames_new
182+
183+
# Video files and frames
184+
/vids
185+
/mumbai_video_frames
186+
/mumbai_vids
187+
/mumbai_frames_3x
188+
/mumbai_frames_flash_3x
189+
/frames
190+
/annotation_frames
191+
/annotation_frames_new
192+
/frames_new
193+
/exif
194+
195+
# Scripts output
196+
/scripts/video_frames
197+
/scripts/output
198+
/out
199+
200+
# GCP and other large data
201+
/gcp
202+
/sampling/bangalore/network
203+
/data/delhi_planet_76.833,28.408_77.338,28.885-shp
204+
205+
# Streamlit secrets
206+
streamlit/.secrets.toml
207+
/streamlit/.streamlit
208+
209+
# LaTeX output
210+
/streetscope_science/tex
211+
/data/annotation_task
212+
# Never commit the giant raw GPS-timeseries CSVs (any city); keep the compact Parquet.
213+
data/*/exif_metadata/*_gps_timeseries*.csv
214+
data/*/exif_metadata/gps_timeseries.csv
215+
# Bangalore/Delhi GPS timeseries is regenerable from committed exif/*.txt -> scratch, don't commit.
216+
data/bangalore/exif_metadata/*_gps_timeseries*.parquet
217+
data/delhi/exif_metadata/*_gps_timeseries*.parquet
218+
# (Mumbai/Navi *_gps_timeseries*.parquet ARE committed: their only raw GPS source.)
219+
# analysis_data is derived: rebuildable offline from committed inputs via `make data`.
220+
data/*/analysis_data.parquet
221+
data/mumbai/validation_batch_input.jsonl
222+
/cache
223+
/scripts/gen_data/cache
224+
/scripts/gen_data/delhi/cache
225+
/scripts/gen_data/hyderabad/cache
226+
/sampling/hyderabad/network
227+
/sampling/delhi/network
228+
*.pbf
229+
230+
# Manuscript source is versioned; LaTeX outputs are not.
231+
/figs/*.html
232+
/ms/build/
233+
/ms/*.aux
234+
/ms/*.bbl
235+
/ms/*.blg
236+
/ms/*.fdb_latexmk
237+
/ms/*.fls
238+
/ms/*.log
239+
/ms/*.maf
240+
/ms/*.mtc
241+
/ms/*.mtc0
242+
/ms/*.out
243+
/ms/*.pdf
244+
/ms/*.synctex.gz
245+
/ms/*.toc
246+
/ms.aux
247+
/ms.bbl
248+
/ms.blg
249+
/ms.fdb_latexmk
250+
/ms.fls
251+
/ms.log
252+
/ms.maf
253+
/ms.mtc
254+
/ms.mtc0
255+
/ms.out
256+
/ms.pdf

‎CHANGELOG.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Changelog
2+
3+
## Unreleased
4+
5+
- Require Pillow 12.3.0; the four analysis datasets and manuscript PDF remain byte-identical
6+
to version 0.1.0.
7+
- Document the repository's manual dependency-review policy.
8+
9+
## 0.1.0 - 2026-08-16
10+
11+
- Release the four-city analysis and manuscript reproduction workflow.
12+
- Reconcile physical-frame aliases before selecting primary annotations.
13+
- Generate manuscript statistics, adult Census benchmarks, figures, and tables from code.
14+
- Add collection-day-clustered intervals, leave-one-day-out checks, inter-rater reliability,
15+
top-code sensitivity, and mobility-gap accounting.
16+
- Standardize percentage, percentage-point, and women-per-1,000-men reporting.

‎Makefile‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Streetscope — analysis-half workflow.
2+
#
3+
# These targets cover the FAIR-GAME half of the pipeline only: parse annotations
4+
# -> assign GPS -> enrich -> build analysis_data -> figures/tables/maps. They never
5+
# re-run the frozen collection half (video processing / fixed-interval frame
6+
# extraction); that is done once via scripts/00_process_videos.py.
7+
8+
PY ?= .venv/bin/python
9+
CITY ?= mumbai # single city for run_pipeline
10+
CITIES ?= mumbai,navi_mumbai,bangalore,delhi # comma-separated for figure/table scripts
11+
SOURCE_DATE_EPOCH ?= 1786838400
12+
export SOURCE_DATE_EPOCH
13+
PYTHONHASHSEED ?= 0
14+
export PYTHONHASHSEED
15+
16+
.PHONY: help install data analyze figures tables irr analysis paper paper-clean test lint ci clean
17+
18+
help:
19+
@echo "make install sync the locked analysis environment"
20+
@echo "make data (re)build analysis_data.parquet for all cities (gitignored, derived)"
21+
@echo "make analyze CITY=mumbai run the analysis half for one city (uses cached GPS index)"
22+
@echo "make figures CITIES=... regenerate EDA + publication figures/tables + maps"
23+
@echo "make tables CITIES=... regenerate publication tables only"
24+
@echo "make irr CITIES=... inter-rater reliability table + console summary"
25+
@echo "make analysis rebuild data and all manuscript figures/tables"
26+
@echo "make paper rebuild the analysis and compile ms/ms.pdf"
27+
@echo "make paper-clean remove LaTeX build files"
28+
@echo "make test run the analysis regression tests"
29+
@echo "make lint ruff format + ruff check --fix over scripts/"
30+
@echo "make ci run formatting, lint, and tests without modifying files"
31+
@echo "make clean remove generated figures and tables"
32+
33+
install:
34+
uv sync
35+
36+
# Rebuild the (gitignored) analysis_data.parquet for every city from committed inputs
37+
# (GPS index + annotations + osm_roads.parquet). Skips GPS-index rebuild and viz.
38+
data:
39+
$(PY) scripts/run_pipeline.py --city all --skip-rebuild-gps --skip-viz
40+
41+
# Build analysis_data for one city, then its figures/tables/maps. Skips the GPS
42+
# index rebuild (cached) and never processes videos.
43+
analyze:
44+
$(PY) scripts/run_pipeline.py --city $(CITY) --skip-rebuild-gps
45+
46+
figures:
47+
$(PY) scripts/09_eda.py --cities $(CITIES)
48+
$(PY) scripts/10_make_publication_outputs.py --cities $(CITIES)
49+
$(PY) scripts/11_make_maps.py --cities $(CITIES)
50+
$(PY) scripts/14_descriptive_patterns.py --cities $(CITIES)
51+
$(PY) scripts/15_gap_accounting.py --cities $(CITIES)
52+
53+
tables:
54+
$(PY) scripts/10_make_publication_outputs.py --cities $(CITIES)
55+
$(PY) scripts/15_gap_accounting.py --cities $(CITIES)
56+
57+
irr:
58+
$(PY) scripts/13_interrater_reliability.py --cities $(CITIES)
59+
60+
analysis: data figures irr
61+
62+
paper: analysis
63+
latexmk -pdf -interaction=nonstopmode -halt-on-error -outdir=ms ms/ms.tex
64+
65+
paper-clean:
66+
latexmk -C -outdir=ms ms/ms.tex
67+
68+
test:
69+
uv run pytest
70+
71+
lint:
72+
uv run ruff format scripts/
73+
uv run ruff check --fix scripts/
74+
75+
ci:
76+
uv run ruff format --check scripts/ tests/
77+
uv run ruff check scripts/ tests/
78+
uv run pytest
79+
80+
clean:
81+
rm -f figs/*.pdf figs/*.png figs/*.html tabs/*.tex tabs/descriptive_patterns.md

0 commit comments

Comments
 (0)