Skip to content

Commit c497487

Browse files
jordancaraballojordancaraballo
authored
Merge pull request #14 from nasa-nccs-hpda/develop
Develop
2 parents e3668d3 + f7b8b77 commit c497487

5 files changed

Lines changed: 238 additions & 8 deletions

File tree

‎.github/workflows/docker-hub-pytorch.yaml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: CI to Dockerhub
2+
3+
on:
4+
push:
5+
branches:
6+
- 'container-devel'
7+
8+
jobs:
9+
docker:
10+
runs-on: ubuntu-latest
11+
steps:
12+
-
13+
name: Checkout
14+
uses: actions/checkout@v3
15+
-
16+
name: Set up QEMU
17+
uses: docker/setup-qemu-action@v2
18+
-
19+
name: Set up Docker Buildx
20+
uses: docker/setup-buildx-action@v2
21+
-
22+
name: Login to Docker Hub
23+
uses: docker/login-action@v2
24+
with:
25+
username: ${{ secrets.DOCKERHUB_USERNAME }}
26+
password: ${{ secrets.DOCKERHUB_TOKEN }}
27+
28+
-
29+
name: Lower github-runner storage
30+
run: |
31+
# Remove software and language runtimes we're not using
32+
sudo rm -rf \
33+
"$AGENT_TOOLSDIRECTORY" \
34+
/opt/google/chrome \
35+
/opt/microsoft/msedge \
36+
/opt/microsoft/powershell \
37+
/opt/pipx \
38+
/usr/lib/mono \
39+
/usr/local/julia* \
40+
/usr/local/lib/android \
41+
/usr/local/lib/node_modules \
42+
/usr/local/share/chromium \
43+
/usr/local/share/powershell \
44+
/usr/share/dotnet \
45+
/usr/share/swift
46+
df -h /
47+
48+
-
49+
name: Build and push
50+
uses: docker/build-push-action@v4
51+
with:
52+
context: .
53+
file: ./requirements/Dockerfile.pytorch
54+
push: true
55+
tags: nasanccs/vhr-toolkit:pytorch

‎README.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,15 @@
88
The CPU-only version of the vhr-toolkit container can be downloaded with:
99

1010
```bash
11-
singularity build --sandbox /lscratch/$USER/container/vhr-toolkit docker://nasanccs/vhr-toolkit:cpu
11+
singularity build --sandbox /lscratch/$USER/container/vhr-toolkit docker://nasanccs/vhr-toolkit:pytorch
12+
```
13+
14+
## Quickstart
15+
16+
To run the entire workflow for a single test scene:
17+
18+
```bash
19+
singularity exec --nv -B /explore,/panfs,/css,/nfs4m /explore/nobackup/projects/ilab/containers/vhr-toolkit-latest python /usr/local/ilab/vhr-toolkit/vhr_toolkit/view/vhrToolkit.py -o vhr-toolkit-output --scenes_in_file tests/catalog_1030010003A81D00.txt | tee catalog_1030010003A81D00.log
1220
```
1321

1422
## Contributing

‎requirements/Dockerfile.pytorch‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
# geedim, plotnine, pylr2
2+
3+
FROM nasanccs/vhr-cloudmask.pytorch:latest
4+
5+
# Arguments to pass to the image
6+
ARG STEREO_URL="https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2"
7+
ARG PROJECT_PATH="/usr/local/ilab"
8+
9+
# Environment
10+
# ENV PYTHONPATH="$PYTHONPATH:/usr/local/ilab"
11+
# why is this??????? really bad practice
12+
# ENV PYTHONPATH="/usr/local/lib/python3.8/dist-packages"
13+
ENV REDIS_PORT="6379"
14+
ENV REDIS_FILE="/etc/profile.d/redis_server.sh"
15+
16+
# Ubuntu needs noninteractive to be forced
17+
ENV DEBIAN_FRONTEND=noninteractive
18+
ENV CPLUS_INCLUDE_PATH="/usr/include/gdal"
19+
ENV C_INCLUDE_PATH="/usr/include/gdal"
20+
# ENV PROJ_LIB="/usr/share/proj"
21+
# ENV PROJ_DATA="/usr/share/proj"
22+
23+
# ENV PYTHONPATH="$PYTHONPATH:/usr/local/ilab"
24+
25+
ENV PYTHONPATH="/usr/local/ilab:/usr/local/ilab/vhr-cloudmask:/usr/local/ilab/evhr:/usr/local/ilab/core:/usr/local/ilab/srlite/srlite"
26+
ENV PYTHONPATH="$PYTHONPATH:/usr/local/ilab/srlite:/usr/local/ilab/pyCCDC:/usr/local/ilab/vhr-toolkit:/usr/local/ilab/vhr-toolkit/vhr-toolkit:/opt/DgStereo/pygeotools:/opt/DgStereo/dgtools"
27+
ENV PYTHONPATH="$PYTHONPATH:/opt/DgStereo/pygeotools:/opt/DgStereo/dgtools"
28+
29+
ENV PATH="$PATH:/opt/DgStereo/dgtools:/opt/DgStereo/pygeotools:/opt/DgStereo/dgtools/dgtools:/opt/StereoPipeline/bin:/opt/DgStereo/evhr"
30+
31+
RUN PROJECT_PATH="/usr/local/ilab" && \
32+
REDIS_FILE="/etc/profile.d/redis_server.sh" && \
33+
echo $PROJECT_PATH && \
34+
mkdir -p $PROJECT_PATH && \
35+
apt-get update && \
36+
DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade && \
37+
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
38+
wget vim curl git procps gcc g++ bzip2 libssl-dev \
39+
libsqlite3-dev libx11-dev libgeos++-dev libproj-dev \
40+
build-essential parallel libdatetime-perl gawk util-linux bc \
41+
python3-tk tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev \
42+
zlib1g-dev liblzma-dev libgirepository1.0-dev libcairo2-dev \
43+
pkg-config gir1.2-gtk-3.0 libnetcdf-dev libhdf4-dev 2to3 && \
44+
DEBIAN_FRONTEND=noninteractive apt-get -y autoremove && \
45+
rm -rf /var/lib/apt/lists/* && \
46+
rm -rf /var/cache/apt
47+
48+
RUN rm -rf /usr/lib/python*/EXTERNALLY-MANAGED && \
49+
pip install pipx && \
50+
pip install rasterio && \
51+
pip install awscli s3fs && \
52+
pip install celery[redis] && \
53+
pip install certifi && \
54+
pip install cryptography && \
55+
pip install Cython && \
56+
pip install dask && \
57+
pip install decorator && \
58+
pip install dill && \
59+
pip install distributed && \
60+
pip install docutils && \
61+
pip install fiona && \
62+
pip install flower && \
63+
pip install geopandas && \
64+
pip install h5py && \
65+
pip install imageio && \
66+
pip install ipykernel && \
67+
pip install ipython && \
68+
pip install ipython-genutils && \
69+
pip install ipywidgets && \
70+
pip install matplotlib && \
71+
pip install netcdf4 && \
72+
pip install networkx && \
73+
pip install notebook && \
74+
pip install numba && \
75+
pip install numexpr && \
76+
pip install numpy && \
77+
pip install pandas && \
78+
pip install pyhdf && \
79+
pip install pyproj && \
80+
pip install Pysal && \
81+
pip install PyYAML && \
82+
pip install rasterio && \
83+
pip install redis && \
84+
pip install requests && \
85+
pip install rioxarray && \
86+
pip install scikit-image && \
87+
pip install scikit-learn && \
88+
pip install scipy && \
89+
pip install seaborn && \
90+
pip install shapely && \
91+
pip install cartopy && \
92+
pip install xarray && \
93+
pip install urllib3 && \
94+
pip install zarr && \
95+
pip install earthengine-api && \
96+
pip install geemap && \
97+
pip install geedim && \
98+
#pip install vhr-cloudmask && \
99+
pip install 2to3 psycopg2-binary && \
100+
pip install plotnine && \
101+
pip install --upgrade pylr2
102+
103+
RUN PROJECT_PATH="/usr/local/ilab" && \
104+
REDIS_FILE="/etc/profile.d/redis_server.sh" && \
105+
echo $PROJECT_PATH && \
106+
mkdir -p $PROJECT_PATH && \
107+
# Add redis-server binary to /usr/local/bin
108+
export pyVer=`python --version | awk -F' ' '{print $2}' | awk -F'.' '{print $1"."$2}'` && \
109+
ln -sf /usr/local/lib/python${pyVer}/dist-packages/redis_server/bin/redis-server /usr/local/bin/redis-server && \
110+
echo "redis-server --daemonize yes --port \$REDIS_PORT" >> $REDIS_FILE && \
111+
echo "export SINGULARITYENV_REDIS_PORTS=\$REDIS_PORT" >> $REDIS_FILE && \
112+
chmod +x $REDIS_FILE
113+
114+
RUN mkdir -p "/usr/local/ilab" "/opt/DgStereo" && \
115+
# EVHR System Dependencies
116+
wget "https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2" \
117+
-O /opt/StereoPipeline.tar.bz2 && \
118+
cd /opt && \
119+
tar -jxf StereoPipeline.tar.bz2 && \
120+
rm /opt/StereoPipeline.tar.bz2 && \
121+
mv /opt/StereoPipeline* /opt/StereoPipeline && \
122+
123+
# EVHR DEM Dependencies
124+
git clone https://github.com/nasa-nccs-hpda/evhr-dgtools.git /opt/DgStereo/dgtools && \
125+
git clone https://github.com/nasa-nccs-hpda/evhr-imview.git /opt/DgStereo/imview && \
126+
git clone https://github.com/nasa-nccs-hpda/evhr-pygeotools.git /opt/DgStereo/pygeotools && \
127+
git clone https://github.com/nasa-nccs-hpda/evhr-dem.git /opt/DgStereo/evhr && \
128+
git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/core.git /usr/local/ilab/core && \
129+
git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/evhr.git /usr/local/ilab/evhr && \
130+
131+
# VHR Toolkit Dependencies
132+
git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/srlite.git /usr/local/ilab/srlite && \
133+
git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/pyCCDC.git /usr/local/ilab/pyCCDC && \
134+
git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/vhr-toolkit.git /usr/local/ilab/vhr-toolkit && \
135+
git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/vhr-cloudmask.pytorch.git /usr/local/ilab/vhr-cloudmask && \
136+
137+
# Fix permissions
138+
chmod 755 -R /opt/* && \
139+
2to3 -w /opt/DgStereo && \
140+
chmod +wx /lib/x86_64-linux-gnu/libc.so.6 && \
141+
chmod a+rwx -R /usr/local/ilab/*
142+
143+
# Create a CLI wrapper so "vhr-toolkit" behaves like a binary
144+
RUN printf '%s\n' \
145+
'#!/usr/bin/env bash' \
146+
'set -euo pipefail' \
147+
'exec python /usr/local/ilab/vhr-toolkit/vhr_toolkit/view/vhrToolkit.py "$@"' \
148+
> /usr/local/bin/vhr-toolkit && \
149+
chmod +x /usr/local/bin/vhr-toolkit
150+
151+
HEALTHCHECK NONE
152+
ENTRYPOINT []
153+
CMD ["/bin/bash"]

‎tests/catalog_1030010003A81D00.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/css/nga/WV02/1B/2010/030/WV02_1030010003A81D00_M1BS_505002120050_01/WV02_20100130130921_1030010003A81D00_10JAN30130921-M1BS-505002120050_01_P001.ntf
2+
/css/nga/WV02/1B/2010/030/WV02_1030010003A81D00_M1BS_505002120050_01/WV02_20100130130922_1030010003A81D00_10JAN30130922-M1BS-505002120050_01_P002.ntf

‎vhr_toolkit/view/vhrToolkit.py‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@ def main():
8181
for toaFile in toas:
8282

8383
logger.info('Processing ' + str(toaFile))
84-
processToaFile(toaFile, toaDir, toaDirNum, outDir, logger)
84+
processToaFile(
85+
toaFile, toaDir, toaDirNum, outDir, args.gee_key, logger)
8586

8687

8788
# -----------------------------------------------------------------------------
@@ -109,7 +110,7 @@ def getBandPairs(toaFile: Path, ccdcFile: Path) -> list:
109110
raise RuntimeError('Unable to map bands for ' + str(ccdcFile))
110111

111112
# How many bands in the ToA?
112-
numBands: int = gdal.Open(toaFile, gdal.GA_ReadOnly).RasterCount
113+
numBands: int = gdal.Open(str(toaFile), gdal.GA_ReadOnly).RasterCount
113114

114115
if numBands == 8:
115116

@@ -129,7 +130,8 @@ def processToaFile(toaFile: Path,
129130
toaDir: Path,
130131
toaDirNum: int,
131132
outDir: Path,
132-
logger: logging.RootLogger) -> None:
133+
geeKey: str = None,
134+
logger: logging.RootLogger = None) -> None:
133135

134136
# ---
135137
# EVHR -> CloudMaskPipeline
@@ -138,7 +140,6 @@ def processToaFile(toaFile: Path,
138140
cMaskDirNum = toaDirNum + 1
139141
cMaskDir = outDir / (str(cMaskDirNum) + '-masks')
140142
cMaskDir.mkdir(exist_ok=True)
141-
cMaskActualOutDir = cMaskDir / '5-toas'
142143

143144
cmpl = CloudMaskPipeline(output_dir=cMaskDir,
144145
inference_regex_list=[str(toaFile)])
@@ -152,13 +153,18 @@ def processToaFile(toaFile: Path,
152153
ccdcDirNum = cMaskDirNum + 1
153154
ccdcDir = outDir / (str(ccdcDirNum) + '-ccdc')
154155
ccdcDir.mkdir(exist_ok=True)
155-
ccdc = CCDCPipeline(input_dir=toaDir, output_dir=ccdcDir)
156+
ccdc = CCDCPipeline(
157+
input_dir=toaDir,
158+
output_dir=ccdcDir,
159+
gee_key=geeKey)
156160
ccdcFile: Path = ccdc.run(toaFile)[0]
157161

158162
# ---
159163
# Get band pairs for SRL.
160164
# ---
165+
print("BEFORE BAND PAIRS")
161166
bandPairs: list = getBandPairs(toaFile, ccdcFile)
167+
print("THIS ARE MY BAND PAIRS")
162168

163169
# ---
164170
# EVHR + Cloud Mask + CCDC -> SRL
@@ -171,15 +177,15 @@ def processToaFile(toaFile: Path,
171177
srl = SrliteWorkflow(output_dir=srlDir,
172178
toa_src=toaDir,
173179
target_dir=ccdcDir,
174-
cloudmask_dir=cMaskActualOutDir,
180+
cloudmask_dir=cMaskDir,
175181
regressor='rma',
176182
debug=1,
177183
pmask='True',
178184
cloudmask='True',
179185
csv='True',
180186
band8='True',
181187
clean='True',
182-
cloudmask_suffix='-toa.cloudmask.tif',
188+
cloudmask_suffix='-toa_cloudmask.tif',
183189
target_suffix='-toa_ccdc.tif',
184190
bandpairs=str(bandPairs),
185191
logger=logger)
@@ -243,6 +249,12 @@ def parseArgs() -> argparse.Namespace:
243249
# ---
244250
# CCDC Parameters
245251
# ---
252+
parser.add_argument('--gee-key',
253+
type=Path,
254+
default=None,
255+
required=False,
256+
dest='gee_key',
257+
help='Path to GEE location')
246258

247259
# ---
248260
# SR-lite Parameters

0 commit comments

Comments
 (0)