Firefly-1960: Hips Extraction

src/firefly/java/edu/caltech/ipac/firefly/data/ServerParams.java

@@ -34,7 +34,6 @@ public class ServerParams {
     public static final String PTARY = "ptAry";
     public static final String WPT_ARY = "wptAry";
     public static final String WL_ARY = "wlAry";
-    public static final String FLUX_UNIT= "fluxUnit";
     public static final String WL_UNIT= "wlUnit";
     public static final String PLANE = "plane";
     public static final String CRO_MULTI_ALL = "cropMultiAll";
@@ -100,6 +99,7 @@ public class ServerParams {
     public static final String ALL = "all";
     public static final String CDS = "cds";
     public static final String IS_HIPS_TILE = "isHipsTile";
+    public static final String HIPS_TILE_URL_ARY = "hipsTileUrlAry";
     public static final String HIPS_SOURCES = "hipsSources";
     public static final String HIPS_LIST_SOURCE= "hipsListSource";
     public static final String HIPS_LIST_SOURCE_NAME= "hipsListSourceName";

src/firefly/java/edu/caltech/ipac/firefly/data/table/MetaConst.java

@@ -88,6 +88,12 @@ public class MetaConst {
      */
     public static final String FITS_FILE_PATH= "FitsFilePath";
 
+    /**
+     * HiPS FITS tile url, the url of the HiPS FITS tile that is associated with this table.
+     * It is used in extraction from a HiPS. The HiPS root of the url will match the hipsUrlRoot of a HiPS WebPlot
+     */
+    public static final String HIPS_TILE_URL= "HiPSTileUrl";
+
     /**
      * Fits Image HDU, The HDU number of a FITS file that is associated with this table
      */

src/firefly/java/edu/caltech/ipac/firefly/server/query/ExtractFromImage.java

@@ -2,23 +2,25 @@
 
 import edu.caltech.ipac.firefly.data.ServerParams;
 import edu.caltech.ipac.firefly.data.TableServerRequest;
+import edu.caltech.ipac.firefly.data.table.MetaConst;
+import edu.caltech.ipac.firefly.server.ServerContext;
 import edu.caltech.ipac.firefly.server.SrvParam;
+import edu.caltech.ipac.firefly.server.servlets.HiPSRetrieve;
 import edu.caltech.ipac.table.DataGroup;
 import edu.caltech.ipac.table.io.FITSExtractToTable;
 import edu.caltech.ipac.visualize.plot.ImagePt;
 import edu.caltech.ipac.visualize.plot.WorldPt;
 import edu.caltech.ipac.visualize.plot.plotdata.FitsExtract;
 import nom.tam.fits.FitsException;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static edu.caltech.ipac.firefly.core.Util.Try;
-
-;
+import static edu.caltech.ipac.util.StringUtils.isEmpty;
 
 /**
  * Extract data from an image and make a table
@@ -32,8 +34,9 @@
                 @ParamDoc(name="wptAry", desc="for point selection, added to the created table"),
                 @ParamDoc(name="wlAry[i]", desc="an array of wavelength to add to the table, added to the created table"),
                 @ParamDoc(name="wlUnit[i]", desc="wavelength units. if defined"),
-                @ParamDoc(name="fluxUnit[i]", desc="flux units. if defined"),
                 @ParamDoc(name="filename[i]", desc="filename on the server"),
+                @ParamDoc(name="hipsTileUrl", desc="z-axis only, url of a HiPS FITS cube tile, used instead of filename"),
+                @ParamDoc(name=ServerParams.HIPS_TILE_URL_ARY, desc="points or line, url of the HiPS FITS tile of each point, used with ptAry, wptAry and plane instead of the indexed parameters"),
                 @ParamDoc(name="refHDUNum[i]", desc="hdu number to extract"),
                 @ParamDoc(name="extractionSizeX", desc="number of the x size of the extract"),
                 @ParamDoc(name="extractionSizeY", desc="number of the y size of the extract"),
@@ -46,6 +49,7 @@ public class ExtractFromImage extends EmbeddedDbProcessor {
     public static final String ID = "ExtractFromImage";
     public static final String EXTRACTION_TYPE = "extractionType";
     public static final String FILENAME = "filename";
+    public static final String HIPS_TILE_URL = "hipsTileUrl";
     public static final String REF_HDU_NUM = "refHDUNum";
     public static final String EXTRACTION_SIZE_X = "extractionSizeX";
     public static final String EXTRACTION_SIZE_Y = "extractionSizeY";
@@ -54,44 +58,77 @@ public class ExtractFromImage extends EmbeddedDbProcessor {
     @Override
     public DataGroup fetchDataGroup(TableServerRequest req) throws DataAccessException {
 
-        String extType = req.getParam(EXTRACTION_TYPE);
+        String extType = req.getParam(EXTRACTION_TYPE, "");
         int ptSizeX = req.getIntParam(EXTRACTION_SIZE_X, 1);
         int ptSizeY = req.getIntParam(EXTRACTION_SIZE_Y, 1);
         FitsExtract.CombineType ct= Enum.valueOf(FitsExtract.CombineType.class,req.getParam(ServerParams.COMBINE_OP,"AVG"));
         FitsExtract.SecondaryHduCombine secondCombine= Enum.valueOf(FitsExtract.SecondaryHduCombine.class,req.getParam("secondaryCombine","AUTO"));
         boolean allMatchingHDUs = req.getBooleanParam(ALL_MATCHING_HDUS, true);
+        var isLine= extType.equals("line");
+        var ptOrLine= extType.equals("points") || isLine;
+        var hipsTiles= req.getParam(ServerParams.HIPS_TILE_URL_ARY)!=null;
+        WorldPt[] wptAry = ptOrLine ? SrvParam.getWorldPtAryFromJson(req.getParam(ServerParams.WPT_ARY)) : null;
         try {
-            if (extType == null || extType.equals("z-axis")) {
-                String filename = req.getParam(FILENAME);
-                int refHduNum = req.getIntParam(REF_HDU_NUM, -1);
+            if (extType.equals("z-axis") || isEmpty(extType)) {
+                String hipsTileUrl = req.getParam(HIPS_TILE_URL);
+                File fitsFile = hipsTileUrl!=null
+                        ? HiPSRetrieve.getHiPSFitsTileFile(hipsTileUrl)
+                        : ServerContext.convertToFile(req.getParam(FILENAME));
+                int refHduNum = hipsTileUrl!=null ? FitsExtract.FIRST_IMAGE_HDU : req.getIntParam(REF_HDU_NUM, FitsExtract.NOT_SPECIFIED);
                 ImagePt pt = ImagePt.parse(req.getParam(ServerParams.PT));
                 WorldPt wpt = WorldPt.parse(req.getParam(ServerParams.WPT));
                 String wlUnit= req.getParam(ServerParams.WL_UNIT);
                 double[] wlAry= SrvParam.getDoubleAryFromJson(req.getParam(ServerParams.WL_ARY));
-                checkZAxisParams(pt, filename, refHduNum);
-                Map<Integer,String> fluxUnit= makeMapOfUnitsFromParam(req);
-                return FITSExtractToTable.getCubeZaxisAsTable(pt, wpt, filename, refHduNum, allMatchingHDUs,
-                        ptSizeX, ct, secondCombine, wlAry,wlUnit,fluxUnit);
+                checkZAxisParams(pt, fitsFile, refHduNum);
+                DataGroup dg= FITSExtractToTable.getCubeZaxisAsTable(pt, wpt, fitsFile, refHduNum, allMatchingHDUs,
+                        ptSizeX, ct, secondCombine, wlAry,wlUnit);
+                if (hipsTileUrl!=null) dg.getTableMeta().addKeyword(MetaConst.HIPS_TILE_URL, hipsTileUrl);
+                return dg;
+            }
+            else if (ptOrLine && hipsTiles) {
+                String[] tileUrlAry= SrvParam.getStringAryFromJson(req.getParam(ServerParams.HIPS_TILE_URL_ARY));
+                return FITSExtractToTable.getHiPSPointsAsTable(
+                        tileUrlAry,
+                        getHiPSTileFiles(tileUrlAry),
+                        SrvParam.getImagePtAryFromJson(req.getParam(ServerParams.PTARY)),
+                        wptAry,
+                        FitsExtract.FIRST_IMAGE_HDU,
+                        req.getIntParam(ServerParams.PLANE, 0),
+                        allMatchingHDUs, ptSizeX, ptSizeY, ct, secondCombine, isLine);
             }
-            else if (extType.equals("line") || extType.equals("points")) {
+            else if (ptOrLine) {
                 var extractParamsList = getDataExtractParams(req);
-                WorldPt[] wptAry = SrvParam.getWorldPtAryFromJson(req.getParam(ServerParams.WPT_ARY));
-                return FITSExtractToTable.getDataSelectAsTable(extractParamsList, wptAry, allMatchingHDUs,
-                        ptSizeX, ptSizeY, ct, secondCombine, extType.equals("line"));
+                return FITSExtractToTable.getDataSelectAsTable(extractParamsList, wptAry,
+                        allMatchingHDUs, ptSizeX, ptSizeY, ct, secondCombine, isLine);
             }
         } catch (IOException | FitsException e) {
             throw new IllegalArgumentException("Could not make a table from extracted data");
         }
         throw new IllegalArgumentException("extractionType must be z-axis, line, or points");
     }
 
+    /**
+     * retrieve the HiPS FITS tiles, each url is retrieved once
+     * @param tileUrlAry the url of the tile for each point
+     * @return the file for each url, null for a tile that could not be retrieved
+     */
+    private static File[] getHiPSTileFiles(String[] tileUrlAry) {
+        if (tileUrlAry==null) throw new IllegalArgumentException(ServerParams.HIPS_TILE_URL_ARY+" is required");
+        Map<String,File> fileByUrl= new HashMap<>();
+        File[] fileAry= new File[tileUrlAry.length];
+        for(int i=0; i<tileUrlAry.length; i++) {
+            fileAry[i]= fileByUrl.computeIfAbsent(tileUrlAry[i], HiPSRetrieve::getHiPSFitsTileFile);
+        }
+        return fileAry;
+    }
+
     private static List<FITSExtractToTable.DataExtractParams> getDataExtractParams(TableServerRequest req) {
         var eList = new ArrayList<FITSExtractToTable.DataExtractParams>();
         String[] titles= SrvParam.getStringAryFromJson( req.getParam("titleAry"));
         if (titles==null) titles= new String[] {""};
         for (int i=0; i<titles.length; i++) {
             String filename = req.getParam(FILENAME+i);
-            int refHduNum = req.getIntParam(REF_HDU_NUM+i, -1);
+            int refHduNum = req.getIntParam(REF_HDU_NUM+i, FitsExtract.NOT_SPECIFIED);
             int plane = req.getIntParam(ServerParams.PLANE+i, -1);
             double[] wlAry= SrvParam.getDoubleAryFromJson(req.getParam(ServerParams.WL_ARY+i));
             String wlUnit= req.getParam(ServerParams.WL_UNIT+i);
@@ -102,30 +139,21 @@ private static List<FITSExtractToTable.DataExtractParams> getDataExtractParams(T
         return eList;
     }
 
-    private static Map<Integer,String> makeMapOfUnitsFromParam(TableServerRequest req) {
-        String[] sAry= SrvParam.getStringAryFromJson(req.getParam(ServerParams.FLUX_UNIT));
-        Map<Integer,String> fluxUnit= new HashMap<>();
-        if (sAry==null) return fluxUnit;
-        for(String entry : sAry) {
-            String[] s= entry.split("=");
-            if (s.length==2) {
-                Try.it(() -> fluxUnit.put(Integer.parseInt(s[0]),s[1]));
-            }
-        }
-        return fluxUnit;
-    }
-
-    private static void checkZAxisParams(ImagePt pt, String filename, int refHduNum) {
+    private static void checkZAxisParams(ImagePt pt, File fitsFile, int refHduNum) {
         if (pt==null) throw new IllegalArgumentException("Point is require for z-axis extraction");
-        if (filename==null) throw new IllegalArgumentException("filename is require for z-axis extraction");
-        if (refHduNum<0) throw new IllegalArgumentException("refHduNum is require for z-axis extraction");
+        if (fitsFile==null) throw new IllegalArgumentException("filename or a retrievable hipsTileUrl is require for z-axis extraction");
+        if (refHduNum==FitsExtract.NOT_SPECIFIED || (refHduNum<0 && refHduNum!=FitsExtract.FIRST_IMAGE_HDU)) {
+            throw new IllegalArgumentException("refHduNum is require for z-axis extraction");
+        }
     }
 
     private static void checkPointParams(ImagePt[] ptAry, int plane, String filename, int refHduNum) {
         if (ptAry==null || ptAry.length==0) throw new IllegalArgumentException("ptAry is require for points extraction");
         if (filename==null) throw new IllegalArgumentException("filename is require for points extraction");
-        if (refHduNum<0) throw new IllegalArgumentException("refHduNum is require for points extraction");
-        if (plane<0) throw new IllegalArgumentException("refHduNum is require for points extraction");
+        if (refHduNum==FitsExtract.NOT_SPECIFIED || refHduNum<0) {
+            throw new IllegalArgumentException("refHduNum is require for points extraction");
+        }
+        if (plane<0) throw new IllegalArgumentException("plane is require for points extraction");
     }
 
 }

src/firefly/java/edu/caltech/ipac/firefly/server/rpc/VisServerCommands.java

@@ -8,7 +8,9 @@
 import edu.caltech.ipac.firefly.server.ServCommand;
 import edu.caltech.ipac.firefly.server.ServerCommandAccess;
 import edu.caltech.ipac.firefly.server.SrvParam;
+import edu.caltech.ipac.firefly.server.servlets.HiPSRetrieve;
 import edu.caltech.ipac.firefly.server.util.Logger;
+import edu.caltech.ipac.firefly.server.visualize.CtxControl;
 import edu.caltech.ipac.firefly.server.visualize.DirectStretchUtils.CompressType;
 import edu.caltech.ipac.firefly.server.visualize.FluxValueUtil;
 import edu.caltech.ipac.firefly.server.visualize.VisJsonSerializer;
@@ -26,6 +28,7 @@
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -201,30 +204,36 @@ private static CompressType getCompressType(SrvParam sp) {
     public static class ExtractionCmd extends ServerCommandAccess.HttpCommand {
         public void processRequest(HttpServletRequest req, HttpServletResponse res, SrvParam sp) throws Exception {
 
-            PlotState state= sp.getState();
+            boolean isHiPSTile= sp.getOptionalBoolean(ServerParams.IS_HIPS_TILE, false);
             String exType= sp.getRequired(ServerParams.EXTRACTION_TYPE);
             boolean useFloat = sp.getOptionalInt(ServerParams.EXTRACTION_FLOAT_SIZE,64)==32;
             int ptSize= sp.getOptionalInt(ServerParams.POINT_SIZE,1);
             int ptSizeX= sp.getOptionalInt(ServerParams.POINT_SIZE_X,1);
             int ptSizeY= sp.getOptionalInt(ServerParams.POINT_SIZE_Y,1);
-            int hduNum= sp.getRequiredInt(ServerParams.HDU_NUM);
+            int hduNum= isHiPSTile ? FitsExtract.FIRST_IMAGE_HDU : sp.getRequiredInt(ServerParams.HDU_NUM);
             FitsExtract.CombineType ct= Enum.valueOf(FitsExtract.CombineType.class,sp.getOptional(ServerParams.COMBINE_OP,"AVG"));
 
             List<Number> extractList= switch (exType) {
                 case "z-axis" ->
-                    VisServerOps.getZAxisAry(state,
+                    VisServerOps.getZAxisAry(getFitsFile(sp,isHiPSTile),
                             sp.getRequiredImagePt(ServerParams.PT),
                             hduNum, ptSize, ct);
 
                 case "line" ->
-                    VisServerOps.getLineDataAry(state,
+                    VisServerOps.getLineDataAry(getFitsFile(sp,isHiPSTile),
                             sp.getRequiredImagePt(ServerParams.PT),
                             sp.getRequiredImagePt(ServerParams.PT2),
                             sp.getRequiredInt(ServerParams.PLANE),
                             hduNum, ptSize, ct);
 
-                case "points" ->
-                        VisServerOps.getPointDataAry(state,
+                case "points" -> isHiPSTile
+                        ? VisServerOps.getHiPSPointDataAry(
+                                SrvParam.getStringAryFromJson(sp.getRequired(ServerParams.HIPS_TILE_URL_ARY)),
+                                sp.getRequiredImagePtAry(ServerParams.PTARY),
+                                hduNum,
+                                sp.getRequiredInt(ServerParams.PLANE),
+                                ptSizeX, ptSizeY, ct)
+                        : VisServerOps.getPointDataAry(getFitsFile(sp,isHiPSTile),
                                 sp.getRequiredImagePtAry(ServerParams.PTARY),
                                 sp.getRequiredInt(ServerParams.PLANE),
                                 hduNum, ptSizeX, ptSizeY, ct);
@@ -253,6 +262,17 @@ public void processRequest(HttpServletRequest req, HttpServletResponse res, SrvP
         }
     }
 
+    /**
+     * @param sp the parameters
+     * @param isHiPSTile if true use the URL parameter of a HiPS FITS tile, otherwise the plot state
+     * @return the FITS file to extract from or null if it is not available
+     */
+    private static File getFitsFile(SrvParam sp, boolean isHiPSTile) {
+        return isHiPSTile
+                ? HiPSRetrieve.getHiPSFitsTileFile(sp.getRequired(ServerParams.URL))
+                : CtxControl.getConfirmedWorkingFitsFile(sp.getState());
+    }
+
     public static class Crop extends ServCommand {
 
         public String doCommand(SrvParam sp) throws IllegalArgumentException {

src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HiPSRetrieve.java

@@ -61,7 +61,17 @@ private record DeadEntry(long time, int status) {};
 
     public static FileInfo retrieveHiPSData(String urlStr, boolean alwaysUseCached) {
         return retrieveHiPSData(urlStr,null, alwaysUseCached);
+    }
 
+    /**
+     * Get a HiPS FITS tile, retrieving it if it is not in the cache. This call will block on the retrieve.
+     * @param fitsTileUrl the url of the HiPS FITS tile
+     * @return the cached FITS file or null if it could not be retrieved
+     */
+    public static File getHiPSFitsTileFile(String fitsTileUrl) {
+        if (fitsTileUrl==null || isDeadFitsUrl(fitsTileUrl)) return null;
+        FileInfo fi= retrieveHiPSData(fitsTileUrl, true);
+        return fi.isSuccess() ? fi.getFile() : null;
     }
 
     public static FileInfo retrieveHiPSData(String urlStr, String fileName, boolean alwaysUseCached) {

src/firefly/java/edu/caltech/ipac/firefly/server/visualize/CtxControl.java

@@ -32,6 +32,31 @@ public static void confirmFiles(PlotState state) throws FailedRequestException {
         if (isFitsFilesMissing(state)) revalidatePlot(state,true); //most of the time the files are there so nothing happens
     }
 
+    /**
+     * Get the NO_BAND working FITS file of a standard (non-HiPS) plot
+     * @param state the plot state
+     * @return the FITS file or null if it could not be confirmed
+     */
+    public static File getConfirmedWorkingFitsFile(PlotState state) {
+        return getConfirmedWorkingFitsFile(state,Band.NO_BAND);
+    }
+
+    /**
+     * Get the working FITS file of a standard (non-HiPS) plot
+     * @param state the plot state
+     * @param band the band of the file
+     * @return the FITS file or null if it could not be confirmed
+     */
+    public static File getConfirmedWorkingFitsFile(PlotState state, Band band) {
+        try {
+            confirmFiles(state);
+            return ServerContext.convertToFile(state.getWorkingFitsFileStr(band));
+        } catch (FailedRequestException e) {
+            Logger.getLogger().warn(e, "could not confirm working FITS file: " + e.getMessage());
+            return null;
+        }
+    }
+
     private static boolean isFitsFilesMissing(PlotState state) {
         return state.isThreeColor() ?
                 Arrays.stream(state.getBands())

src/firefly/java/edu/caltech/ipac/firefly/server/visualize/FitsCacher.java

@@ -249,7 +249,9 @@ private static DirectFileAccess makeDirectFileAccessData(FitsRead fr) {
         var blankVal= FitsReadUtil.getBlankValue(h);
         var blankValStr= Double.isNaN(blankVal) ? "" : blankVal+"";
         return new DirectFileAccess(
-                fr.getHduNumber(), fr.isCube(), FitsReadUtil.getNaxis3(h), -1, dataOffset, h.getIntValue(SPOT_BP),
+                FitsReadUtil.getExtNameOrType(h),
+                fr.getHduNumber(), fr.isCube(), FitsReadUtil.getNaxis3(h), FitsReadUtil.getNaxis(h),
+                dataOffset, h.getIntValue(SPOT_BP),
                 FitsReadUtil.getNaxis1(h), FitsReadUtil.getNaxis2(h), FitsReadUtil.getNaxis3(h),
                 FitsReadUtil.getCdelt2(h), FitsReadUtil.getBUnit(h,"---"),
                 FitsReadUtil.getBscale(h), FitsReadUtil.getBzero(h),