IRSA-8042: Fixes a votable with a fix extension being recognized wrong on linux

 - FormatUtil.getMimeType() test for votable
 - removed Files.probeContentType(Path.of(inFile)) since it is platform specific
 - better use of tile extentions
 - added some test
 - SpectrumMetaInspector will recognize HOPS / spitzer

Author: robyww

src/firefly/java/edu/caltech/ipac/firefly/server/util/QueryUtil.java

@@ -240,7 +240,8 @@ public static File resolveFileFromSource(String source, boolean checkForUpdates,
     public static File tmpFileForUrl(String urlStr, String prefix, File dir) {
         URL url = makeUrl(urlStr);
         String pExt = url == null ? null : FileUtil.getExtension(url.getPath().replaceFirst("^.*/", ""), true);
-        String ext = isEmpty(pExt) ? ".ul" : "." + pExt;
+        String pQuery = url == null ? null : url.getQuery();
+        String ext = isEmpty(pExt) || !isEmpty(pQuery) ? ".ul" : "." + pExt;
         return Util.Try.it(() -> File.createTempFile(prefix, ext, dir)).get();
     }
 

src/firefly/java/edu/caltech/ipac/table/io/SpectrumMetaInspector.java

@@ -179,6 +179,8 @@ else if (!spectrumHint) { // determine if we are to continue or just return
         //    - guessing columns or checking utype
         //    - checking utype only
 
+        fixAnyBadDataGroup(dg);
+
         List<DataType> dtAry= Arrays.asList(dg.getDataDefinitions());
         List<GroupInfo> groupInfosList= new ArrayList<>();
         var nonStandardSpecType= dg.getAttribute("SPEC_TYPE","").toLowerCase();
@@ -486,5 +488,24 @@ public static void insertSpectralData(DataGroup dg, InsertEntry wl, InsertEntry
         meta.addKeyword(TableMeta.UTYPE,SPEC_SPECTRUM);
     }
 
+    static void fixAnyBadDataGroup(DataGroup dg) {
+        var keyWords= dg.getTableMeta().getKeywords();
+        var isHopsSpec= keyWords.stream().filter(s -> (s.getKey()==null && s.getValue().startsWith("HOPS"))).count()==1 &&
+                        keyWords.stream().filter(s -> (s.getKey()==null && s.getValue().startsWith("Spitzer"))).count()==1;
+        if (isHopsSpec) { // try to fix it
+            var dtAry= dg.getDataDefinitions();
+            for (DataType dt : dtAry){
+                if (dt.getDataType()==String.class) {
+                    dt.setDataType(Double.class);
+                    dt.setUnits(dt.getTypeDesc());
+                    dt.setTypeDesc("double");
+                }
+            }
+        }
+
+        // add others here
+
+    }
+
     public record InsertEntry(String column, String unit) {}
 }

src/firefly/java/edu/caltech/ipac/util/FormatUtil.java

@@ -9,19 +9,19 @@
 
 import edu.caltech.ipac.firefly.server.db.DuckDbReadable;
 import edu.caltech.ipac.firefly.server.util.Logger;
-import nom.tam.fits.Fits;
-import nom.tam.fits.FitsException;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 
 import javax.annotation.Nonnull;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.EnumSet;
-import java.util.Set;
 import java.io.BufferedReader;
 import java.io.CharArrayReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
+import java.util.EnumSet;
+import java.util.Set;
+import java.util.zip.GZIPInputStream;
 
 import static edu.caltech.ipac.firefly.core.Util.Opt.ifNotNull;
 import static edu.caltech.ipac.table.IpacTableUtil.isIpacTable;
@@ -36,6 +36,7 @@
 public class FormatUtil {
     private static final int SAMPLE_SIZE = (int) (8 * FileUtil.K);
     private static final Logger.LoggerImpl LOGGER = Logger.getLogger();
+    private static final int[] FITS_MAGIC= {'S','I','M','P','L','E',' ',' ','='};
 
     public record MimeDesc(String mime, String desc) {}
 
@@ -114,7 +115,7 @@ public static Format fromMime(String mime) {
 
     /**
      * Detects the MIME type of file using magic-byte inspection of the file header,
-     * falling back to OS-level and extension-based guessing for unrecognized types.
+     * falling back to extension-based guessing for unrecognized types.
      * This is platform-independent for all binary formats handled here; text-based
      * formats (FITS text headers, VOTable, IPAC table, etc.) are handled downstream
      * by {@link FormatUtil#detect}.
@@ -159,17 +160,21 @@ public static MimeDesc getMimeType(String inFile) {
             if (magic(hdr, 257, 'u','s','t','a','r'))
                 return new MimeDesc(TAR.mime(), "POSIX tar archive");
             // Binary FITS: header block starts with "SIMPLE  ="
-            if (magic(hdr, 0, 'S','I','M','P','L','E',' ',' ','='))
+            if (magic(hdr, 0, FITS_MAGIC))
                 return new MimeDesc(FITS.mime(), "FITS image data");
 
+            var hdrStr= new String(hdr);
+            if ((hdrStr.startsWith("<?xml") && hdrStr.contains("<VOTABLE")) || hdrStr.startsWith("<VOTABLE")) { // this is not a magic number but close and might fit in the 264 bytes
+                return new MimeDesc(VO_TABLE.mime(), "VO Table");
+            }
+
         } catch (Exception ex) {
             Logger.getLogger().error(ex, "Failed to read header for mime detection: " + inFile);
         }
 
-        // Fall back to OS content probing and extension-based guess for text formats
+        // Fall back to extension-based guess for text formats
         try {
-            String mime = Files.probeContentType(Path.of(inFile));
-            if (mime == null) mime = java.net.URLConnection.guessContentTypeFromName(inFile);
+            var mime = java.net.URLConnection.guessContentTypeFromName(inFile);
             if (mime != null) return new MimeDesc(mime, mime);
         } catch (Exception ex) {
             Logger.getLogger().error(ex, "Failed to detect mime type for: " + inFile);
@@ -208,14 +213,8 @@ public static Format detect(File inFile) throws IOException {
         }
 
 
-        if ((format == BZIP2 || format == GZIP) &&
-                inFile.getName().toLowerCase().contains("fit")) {
-            // special case and a very heavy operation: we can handle fits bz2 or gzip files but don't try unless we are pretty sure
-            // in archives: legacy files are often named a.fits.gz or a.fits.bz2
-            try (var ignored = new Fits(inFile)) {
-                return FITS;
-            }
-            catch (FitsException ignore) {}
+        if (isCompressedFits(inFile, format)) { // archives often serve a.fits.gz or a.fits.bz2; check the decompressed header
+            return FITS;
         }
 
         format = ifNotNull(guessBySamplingContent(inFile)).getOrElse(format);
@@ -307,7 +306,7 @@ private static Format guessBySamplingContent(File inf) throws IOException {
                     //NAME-RESOLVER: xxx
                     return FIXEDTARGETS;
                 } else if (line.startsWith("<VOTABLE") ||
-                        (line.contains("<?xml") && line.contains("<VOTABLE "))) {
+                        (line.contains("<?xml") && line.contains("<VOTABLE"))) {
                     return VO_TABLE;
                 } else if (isUwsEl(line)) {
                     return UWS;
@@ -325,6 +324,20 @@ private static boolean isUwsEl(String line) {
         return isUws && line.matches("<(.+:)?job .*");
     }
 
+    /**
+     * Check the first bytes of a gzip or bzip2 file, after decompressing, for the FITS "SIMPLE  =" keyword
+     */
+    private static boolean isCompressedFits(File inFile, Format format) {
+        if (format != BZIP2 && format != GZIP) return false;
+        try (var fis = new FileInputStream(inFile);
+             InputStream in = format == GZIP ? new GZIPInputStream(fis) : new BZip2CompressorInputStream(fis)) {
+            byte[] hdr = in.readNBytes(9);
+            return magic(hdr, 0, FITS_MAGIC);
+        } catch (IOException e) {
+            return false;
+        }
+    }
+
     /** Returns true when {@code data[offset..]} starts with the given byte values. */
     private static boolean magic(byte[] data, int offset, int... expected) {
         if (data.length < offset + expected.length) return false;

src/firefly/java/edu/caltech/ipac/util/download/RetrieveUtil.java

@@ -16,6 +16,7 @@
 import java.util.Objects;
 import java.util.stream.Stream;
 
+import static edu.caltech.ipac.util.StringUtils.isEmpty;
 import static edu.caltech.ipac.util.download.UriRef.ResourceType.GcsCloud;
 import static edu.caltech.ipac.util.download.UriRef.ResourceType.OnPrimUrl;
 import static edu.caltech.ipac.util.download.UriRef.ResourceType.S3Cloud;
@@ -94,11 +95,13 @@ public static String makeUniqueFileName(UriRef uri, String userLoginName, String
         String path;
         String outLoc;
         String prefix;
+        String query= null;
 
         switch (uri.ref()) {
             case URL url -> {
                 loc= url.getHost();
                 path= url.getFile();
+                query= url.getQuery();
                 outLoc = FileUtil.makeShortHostName(loc);
                 prefix= "URL-";
             }
@@ -134,7 +137,7 @@ public static String makeUniqueFileName(UriRef uri, String userLoginName, String
         retval = prefix + outLoc + "-" + originalHashCode + baseKey;
         //note: "=","," signs causes problem in download servlet.
         retval = retval.replaceAll("[ :\\[\\]/\\\\|*?<>=,]", "-");
-        String ext= FileUtil.getExtension(fileStr,true);
+        String ext= isEmpty(query) ? FileUtil.getExtension(fileStr,true) : "";
         var fileExt= EXT_LIST.contains(ext) ? ext : EXT_LIST.getFirst();
         return retval + "." + fileExt;
     }

src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java

@@ -134,17 +134,12 @@ public static Map<String,List<String>> getQueryParams(URL url) {
         var paramList= url.getQuery().split("&");
         var queryParamMap = new HashMap<String, List<String>>();
         for(var p : paramList) {
-            var parts = p.split("=");
-            if (parts.length == 2) {
-                var key= parts[0].trim();
-                var value= parts[1].trim();
-                var valueList= queryParamMap.get(key);
-                if (valueList == null) {
-                   valueList = new ArrayList<>();
-                   queryParamMap.put(key, valueList);
-                }
-                valueList.add(value);
-            }
+            var parts = p.split("=", 2);   // split on the first "=" only, so values can contain "=" (e.g. base64 padding)
+            var key= parts[0].trim();
+            if (key.isEmpty()) continue;
+            var value= parts.length == 2 ? parts[1].trim() : "true";  // a parameter with no value, e.g. ?flag
+            var valueList = queryParamMap.computeIfAbsent(key, k -> new ArrayList<>());
+            valueList.add(value);
         }
         return queryParamMap;
     }
@@ -161,7 +156,7 @@ public static String firstParamValUsingKeyList(Map<String, List<String>> params,
         return foundKey!=null ? getFirstVal(params, foundKey) : null;
     }
 
-    private static int codeFromException(Exception e) {
+    static int codeFromException(Exception e) {   // package-private for testing
         return switch (e) {
             case SSLException ignored -> 495;
             case SocketTimeoutException ignored -> HttpURLConnection.HTTP_CLIENT_TIMEOUT;

src/firefly/test/edu/caltech/ipac/firefly/server/util/QueryUtilTest.java

@@ -0,0 +1,36 @@
+/*
+ * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
+ */
+
+package edu.caltech.ipac.firefly.server.util;
+
+import edu.caltech.ipac.firefly.ConfigTest;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import static org.junit.Assert.assertTrue;
+
+public class QueryUtilTest extends ConfigTest {
+
+    @Test
+    public void tmpFileForUrl() throws IOException {
+        File dir = Files.createTempDirectory("queryUtilTest").toFile();
+        dir.deleteOnExit();
+
+        assertTmpExt(dir, "http://h/a/x.fits.gz", ".fits.gz");
+        assertTmpExt(dir, "http://h/a/x.csv", ".csv");
+        // a query string makes the path extension unreliable
+        assertTmpExt(dir, "http://h/a/x.fits.gz?token=1", ".ul");
+        assertTmpExt(dir, "http://h/cgi/get.py?file=a.csv", ".ul");
+        assertTmpExt(dir, "http://h/a/noext", ".ul");
+    }
+
+    private static void assertTmpExt(File dir, String url, String ext) {
+        File f = QueryUtil.tmpFileForUrl(url, "req_", dir);
+        f.deleteOnExit();
+        assertTrue(url + " -> " + f.getName(), f.getName().endsWith(ext));
+    }
+}

src/firefly/test/edu/caltech/ipac/util/FormatUtilTest.java

@@ -13,10 +13,25 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-
-import static edu.caltech.ipac.util.FormatUtil.Format.*;
+import java.nio.file.StandardCopyOption;
+import java.util.zip.GZIPOutputStream;
+
+import static edu.caltech.ipac.util.FormatUtil.Format.CSV;
+import static edu.caltech.ipac.util.FormatUtil.Format.FITS;
+import static edu.caltech.ipac.util.FormatUtil.Format.GZIP;
+import static edu.caltech.ipac.util.FormatUtil.Format.HTML;
+import static edu.caltech.ipac.util.FormatUtil.Format.IPACTABLE;
+import static edu.caltech.ipac.util.FormatUtil.Format.JSON;
+import static edu.caltech.ipac.util.FormatUtil.Format.PARQUET;
+import static edu.caltech.ipac.util.FormatUtil.Format.PNG;
+import static edu.caltech.ipac.util.FormatUtil.Format.REGION;
+import static edu.caltech.ipac.util.FormatUtil.Format.TEXT;
+import static edu.caltech.ipac.util.FormatUtil.Format.TSV;
+import static edu.caltech.ipac.util.FormatUtil.Format.UNKNOWN;
+import static edu.caltech.ipac.util.FormatUtil.Format.VO_TABLE;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -204,18 +219,57 @@ public void badMixedDelimiters() throws IOException {
     }
 
     @Test
-    public void mimeType() {
+    public void mimeType() throws IOException {
         File tfile = FileLoader.resolveFile("FileUpload-samples/VOTable/binary/binary_gaia.xml");
-        assertEquals(tfile.getName(), "application/xml", FormatUtil.getMimeType(tfile).mime());
+        assertEquals(tfile.getName(), VO_TABLE.mime(), FormatUtil.getMimeType(tfile).mime());
 
         tfile = FileLoader.resolveFile("FileUpload-samples/VOTable/fits/starlinkVOfits.xml");
-        assertEquals(tfile.getName(), "application/xml", FormatUtil.getMimeType(tfile).mime());
+        assertEquals(tfile.getName(), VO_TABLE.mime(), FormatUtil.getMimeType(tfile).mime());
 
         tfile = FileLoader.resolveFile("edu/caltech/ipac/firefly/server/query/ptf-lc.tbl");
         assertEquals(tfile.getName(), UNKNOWN.mime(), FormatUtil.getMimeType(tfile).mime());
 
         tfile = FileLoader.resolveFile("stripe82-testfits/calexp-i-0-366,0.fits.gz");
         String mtype = FormatUtil.getMimeType(tfile).mime();
         assertEquals(tfile.getName(), GZIP.mime(), mtype);
+        assertEquals(tfile.getName(), FITS, FormatUtil.detect(tfile));
+
+        // gzipped FITS must be detected by content, not by a "fit" in the file name (e.g. temp files for urls with a query string)
+        Path tmp = Files.createTempFile("req_", ".ul");
+        tmp.toFile().deleteOnExit();
+        Files.copy(tfile.toPath(), tmp, StandardCopyOption.REPLACE_EXISTING);
+        assertEquals("gzipped fits named .ul", FITS, FormatUtil.detect(tmp.toFile()));
+    }
+
+    @Test
+    public void extensionDoesNotDecideFormat() throws IOException {
+        // a .fits name must not decide the format; on linux /etc/mime.types maps .fits to image/fits
+        Path tmp = Files.createTempFile("test", ".fits");
+        tmp.toFile().deleteOnExit();
+        Files.writeString(tmp, "a,b,c\n1,2,3\n4,5,6\n");
+        assertEquals("csv named .fits", CSV, FormatUtil.detect(tmp.toFile()));
+    }
+
+    @Test
+    public void voTableWithWrongExtension() throws IOException {
+        File tfile = FileLoader.resolveFile("FileUpload-samples/VOTable/binary/binary_gaia.xml");
+        for (String ext : new String[] {".csv", ".json"}) {
+            Path tmp = Files.createTempFile("test", ext);
+            tmp.toFile().deleteOnExit();
+            Files.copy(tfile.toPath(), tmp, StandardCopyOption.REPLACE_EXISTING);
+            assertEquals("votable named " + ext, VO_TABLE.mime(), FormatUtil.getMimeType(tmp.toFile()).mime());
+            assertEquals("votable named " + ext, VO_TABLE, FormatUtil.detect(tmp.toFile()));
+        }
+    }
+
+    @Test
+    public void gzipNotFitsNamedFits() throws IOException {
+        // the name says fits but the decompressed content does not
+        Path tmp = Files.createTempFile("test", ".fits.gz");
+        tmp.toFile().deleteOnExit();
+        try (OutputStream out = new GZIPOutputStream(Files.newOutputStream(tmp))) {
+            out.write("a,b,c\n1,2,3\n".getBytes());
+        }
+        assertEquals("gzipped csv named .fits.gz", GZIP, FormatUtil.detect(tmp.toFile()));
     }
 }