Firefly-2137: use a non-openssl version of redis for firefly standalone

Author: robyww

bin/install.sh

@@ -270,7 +270,8 @@ fi
 # --------------------------
 
 requiredFiles=("standalone_cleanup.sh" "$startScript" "startFireflyServer.sh" \
-               "stopFireflyServer.sh" "statusFireflyServer.sh" "javaInstaller.sh" "updater.sh" "common.sh")
+               "stopFireflyServer.sh" "statusFireflyServer.sh" "javaInstaller.sh" "updater.sh" "common.sh" \
+               "redis/osx/redis-server-7.2-darwin-arm64")
 missingFiles=""
 for f in "${requiredFiles[@]}"; do
   if [ ! -f "$applicationDir/$f" ]; then
@@ -294,6 +295,13 @@ chmod 775 "$applicationDir/standalone_cleanup.sh" \
           "$applicationDir/install.sh"
 /bin/mv "$applicationDir/updater.sh" "$applicationRoot"
 
+# redis for macOS (see assets/redis/osx/redis-info.md). unzip marks the files of a browser-downloaded zip as
+# quarantined, and macOS kills a quarantined binary that is not notarized
+if [[ "$(uname)" == "Darwin" ]]; then
+  chmod +x "$applicationDir/redis/osx/redis-server-7.2-darwin-arm64"
+  xattr -dr com.apple.quarantine "$applicationDir/redis" 2> /dev/null
+fi
+
 cp "$applicationDir/$startScript" "$binDir"
 chmod +x "$binDir/$startScript"
 

src/firefly/java/edu/caltech/ipac/firefly/core/RedisService.java

@@ -5,6 +5,7 @@
 package edu.caltech.ipac.firefly.core;
 
 import edu.caltech.ipac.firefly.data.ServerEvent;
+import edu.caltech.ipac.firefly.server.ServerContext;
 import edu.caltech.ipac.firefly.server.events.FluxAction;
 import edu.caltech.ipac.firefly.server.events.ServerEventManager;
 import edu.caltech.ipac.firefly.server.servlets.ServerStatus;
@@ -25,6 +26,7 @@
 import io.lettuce.core.event.connection.ConnectionDeactivatedEvent;
 import io.lettuce.core.resource.DefaultClientResources;
 import redis.embedded.RedisServer;
+import redis.embedded.core.ExecutableProvider;
 
 import java.io.File;
 import java.security.MessageDigest;
@@ -162,15 +164,36 @@ private static void tryConnect() throws Exception{
         }
     }
 
+    /**
+     * Redis without TLS packaged with standalone Firefly for Apple Silicon (see assets/redis/osx/redis-info.md).
+     * The Redis bundled in embedded-redis for Apple Silicon needs Homebrew's OpenSSL.
+     * Returns null when not applicable
+     */
+    private static ExecutableProvider packagedRedis7Provider() {
+        boolean appleSilicon =
+                System.getProperty("os.name", "").startsWith("Mac")
+                && "aarch64".equals(System.getProperty("os.arch"));
+        String path = System.getProperty(ExecutableProvider.PROPERTY_EXECUTABLE_LOCATION);
+        if (!appleSilicon || path == null) return null;
+        if (!new File(path).canExecute()) {
+            LOG.warn("Redis build without TLS not found or not executable: " + path + ", using the bundled Redis");
+            return null;
+        }
+        return ExecutableProvider.newSystemPropertyProvider();
+    }
+
     private static void startLocal() throws Exception {
         new File(DB_DIR).mkdirs();
-        RedisServer localRedis = RedisServer.newRedisServer()
+        var execProvider = ServerContext.isStandalone() ? packagedRedis7Provider() : null;
+        var dbFile = execProvider==null ? "redis.rdb" : "redis-7.2.rdb";   // Redis 6.2 cannot read an rdb written by 7.2
+        var builder = RedisServer.newRedisServer()
                 .port(REDIS_PORT)
                 .setting("maxmemory %s".formatted(LOCAL_MAX_MEM))
                 .setting("dir %s".formatted(DB_DIR))
-                .setting("dbfilename redis.rdb")
-                .setting("save 600 1")
-                .build();
+                .setting("dbfilename %s".formatted(dbFile))
+                .setting("save 600 1");
+        if (execProvider != null) builder = builder.executableProvider(execProvider);
+        RedisServer localRedis = builder.build();
         localRedis.start();
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             try {

src/firefly/java/edu/caltech/ipac/firefly/server/ServerContext.java

@@ -88,6 +88,8 @@ public class ServerContext {
 
 
     private final static String cloudEnvironmentDefault = System.getProperty("cloud.environment", "detect"); // can be "GCP", "AWS", "ON_PREM" or "detect"
+
+
     private final static UriRef.CloudEnvironment cloudEnvironment= findEnvironment();
     private static RequestOwnerThreadLocal owner = new RequestOwnerThreadLocal();
     private static String webappConfigPath;
@@ -843,6 +845,10 @@ public static boolean isInUploadDir(File f) {
     public static UriRef.CloudEnvironment getCloudEnvironment() {return cloudEnvironment;}
     public static boolean isRunningInCloud() {return cloudEnvironment!= UriRef.CloudEnvironment.ON_PREM;}
 
+    public static boolean isStandalone() {
+        return AppProperties.getBooleanProperty("OP_standaloneEnabled", false);
+    }
+
 
     private static class AssertLogger implements Assert.Logger {
         private final Logger.LoggerImpl _log= Logger.getLogger();

src/standalone/assets/common.sh

@@ -2,7 +2,7 @@
 # common.sh: shared variables for the Firefly standalone runtime scripts.
 # Must be sourced with $INSTALL_DIR already set (every caller needs it anyway to
 # locate this file). Defines: applicationDir, fireflyDir, binDir, configJsonFile,
-# JQ, pidFile, portFile, redisDbDir, ffPort, redisPort.
+# JQ, pidFile, portFile, redisDbDir, redisMacExecFile, ffPort, redisPort.
 # --------------------------
 
 # shellcheck disable=SC2034
@@ -13,6 +13,7 @@ configJsonFile="$fireflyDir/config.json"
 pidFile="$fireflyDir/pid.txt"
 portFile="$fireflyDir/port.txt"
 redisDbDir="${fireflyDir}/server/temp/redis"
+redisMacExecFile="${applicationDir}/redis/osx/redis-server-7.2-darwin-arm64"
 JQ=$(which jq || echo "$binDir/jq")
 
 if [[ -f "$portFile" && -s "$portFile" && -r "$portFile" ]]; then

src/standalone/assets/redis/osx/redis-info.md

@@ -0,0 +1,55 @@
+# Redis for macOS (Apple Silicon), no-TLS build
+
+`redis-server-7.2-darwin-arm64` is Redis 7.2.3 for Apple Silicon, built without TLS, so it has no
+OpenSSL dependency (`otool -L` lists only `libSystem`).
+
+## Why it is here
+
+The Redis binary bundled in the `embedded-redis` library for Apple Silicon links Homebrew's OpenSSL
+and will not start on a Mac without it:
+
+    dyld: Library not loaded: /opt/homebrew/opt/openssl@3/lib/libssl.3.dylib
+
+Standalone Firefly (`OP_standaloneEnabled=true`) on Apple Silicon uses this binary instead.
+Linux, Intel Macs and Docker are unchanged. Firefly does not use Redis TLS.
+
+## Where it came from
+
+- A third-party build ("HankCP") distributed in the embedded-redis project, which is also the
+  library's `ExecutableProvider.REDIS_7_2_MACOSX_14_SONOMA_HANKCP`:
+  https://github.com/codemonstur/embedded-redis/tree/master/src/main/binaries
+- `./redis-server-7.2-darwin-arm64 --version` reports `v=7.2.3 sha=7f4bae81:0`. That commit is the
+  official redis/redis tag 7.2.3, and `:0` means a clean checkout.
+- Built on macOS 14 (minimum macOS 14.0), ad-hoc signed, 2,338,096 bytes.
+- SHA-256: `a4a0078c060895140d8277877cec64ef414702af1fbc53dc3b3d7f1d6700dced`
+
+## How to get it
+
+The URL below is pinned to a commit, so the content cannot change:
+
+    curl -L -o redis-server-7.2-darwin-arm64 \
+      https://github.com/codemonstur/embedded-redis/raw/4a83d18637478bfffef191648d6e9c36a708a347/src/main/binaries/redis-server-7.2-darwin-arm64
+    shasum -a 256 redis-server-7.2-darwin-arm64     # must match the SHA-256 above
+    chmod +x redis-server-7.2-darwin-arm64
+    otool -L redis-server-7.2-darwin-arm64          # only /usr/lib/libSystem.B.dylib
+    ./redis-server-7.2-darwin-arm64 --version       # Redis server v=7.2.3 sha=7f4bae81:0
+
+Use curl, not a browser. A browser adds `com.apple.quarantine`, and macOS kills a non-notarized
+binary that has it (the process just dies, exit 137). If that happened:
+
+    xattr -d com.apple.quarantine redis-server-7.2-darwin-arm64
+
+## License
+
+Redis 7.2.3 is BSD-3-Clause. The license text must accompany binary distributions:
+
+    Copyright (c) 2006-2020, Salvatore Sanfilippo
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+        * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+        * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+        * Neither the name of Redis nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

src/standalone/assets/redis/osx/redis-server-7.2-darwin-arm64

@@ -265,12 +265,14 @@ if [[ "$name" == "Darwin" ]]; then
    nameParam='-Xdock:name=Firefly Server'
    runAsDesktopApplication="true"
    headless="false"
+   redisExecParam="-Dembedded.redis.executable=${redisMacExecFile}"
    #dockIcon="-Xdock:icon=${applicationDir}/fireflyDockIcon.png"  --- keep around if we decide to read dock
 else
    splash=
    nameParam="-DdockPlaceHolder="
    runAsDesktopApplication="false"
    headless="true"
+   redisExecParam=
    #dockIcon=
 fi
 
@@ -296,6 +298,7 @@ PROPS=" \
   -Djava.awt.headless=${headless} \
   -Dvisualize.fits.search.path=${HOME} \
   -Dredis.db.dir=${redisDbDir} \
+  ${redisExecParam} \
   -Djava.io.tmpdir=${fireflyServer}/temp \
   -Dalerts.dir=${fireflyServer}/alerts \
   -Dserver_config_dir=${serverConfigDir} \

src/standalone/assets/startFireflyServer.sh

@@ -33,9 +33,11 @@ stopOrphans() {
     fi
     kill -9 "$pid" 2> /dev/null
   done
-  redisPids=$(pgrep -f "${redisDbDir}-")
+  redisPids=$(pgrep -f "${redisDbDir}-|${redisMacExecFile}")
   for pid in $redisPids; do
-    zombieRedisPort=$(ps -o args= -p "$pid" 2> /dev/null | grep -oE '[0-9]+$')
+    # Redis rewrites its process title to "<path> <bind-address>:<port>" followed by padding spaces,
+    # so take the digits after the last ":" and allow trailing spaces
+    zombieRedisPort=$(ps -o args= -p "$pid" 2> /dev/null | sed -nE 's/.*:([0-9]+) *$/\1/p')
     echo "Stopping orphaned redis-server process at process id $pid on port ${zombieRedisPort:-unknown}"
     kill -9 "$pid" 2> /dev/null
   done