Merge branch 'development' into serial-honcho

Author: baltzell

.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-java@v5.7.0
+      - uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -91,7 +91,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-java@v5.7.0
+      - uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -137,7 +137,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-java@v5.7.0
+      - uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ matrix.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -161,7 +161,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -170,7 +170,7 @@ jobs:
         with:
           name: build_ubuntu-latest
       - name: untar build
-        run: tar xzvf coatjava.tar.gz
+        run: tar xzf coatjava.tar.gz
       - name: spotbugs
         run: ./build-coatjava.sh --spotbugs --no-maps --no-progress
 
@@ -180,7 +180,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -194,7 +194,7 @@ jobs:
           path: |
             clas_018779.evio.00001
       - name: untar build
-        run: tar xzvf coatjava.tar.gz
+        run: tar xzf coatjava.tar.gz
       - name: run test
         run: |
           ls -lhtr
@@ -209,7 +209,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -224,8 +224,8 @@ jobs:
             clas_018779.evio.00001
       - name: untar build
         run: |
-          tar xzvf clara.tar.gz
-          tar xzvf coatjava.tar.gz
+          tar xzf clara.tar.gz
+          tar xzf coatjava.tar.gz
       - run: ls
       - name: run test
         run: ./bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 500 -c ./clara -o ./tmp ./clas_018779.evio.00001
@@ -267,7 +267,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -277,8 +277,8 @@ jobs:
           name: build_${{ matrix.runner }}
       - name: untar build
         run: |
-          tar xzvf coatjava.tar.gz
-          tar xzvf clara.tar.gz
+          tar xzf coatjava.tar.gz
+          tar xzf clara.tar.gz
       - name: run test
         run: |
           git lfs install
@@ -295,7 +295,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -305,8 +305,8 @@ jobs:
           name: build_macos
       - name: untar build
         run: |
-          tar xzvf coatjava.tar.gz
-          tar xzvf clara.tar.gz
+          tar xzf coatjava.tar.gz
+          tar xzf clara.tar.gz
       - name: run test
         run: |
           git lfs install
@@ -320,7 +320,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -333,7 +333,7 @@ jobs:
         with:
           name: build_ubuntu-latest
       - name: untar build
-        run: tar xzvf coatjava.tar.gz
+        run: tar xzf coatjava.tar.gz
       - name: test run-groovy
         run: coatjava/bin/run-groovy validation/advanced-tests/test-run-groovy.groovy
 
@@ -343,7 +343,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -355,7 +355,7 @@ jobs:
         with:
           name: build_ubuntu-latest
       - name: untar build
-        run: tar xzvf coatjava.tar.gz
+        run: tar xzf coatjava.tar.gz
       - name: hipo2npz
         run: ./coatjava/bin/hipo2npz rec.hipo rec.npz
       - name: hipo2npz-dump
@@ -384,7 +384,7 @@ jobs:
           path: pages/jacoco
       ### javadoc
       - name: set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}

bin/postprocess2

@@ -6,5 +6,5 @@ export MALLOC_ARENA_MAX=1
 
 java ${JAVA_OPTS-} -Xmx768m -Xms768m -XX:+UseSerialGC \
     -cp ${COATJAVA_CLASSPATH:-''} \
-    org.jlab.analysis.postprocess.Processor \
+    org.jlab.detector.serial.PostProcessor \
     $*

common-tools/clara-io/pom.xml

@@ -48,12 +48,6 @@
       <version>14.2.0-SNAPSHOT</version>
     </dependency>
 
-    <dependency>
-      <groupId>org.jlab.clas</groupId>
-      <artifactId>clas-analysis</artifactId>
-      <version>14.2.0-SNAPSHOT</version>
-    </dependency> 
-
     <dependency>
       <groupId>org.jlab.clas</groupId>
       <artifactId>clas-utils</artifactId>

common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java

@@ -3,7 +3,8 @@
 import java.io.File;
 import java.nio.file.Path;
 import java.util.List;
-import org.jlab.analysis.postprocess.Processor;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import org.jlab.clara.std.services.EventWriterException;
 import org.jlab.detector.calib.utils.ConstantsManager;
 import org.jlab.detector.helicity.HelicitySequenceDelayed;
@@ -97,8 +98,8 @@ private int getRunNumber() {
     private void postprocess() {
         int d = conman.getConstants(getRunNumber(), "/runcontrol/helicity").getIntValue("delay",0,0,0);
         HelicitySequenceDelayed helicity = new HelicitySequenceDelayed(d);
-        helicity.addStream(serial.getHelicities());
-        Processor p = new Processor(List.of(filename), fullSchema, helicity, serial.getScalers());
+        helicity.addStream(helicities);
+        PostProcessor p = new PostProcessor(List.of(filename), fullSchema, helicity, scalers);
         HipoReader r = new HipoReader();
         r.open(filename);
         Event e = new Event();

common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/RebuildScalers.java

@@ -9,6 +9,7 @@
 import org.jlab.detector.scalers.DaqScalers;
 import org.jlab.detector.helicity.HelicitySequenceManager;
 import org.jlab.detector.scalers.DaqScalersSequence;
+import org.jlab.detector.serial.SerialUtil;
 import org.jlab.jnp.hipo4.data.Bank;
 import org.jlab.jnp.hipo4.data.Event;
 import org.jlab.jnp.hipo4.io.HipoReader;
@@ -122,7 +123,7 @@ else if (seq != null) {
                     runScalerBank = ds.createRunBank(writer.getSchemaFactory());
                     helScalerBank = ds.createHelicityBank(writer.getSchemaFactory());
 
-                    Util.assignScalerHelicity(event, helScalerBank, helSeq);
+                    SerialUtil.assignScalerHelicity(event, helScalerBank, helSeq);
 
                     // put modified HEL/RUN::scaler back in the event:
                     event.write(runScalerBank);

common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Tag1ToEvent.java

@@ -1,5 +1,6 @@
 package org.jlab.analysis.postprocess;
 
+import org.jlab.detector.serial.PostProcessor;
 import java.util.TreeMap;
 import java.util.logging.Logger;
 import org.jlab.clas.reco.ReconstructionEngine;
@@ -13,6 +14,7 @@
 import org.jlab.detector.scalers.DaqScalersSequence;
 import org.jlab.detector.helicity.HelicityBit;
 import org.jlab.detector.helicity.HelicitySequenceDelayed;
+import org.jlab.detector.serial.SerialUtil;
 import org.jlab.jnp.hipo4.data.SchemaFactory;
 import org.jlab.utils.groups.IndexedTable;
 import org.jlab.utils.options.OptionParser;
@@ -81,7 +83,7 @@ public static void main(String[] args) {
             LOGGER.info("\n>>> Initializing helicity configuration from CCDB ...\n");
             ConstantsManager conman = new ConstantsManager();
             conman.init("/runcontrol/hwp","/runcontrol/helicity");
-            final int run = Util.getRunNumber(parser.getInputList().get(0));
+            final int run = SerialUtil.getRunNumber(parser.getInputList().get(0));
             IndexedTable helTable = conman.getConstants(run, "/runcontrol/helicity");
 
             // Initialize the scaler sequence from tag-1 events:
@@ -102,7 +104,7 @@ public static void main(String[] args) {
             }
 
             // Initialize the unix-event map:
-            TreeMap<Integer,Integer> eventUnix = Processor.getEventUnixMap(schema, parser.getInputList());
+            TreeMap<Integer,Integer> eventUnix = PostProcessor.getEventUnixMap(schema, parser.getInputList());
 
             // Loop over the input HIPO files:
             LOGGER.info("\n>>> Starting post-processing ...\n");
@@ -138,7 +140,7 @@ public static void main(String[] args) {
                     if (doHelicityDelay) {
                         recEventBank.putByte("helicity",0,hb.value());
                         recEventBank.putByte("helicityRaw",0,hbraw.value());
-                        Util.assignScalerHelicity(runConfigBank.getLong("timestamp",0), helScalerBank, helSeq);
+                        SerialUtil.assignScalerHelicity(runConfigBank.getLong("timestamp",0), helScalerBank, helSeq);
                     }
 
                     // Write beam charge to REC::Event:
@@ -169,7 +171,7 @@ public static void main(String[] args) {
                     writer.addEvent(event, event.getEventTag());
 
                     // Copy config banks to new, tag-1 events:
-                    Util.createTag1Events(writer, event, configEvent, configBanks);
+                    SerialUtil.createTag1Events(writer, event, configEvent, configBanks);
                 }
 
                 reader.close();

…jlab/analysis/postprocess/Processor.java …/jlab/detector/serial/PostProcessor.javacommon-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java renamed to common-tools/clas-detector/src/main/java/org/jlab/detector/serial/PostProcessor.java common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java renamed to common-tools/clas-detector/src/main/java/org/jlab/detector/serial/PostProcessor.java

@@ -1,4 +1,4 @@
-package org.jlab.analysis.postprocess;
+package org.jlab.detector.serial;
 
 import java.util.List;
 import java.util.TreeMap;
@@ -24,7 +24,7 @@
  *
  * @author baltzell
  */
-public class Processor {
+public class PostProcessor {
 
     public static final String CCDB_TABLES[] = {"/runcontrol/fcup","/runcontrol/slm",
         "/runcontrol/helicity","/daq/config/scalers/dsc1","/runcontrol/hwp"};
@@ -37,7 +37,7 @@ public class Processor {
     private HelicitySequenceDelayed helicitySequence = null;
     private TreeMap<Integer,Integer> eventUnix = null;
 
-    public Processor(List<String> files, boolean restream, boolean rebuild) {
+    public PostProcessor(List<String> files, boolean restream, boolean rebuild) {
         HipoReader r = new HipoReader();
         r.open(files.get(0));
         schemaFactory = r.getSchemaFactory();
@@ -46,13 +46,13 @@ public Processor(List<String> files, boolean restream, boolean rebuild) {
         recEvent = new Bank(schemaFactory.getSchema("REC::Event"));
         conman = new ConstantsManager();
         conman.init(CCDB_TABLES);
-        helicitySequence = Util.getHelicity(files, schemaFactory, restream, conman);
+        helicitySequence = SerialUtil.getHelicity(files, schemaFactory, restream, conman);
         if (rebuild) chargeSequence = DaqScalersSequence.rebuildSequence(1, conman, files);
         else chargeSequence = DaqScalersSequence.readSequence(files);
         eventUnix = getEventUnixMap(schemaFactory, files); 
     }
 
-    public Processor(List<String> files, SchemaFactory schema, HelicitySequenceDelayed h, DaqScalersSequence s) {
+    public PostProcessor(List<String> files, SchemaFactory schema, HelicitySequenceDelayed h, DaqScalersSequence s) {
         schemaFactory = schema;
         helicitySequence = h;
         chargeSequence = s;
@@ -102,7 +102,7 @@ private void processEventHelicity(DataEvent event, DataBank runcfg, DataBank rec
         DataBank helScaler = event.getBank("HEL::scaler");
         if (helScaler.rows()>0) {
             event.removeBank("HEL::scaler");
-            Util.assignScalerHelicity(runcfg.getLong("timestamp",0), ((HipoDataBank)helScaler).getBank(), helicitySequence);
+            SerialUtil.assignScalerHelicity(runcfg.getLong("timestamp",0), ((HipoDataBank)helScaler).getBank(), helicitySequence);
             event.appendBank(helScaler);
         }
     }
@@ -122,7 +122,7 @@ private void processEventHelicity(Event event, Bank runcfg, Bank recevt) {
         event.read(helScaler);
         if (helScaler.getRows()>0) {
             event.remove(schemaFactory.getSchema("HEL::scaler"));
-            Util.assignScalerHelicity(runcfg.getLong("timestamp",0), helScaler, helicitySequence);
+            SerialUtil.assignScalerHelicity(runcfg.getLong("timestamp",0), helScaler, helicitySequence);
             event.write(helScaler);
         }
     }
@@ -247,7 +247,7 @@ public static void main(String args[]) {
         boolean restream = !o.getOption("-f").isDefault();
         boolean rebuild = !o.getOption("-c").isDefault();
 
-        Processor post = new Processor(o.getInputList(), restream, rebuild);
+        PostProcessor post = new PostProcessor(o.getInputList(), restream, rebuild);
 
         HipoWriterSorted writer = null;
 

…/org/jlab/analysis/postprocess/Util.java …org/jlab/detector/serial/SerialUtil.javacommon-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Util.java renamed to common-tools/clas-detector/src/main/java/org/jlab/detector/serial/SerialUtil.java common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Util.java renamed to common-tools/clas-detector/src/main/java/org/jlab/detector/serial/SerialUtil.java

@@ -1,4 +1,4 @@
-package org.jlab.analysis.postprocess;
+package org.jlab.detector.serial;
 
 import java.sql.Time;
 import java.util.Arrays;
@@ -26,9 +26,9 @@
  * Static utility methods for postprocessing.
  * @author baltzell
  */
-class Util {
+public class SerialUtil {
 
-    static final Logger logger = Logger.getLogger(Util.class.getName());
+    static final Logger logger = Logger.getLogger(SerialUtil.class.getName());
 
     /**
      * Assign the delay-corrected helicity to the HEL::scaler bank's rows

common-tools/clas-io/src/main/java/org/jlab/utils/ClaraYaml.java

@@ -143,9 +143,11 @@ public JSONObject filter(String serviceName) {
 
     /**
      * Emulate the way CLARA parses the full YAML and presents it in EngineData.
-     * The "global" and "service" subsections in the "configuration" section get
-     * squashed into one namespace, and service-specific keys override any
-     * globals of the same name.
+     *
+     * The YAML's "global" and "services" configuration sections get squashed
+     * into one namespace, with service-specific parameters overriding globals
+     * of the same name.  Also, for the special services named "reader" and
+     * "writer", the "io-services" section is searched instead of "services".
      *
      * @param claraJson the full CLARA YAML contents
      * @param serviceName the name of the service in CLARA YAML (not class name)
@@ -161,9 +163,10 @@ public static JSONObject filter(JSONObject claraJson, String serviceName) {
                     ret.accumulate(key, globals.getString(key));
                 }
             }
-            if (config.has("services")) {
-                if (config.getJSONObject("services").has(serviceName)) {
-                    JSONObject service = config.getJSONObject("services").getJSONObject(serviceName);
+	    String section = serviceName.equals("reader") || serviceName.equals("writer") ? "io-services" : "services";
+            if (config.has(section)) {
+                if (config.getJSONObject(section).has(serviceName)) {
+                    JSONObject service = config.getJSONObject(section).getJSONObject(serviceName);
                     for (String key : service.keySet()) {
                         ret.put(key, service.getString(key));
                     }