Merge branch 'development' into occupancy

Author: baltzell

.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-java@v5.7.0
+      - uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -91,7 +91,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-java@v5.7.0
+      - uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -137,7 +137,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-java@v5.7.0
+      - uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ matrix.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -161,7 +161,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -170,7 +170,7 @@ jobs:
         with:
           name: build_ubuntu-latest
       - name: untar build
-        run: tar xzvf coatjava.tar.gz
+        run: tar xzf coatjava.tar.gz
       - name: spotbugs
         run: ./build-coatjava.sh --spotbugs --no-maps --no-progress
 
@@ -180,7 +180,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -194,7 +194,7 @@ jobs:
           path: |
             clas_018779.evio.00001
       - name: untar build
-        run: tar xzvf coatjava.tar.gz
+        run: tar xzf coatjava.tar.gz
       - name: run test
         run: |
           ls -lhtr
@@ -209,7 +209,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -224,8 +224,8 @@ jobs:
             clas_018779.evio.00001
       - name: untar build
         run: |
-          tar xzvf clara.tar.gz
-          tar xzvf coatjava.tar.gz
+          tar xzf clara.tar.gz
+          tar xzf coatjava.tar.gz
       - run: ls
       - name: run test
         run: ./bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 500 -c ./clara -o ./tmp ./clas_018779.evio.00001
@@ -267,7 +267,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -277,8 +277,8 @@ jobs:
           name: build_${{ matrix.runner }}
       - name: untar build
         run: |
-          tar xzvf coatjava.tar.gz
-          tar xzvf clara.tar.gz
+          tar xzf coatjava.tar.gz
+          tar xzf clara.tar.gz
       - name: run test
         run: |
           git lfs install
@@ -295,7 +295,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -305,8 +305,8 @@ jobs:
           name: build_macos
       - name: untar build
         run: |
-          tar xzvf coatjava.tar.gz
-          tar xzvf clara.tar.gz
+          tar xzf coatjava.tar.gz
+          tar xzf clara.tar.gz
       - name: run test
         run: |
           git lfs install
@@ -320,7 +320,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -333,7 +333,7 @@ jobs:
         with:
           name: build_ubuntu-latest
       - name: untar build
-        run: tar xzvf coatjava.tar.gz
+        run: tar xzf coatjava.tar.gz
       - name: test run-groovy
         run: coatjava/bin/run-groovy validation/advanced-tests/test-run-groovy.groovy
 
@@ -343,7 +343,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -355,7 +355,7 @@ jobs:
         with:
           name: build_ubuntu-latest
       - name: untar build
-        run: tar xzvf coatjava.tar.gz
+        run: tar xzf coatjava.tar.gz
       - name: hipo2npz
         run: ./coatjava/bin/hipo2npz rec.hipo rec.npz
       - name: hipo2npz-dump
@@ -384,7 +384,7 @@ jobs:
           path: pages/jacoco
       ### javadoc
       - name: set up JDK
-        uses: actions/setup-java@v5.7.0
+        uses: actions/setup-java@v6.0.0
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
@@ -440,7 +440,7 @@ jobs:
           mv coatjava{,-${{ env.TAG_NAME }}}
           tar czf coatjava-${{ env.TAG_NAME }}{.tar.gz,}
       - name: release
-        uses: softprops/action-gh-release@v3.0.2
+        uses: softprops/action-gh-release@v3.0.3
         with:
           files: coatjava-${{ env.TAG_NAME }}.tar.gz
       - name: open issue if failed

.gitignore

@@ -8,6 +8,11 @@ target
 tmp
 /common-tools/clas-io/src/main/java/org/jlab/io/banks
 
+# data files
+*.evio
+*.hipo
+!/etc/data/test/*.hipo
+
 # deployment
 /myLocalMvnRepo
 /coatjava-*.tar.gz
@@ -50,7 +55,6 @@ hs_err_pid*
 
 # no log files:
 *.log
-*.evio 
 
 bin/evio2hipotest
 export.sh

bin/postprocess2

@@ -6,5 +6,5 @@ export MALLOC_ARENA_MAX=1
 
 java ${JAVA_OPTS-} -Xmx768m -Xms768m -XX:+UseSerialGC \
     -cp ${COATJAVA_CLASSPATH:-''} \
-    org.jlab.analysis.postprocess.Processor \
+    org.jlab.detector.serial.PostProcessor \
     $*

common-tools/clara-io/pom.xml

@@ -48,12 +48,6 @@
       <version>14.2.0-SNAPSHOT</version>
     </dependency>
 
-    <dependency>
-      <groupId>org.jlab.clas</groupId>
-      <artifactId>clas-analysis</artifactId>
-      <version>14.2.0-SNAPSHOT</version>
-    </dependency> 
-
     <dependency>
       <groupId>org.jlab.clas</groupId>
       <artifactId>clas-utils</artifactId>

common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java

@@ -5,14 +5,11 @@
 import java.util.List;
 import java.util.TreeMap;
 import java.util.TreeSet;
-import org.jlab.analysis.postprocess.Processor;
 import org.jlab.clara.std.services.EventWriterException;
 import org.jlab.detector.calib.utils.ConstantsManager;
-import org.jlab.detector.decode.CLASDecoder4;
-import org.jlab.detector.helicity.HelicitySequence;
 import org.jlab.detector.helicity.HelicitySequenceDelayed;
-import org.jlab.detector.helicity.HelicityState;
-import org.jlab.detector.scalers.DaqScalersSequence;
+import org.jlab.detector.serial.SerialHoncho;
+import org.jlab.detector.serial.PostProcessor;
 import org.jlab.jnp.hipo4.data.Bank;
 import org.jlab.jnp.hipo4.data.Event;
 import org.jlab.jnp.hipo4.data.SchemaFactory;
@@ -33,34 +30,22 @@
  */
 public class Clas12Writer extends HipoToHipoWriter {
 
-    static final String[] TAG1BANKS = {"RUN::scaler","HEL::scaler","RAW::scaler","RAW::epics","HEL::flip","COAT::config"};
-
-    Bank[] tag1banks;
+    SerialHoncho serial;
     Bank runConfig;
-    Bank helicityAdc;
     ConstantsManager conman;
-    TreeMap<Integer,Integer> eventUnix;
-    TreeSet<HelicityState> helicities;
-    DaqScalersSequence scalers;
     SchemaFactory fullSchema;
     boolean postprocess;
 
     private void init(JSONObject opts) {
         fullSchema = new SchemaFactory();
         fullSchema.initFromDirectory(FileUtils.getEnvironmentPath("CLAS12DIR","etc/bankdefs/hipo4"));
+        serial = new SerialHoncho(fullSchema);
         runConfig = new Bank(fullSchema.getSchema("RUN::config"));
-        helicityAdc = new Bank(fullSchema.getSchema("HEL::adc"));
-        helicities = new TreeSet<>();
-        scalers = new DaqScalersSequence(fullSchema);
         conman = new ConstantsManager();
-        eventUnix = new TreeMap<>();
         conman.init("/runcontrol/hwp","/runcontrol/helicity");
         postprocess = opts.optBoolean("postprocess", false);
         if (opts.has("variation")) conman.setVariation(opts.getString("variation"));
         if (opts.has("timestamp")) conman.setTimeStamp(opts.getString("timestamp"));
-        tag1banks = new Bank[TAG1BANKS.length];
-        for (int i=0; i<tag1banks.length; ++i)
-            tag1banks[i] = new Bank(fullSchema.getSchema(TAG1BANKS[i]));
     }
 
     @Override
@@ -78,29 +63,17 @@ protected HipoWriterSorted createWriter(Path file, JSONObject opts) throws Event
 
     @Override
     protected void writeEvent(Object event) throws EventWriterException {
-        scalers.add((Event)event);
-        ((Event)event).read(runConfig);
-        ((Event)event).read(helicityAdc);
-        if (runConfig.getRows() > 0) {
-            int unix = runConfig.getInt("unixtime",0);
-            int evno = runConfig.getInt("event",0);
-            if (unix > 0 && evno > 0) eventUnix.put(evno, unix);
-        }
-        helicities.add(HelicityState.createFromFadcBank(helicityAdc, runConfig, conman));
-        Event t = CLASDecoder4.createTaggedEvent((Event)event, runConfig, tag1banks);
+        Event t = serial.read((Event)event);
         if (!t.isEmpty()) writer.addEvent(t, 1);
         super.writeEvent(event);
     }
 
     @Override
     protected void closeWriter() {
-        HelicitySequence.writeFlips(fullSchema, writer, helicities);
-        writer.addEvent(getUnixEvent(runConfig),1);
+        serial.finish(writer);
         super.closeWriter();
         if (postprocess) postprocess();
-        // keep the latest helicity/scaler reading for the next file:
-        while (helicities.size() > 60) helicities.pollFirst();
-        scalers.clear(10);
+        serial.clear();
     }
 
     /**
@@ -120,35 +93,14 @@ private int getRunNumber() {
         return 0;
     }
 
-    /**
-     * Get a new event with a RUN::unix bank containing event-timestamp mapping,
-     * and the latest RUN::config bank.
-     * @param config
-     * @return 
-     */
-    private Event getUnixEvent(Bank config) {
-        Bank unix = new Bank(fullSchema.getSchema("RUN::unix"));
-        unix.setRows(eventUnix.size());
-        int row = 0;
-        for (int evno : eventUnix.keySet()) {
-            unix.putInt("event", row, evno);
-            unix.putInt("unixtime",row, eventUnix.get(evno));
-            row++;
-        }
-        Event e = new Event();
-        e.write(config);
-        e.write(unix);
-        return e;
-    }
-
     /**
      * Copy helicity/charge tag-1 information to all events.
      */
     private void postprocess() {
         int d = conman.getConstants(getRunNumber(), "/runcontrol/helicity").getIntValue("delay",0,0,0);
         HelicitySequenceDelayed helicity = new HelicitySequenceDelayed(d);
-        helicity.addStream(helicities);
-        Processor p = new Processor(List.of(filename), fullSchema, helicity, scalers);
+        helicity.addStream(serial.getHelicities());
+        PostProcessor p = new PostProcessor(List.of(filename), fullSchema, helicity, serial.getScalers());
         HipoReader r = new HipoReader();
         r.open(filename);
         Event e = new Event();

common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/RebuildScalers.java

@@ -9,6 +9,7 @@
 import org.jlab.detector.scalers.DaqScalers;
 import org.jlab.detector.helicity.HelicitySequenceManager;
 import org.jlab.detector.scalers.DaqScalersSequence;
+import org.jlab.detector.serial.SerialUtil;
 import org.jlab.jnp.hipo4.data.Bank;
 import org.jlab.jnp.hipo4.data.Event;
 import org.jlab.jnp.hipo4.io.HipoReader;
@@ -122,7 +123,7 @@ else if (seq != null) {
                     runScalerBank = ds.createRunBank(writer.getSchemaFactory());
                     helScalerBank = ds.createHelicityBank(writer.getSchemaFactory());
 
-                    Util.assignScalerHelicity(event, helScalerBank, helSeq);
+                    SerialUtil.assignScalerHelicity(event, helScalerBank, helSeq);
 
                     // put modified HEL/RUN::scaler back in the event:
                     event.write(runScalerBank);