bootstrap_rec3D bugfix

Author: hael

doc/implementation_notes/pcg_priors.md

@@ -3444,6 +3444,35 @@ the acceptable-looking outputs do not validate the prior.
    - `bootstrap_rec3D` imports `NU_AUTOTARGET_MIN/MAX` instead of
      duplicating the bounds.
 
+10. **Final-reconstruction stage made standalone (2026-09-07).**
+   - msp1 residual-pass crash reported after the 09:25 fix carried exactly
+     the same code addresses as the 09:18 log (0x12ac0b2
+     choose_and_run_strategy, 0x12b2812 refine3d_exec._omp_fn.0, 0x166a8d3
+     alloc_hash): same binary, i.e. the pre-fix shared-orientation race, not
+     a second race. Rebuild `simple_private_exec` on that machine; addr2line
+     on 0x12ac0b2 must no longer name the old choose_and_run_strategy line.
+   - Export gate: one event, the master's own THROW at the cold base solve
+     of the final reconstruction on the residual sigmas (dot(p,Hp)=NaN at CG
+     iteration 1). Its backtrace appears before the "10 PART(S) COMPLETED"
+     line only because stderr is unbuffered; no worker died. D is validated
+     finite by update_lambda_from_density, so the NaN enters through the RHS
+     or the preconditioner floor. The bootstrap reconstruction on the
+     image-power STAR (iteration N) solved; the final one on the residual
+     STAR (N+1) did not, so the residual groups are the suspect (scale, tiny
+     shells). Diagnosis needs sigma2_it_N.star vs sigma2_it_N+1.star from
+     the run directory.
+   - `bootstrap_rec3D` moved to simple_commanders_refine3D and now owns the
+     complete sequence: image-power seed (N), euclid ML bootstrap map,
+     refine=sigma residual pass against it, consolidation (N+1), final euclid
+     ML map on the residual sigmas; it returns vol1..N and which_iter=N+1 on
+     the command line. abinitio3D's calc_final_rec (takes the commander as
+     class(commander_base) from the abinitio commander) and refine3D_auto
+     call it and carry no copy. Standalone reproduction of the failing stage
+     on any project with 3D orientations:
+     `simple_exec prg=bootstrap_rec3D projfile=... pgrp=... mskdiam=...
+     nparts=... nthr=... rec_backend=pcg` (UI now exposes rec_backend,
+     maxits_pcg, rtol).
+
 ## 11. The NU machinery as the prior infrastructure
 
 The nonuniform-regularization machinery

doc/policies/abinitio3D_policy.md

@@ -135,9 +135,15 @@ iteration and sets `sigma_transition_ready=yes` on the stage command line. The
 starting reconstruction then runs as planned, and the stage's first euclid
 iteration initializes its workers from that STAR, emits per-particle sigma
 files in its own partition layout and replaces the seed with residual sigmas.
-The final reconstruction at original sampling seeds the same way through
-`bootstrap_rec3D`, then upgrades the seed with one residual sigma pass
-(`refine=sigma`) before the shipped euclid ML reconstruction.
+The final reconstruction at original sampling is one program call,
+`bootstrap_rec3D`, which owns the complete sequence: the same image-power
+seed, a euclid ML bootstrap map on it, one residual sigma pass
+(`refine=sigma`) against that map, consolidation of the residual groups as the
+next iteration and the shipped euclid ML reconstruction on them. Because the
+program takes any project with 3D orientations, it is also the standalone
+test for this stage (`simple_exec prg=bootstrap_rec3D projfile=... pgrp=...
+mskdiam=... nparts=... nthr=... rec_backend=...`), so failures in the final
+reconstruction can be reproduced in minutes rather than after a full run.
 
 ## 4. Low-Pass and Cropping
 

doc/policies/refine3D_policy.md

@@ -184,8 +184,12 @@ an empty directory, and every final reconstruction at a new sampling. Since
   residual pass: `refine=sigma` against the seeded map at the final sampling
   (no search, no volume assembly, no orientation output, alignment docs are
   not merged), consolidated as the next iteration, then the shipped euclid
-  ML reconstruction runs on the residual sigmas. `bootstrap_rec3D` is now
-  the seed plus a single euclid ML pass.
+  ML reconstruction runs on the residual sigmas. Since 2026-09-07
+  `bootstrap_rec3D` (module `simple_commanders_refine3D`) owns this whole
+  sequence: seed, bootstrap map, residual pass, consolidation, final map;
+  abinitio3D's `calc_final_rec` and refine3D_auto call it and carry no copy
+  of the sequence. It runs standalone on any project with 3D orientations
+  and is the test entry point for the final-reconstruction stage.
 
 ## 6. Reference Preparation
 

src/main/commanders/simple/simple_commanders_abinitio.f90

@@ -7,7 +7,7 @@ module simple_commanders_abinitio
 use simple_procimgstk,              only: shift_imgfile
 use simple_commanders_project_core, only: commander_selection
 use simple_commanders_reproject,    only: commander_reproject
-use simple_commanders_refine3D,     only: commander_refine3D, commander_refine3D_states
+use simple_commanders_refine3D,     only: commander_refine3D, commander_refine3D_states, commander_bootstrap_rec3D
 use simple_commanders_rec,          only: commander_rec3D
 use simple_cluster_seed,            only: gen_labelling
 use simple_refine3D_fnames,         only: refine3D_startvol_fname, refine3D_startvol_half_fname, &
@@ -40,6 +40,7 @@ subroutine exec_abinitio3D_cavgs( self, cline )
         ! shared-mem commanders
         type(commander_refine3D)  :: xrefine3D
         type(commander_rec3D)     :: xrec3D
+        type(commander_bootstrap_rec3D) :: xbootstrap_rec3D
         type(commander_reproject) :: xreproject
         ! other
         type(string)              :: stk, orig_stk, shifted_stk, stk_even, stk_odd, ext
@@ -293,7 +294,7 @@ subroutine exec_abinitio3D_cavgs( self, cline )
             if( params%nstates > 1 ) call conv_eo_states(work_proj%os_ptcl3D)
             call conv_eo(work_proj%os_ptcl3D)
             ! calculate 3D reconstruction at original sampling
-            call calc_final_rec(params, work_proj, work_projfile, xrec3D, xrefine3D, l_postprocess=.false.)
+            call calc_final_rec(params, work_proj, work_projfile, xrec3D, xbootstrap_rec3D, l_postprocess=.false.)
             ! final raw and low-pass diagnostic 3D reconstruction outputs
             call write_final_rec_outputs(params, work_proj, lpinfo(nstages_ini3D)%lp)
             ! add rec_final to os_out
@@ -506,6 +507,7 @@ subroutine exec_abinitio3D( self, cline )
         type(commander_refine3D)        :: xrefine3D
         type(commander_refine3D_states) :: xrefine3D_states
         type(commander_rec3D)           :: xrec3D
+        type(commander_bootstrap_rec3D) :: xbootstrap_rec3D
         ! other
         integer,            allocatable :: tmpinds(:), clsinds(:), pinds(:), cls_states(:)
         type(class_sample), allocatable :: clssmp(:)
@@ -973,7 +975,7 @@ subroutine exec_abinitio3D( self, cline )
                     call ensure_multistate_particle_assignments
             end select
             ! calculate 3D reconstruction at original sampling
-            call calc_final_rec(params, spproj, params%projfile, xrec3D, xrefine3D, l_postprocess=.true.)
+            call calc_final_rec(params, spproj, params%projfile, xrec3D, xbootstrap_rec3D, l_postprocess=.true.)
             ! for visualization
             call gen_ortho_reprojs4viz(params, spproj)
             ! final raw and low-pass diagnostic 3D reconstruction outputs

src/main/commanders/simple/simple_commanders_rec.f90

@@ -3,7 +3,6 @@ module simple_commanders_rec
 use simple_commanders_api
 use simple_matcher_2Dprep
 use simple_matcher_3Drec, only: calc_3Drec, calc_projdir3Drec
-use simple_refine3D_fnames, only: refine3D_fsc_fname, refine3D_state_halfvol_fname, refine3D_state_vol_fname
 use simple_sigma2_files, only: load_sigma2_groups
 implicit none
 #include "simple_local_flags.inc"
@@ -13,11 +12,6 @@ module simple_commanders_rec
     procedure :: execute => exec_rec3D
 end type commander_rec3D
 
-type, extends(commander_base) :: commander_bootstrap_rec3D
-  contains
-    procedure :: execute => exec_bootstrap_rec3D
-end type commander_bootstrap_rec3D
-
 type, extends(commander_base) :: commander_rec3D_worker
   contains
     procedure :: execute      => exec_rec3D_distr_worker
@@ -59,164 +53,6 @@ subroutine exec_rec3D( self, cline )
         if( allocated(strategy) ) deallocate(strategy)
     end subroutine exec_rec3D
 
-    subroutine exec_bootstrap_rec3D( self, cline )
-        use simple_commanders_euclid, only: commander_calc_pspec
-        class(commander_bootstrap_rec3D), intent(inout) :: self
-        class(cmdline),                  intent(inout) :: cline
-        type(commander_rec3D) :: xrec3D
-        type(commander_calc_pspec) :: xcalc_pspec
-        type(cmdline)         :: cline_reg, cline_pspec
-        type(parameters)      :: params
-        integer               :: state, which_iter
-        if( .not. cline%defined('mkdir')       ) call cline%set('mkdir',       'yes')
-        call cline%set('oritype', 'ptcl3D')
-        if( .not. cline%defined('nstates')     ) call cline%set('nstates',          1)
-        call warn_for_forced_bootstrap_overrides(cline)
-        call cline%set('sigma_est', 'global')
-        if( .not. cline%defined('which_iter')  ) call cline%set('which_iter',       1)
-        if( .not. cline%defined('postprocess') ) call cline%set('postprocess',  'yes')
-        if( .not. cline%defined('combine_eo')  ) call cline%set('combine_eo',    'no')
-        if( .not. cline%defined('envfsc')      ) call cline%set('envfsc',        'no')
-        call cline%delete('objfun')
-        call cline%delete('ml_reg')
-        call params%new(cline)
-        which_iter = max(1, params%which_iter)
-        call cline%set('which_iter', which_iter)
-        call cline%set('mkdir', 'no') ! child reconstruct3D calls must not create nested run directories
-        ! One sigma2 basis for every bootstrap (2026-09-06): the particle
-        ! power spectra, exactly what a fresh refinement seeds from, written
-        ! as the grouped STAR of which_iter (legacy store) or the registered
-        ! canonical state. The former half-map power estimator sat on a
-        ! different basis than the residual sigmas a refinement then computes
-        ! and conditioned the euclid system markedly worse (bgal residual
-        ! 0.23 vs 0.08, refine3D_auto startup record). With the seed in hand
-        ! the reconstruction is a single euclid ML-regularized pass; callers
-        ! that ship a final map upgrade the seed with a residual pass
-        ! (simple_sigma2_bootstrap).
-        cline_pspec = cline
-        call cline_pspec%set('prg',       'calc_pspec')
-        call cline_pspec%set('mkdir',              'no')
-        call cline_pspec%set('objfun',       'euclid')
-        call cline_pspec%set('sigma_est',    'global')
-        call cline_pspec%set('which_iter',  which_iter)
-        call cline_pspec%delete('postprocess')
-        call cline_pspec%delete('combine_eo')
-        call cline_pspec%delete('rec_backend')
-        call cline_pspec%delete('maxits_pcg')
-        call cline_pspec%delete('rtol')
-        call cline_pspec%delete('trail_seed')
-        call cline_pspec%delete('outfile')
-        write(logfhandle,'(A,I0)') '>>> BOOTSTRAP_REC3D SIGMA2 FROM PARTICLE POWER SPECTRA, ITERATION ', which_iter
-        call xcalc_pspec%execute(cline_pspec)
-        call cline_pspec%kill
-        cline_reg = cline
-        call prepare_bootstrap_rec_cline(cline_reg, l_regularized=.true.)
-        write(logfhandle,'(A)') '>>> BOOTSTRAP_REC3D: EUCLID ML-REGULARIZED RECONSTRUCTION'
-        call xrec3D%execute(cline_reg)
-        call register_bootstrap_rec_outputs()
-        do state = 1, params%nstates
-            call cline%set('vol'//int2str(state), refine3D_state_vol_fname(state))
-        enddo
-        call cline_reg%kill
-        call simple_end('**** SIMPLE_BOOTSTRAP_REC3D NORMAL STOP ****', print_simple=.false.)
-
-    contains
-
-        subroutine prepare_bootstrap_rec_cline( cline_rec, l_regularized )
-            class(cmdline), intent(inout) :: cline_rec
-            logical,        intent(in)    :: l_regularized
-            integer :: state
-            call cline_rec%set('prg',       'reconstruct3D')
-            call cline_rec%set('mkdir',              'no')
-            call cline_rec%set('oritype', params%oritype)
-            call cline_rec%set('nstates', params%nstates)
-            call cline_rec%set('sigma_est',     'global')
-            call cline_rec%set('which_iter', which_iter)
-            call cline_rec%set('trail_rec',        'no')
-            call cline_rec%set('combine_eo',       'no')
-            call cline_rec%delete('refine')
-            call cline_rec%delete('update_frac')
-            call cline_rec%delete('fillin')
-            call cline_rec%delete('objfun_den')
-            call cline_rec%delete('objfun_den_w')
-            call cline_rec%delete('ufrac_trec')
-            call cline_rec%delete('endit')
-            call cline_rec%delete('vol_even')
-            call cline_rec%delete('vol_odd')
-            call cline_rec%delete('refs')
-            call cline_rec%delete('refs_even')
-            call cline_rec%delete('refs_odd')
-            do state = 1, params%nstates
-                call cline_rec%delete('vol'//int2str(state))
-            enddo
-            if( l_regularized )then
-                call cline_rec%set('objfun', 'euclid')
-                call cline_rec%set('ml_reg',    'yes')
-            else
-                call cline_rec%set('objfun',       'cc')
-                call cline_rec%set('ml_reg',       'no')
-                call cline_rec%set('postprocess',  'no')
-                call cline_rec%set('filt_mode',    'none')
-                call cline_rec%set('automsk',      'no')
-            endif
-        end subroutine prepare_bootstrap_rec_cline
-
-
-        subroutine register_bootstrap_rec_outputs()
-            type(sp_project) :: spproj
-            type(string)     :: volname, fscname
-            integer          :: state, pop
-            character(len=16) :: imgkind
-            call spproj%read_segment('out', params%projfile)
-            call spproj%read_segment(params%oritype, params%projfile)
-            select case(trim(params%oritype))
-                case('cls3D')
-                    imgkind = 'vol_cavg'
-                case DEFAULT
-                    imgkind = 'vol'
-            end select
-            do state = 1, params%nstates
-                select case(trim(params%oritype))
-                    case('cls3D')
-                        pop = spproj%os_cls3D%get_pop(state, 'state')
-                    case DEFAULT
-                        pop = spproj%os_ptcl3D%get_pop(state, 'state')
-                end select
-                if( pop == 0 )cycle
-                volname = refine3D_state_vol_fname(state)
-                if( .not. file_exists(volname) )then
-                    call volname%kill
-                    cycle
-                endif
-                fscname = refine3D_fsc_fname(state)
-                ! params%box_crop/smpd_crop are the effective reconstruction
-                ! sampling resolved by params%new or explicitly pinned by
-                ! callers that must avoid staged downsampling leakage.
-                call spproj%add_vol2os_out(volname, params%smpd_crop, state, trim(imgkind), pop=pop)
-                if( file_exists(fscname) ) call spproj%add_fsc2os_out(fscname, state, params%box_crop)
-                call volname%kill
-                call fscname%kill
-            enddo
-            call spproj%write_segment_inside('out', params%projfile)
-            call spproj%kill
-        end subroutine register_bootstrap_rec_outputs
-
-
-        subroutine warn_for_forced_bootstrap_overrides( cline_in )
-            class(cmdline), intent(inout) :: cline_in
-            type(string) :: val
-            if( cline_in%defined('sigma_est') )then
-                val = cline_in%get_carg('sigma_est')
-                if( val%to_char().ne.'global' )then
-                    THROW_WARN('bootstrap_rec3D enforces sigma_est=global; ignoring input sigma_est='//val%to_char())
-                endif
-                call val%kill
-            endif
-            if( cline_in%defined('objfun') ) THROW_WARN('bootstrap_rec3D controls objfun internally; ignoring input objfun')
-            if( cline_in%defined('ml_reg') ) THROW_WARN('bootstrap_rec3D controls ml_reg internally; ignoring input ml_reg')
-        end subroutine warn_for_forced_bootstrap_overrides
-    end subroutine exec_bootstrap_rec3D
-
     subroutine exec_rec3D_distr_worker( self, cline )
         use simple_rec3D_pcg_strategy, only: execute_rec3D_pcg_worker
         class(commander_rec3D_worker), intent(inout) :: self