updating docs and tests

Author: carsen-stringer

.github/workflows/test_and_deploy.yml

@@ -21,8 +21,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest, windows-latest]
-        python-version: [3.8]
+        platform: [ubuntu-latest] #, windows-latest]
+        python-version: [3.11]
 
     steps:
       - uses: actions/checkout@v4

conftest.py

@@ -18,7 +18,7 @@ def data_dir():
     data_path.mkdir(exist_ok=True)
     download_cached_inputs(data_path)
     cached_outputs = data_path.joinpath('test_outputs')
-    cached_outputs_url = 'https://osf.io/download/6984d8cfe14e1c4246442669'
+    cached_outputs_url = 'https://osf.io/download/69890166f7d481f73ae24adb'
     if not os.path.exists(cached_outputs):
         extract_zip(data_path.joinpath('test_outputs.zip'), cached_outputs_url, data_path)
     return data_path
@@ -32,7 +32,7 @@ def test_settings(tmpdir, data_dir):
 def download_cached_inputs(data_path):
     """ Downloads test_input data if not present on machine. This function was created so it can also be used by scripts/generate_test_data.py."""
     cached_inputs = data_path.joinpath('test_inputs')
-    cached_inputs_url = 'https://osf.io/download/6984d9ca54398fbd42f5575d'
+    cached_inputs_url = 'https://osf.io/download/69890549bffbac83ffe2530a'
     if not os.path.exists(cached_inputs):
         print('Cached inputs not found. Downloading now...')
         extract_zip(data_path.joinpath('test_inputs.zip'), cached_inputs_url, data_path)

mkdocs.yml

@@ -1,4 +1,8 @@
 site_name: Suite2p Documentation
+
+repo_name: mouseland/suite2p
+repo_url: https://github.com/mouseland/suite2p
+
 theme:
   name: material
   logo: _static/favicon.ico

scripts/generate_test_data.py

@@ -1,21 +1,4 @@
 import os
-<<<<<<< HEAD
-import suite2p 
-import shutil
-import numpy as np 
-
-from pathlib import Path
-from conftest import initialize_ops, download_cached_inputs #Guarantees that tests and this script use the same ops
-from tests.regression.utils import FullPipelineTestUtils, DetectionTestUtils, ExtractionTestUtils
-from suite2p.extraction import masks
-
-"""
-IMPORTANT: When running this script, make sure to use it in the scripts directory 
-(e.g., suite2p/scripts). The generated test data will be placed in the directory 
-suite2p/scripts/test_data. Take the directories in this folder and replace the directories 
-with the same name in suite2p/data/test_data (e.g.,replace suite2p/data/test_data/1plane1chan1500 with suite2p/scripts/test_data/1plane1chan1500). 
-"""
-=======
 import suite2p
 import shutil
 import numpy as np
@@ -107,7 +90,6 @@ class TestDataConfigs:
         'baseline_methods': ['maximin', 'constant', 'constant_prctile'],
         'subdirs': ['1plane1chan', '2plane2chan']
     }
->>>>>>> suite2p_dev/tomerge
 
 current_dir = Path(os.getcwd())
 # Assumes the input file has already been downloaded

tests/regression/test_classification_pipeline.py

@@ -13,18 +13,10 @@ def get_stat_iscell(data_dir_path):
     return stat, expected_output
 
 
-<<<<<<< HEAD
-def test_classification_output(test_ops, data_dir):
-    """
-    Regression test that checks to see if the main_classify function works. Only checks iscell output.
-    """
-    test_ops['save_path'] = test_ops['save_path0']
-=======
 def test_classification_output(test_settings, data_dir):
     """
     Regression test that checks to see if the main_classify function works. Only checks iscell output.
     """
->>>>>>> suite2p_dev/tomerge
     stat, expected_output = get_stat_iscell(data_dir)
     iscell = classification.classify(stat, classfile=classification.builtin_classfile)
     assert np.allclose(iscell, expected_output, atol=1e-1)

tests/regression/test_detection_pipeline.py

@@ -1,75 +1,5 @@
 """
 Tests for the Suite2p Detection module.
-<<<<<<< HEAD
-"""
-from pathlib import Path
-import numpy as np
-import utils
-from suite2p import detection
-from suite2p.extraction import masks
-
-def detect_wrapper(ops):
-    """
-    Calls the main detect function and compares output dictionaries (cell_pix, cell_masks,
-    neuropil_masks, stat) with prior output dicts.
-    """
-    for i in range(len(ops)):
-        op = ops[i]
-        op['neuropil_extract'] = True
-        op, stat = detection.detect(ops=op)
-        output_check = np.load(
-            op['data_path'][0].parent.joinpath(f"test_outputs/detection/expected_detect_output_{ op['nplanes'] }p{ op['nchannels'] }c{ i }.npy"),
-            allow_pickle=True
-        )[()]
-        #assert np.array_equal(output_check['cell_pix'], cell_pix)
-        cell_masks, neuropil_masks = masks.create_masks(stat, op['Ly'], op['Lx'], ops=op)
-        assert all(np.allclose(a, b, rtol=1e-4, atol=5e-2) for a, b in zip(cell_masks, output_check['cell_masks']))
-        assert all(np.allclose(a, b, rtol=1e-4, atol=5e-2) for a, b in zip(neuropil_masks, output_check['neuropil_masks']))
-        for gt_dict, output_dict in zip(stat, output_check['stat']):
-            for k in gt_dict.keys():
-                if k=='ypix' or k=='xpix' or k=='lam':
-                    assert np.allclose(gt_dict[k], output_dict[k], rtol=1e-4, atol=5e-2)
-
-def test_detection_output_1plane1chan(test_ops):
-    test_ops.update({
-        'tiff_list': ['input.tif'],
-    })
-    ops = utils.DetectionTestUtils.prepare(
-        test_ops,
-        [[test_ops['data_path'][0].joinpath('detection/pre_registered.npy')]],
-        (404, 360)
-    )
-    detect_wrapper(ops)
-
-
-def test_detection_output_2plane2chan(test_ops):
-    test_ops.update({
-        'nchannels': 2,
-        'nplanes': 2,
-    })
-    detection_dir = test_ops['data_path'][0].joinpath('detection')
-    ops = utils.DetectionTestUtils.prepare(
-        test_ops,
-        [
-            [detection_dir.joinpath('pre_registered01.npy'), detection_dir.joinpath('pre_registered02.npy')],
-            [detection_dir.joinpath('pre_registered11.npy'), detection_dir.joinpath('pre_registered12.npy')]
-        ]
-        , (404, 360),
-    )
-    ops[0]['meanImg_chan2'] = np.load(detection_dir.joinpath('meanImg_chan2p0.npy'))
-    ops[1]['meanImg_chan2'] = np.load(detection_dir.joinpath('meanImg_chan2p1.npy'))
-    detect_wrapper(ops)
-    nplanes = test_ops['nplanes']
-
-    outputs_to_check = ['redcell']
-    # rely on the 2plane2chan1500's redcell
-    for i in range(nplanes):
-        assert all(utils.compare_list_of_outputs(
-            outputs_to_check,
-            utils.get_list_of_data(outputs_to_check, test_ops['data_path'][0].parent.joinpath(f"test_outputs/detection/suite2p/plane{i}")),
-            utils.get_list_of_data(outputs_to_check, Path(test_ops['save_path0']).joinpath(f"suite2p/plane{i}")),
-        ))
-=======
 Structured to match generate_test_data.py pattern for true regression testing.
 """
 from pathlib import Path
@@ -162,4 +92,3 @@ def test_detection_output_2plane2chan(test_settings):
             for k in gt_dict.keys():
                 if k in ['ypix', 'xpix', 'lam']:
                     assert np.allclose(gt_dict[k], output_dict[k], rtol=1e-4, atol=5e-2)
->>>>>>> suite2p_dev/tomerge

tests/regression/test_extraction_pipeline.py

@@ -1,54 +1,5 @@
 """
 Tests for the Suite2p Extraction module.
-<<<<<<< HEAD
-"""
-import numpy as np
-from suite2p import extraction
-from suite2p.io import BinaryFile
-
-from pathlib import Path
-import utils
-
-def extract_wrapper(ops):
-    for plane in range(ops[0]['nplanes']):
-        curr_op = ops[plane]
-        plane_dir = Path(curr_op['save_path0']).joinpath(f'suite2p/plane{plane}')
-        plane_dir.mkdir(exist_ok=True, parents=True)
-        extract_input = np.load(
-            curr_op['data_path'][0].parent.joinpath(
-                'test_outputs',
-                'detection',
-                'expected_detect_output_{0}p{1}c{2}.npy'.format(curr_op['nplanes'], curr_op['nchannels'], plane)),
-            allow_pickle=True
-        )[()]
-        #extraction.create_masks_and_extract(curr_op, extract_input['stat'])
-        stat, F, Fneu, F_chan2, Fneu_chan2 = extraction.create_masks_and_extract(
-            curr_op,
-            extract_input['stat'],
-            extract_input['cell_masks'],
-            extract_input['neuropil_masks']
-        )
-        dF = F - curr_op['neucoeff'] * Fneu
-        dF = extraction.preprocess(
-            F=dF,
-            baseline=curr_op['baseline'],
-            win_baseline=curr_op['win_baseline'],
-            sig_baseline=curr_op['sig_baseline'],
-            fs=curr_op['fs'],
-            prctile_baseline=curr_op['prctile_baseline']
-        )
-        spks = extraction.oasis(F=dF, batch_size=curr_op['batch_size'], tau=curr_op['tau'], fs=curr_op['fs'])
-        np.save(plane_dir.joinpath('ops.npy'), curr_op)
-        np.save(plane_dir.joinpath('stat.npy'), stat)
-        np.save(plane_dir.joinpath('F.npy'), F)
-        np.save(plane_dir.joinpath('Fneu.npy'), Fneu)
-        np.save(plane_dir.joinpath('F_chan2.npy'), F_chan2)
-        np.save(plane_dir.joinpath('Fneu_chan2.npy'), Fneu_chan2)
-        np.save(plane_dir.joinpath('spks.npy'), spks)
-
-
-def run_preprocess(f: np.ndarray, test_ops):
-=======
 Structured to match generate_test_data.py pattern for true regression testing.
 """
 import numpy as np
@@ -71,74 +22,11 @@ def test_pre_process_baseline(test_settings):
     f = np.load(op['data_path'][0].parent.joinpath('test_outputs/1plane1chan1500/suite2p/plane0/F.npy'))
 
     # Test all baseline methods like generate_test_data.py
->>>>>>> suite2p_dev/tomerge
     baseline_vals = ['maximin', 'constant', 'constant_prctile']
     for bv in baseline_vals:
         pre_f = extraction.preprocess(
             F=f,
             baseline=bv,
-<<<<<<< HEAD
-            win_baseline=test_ops['win_baseline'],
-            sig_baseline=test_ops['sig_baseline'],
-            fs=test_ops['fs'],
-            prctile_baseline=test_ops['prctile_baseline']
-        )
-        test_f = np.load(test_ops['data_path'][0].parent.joinpath('test_outputs/extraction/{}_f.npy'.format(bv)))
-        yield np.allclose(pre_f, test_f, rtol=1e-4, atol=5e-2)
-
-
-def test_pre_process_baseline(test_ops):
-    f = np.load(test_ops['data_path'][0].parent.joinpath('test_outputs/1plane1chan1500/suite2p/plane0/F.npy'))
-    assert all(run_preprocess(f, test_ops))
-
-
-def test_extraction_output_1plane1chan(test_ops):
-    test_ops.update({
-        'tiff_list': ['input.tif'],
-    })
-    ops = utils.ExtractionTestUtils.prepare(
-        test_ops,
-        [[test_ops['data_path'][0].joinpath('detection/pre_registered.npy')]],
-        (404, 360)
-    )
-    extract_wrapper(ops)
-    ops = ops[0]
-    nplanes = ops['nplanes']
-    outputs_to_check = ['F', 'Fneu', 'stat', 'spks']
-    for i in range(nplanes):
-        assert all(utils.compare_list_of_outputs(
-            outputs_to_check,
-            utils.get_list_of_data(outputs_to_check, Path(ops['data_path'][0]).parent.joinpath(f"test_outputs/extraction/1plane1chan/plane0")),
-            utils.get_list_of_data(outputs_to_check, Path(ops['save_path0']).joinpath(f"suite2p/plane0")),
-        ))
-
-
-def test_extraction_output_2plane2chan(test_ops):
-    test_ops.update({
-        'nchannels': 2,
-        'nplanes': 2,
-        'tiff_list': ['input.tif'],
-    })
-    detection_dir = test_ops['data_path'][0].joinpath('detection')
-    ops = utils.ExtractionTestUtils.prepare(
-        test_ops,
-        [
-            [detection_dir.joinpath('pre_registered01.npy'), detection_dir.joinpath('pre_registered02.npy')],
-            [detection_dir.joinpath('pre_registered11.npy'), detection_dir.joinpath('pre_registered12.npy')]
-        ]
-        , (404, 360),
-    )
-    ops[0]['meanImg_chan2'] = np.load(detection_dir.joinpath('meanImg_chan2p0.npy'))
-    ops[1]['meanImg_chan2'] = np.load(detection_dir.joinpath('meanImg_chan2p1.npy'))
-    extract_wrapper(ops)
-    outputs_to_check = ['F', 'Fneu', 'F_chan2', 'Fneu_chan2', 'stat', 'spks']
-    for i in range(len(ops)):
-        assert all(utils.compare_list_of_outputs(
-            outputs_to_check,
-            utils.get_list_of_data(outputs_to_check, Path(ops[i]['data_path'][0].parent.joinpath(f"test_outputs/extraction/2plane2chan/plane{i}"))),
-            utils.get_list_of_data(outputs_to_check, Path(ops[i]['save_path0']).joinpath(f"suite2p/plane{i}")),
-        ))
-=======
             win_baseline=op['dcnv_preprocess']['win_baseline'],
             sig_baseline=op['dcnv_preprocess']['sig_baseline'],
             fs=op['fs'],
@@ -266,4 +154,3 @@ def test_extraction_output_2plane2chan(test_settings):
         assert np.allclose(F_chan2, expected_F_chan2, rtol=1e-4, atol=5e-2)
         assert np.allclose(Fneu_chan2, expected_Fneu_chan2, rtol=1e-4, atol=5e-2)
         assert np.allclose(spks, expected_spks, rtol=1e-4, atol=5e-2)
->>>>>>> suite2p_dev/tomerge