data processor builds

Author: seohyonkim

src/api/comp_control_method.yaml

@@ -29,5 +29,5 @@ test_resources:
     path: /common/component_tests/run_and_check_output.py
   - type: python_script
     path: /common/component_tests/check_config.py
-  - path: /resources_test/task_template/cxg_mouse_pancreas_atlas
-    dest: resources_test/task_template/cxg_mouse_pancreas_atlas
+  - path: /resources_test/common/cxg_mouse_pancreas_atlas
+    dest: resources_test/common/cxg_mouse_pancreas_atlas

src/api/comp_data_processor.yaml

@@ -11,7 +11,7 @@ arguments:
     __merge__: file_common_dataset.yaml
     direction: input
     required: true
-  - name: "--output"
+  - name: "--output_dataset"
     __merge__: file_dataset.yaml
     direction: output
     required: true

src/api/comp_method.yaml

@@ -20,5 +20,5 @@ test_resources:
     path: /common/component_tests/run_and_check_output.py
   - type: python_script
     path: /common/component_tests/check_config.py
-  - path: /resources_test/task_template/cxg_mouse_pancreas_atlas
-    dest: resources_test/task_template/cxg_mouse_pancreas_atlas
+  - path: /resources_test/common/cxg_mouse_pancreas_atlas
+    dest: resources_test/common/cxg_mouse_pancreas_atlas

src/api/comp_metric.yaml

@@ -24,5 +24,5 @@ test_resources:
     path: /common/component_tests/run_and_check_output.py
   - type: python_script
     path: /common/component_tests/check_config.py
-  - path: /resources_test/task_template/cxg_mouse_pancreas_atlas
-    dest: resources_test/task_template/cxg_mouse_pancreas_atlas
+  - path: /resources_test/common/cxg_mouse_pancreas_atlas
+    dest: resources_test/common/cxg_mouse_pancreas_atlas

src/api/file_dataset.yaml

@@ -1,42 +1,28 @@
-#TODO: Change to the required and/or optional fields of the anndata
 type: file
-example: "resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad" # maybe adjust this after syncing resources
+example: "resources_test/task_spatial_trajectory_inference/cxg_mouse_pancreas_atlas/dataset.h5ad"
 label: "Input data"
-summary: "The input data for the spatial TI task"
+summary: "The input data for the spatial TI methods"
 info:
   format:
     type: h5ad
-    layers: 
+    layers:
       - type: integer
         name: counts
         description: Raw counts
         required: true
-      - type: double
-        name: normalized
-        description: Normalized counts
-        required: true
     obs:
       - type: string
-        name: label
-        description: Ground truth cell type labels
+        name: cell_type
+        description: Cell type information
         required: true
       - type: string
         name: batch
         description: Batch information
         required: true
-    var:
-      - type: boolean
-        name: hvg
-        description: Whether or not the feature is considered to be a 'highly variable gene'
-        required: true
-      - type: double
-        name: hvg_score
-        description: A ranking of the features by hvg.
-        required: true
     obsm:
       - type: double
-        name: X_pca
-        description: The resulting PCA embedding.
+        name: X_spatial
+        description: Spatial coordinates of each cell/spot.
         required: true
     uns:
       - type: string
@@ -46,4 +32,4 @@ info:
       - type: string
         name: normalization_id
         description: "Which normalization was used"
-        required: true
+        required: true

src/api/file_solution.yaml

@@ -1,42 +1,32 @@
-#TODO: Change to the required and/or optional fields of the anndata
 type: file
-example: "resources_test/task_template/cxg_mouse_pancreas_atlas/solution.h5ad"
+example: "resources_test/task_spatial_trajectory_inference/cxg_mouse_pancreas_atlas/solution.h5ad"
 label: "Solution"
-summary: "The solution for the test data"
+summary: "The solution (ground truth) for the test data"
 info:
   format:
     type: h5ad
-    layers: 
+    layers:
       - type: integer
         name: counts
         description: Raw counts
         required: true
-      - type: double
-        name: normalized
-        description: Normalized counts
-        required: true
     obs:
       - type: string
-        name: label
-        description: Ground truth cell type labels
+        name: cell_type
+        description: Cell type information
         required: true
       - type: string
         name: batch
         description: Batch information
         required: true
-    var:
-      - type: boolean
-        name: hvg
-        description: Whether or not the feature is considered to be a 'highly variable gene'
-        required: true
       - type: double
-        name: hvg_score
-        description: A ranking of the features by hvg.
+        name: pseudotime_true
+        description: Ground truth pseudotime for each cell/spot.
         required: true
     obsm:
       - type: double
-        name: X_pca
-        description: The resulting PCA embedding.
+        name: X_spatial
+        description: Spatial coordinates of each cell/spot.
         required: true
     uns:
       - type: string

src/data_processors/process_dataset/config.vsh.yaml

@@ -1,23 +1,43 @@
 __merge__: ../../api/comp_data_processor.yaml
 name: process_dataset
-arguments:
-  - name: "--method"
-    type: "string"
-    description: "The process method to assign train/test."
-    choices: ["batch", "random"]
-    default: "batch"
-  - name: "--obs_label"
-    type: "string"
-    description: "Which .obs slot to use as label."
-    default: "cell_type"
-  - name: "--obs_batch"
-    type: "string"
-    description: "Which .obs slot to use as batch covariate."
-    default: "batch"
-  - name: "--seed"
-    type: "integer"
-    description: "A seed for the subsampling."
-    example: 123
+namespace: data_processors
+
+info:
+  label: Process dataset
+  summary: "Filter and split a common dataset into task-specific data and solution files."
+
+argument_groups:
+  - name: Parameters
+    arguments:
+      - name: "--dataset_id"
+        type: "string"
+        description: "New dataset ID"
+        required: false
+      - name: "--obs_label"
+        type: "string"
+        description: "Which .obs slot to use as label."
+        default: "cell_type"
+      - name: "--obs_batch"
+        type: "string"
+        description: "Which .obs slot to use as batch covariate."
+        default: "batch"
+      - name: "--obs_ptime"
+        type: "string"
+        description: "Which .obs slot to use as pseudotime covariate."
+        default: "pseudotime_true"
+      - name: "--obsm_spatial"
+        type: "string"
+        description: "Which .obsm slot to use as spatial metadata."
+        default: "X_spatial"
+      - name: "--layer_counts"
+        type: "string"
+        description: "Which .layers slot to use as counts."
+        default: "counts"
+      - name: "--seed"
+        type: "integer"
+        description: "The seed for." # TODO for what..?
+        default: 1
+
 resources:
   - type: python_script
     path: script.py
@@ -31,4 +51,8 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [highmem, midcpu, midtime]
+      label: [highmem, midcpu, midtime]
+
+test_resources:
+  - path: /resources_test/common/cxg_mouse_pancreas_atlas
+    dest: resources_test/common/cxg_mouse_pancreas_atlas

src/data_processors/process_dataset/script.py

@@ -7,12 +7,13 @@
 ## VIASH START
 par = {
     'input': 'resources_test/common/cxg_mouse_pancreas_atlas/dataset.h5ad',
-    'method': 'batch',
     'seed': None,
     'obs_batch': 'batch',
     'obs_label': 'cell_type',
-    'output_train': 'train.h5ad',
-    'output_test': 'test.h5ad',
+    'obs_ptime': 'pseudotime_true',
+    'obsm_spatial': 'X_spatial',
+    'layer_counts': 'counts',
+    'output_dataset': 'dataset.h5ad',
     'output_solution': 'solution.h5ad'
 }
 meta = {
@@ -25,62 +26,51 @@
 sys.path.append(meta['resources_dir'])
 from subset_h5ad_by_format import subset_h5ad_by_format
 
+# read viash config
 config = op.project.read_viash_config(meta["config"])
 
 # set seed if need be
 if par["seed"]:
-    print(f">> Setting seed to {par['seed']}")
+    print(f">> Setting seed to {par['seed']}", flush=True)
     random.seed(par["seed"])
 
+# read the dataset
 print(">> Load data", flush=True)
-adata = ad.read_h5ad(par["input"])
-print("input:", adata)
+input = ad.read_h5ad(par['input'])
+print("input:", input, flush=True)
 
-print(f">> Process data using {par['method']} method")
-if par["method"] == "batch":
-    batch_info = adata.obs[par["obs_batch"]]
-    batch_categories = batch_info.dtype.categories
-    test_batches = random.sample(list(batch_categories), 1)
-    is_test = [ x in test_batches for x in batch_info ]
-elif par["method"] == "random":
-    train_ix = np.random.choice(adata.n_obs, round(adata.n_obs * 0.8), replace=False)
-    is_test = [ not x in train_ix for x in range(0, adata.n_obs) ]
-
-# subset the different adatas
-print(">> Figuring which data needs to be copied to which output file", flush=True)
-# use par arguments to look for label and batch value in different slots
+# map the source slots of the common dataset onto the dest slots expected by the
+# task-specific file formats (file_dataset.yaml / file_solution.yaml)
 slot_mapping = {
+    "layers": {
+        "counts": par["layer_counts"],
+    },
     "obs": {
-        "label": par["obs_label"],
+        "cell_type": par["obs_label"],
         "batch": par["obs_batch"],
-    }
+        "pseudotime_true": par["obs_ptime"],
+    },
+    "obsm": {
+        "X_spatial": par["obsm_spatial"],
+    },
 }
 
-print(">> Creating train data", flush=True)
-output_train = subset_h5ad_by_format(
-    adata[[not x for x in is_test]],
-    config,
-    "output_train",
-    slot_mapping
-)
-
-print(">> Creating test data", flush=True)
-output_test = subset_h5ad_by_format(
-    adata[is_test],
+print(">> Creating input data for the methods", flush=True)
+output_dataset = subset_h5ad_by_format(
+    input,
     config,
-    "output_test",
+    "output_dataset",
     slot_mapping
 )
 
 print(">> Creating solution data", flush=True)
 output_solution = subset_h5ad_by_format(
-    adata[is_test],
+    input,
     config,
     "output_solution",
     slot_mapping
 )
 
 print(">> Writing data", flush=True)
-output_train.write_h5ad(par["output_train"])
-output_test.write_h5ad(par["output_test"])
+output_dataset.write_h5ad(par["output_dataset"])
 output_solution.write_h5ad(par["output_solution"])