Add random_labels negative control (#22)

* add random_labels control method

* seed np.random in process_dataset

* write metric_values as a list

* update changelog

Author: rcannood

CHANGELOG.md

@@ -14,6 +14,8 @@
 
 * Added `metrics/accuracy` component (PR #5).
 
+* Added `control_methods/random_labels` component (PR #22).
+
 ## MAJOR CHANGES
 
 * Updated `api` files (PR #5).
@@ -43,3 +45,7 @@
 
 ## BUGFIXES
 
+* `process_dataset`: also seed `np.random` when `--seed` is set (PR #22).
+
+* `accuracy`: write `metric_values` as a list to match `metric_ids` (PR #22).
+

src/control_methods/random_labels/config.vsh.yaml

@@ -0,0 +1,45 @@
+# The API specifies which type of component this is.
+# It contains specifications for:
+#   - The input/output files
+#   - Common parameters
+#   - A unit test
+__merge__: ../../api/comp_control_method.yaml
+
+# A unique identifier for your component (required).
+# Can contain only lowercase letters or underscores.
+name: random_labels
+
+# A relatively short label, used when rendering visualisations (required)
+label: Random Labels
+# A one sentence summary of how this method works (required). Used when
+# rendering summary tables.
+summary: "a negative control, where the labels are randomly predicted."
+# A multi-line description of how this component works (required). Used
+# when rendering reference documentation.
+description: |
+  A negative control, where the labels are randomly predicted based on the
+  label distribution of the training data, without looking at the input data.
+
+# Metadata for your component
+info:
+  # Which normalisation method this component prefers to use (required).
+  preferred_normalization: counts
+
+# Resources required to run the component
+resources:
+  # The script of your component (required)
+  - type: python_script
+    path: script.py
+
+engines:
+  # Specifications for the Docker image for this component.
+  - type: docker
+    image: openproblems/base_python:1
+
+runners:
+  # This platform allows running the component natively
+  - type: executable
+  # Allows turning the component into a Nextflow module / pipeline.
+  - type: nextflow
+    directives:
+      label: [midtime, lowmem, lowcpu]

src/control_methods/random_labels/script.py

@@ -0,0 +1,47 @@
+import anndata as ad
+import numpy as np
+
+## VIASH START
+# Note: this section is auto-generated by viash at runtime. To edit it, make changes
+# in config.vsh.yaml and then run `viash config inject config.vsh.yaml`.
+par = {
+  'input_train': 'resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad',
+  'input_test': 'resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad',
+  'input_solution': 'resources_test/task_template/cxg_mouse_pancreas_atlas/solution.h5ad',
+  'output': 'output.h5ad'
+}
+meta = {
+  'name': 'random_labels'
+}
+## VIASH END
+
+print('Reading input files', flush=True)
+input_train = ad.read_h5ad(par['input_train'])
+input_test = ad.read_h5ad(par['input_test'])
+
+print('Compute label distribution', flush=True)
+label_distribution = input_train.obs["label"].value_counts()
+label_distribution = label_distribution / label_distribution.sum()
+
+print('Generate predictions', flush=True)
+obs_label_pred = np.random.choice(
+  label_distribution.index,
+  size=input_test.n_obs,
+  replace=True,
+  p=label_distribution
+)
+
+print("Write output AnnData to file", flush=True)
+output = ad.AnnData(
+  uns={
+    'dataset_id': input_train.uns['dataset_id'],
+    'normalization_id': input_train.uns['normalization_id'],
+    'method_id': meta['name']
+  },
+  obs={
+    'label_pred': obs_label_pred
+  }
+)
+output.obs_names = input_test.obs_names
+
+output.write_h5ad(par['output'], compression='gzip')

src/data_processors/process_dataset/script.py

@@ -28,9 +28,10 @@
 config = op.project.read_viash_config(meta["config"])
 
 # set seed if need be
-if par["seed"]:
+if par["seed"] is not None:
     print(f">> Setting seed to {par['seed']}")
     random.seed(par["seed"])
+    np.random.seed(par["seed"])
 
 print(">> Load data", flush=True)
 adata = ad.read_h5ad(par["input"])

src/metrics/accuracy/script.py

@@ -32,7 +32,7 @@
 # metric_ids and metric_values can have length > 1
 # but should be of equal length
 uns_metric_ids = [ 'accuracy' ]
-uns_metric_values = np.mean(input_solution.obs["label"] == input_prediction.obs["label_pred"])
+uns_metric_values = [ np.mean(input_solution.obs["label"] == input_prediction.obs["label_pred"]) ]
 
 print("Write output AnnData to file", flush=True)
 output = ad.AnnData(

src/workflows/run_benchmark/config.vsh.yaml

@@ -96,6 +96,7 @@ dependencies:
   - name: utils/extract_uns_metadata
     repository: openproblems
   - name: control_methods/true_labels
+  - name: control_methods/random_labels
   - name: methods/logistic_regression
   - name: metrics/accuracy
 

src/workflows/run_benchmark/main.nf

@@ -10,6 +10,7 @@ workflow auto {
 // construct list of methods and control methods
 methods = [
   true_labels,
+  random_labels,
   logistic_regression
 ]