Skip to content

Commit 73b261f

Browse files
ch99lch99l
authored
update bambu version and github workflows (#38) (#40)
1 parent 6e11291 commit 73b261f

11 files changed

Lines changed: 130 additions & 47 deletions

File tree

‎.github/workflows/build_container.yml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
name: Build R container
2+
3+
# The image tag is read from conf/containers.config, so that single line is both what the
4+
# pipeline pulls and what this workflow publishes -- the two cannot drift.
5+
# Runs on pushes to a feature branch, whenever the container changes.
6+
7+
on:
8+
push:
9+
paths:
10+
- 'containers/r/**'
11+
- 'conf/containers.config'
12+
- '.github/workflows/build_container.yml'
13+
workflow_dispatch:
14+
inputs:
15+
force:
16+
description: 'Rebuild and overwrite the tag even on main/devel'
17+
type: boolean
18+
default: false
19+
20+
concurrency:
21+
group: ${{ github.workflow }}-${{ github.ref }}
22+
cancel-in-progress: true
23+
24+
jobs:
25+
build:
26+
name: Build and publish
27+
runs-on: ubuntu-latest
28+
permissions:
29+
contents: read
30+
packages: write
31+
32+
steps:
33+
- uses: actions/checkout@v4
34+
35+
- name: Resolve image from conf/containers.config
36+
id: image
37+
run: |
38+
IMAGE=$(sed -n "s|.*withLabel: *'r'.*container *= *\"\([^\"]*\)\".*|\1|p" conf/containers.config)
39+
if [ -z "$IMAGE" ]; then
40+
echo "::error::could not parse the 'r' container from conf/containers.config"
41+
exit 1
42+
fi
43+
case "$IMAGE" in
44+
ghcr.io/goekelab/*) ;;
45+
*) echo "::error::refusing to push outside ghcr.io/goekelab: $IMAGE"; exit 1 ;;
46+
esac
47+
echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
48+
echo "Resolved container: $IMAGE"
49+
50+
- name: Refuse to overwrite a tag main or devel is pinned to
51+
if: github.ref_name != 'main' && github.ref_name != 'devel'
52+
run: |
53+
IMAGE="${{ steps.image.outputs.image }}"
54+
for BASE in main devel; do
55+
git fetch --no-tags --depth=1 origin "$BASE" 2>/dev/null || continue
56+
PINNED=$(git show "FETCH_HEAD:conf/containers.config" 2>/dev/null \
57+
| sed -n "s|.*withLabel: *'r'.*container *= *\"\([^\"]*\)\".*|\1|p")
58+
if [ "$IMAGE" = "$PINNED" ]; then
59+
echo "::error::$BASE is pinned to $IMAGE -- bump the tag in conf/containers.config before rebuilding it"
60+
exit 1
61+
fi
62+
done
63+
64+
- uses: docker/login-action@v3
65+
with:
66+
registry: ghcr.io
67+
username: ${{ github.actor }}
68+
password: ${{ secrets.GITHUB_TOKEN }}
69+
70+
- name: Decide whether to build
71+
id: decide
72+
run: |
73+
IMAGE="${{ steps.image.outputs.image }}"
74+
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then EXISTS=true; else EXISTS=false; fi
75+
76+
if [ "${{ inputs.force }}" = "true" ]; then
77+
BUILD=true
78+
elif [ "${{ github.ref_name }}" = "main" ] || [ "${{ github.ref_name }}" = "devel" ]; then
79+
# Released tags are immutable: the merge that brought the bump in has already
80+
# published it from the feature branch, so there is nothing left to do
81+
[ "$EXISTS" = "false" ] && BUILD=true || BUILD=false
82+
else
83+
# Feature branch: the tag is unreleased (guarded above), so rebuild freely
84+
BUILD=true
85+
fi
86+
87+
echo "build=$BUILD" >> "$GITHUB_OUTPUT"
88+
if [ "$BUILD" = "false" ]; then
89+
echo "::notice::$IMAGE is already published -- skipping build"
90+
fi
91+
92+
- uses: docker/setup-buildx-action@v3
93+
if: steps.decide.outputs.build == 'true'
94+
95+
- uses: docker/build-push-action@v6
96+
if: steps.decide.outputs.build == 'true'
97+
with:
98+
context: containers/r
99+
push: true
100+
tags: ${{ steps.image.outputs.image }}
101+
cache-from: type=gha
102+
cache-to: type=gha,mode=max

‎.github/workflows/smoke_test.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
name: Smoke Tests
22

33
on:
4-
push:
5-
branches: [main, devel]
64
pull_request:
75
branches: [main, devel]
6+
workflow_dispatch:
7+
8+
concurrency:
9+
group: ${{ github.workflow }}-${{ github.ref }}
10+
cancel-in-progress: true
811

912
jobs:
1013
smoke_test:

‎conf/containers.config‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
process {
2-
withLabel: 'r' { container = "ghcr.io/ch99l/bambu-pipe-r:latest" }
2+
withLabel: 'r' { container = "ghcr.io/goekelab/bambu-pipe-r:1.0.0" }
33
withLabel: 'spaceranger' { container = "quay.io/nf-core/spaceranger:9c5e7dc93c32448e" }
44
withLabel: 'minimap2_samtools' { container = "community.wave.seqera.io/library/minimap2_samtools:b09096fc890429ce" }
55
withLabel: 'preprocess' { container = "community.wave.seqera.io/library/chopper_cutadapt_flexiplex_pigz:077c3bc67452482c" }

‎containers/r/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ RUN apt-get update && apt-get install -y libcurl4-openssl-dev procps git \
77
ARG MAMBA_DOCKERFILE_ACTIVATE=1
88

99
# Install bioconductor version of bambu for its dependencies
10+
# TODO: data.table 1.17.8 installation is a temp fix for clustered_EM
1011
RUN micromamba install -y -n base -c conda-forge -c bioconda \
1112
r-base=4.5.3 \
1213
r-devtools=2.5.2 \
@@ -15,11 +16,12 @@ RUN micromamba install -y -n base -c conda-forge -c bioconda \
1516
bioconductor-dropletutils=1.30.0 \
1617
r-seurat=5.4.0 \
1718
r-harmony=2.0.2 \
19+
r-data.table=1.17.8 \
1820
gxx=15.2.0 \
1921
&& micromamba clean --all --yes
2022

2123
# Clone bambu single cell feature branch
22-
RUN git clone --branch exportNewFunctions \
24+
RUN git clone --branch devel_pre_v4 \
2325
https://github.com/GoekeLab/bambu.git /opt/bambu \
2426
&& R -e 'devtools::install("/opt/bambu")'
2527

‎main.nf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ workflow {
8888

8989
if (params.quantification_mode != 'no_quant') {
9090
if (params.quantification_mode == 'EM_clusters') {
91-
CLUSTERING(BAMBU_TRANSCRIPT_DISCOVERY.out.se_gene_counts, BAMBU_TRANSCRIPT_DISCOVERY.out.sample_names, ch_n_samples)
91+
CLUSTERING(BAMBU_TRANSCRIPT_DISCOVERY.out.se_gene_counts, ch_n_samples)
9292
ch_clusters = CLUSTERING.out.clusters.map { clusters -> [true, clusters] } // flag to indicate that clustering was performed
9393
} else {
9494
ch_clusters = channel.value([false, []]) // flag to indicate that clustering was not performed

‎modules/bambu/EM_quant.nf‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,11 @@ process BAMBU_EM{
3232
degBias <- !is.null(clusters)
3333
3434
se <- bambu.singlecell(
35-
reads = NULL,
35+
reads = quantData,
36+
output = if (is.null(clusters)) "EM" else "clusteredEM",
3637
annotations = extendedAnno,
3738
genome = "$genome",
38-
quantData = quantData,
39-
assignDist = FALSE,
4039
ncore = $task.cpus,
41-
discovery = FALSE,
42-
quant = TRUE,
4340
verbose = FALSE,
4441
opt.em = list(degradationBias = degBias),
4542
clusters = clusters

‎modules/bambu/construct_read_class.nf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ process BAMBU_CONSTRUCT_READ_CLASS{
2222
annotation <- readRDS("$bambu_annotation")
2323
# Rename BAM to val(sample) so bambu derives sampleName from val(sample), regardless of the original filename
2424
file.symlink("$bam", "${sample}.bam")
25-
readClassFile <- bambu.singlecell(reads = "${sample}.bam", annotations = annotation, genome = "$genome",
26-
ncore = $task.cpus, discovery = FALSE, quant = FALSE, verbose = FALSE, assignDist = FALSE,
25+
readClassFile <- bambu.singlecell(reads = "${sample}.bam", output = "readClasses",
26+
annotations = annotation, genome = "$genome", ncore = $task.cpus, verbose = FALSE,
2727
processByChromosome = as.logical("$params.process_by_chromosome"), yieldSize = 10000000)
2828
saveRDS(readClassFile[[1]], "${sample}_read_class.rds")
2929

‎modules/bambu/transcript_discovery.nf‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ process BAMBU_TRANSCRIPT_DISCOVERY{
2020
path ('gene_counts/se_gene_counts.rds'), emit: se_gene_counts
2121
path ('extended_annotations.rds'), emit: extended_annotations
2222
path ('extended_annotations.gtf')
23-
path ('sample_names.rds'), emit: sample_names
2423
path ('unique_counts')
2524
path ('gene_counts')
2625
path "versions.yml", topic: 'versions'
@@ -33,26 +32,26 @@ process BAMBU_TRANSCRIPT_DISCOVERY{
3332
source(Sys.which("save_counts.R"))
3433
3534
annotation <- readRDS("$bambu_annotation")
36-
readClassFile <- strsplit("${rds_files.join(',')}", ",")[[1]]
3735
sampleNames <- strsplit("${sample.join(',')}", ",")[[1]]
36+
readClassFile <- setNames(strsplit("${rds_files.join(',')}", ",")[[1]], sampleNames)
3837
sampleData <- strsplit("${spatial_metadata_files.join(',')}", ",")[[1]]
3938
chemistry <- setNames(strsplit("${meta.collect { m -> m.chemistry }.join(',')}", ",")[[1]], sampleNames)
4039
technology <- setNames(strsplit("${meta.collect { m -> m.technology }.join(',')}", ",")[[1]], sampleNames)
4140
4241
# Transcript discovery
43-
extendedAnno <- bambu.singlecell(reads = readClassFile, annotations = annotation, genome = "$genome", ncore = $task.cpus,
44-
discovery = TRUE, quant = FALSE, verbose = FALSE, assignDist = FALSE, NDR = $ndr)
42+
extendedAnno <- bambu.singlecell(reads = readClassFile, output = "extendedAnnotations",
43+
annotations = annotation, genome = "$genome", ncore = $task.cpus, verbose = FALSE, NDR = $ndr)
4544
saveRDS(extendedAnno, "extended_annotations.rds")
4645
writeToGTF(extendedAnno, "extended_annotations.gtf")
4746
48-
# Quantification without EM
47+
# Read to transcript assignment
4948
sampleData <- if (any(startsWith(chemistry, "visium-v"))) sampleData else NULL # Add spatial metadata for visium samples
50-
quantData <- bambu.singlecell(reads = readClassFile, annotations = extendedAnno, genome = "$genome", ncore = $task.cpus,
51-
discovery = FALSE, quant = FALSE, verbose = FALSE, opt.em = list(degradationBias = FALSE), assignDist = TRUE, sampleData = sampleData)
49+
quantData <- bambu.singlecell(reads = readClassFile, output = "quantData",
50+
annotations = extendedAnno, genome = "$genome", ncore = $task.cpus, verbose = FALSE, sampleData = sampleData)
5251
saveRDS(quantData, "quant_data.rds")
5352
54-
# Generate unique counts SE from quantData
55-
seDiscovery <- generateUniqueCountsSEFromQuantData(quantData, extendedAnno)
53+
# Quantification without EM
54+
seDiscovery <- bambu.singlecell(reads = quantData, output = "uniqueCounts", annotations = extendedAnno)
5655
colData(seDiscovery)\$chemistry <- unname(chemistry[colData(seDiscovery)\$sampleName]) # Add chemistry into colData (for subsequent batch correction)
5756
colData(seDiscovery)\$technology <- unname(technology[colData(seDiscovery)\$sampleName]) # Add technology into colData (for subsequent batch correction)
5857
save_counts(seDiscovery, "unique_counts", "Transcript Expression")
@@ -61,9 +60,6 @@ process BAMBU_TRANSCRIPT_DISCOVERY{
6160
seDiscovery.gene <- transcriptToGeneExpression(seDiscovery)
6261
save_counts(seDiscovery.gene, "gene_counts")
6362
64-
# Save sampleNames (required for multi-sample Seurat clustering)
65-
saveRDS(sampleNames, "sample_names.rds")
66-
6763
writeLines(c('"${task.process}":', paste0(' R: ', R.Version()\$version.string), paste0(' bambu: ', as.character(packageVersion("bambu")))), "versions.yml")
6864
"""
6965
}

‎modules/seurat/multi_sample.nf‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ process SEURAT_MULTI_SAMPLE {
88

99
input:
1010
path(se)
11-
path(sample_names)
1211

1312
output:
1413
path ('clusters.rds'), emit: clusters
@@ -19,7 +18,6 @@ process SEURAT_MULTI_SAMPLE {
1918
"""
2019
#!/usr/bin/env Rscript
2120
library(SummarizedExperiment)
22-
library(IRanges)
2321
library(Seurat)
2422
2523
# Extract gene count matrix and colData metadata
@@ -55,24 +53,13 @@ process SEURAT_MULTI_SAMPLE {
5553
cellMix <- FindClusters(cellMix, resolution = $params.resolution, cluster.name = "harmony_clusters")
5654
saveRDS(cellMix, "seurat_obj.rds")
5755
58-
# Build ordered list of CompressedCharacterLists, one per sample, in quantData order
59-
allBarcodes <- names(cellMix\$harmony_clusters)
60-
clusterLabels <- paste0("cluster", as.character(cellMix\$harmony_clusters))
61-
sampleNames <- readRDS("$sample_names") # sampleNames contain the order of the samples in quantData
62-
63-
clusters <- setNames(lapply(sampleNames, function(s) {
64-
idx <- which(cellMix\$sample == s)
65-
sampleBarcode <- allBarcodes[idx]
66-
clusterLabel <- paste0(s, "_", clusterLabels[idx])
67-
splitAsList(sampleBarcode, clusterLabel)
68-
}), sampleNames)
56+
clusters <- setNames(paste0("cluster_", cellMix\$harmony_clusters), names(cellMix\$harmony_clusters))
6957
saveRDS(clusters, "clusters.rds")
7058
7159
writeLines(c(
7260
'"${task.process}":',
7361
paste0(' R: ', R.Version()\$version.string),
7462
paste0(' seurat: ', as.character(packageVersion("Seurat"))),
75-
paste0(' IRanges: ', as.character(packageVersion("IRanges"))),
7663
paste0(' SummarizedExperiment: ', as.character(packageVersion("SummarizedExperiment")))
7764
), "versions.yml")
7865
"""

‎modules/seurat/single_sample.nf‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ process SEURAT_SINGLE_SAMPLE {
1818
"""
1919
#!/usr/bin/env Rscript
2020
library(SummarizedExperiment)
21-
library(IRanges)
2221
library(Seurat)
2322
2423
se <- readRDS("$se")
@@ -34,19 +33,17 @@ process SEURAT_SINGLE_SAMPLE {
3433
cellMix <- RunPCA(cellMix, features = VariableFeatures(object = cellMix), npcs = npcs)
3534
dim <- ifelse(dim >= dim(cellMix@reductions\$pca)[2], dim(cellMix@reductions\$pca)[2], dim)
3635
cellMix <- FindNeighbors(cellMix, dims = 1:dim)
37-
cellMix <- FindClusters(cellMix, resolution = $params.resolution)
36+
cellMix <- FindClusters(cellMix, resolution = $params.resolution, cluster.name = "clusters")
37+
3838
saveRDS(cellMix, "seurat_obj.rds")
3939
40-
x <- setNames(names(cellMix@active.ident), cellMix@active.ident)
41-
clusters <- list(splitAsList(unname(x), paste0("cluster", names(x))))
42-
43-
saveRDS(cellMix, "cell_mix.rds")
40+
clusters <- setNames(paste0("cluster_", cellMix\$clusters), names(cellMix\$clusters))
4441
saveRDS(clusters, "clusters.rds")
42+
4543
writeLines(c(
4644
'"${task.process}":',
4745
paste0(' R: ', R.Version()\$version.string),
4846
paste0(' seurat: ', as.character(packageVersion("Seurat"))),
49-
paste0(' IRanges: ', as.character(packageVersion("IRanges"))),
5047
paste0(' SummarizedExperiment: ', as.character(packageVersion("SummarizedExperiment")))
5148
), "versions.yml")
5249
"""

0 commit comments

Comments
 (0)