split/metaanalysis_split_instructions.txt at master · nikbaya/split · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
# Meta-analysis/split pipeline cloud submission instructions

#–––––––––––– Preprocessing: ––––––––––––
# Submit a job with the three preprocess function calls at the bottom of the script uncommented.
# If the job gets stuck, try submitting jobs with only one preprocess function call uncommented at a time.
# If jobs with only one uncommented preprocess function call still don't work, try increasing the number of workers (may need to go up to 100 workers).
# Below I show how I would submit jobs with only one preprocess function call uncommented.

#–––––– preprocess1 (only uncomment preprocess1 at the bottom of the script meta_split.py) ––––––

cd ${directory containing meta_split.py}
cluster start ${clustername} --num-workers 20 --num-preemptible-workers 100 --max-idle 10m
cluster submit ${clustername} meta_split.py --args "--phen ${phen} --n_chunks ${n_chunks} --batch ${batch} --variant_set ${variant_set}"

# read code for descriptions of each argument passed to the script
# if the job takes too long or seems to be stuck, try restarting the cluster and setting num-preemptibleworkers to 0 and increasing num-workers to a higher number (up to 100)


#–––––– preprocess2 (only uncomment preprocess2 at the bottom of the script meta_split.py) ––––––

cluster start ${clustername} --num-workers 20 --num-preemptible-workers 100 --max-idle 10m
cluster submit ${clustername} meta_split.py --args "--phen ${phen} --n_chunks ${n_chunks} --batch ${batch} --variant_set ${variant_set}"

# if the job takes too long or seems to be stuck, try restarting the cluster and setting num-preemptibleworkers to 0 and increasing num-workers to a higher number (up to 100)

#–––––– preprocess3 (only uncomment preprocess3 at the bottom of the script meta_split.py) ––––––

cluster start ${clustername} --num-workers 50
cluster submit ${clustername} meta_split.py --args "--phen ${phen} --n_chunks ${n_chunks} --batch ${batch} --variant_set ${variant_set}"

# NOTE: DO NOT USE PREEMPTIBLE WORKERS FOR THIS STEP
# You may find that you will need to increase num-workers to 100 if the job gets stuck


#–––––––––––– Meta-split: ––––––––––––
# Be careful when running a job with both metasplit1 and metasplit2 uncommented in the same script. I typically run them separately.


#–––––– metasplit1 (only uncomment metasplit1 at the bottom of the script meta_split.py) ––––––

cluster start ${clustername} --num-workers 20 --max-idle 10m
cluster submit ${clustername} meta_split.py --args "--phen ${phen} --n_chunks ${n_chunks} --batch ${batch} --variant_set ${variant_set}"

# Takes about 1 hour to finish using 20 n1-standard-8 workers


#–––––– metasplit2 (only uncomment metasplit2 at the bottom of the script meta_split.py) ––––––

cluster start ${clustername} --num-workers 20 --max-idle 10m
cluster submit ${clustername} meta_split.py --args "--phen ${phen} --n_chunks ${n_chunks} --batch ${batch} --variant_set ${variant_set}"

# Takes 40 n1-standard-8 workers about 5 hours. Takes 20 n1-standard-8 workers slightly more than 8 hours.
# You can try playing around with the number of workers to suit your needs. Typically, fewer workers takes longer but are more cost-effective.
# However, I generally wouldn't recommend using fewer than 20 workers because this may cause the job to become stuck.


#–––––– metasplit3 (only uncomment metasplit3 at the bottom of the script meta_split.py) ––––––
cluster start ${clustername} --num-workers 2 --max-idle 10m
cluster submit ${clustername} meta_split.py --args "--phen ${phen} --n_chunks ${n_chunks} --batch ${batch} --variant_set ${variant_set}"

# NOTE: I would recommend using meta_split_parallel.py instead of this last function. It will be much faster because the meta-analysis of the permutations can be hard parallelized over clusters without an increase in cost.


#–––––––––––– meta_split_parallel.py: ––––––––––––
# Substitute this script for the last step in the meta-analysis/split pipeline. It is much faster because it hard parallelizes tasks over clusters.

# (Step 1): Start up your clusters using a parallel for-loop

maxi=5
for i in `seq 1 ${maxi}`; do
	cluster start ${clusterprefix}-$i --max-idle 10m &
done

# NOTE: Wait for the clusters to all be running before proceeding to the next step
# NOTE: However, don't let the clusters sit idle for more than 10 minutes because they will automatically be deleted

# (Step 2): Submit tasks using parallel for-loop

cd ${directory containing meta_split_parallel.py}
maxi=5
for i in `seq 1 ${maxi}`; do
	cluster submit ${clusterprefix}-$i meta_split_parallel.py --args "--phen ${phen} --n_chunks ${300} --batch ${batch} --reps 100 --parsplit ${maxi} --paridx $i" &
done