aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile.am4
-rw-r--r--scripts/turbo-index-lsf (renamed from scripts/turbo-index)5
-rwxr-xr-xscripts/turbo-index-slurm85
3 files changed, 90 insertions, 4 deletions
diff --git a/Makefile.am b/Makefile.am
index ba7078a1..49bb9fc8 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -181,9 +181,9 @@ script_DATA = scripts/alternate-stream scripts/cell-please \
scripts/find-pairs scripts/plot-cc-and-scale.R \
scripts/ave-resolution scripts/crystal-frame-number \
scripts/plot-radius-resolution \
- scripts/detector-shift scripts/turbo-index \
+ scripts/detector-shift scripts/turbo-index-lsf \
scripts/gaincal-to-saturation-map scripts/move-entire-detector \
- scripts/split-by-mask
+ scripts/split-by-mask scripts/turbo-index-slurm
EXTRA_DIST += $(script_DATA)
diff --git a/scripts/turbo-index b/scripts/turbo-index-lsf
index c11f521e..00e4ec15 100644
--- a/scripts/turbo-index
+++ b/scripts/turbo-index-lsf
@@ -3,9 +3,9 @@
RUN=$1
NOSAMPLE=`echo $RUN | sed -e 's/\-.*$//'`
-GEOM=<name of geometry file>
+GEOM=my.geom # Name of your geometry file
-find <path to CXI files>/$RUN -name '*.cxi' > files-${RUN}.lst
+find /path/to/CXI/files/$RUN -name '*.cxi' > files-${RUN}.lst # Set location of files
list_events -i files-${RUN}.lst -g $GEOM -o events-${RUN}.lst
wc -l events-${RUN}.lst
rm -f split-events-${RUN}.lst files-${RUN}.lst
@@ -18,6 +18,7 @@ for FILE in split-events-${RUN}.lst*; do
NAME=`echo $FILE | sed -e "s/split-events-${RUN}.lst/${NOSAMPLE}-/"`
echo "$NAME: $FILE ---> $STREAM"
+ # Set indexing parameters here
bsub -q psanaq -o $NAME.log -J $NAME -n 12 -R "span[hosts=1]" \
indexamajig \
-i $FILE -o $STREAM -j 32 -g $GEOM --peaks=cxi
diff --git a/scripts/turbo-index-slurm b/scripts/turbo-index-slurm
new file mode 100755
index 00000000..a0f1bec5
--- /dev/null
+++ b/scripts/turbo-index-slurm
@@ -0,0 +1,85 @@
+#!/bin/sh
+
+# Split a large indexing job into many small tasks and submit using SLURM
+
+# ./turbo-index my-files.lst label my.geom /location/for/streams
+
+# Copyright © 2016-2017 Deutsches Elektronen-Synchrotron DESY,
+# a research centre of the Helmholtz Association.
+#
+# Authors:
+# 2016 Steve Aplin <steve.aplin@desy.de>
+# 2016-2017 Thomas White <taw@physics.org>
+
+SPLIT=1000 # Size of job chunks
+MAIL=you@example.org # Email address for SLURM notifications
+
+INPUT=$1
+RUN=$2
+GEOM=$3
+STREAMDIR=$4
+
+# Set up environment here if necessary
+#source /path/to/crystfel/setup.sh
+
+# Generate event list from file above
+list_events -i $INPUT -g $GEOM -o events-${RUN}.lst
+if [ $? != 0 ]; then
+ echo "list_events failed"
+ exit 1
+fi
+# If you are using single-event files instead of multi-event ("CXI") ones,
+# comment out the above lines and uncomment the following one:
+#cp $INPUT events-${RUN}.lst
+
+# Count total number of events
+wc -l events-${RUN}.lst
+
+# Split the events up, will create files with $SPLIT lines
+split -a 3 -d -l $SPLIT events-${RUN}.lst split-events-${RUN}.lst
+
+# Clean up
+rm -f events-${RUN}.lst
+
+# Loop over the event list files, and submit a batch job for each of them
+for FILE in split-events-${RUN}.lst*; do
+
+ # Stream file is the output of crystfel
+ STREAM=`echo $FILE | sed -e "s/split-events-${RUN}.lst/${RUN}.stream/"`
+
+ # Job name
+ NAME=`echo $FILE | sed -e "s/split-events-${RUN}.lst/${RUN}-/"`
+
+ echo "$NAME: $FILE ---> $STREAM"
+
+ SLURMFILE="${NAME}.sh"
+
+ echo "#!/bin/sh" > $SLURMFILE
+ echo >> $SLURMFILE
+
+ echo "#SBATCH --partition=mypartition" >> $SLURMFILE # Set your partition here
+ echo "#SBATCH --time=01:00:00" >> $SLURMFILE
+ echo "#SBATCH --nodes=1" >> $SLURMFILE
+ echo "#SBATCH --nice=100" >> $SLURMFILE # Set priority very low to allow other jobs through
+ echo >> $SLURMFILE
+
+ echo "#SBATCH --workdir $PWD" >> $SLURMFILE
+ echo "#SBATCH --job-name $NAME" >> $SLURMFILE
+ echo "#SBATCH --output $NAME-%N-%j.out" >> $SLURMFILE
+ echo "#SBATCH --error $NAME-%N-%j.err" >> $SLURMFILE
+ echo "#SBATCH --mail-type END" >> $SLURMFILE
+ echo "#SBATCH --mail-user $MAIL" >> $SLURMFILE
+ echo >> $SLURMFILE
+
+ echo "#source /path/to/crystfel/setup.sh" >> $SLURMFILE # Set up environment here (again) if necessary
+ echo >> $SLURMFILE
+
+ command="indexamajig -i $FILE -o $STREAMDIR/$STREAM"
+ command="$command -j \`nproc\` -g $GEOM"
+ #command="$command --peaks=zaef" # Indexing parameters here
+
+ echo $command >> $SLURMFILE
+
+ sbatch $SLURMFILE
+
+done