SLURM: Move all sbatch parameters into script

This allows the user to re-run any job, if they want, or at least easily
use the GUI's work as a template.

8 files changed, 123 insertions, 73 deletions

diff --git a/src/gui_ambi.c b/src/gui_ambi.c
index d07efcf5..1201234c 100644
--- a/src/gui_ambi.c
+++ b/src/gui_ambi.c
@@ -483,7 +483,8 @@ int write_ambigator_script(const char *filename,
                            const char *stderr_filename,
                            const char *fg_filename,
                            const char *intermediate_rel_filename,
-                           const char *harvest_filename)
+                           const char *harvest_filename,
+                           const char *prologue)
 {
 	FILE *fh;
 	int i;
@@ -492,6 +493,7 @@ int write_ambigator_script(const char *filename,
 	if ( fh == NULL ) return 1;
 
 	fprintf(fh, "#!/bin/sh\n");
+	fprintf(fh, "%s", prologue);
 
 	fprintf(fh, "cat \\\n");
 	for ( i=0; i<input->n_streams; i++ ) {
diff --git a/src/gui_ambi.h b/src/gui_ambi.h
index 06bafe64..fcc7762b 100644
--- a/src/gui_ambi.h
+++ b/src/gui_ambi.h
@@ -45,7 +45,8 @@ extern int write_ambigator_script(const char *filename,
                                   const char *stderr_filename,
                                   const char *fg_filename,
                                   const char *intermediate_rel_filename,
-                                  const char *harvest_filename);
+                                  const char *harvest_filename,
+                                  const char *prologue);
 
 extern double read_ambigator_progress(char *logfile_str, int niter);
 
diff --git a/src/gui_backend_local.c b/src/gui_backend_local.c
index b7c4b886..03761474 100644
--- a/src/gui_backend_local.c
+++ b/src/gui_backend_local.c
@@ -409,7 +409,7 @@ static void *run_ambi(const char *job_title,
 	                             stdout_rel_filename, stderr_rel_filename,
 	                             fg_rel_filename,
 	                             intermediate_rel_filename,
-	                             harvest_rel_filename) )
+	                             harvest_rel_filename, "") )
 	{
 		char *args[3];
 		args[0] = "sh";
@@ -472,7 +472,7 @@ static void *run_merging(const char *job_title,
 	                         &proj->merging_params, output_rel_filename,
 	                         stdout_rel_filename, stderr_rel_filename,
 	                         harvest_rel_filename,
-	                         log_folder_rel) )
+	                         log_folder_rel, "") )
 	{
 		char *args[3];
 		enum gui_job_type type;
@@ -568,7 +568,7 @@ static void *run_indexing(const char *job_title,
 	                               &proj->peak_search_params,
 	                               &proj->indexing_params,
 	                               wavelength_estimate,
-	                               clen_estimate) )
+	                               clen_estimate, "") )
 	{
 		char *args[3];
 		args[0] = "sh";
diff --git a/src/gui_backend_slurm.c b/src/gui_backend_slurm.c
index dddff8dd..3961a92d 100644
--- a/src/gui_backend_slurm.c
+++ b/src/gui_backend_slurm.c
@@ -613,68 +613,89 @@ static int empty(const char *str)
 }
 
 
-static struct slurm_job *start_slurm_job(enum gui_job_type type,
-                                         const char *script_filename,
-                                         const char *jobname,
-                                         const char *array_inx,
-                                         GFile *workdir,
-                                         const char *stdout_filename,
-                                         const char *stderr_filename,
-                                         struct slurm_common_opts *opts)
+static char *add_bits(char *old, const char *new1, const char *new2)
 {
-	char time_limit[64];
-	const gchar *args[64];
-	GError *error = NULL;
-	GSubprocess *sp;
-	char buf[256];
-	gsize bytes_read;
-	int n = 0;
+	size_t len;
+	char *nn;
 
-	snprintf(time_limit, 63, "%i", opts->time_limit);
+	if ( old == NULL ) return NULL;
+
+	len = strlen(new1) + strlen(new2) + strlen(old) + 12;
+	nn = malloc(len);
+	if ( nn == NULL ) {
+		ERROR("Failed to expand string for #SBATCH bits.\n");
+		return NULL;
+	}
+
+	strcpy(nn, old);
+	strcat(nn, "\n#SBATCH ");
+	strcat(nn, new1);
+	strcat(nn, " ");
+	strcat(nn, new2);
+	free(old);
+	return nn;
+}
+
+
+static char *sbatch_bits(struct slurm_common_opts *opts,
+                         const char *jobname,
+                         const char *array_inx,
+                         const char *stdout_filename,
+                         const char *stderr_filename)
+{
+	char time_limit[64];
+	char *str = strdup("");
 
-	args[n++] = "sbatch";
 	if ( !empty(array_inx) ) {
-		args[n++] = "--array";
-		args[n++] = array_inx;
+		str = add_bits(str, "--array", array_inx);
 	}
-	args[n++] = "--job-name";
-	args[n++] = jobname;
-	args[n++] = "--output";
-	args[n++] = stdout_filename;
-	args[n++] = "--error";
-	args[n++] = stderr_filename;
-	args[n++] = "--time";
-	args[n++] = time_limit;
+	str = add_bits(str, "--job-name", jobname);
+	str = add_bits(str, "--output", stdout_filename);
+	str = add_bits(str, "--error", stderr_filename);
+	snprintf(time_limit, 63, "%i", opts->time_limit);
+	str = add_bits(str, "--time", time_limit);
 	if ( !empty(opts->email_address) ) {
-		args[n++] = "--mail-user";
-		args[n++] = opts->email_address;
+		str = add_bits(str, "--mail-user", opts->email_address);
 	}
 	if ( !empty(opts->partition) ) {
-		args[n++] = "--partition";
-		args[n++] = opts->partition;
+		str = add_bits(str, "--partition", opts->partition);
 	}
 	if ( !empty(opts->constraint) ) {
-		args[n++] = "--constraint";
-		args[n++] = opts->constraint;
+		str = add_bits(str, "--constraint", opts->constraint);
 	}
 	if ( !empty(opts->account) ) {
-		args[n++] = "--account";
-		args[n++] = opts->account;
+		str = add_bits(str, "--account", opts->account);
 	}
 	if ( !empty(opts->reservation) ) {
-		args[n++] = "--reservation";
-		args[n++] = opts->reservation;
+		str = add_bits(str, "--reservation", opts->reservation);
 	}
 	if ( !empty(opts->qos) ) {
-		args[n++] = "--qos";
-		args[n++] = opts->qos;
+		str = add_bits(str, "--qos", opts->qos);
 	}
-	args[n++] = "--nodes=1";
-	args[n++] = "--mail-type=FAIL";
-	args[n++] = "--comment";
-	args[n++] = "Submitted via CrystFEL GUI";
-	args[n++] = script_filename;
-	args[n++] = NULL;
+	str = add_bits(str, "--nodes", "1");
+	str = add_bits(str, "--mail-type", "FAIL\n\n");
+
+	return str;
+}
+
+
+static struct slurm_job *start_slurm_job(enum gui_job_type type,
+                                         const char *script_filename,
+                                         const char *jobname,
+                                         GFile *workdir,
+                                         const char *stderr_filename)
+{
+	const gchar *args[5];
+	GError *error = NULL;
+	GSubprocess *sp;
+	char buf[256];
+	gsize bytes_read;
+
+	args[0] = "sbatch";
+	args[1] = "--comment";
+	args[2] = "Submitted via CrystFEL GUI";
+	args[3] = script_filename;
+	args[4] = NULL;
 
 	sp = g_subprocess_newv(args, G_SUBPROCESS_FLAGS_STDOUT_PIPE
 	                           | G_SUBPROCESS_FLAGS_STDERR_MERGE, &error);
@@ -776,6 +797,7 @@ static void *run_indexing(const char *job_title,
 	gchar *files_rel_filename;
 	gchar *stream_rel_filename;
 	gchar *harvest_rel_filename;
+	char *slurm_prologue;
 
 	workdir = make_job_folder(job_title, job_notes);
 	if ( workdir == NULL ) return NULL;
@@ -821,6 +843,9 @@ static void *run_indexing(const char *job_title,
 	stderr_rel_filename = relative_to_cwd(workdir, "stderr-%a.log");
 	harvest_rel_filename = relative_to_cwd(workdir, "parameters.json");
 
+	slurm_prologue = sbatch_bits(&opts->common, job_title, array_inx,
+	                             stdout_rel_filename, stderr_rel_filename);
+
 	if ( !write_indexamajig_script(sc_rel_filename,
 	                               proj->geom_filename,
 	                               "`nproc`",
@@ -832,20 +857,20 @@ static void *run_indexing(const char *job_title,
 	                               &proj->peak_search_params,
 	                               &proj->indexing_params,
 	                               wavelength_estimate,
-	                               clen_estimate) )
+	                               clen_estimate,
+	                               slurm_prologue) )
 	{
+		/* The stderr filename isn't used by indexing_progress() in the
+		 * Slurm backend - it knows where to find the files. */
 		job = start_slurm_job(GUI_JOB_INDEXING,
 		                      sc_rel_filename,
-		                      job_title,
-		                      array_inx,
-		                      workdir,
-		                      stdout_rel_filename,
-		                      stderr_rel_filename,
-		                      &opts->common);
+		                      job_title, workdir, "notused");
 	} else {
 		job = NULL;
 	}
 
+	free(slurm_prologue);
+
 	if ( job != NULL ) {
 		job->n_frames = proj->n_frames;
 		job->n_blocks = n_blocks;
@@ -1023,6 +1048,7 @@ static void *run_ambi(const char *job_title,
 	char *fg_rel_filename;
 	char *intermediate_rel_filename;
 	char *harvest_rel_filename;
+	char *slurm_prologue;
 
 	workdir = make_job_folder(job_title, job_notes);
 	if ( workdir == NULL ) return NULL;
@@ -1035,21 +1061,26 @@ static void *run_ambi(const char *job_title,
 	intermediate_rel_filename = relative_to_cwd(workdir, "ambigator-input.stream");
 	harvest_rel_filename = relative_to_cwd(workdir, "parameters.json");
 
+	slurm_prologue = sbatch_bits(&opts->common, job_title, NULL,
+	                             stdout_rel_filename, stderr_rel_filename);
+
 	if ( !write_ambigator_script(sc_rel_filename, input, "`nproc`",
 	                             &proj->ambi_params, stream_rel_filename,
 	                             stdout_rel_filename, stderr_rel_filename,
 	                             fg_rel_filename,
 	                             intermediate_rel_filename,
-	                             harvest_rel_filename) )
+	                             harvest_rel_filename,
+	                             slurm_prologue) )
 	{
 		job = start_slurm_job(GUI_JOB_AMBIGATOR,
-		                      sc_rel_filename, job_title, NULL, workdir,
-		                      stdout_rel_filename, stderr_rel_filename,
-		                      &opts->common);
+		                      sc_rel_filename, job_title, NULL,
+		                      stderr_rel_filename);
 		job->niter = proj->ambi_params.niter;
 	} else {
 		job = NULL;
 	}
+
+	free(slurm_prologue);
 	g_free(sc_rel_filename);
 	g_free(stdout_rel_filename);
 	g_free(stderr_rel_filename);
@@ -1079,6 +1110,7 @@ static void *run_merging(const char *job_title,
 	char *stderr_rel_filename;
 	char *harvest_rel_filename;
 	char *log_folder_rel;
+	char *slurm_prologue;
 
 	workdir = make_job_folder(job_title, job_notes);
 	if ( workdir == NULL ) return NULL;
@@ -1090,11 +1122,15 @@ static void *run_merging(const char *job_title,
 	harvest_rel_filename = relative_to_cwd(workdir, "parameters.json");
 	log_folder_rel = relative_to_cwd(workdir, "pr-logs");
 
+	slurm_prologue = sbatch_bits(&opts->common, job_title, NULL,
+	                             stdout_rel_filename, stderr_rel_filename);
+
 	if ( !write_merge_script(sc_rel_filename, input, "`nproc`",
 	                         &proj->merging_params, output_rel_filename,
 	                         stdout_rel_filename, stderr_rel_filename,
 	                         harvest_rel_filename,
-	                         log_folder_rel) )
+	                         log_folder_rel,
+	                         slurm_prologue) )
 	{
 		enum gui_job_type type;
 		if ( strcmp(proj->merging_params.model, "process_hkl") == 0 ) {
@@ -1107,13 +1143,13 @@ static void *run_merging(const char *job_title,
 			type = GUI_JOB_PARTIALATOR;
 		}
 		job = start_slurm_job(type, sc_rel_filename, job_title, NULL,
-		                      workdir, stdout_rel_filename,
-		                      stderr_rel_filename,
-		                      &opts->common);
+		                      stderr_rel_filename);
 	} else {
 		job = NULL;
 	}
 
+	free(slurm_prologue);
+
 	if ( job != NULL ) {
 
 		char *hkl1;
diff --git a/src/gui_index.c b/src/gui_index.c
index d690737d..aa2301ef 100644
--- a/src/gui_index.c
+++ b/src/gui_index.c
@@ -1083,7 +1083,8 @@ int write_indexamajig_script(const char *script_filename,
                              struct peak_params *peak_search_params,
                              struct index_params *indexing_params,
                              double wavelength_estimate,
-                             double clen_estimate)
+                             double clen_estimate,
+                             const char *prologue)
 {
 	FILE *fh;
 	int i;
@@ -1105,6 +1106,7 @@ int write_indexamajig_script(const char *script_filename,
 	if ( fh == NULL ) return 1;
 
 	fprintf(fh, "#!/bin/sh\n");
+	fprintf(fh, "%s", prologue);
 
 	i = 0;
 	while ( cmdline[i] != NULL ) {
diff --git a/src/gui_index.h b/src/gui_index.h
index b2b18939..2f5c1a56 100644
--- a/src/gui_index.h
+++ b/src/gui_index.h
@@ -55,6 +55,7 @@ extern int write_indexamajig_script(const char *script_filename,
                                     struct peak_params *peak_search_params,
                                     struct index_params *indexing_params,
                                     double wavelength_estimate,
-                                    double clen_estimate);
+                                    double clen_estimate,
+                                    const char *prologue);
 
 #endif
diff --git a/src/gui_merge.c b/src/gui_merge.c
index 391957d7..604f2591 100644
--- a/src/gui_merge.c
+++ b/src/gui_merge.c
@@ -355,7 +355,8 @@ static int write_partialator_script(const char *filename,
                                     const char *stdout_filename,
                                     const char *stderr_filename,
                                     const char *harvest_filename,
-                                    const char *log_folder)
+                                    const char *log_folder,
+                                    const char *prologue)
 {
 	FILE *fh;
 	int i;
@@ -364,6 +365,7 @@ static int write_partialator_script(const char *filename,
 	if ( fh == NULL ) return 1;
 
 	fprintf(fh, "#!/bin/sh\n");
+	fprintf(fh, "%s", prologue);
 
 	fprintf(fh, "partialator \\\n");
 
@@ -461,7 +463,8 @@ static int write_process_hkl_script(const char *filename,
                                     struct merging_params *params,
                                     const char *out_hkl,
                                     const char *stdout_filename,
-                                    const char *stderr_filename)
+                                    const char *stderr_filename,
+                                    const char *prologue)
 {
 	FILE *fh;
 
@@ -469,6 +472,7 @@ static int write_process_hkl_script(const char *filename,
 	if ( fh == NULL ) return 1;
 
 	fprintf(fh, "#!/bin/sh\n");
+	fprintf(fh, "%s", prologue);
 
 	add_process_hkl(fh, input, params, out_hkl,
 	                stdout_filename, stderr_filename, "", "");
@@ -490,20 +494,23 @@ int write_merge_script(const char *filename,
                        const char *stdout_filename,
                        const char *stderr_filename,
                        const char *harvest_filename,
-                       const char *log_folder)
+                       const char *log_folder,
+                       const char *prologue)
 {
 	if ( strcmp(params->model, "process_hkl") == 0 ) {
 		return write_process_hkl_script(filename, input,
 		                                params, out_hkl,
 		                                stdout_filename,
-		                                stderr_filename);
+		                                stderr_filename,
+		                                prologue);
 	} else {
 		return write_partialator_script(filename, input, n_thread_str,
 		                                params, out_hkl,
 		                                stdout_filename,
 		                                stderr_filename,
 		                                harvest_filename,
-		                                log_folder);
+		                                log_folder,
+		                                prologue);
 	}
 }
 
diff --git a/src/gui_merge.h b/src/gui_merge.h
index 1bd86a2b..6b9c8a66 100644
--- a/src/gui_merge.h
+++ b/src/gui_merge.h
@@ -45,7 +45,8 @@ extern int write_merge_script(const char *filename,
                               const char *stdout_filename,
                               const char *stderr_filename,
                               const char *harvest_filename,
-                              const char *log_folder);
+                              const char *log_folder,
+                              const char *prologue);
 
 extern double read_merge_progress(const char *logfile_str,
                                   enum gui_job_type type);