Add header caching layer

This simultaneously implements --copy-header/--copy-hdf5-field and gives
access to metadata (e.g. detector position or wavelength) when loading
from a stream with header-specified values.

7 files changed, 236 insertions, 23 deletions

diff --git a/libcrystfel/src/datatemplate.c b/libcrystfel/src/datatemplate.c
index 290b9227..185847ea 100644
--- a/libcrystfel/src/datatemplate.c
+++ b/libcrystfel/src/datatemplate.c
@@ -1047,6 +1047,7 @@ DataTemplate *data_template_new_from_string(const char *string_in)
 	dt->peak_list = NULL;
 	dt->shift_x_from = NULL;
 	dt->shift_y_from = NULL;
+	dt->n_headers_to_copy = 0;
 
 	/* The default defaults... */
 	defaults.orig_min_fs = -1;
@@ -1489,6 +1490,10 @@ void data_template_free(DataTemplate *dt)
 		}
 	}
 
+	for ( i=0; i<dt->n_headers_to_copy; i++ ) {
+		free(dt->headers_to_copy[i]);
+	}
+
 	free(dt->wavelength_from);
 	free(dt->peak_list);
 
@@ -1574,8 +1579,12 @@ int data_template_panel_name_to_number(const DataTemplate *dt,
 void data_template_add_copy_header(DataTemplate *dt,
                                    const char *header)
 {
-	/* FIXME: Add "header" to list of things to copy */
-	STATUS("Adding %s\n", header);
+	if ( dt->n_headers_to_copy >= MAX_COPY_HEADERS ) {
+		ERROR("Too many extra headers to copy\n");
+		return;
+	}
+
+	dt->headers_to_copy[dt->n_headers_to_copy++] = strdup(header);
 }
 
 
diff --git a/libcrystfel/src/datatemplate_priv.h b/libcrystfel/src/datatemplate_priv.h
index 71569273..9d9485bf 100644
--- a/libcrystfel/src/datatemplate_priv.h
+++ b/libcrystfel/src/datatemplate_priv.h
@@ -78,6 +78,9 @@ enum peak_layout
 #define DIM_UNDEFINED (-3)
 #define DIM_PLACEHOLDER (-4)
 
+/* Maximum number of headers to cache, in addition to anything already
+ * referenced by the DataTemplate */
+#define MAX_COPY_HEADERS (32)
 
 /* Maximum number of masks per panel */
 #define MAX_MASKS (8)
@@ -232,6 +235,9 @@ struct _datatemplate
 	/* Shift of whole detector, in m */
 	char                      *shift_x_from;
 	char                      *shift_y_from;
+
+	char                      *headers_to_copy[MAX_COPY_HEADERS];
+	int                        n_headers_to_copy;
 };
 
 extern double convert_to_m(double val, int units);
diff --git a/libcrystfel/src/image-hdf5.c b/libcrystfel/src/image-hdf5.c
index 16f20e8a..25e9d3f7 100644
--- a/libcrystfel/src/image-hdf5.c
+++ b/libcrystfel/src/image-hdf5.c
@@ -615,7 +615,7 @@ int image_hdf5_read_mask(struct panel_template *p,
 
 
 double image_hdf5_get_value(const char *name, const char *filename,
-                            const char *event)
+                            const char *event, char *ptype)
 {
 	hid_t dh;
 	hid_t type;
@@ -667,7 +667,11 @@ double image_hdf5_get_value(const char *name, const char *filename,
 	type = H5Dget_type(dh);
 	class = H5Tget_class(type);
 
-	if ( (class != H5T_FLOAT) && (class != H5T_INTEGER) ) {
+	if ( class == H5T_FLOAT ) {
+		*ptype = 'f';
+	} else if ( class == H5T_INTEGER ) {
+		*ptype = 'i';
+	} else {
 		ERROR("Not a floating point or integer value.\n");
 		close_hdf5(fh);
 		return NAN;
@@ -691,16 +695,35 @@ double image_hdf5_get_value(const char *name, const char *filename,
 	ms = H5Screate_simple(1, msdims, NULL);
 
 	if ( ndims == 0 ) {
+
 		/* Easy case, because value is a scalar */
-		r = H5Dread(dh, H5T_NATIVE_DOUBLE, ms, sh, H5P_DEFAULT, &val);
-		if ( r < 0 )  {
-			ERROR("Couldn't read scalar value from %s.\n",
-			      subst_name);
-			free(subst_name);
-			close_hdf5(fh);
-			return NAN;
+
+		if ( class == H5T_FLOAT ) {
+
+			r = H5Dread(dh, H5T_NATIVE_DOUBLE, ms, sh, H5P_DEFAULT, &val);
+			if ( r < 0 )  {
+				ERROR("Couldn't read scalar value from %s.\n",
+				      subst_name);
+				free(subst_name);
+				close_hdf5(fh);
+				return NAN;
+			}
+			return val;
+
+		} else {
+
+			int vali;
+			r = H5Dread(dh, H5T_NATIVE_INT, ms, sh, H5P_DEFAULT, &vali);
+			if ( r < 0 )  {
+				ERROR("Couldn't read scalar value from %s.\n",
+				      subst_name);
+				free(subst_name);
+				close_hdf5(fh);
+				return NAN;
+			}
+			return vali;
+
 		}
-		return val;
 	}
 
 	dim_vals = read_dim_parts(event, &n_dim_vals);
diff --git a/libcrystfel/src/image-hdf5.h b/libcrystfel/src/image-hdf5.h
index efb8a3b7..96b21558 100644
--- a/libcrystfel/src/image-hdf5.h
+++ b/libcrystfel/src/image-hdf5.h
@@ -7,7 +7,7 @@
  *                       a research centre of the Helmholtz Association.
  *
  * Authors:
- *   2020 Thomas White <taw@physics.org>
+ *   2020-2021 Thomas White <taw@physics.org>
  *
  * This file is part of CrystFEL.
  *
@@ -36,7 +36,8 @@
 
 extern double image_hdf5_get_value(const char *from,
                                    const char *filename,
-                                   const char *ev);
+                                   const char *ev,
+                                   char *type);
 
 extern int image_hdf5_read(struct image *image,
                            const DataTemplate *dtempl,
diff --git a/libcrystfel/src/image.c b/libcrystfel/src/image.c
index 99b0891e..8d9f1640 100644
--- a/libcrystfel/src/image.c
+++ b/libcrystfel/src/image.c
@@ -281,11 +281,72 @@ void free_all_crystals(struct image *image)
 }
 
 
+static struct header_cache_entry *find_cache_entry(struct image *image,
+                                                   const char *name)
+{
+	int i;
+	for ( i=0; i<image->n_cached_headers; i++ ) {
+		if ( strcmp(name, image->header_cache[i]->header_name) == 0 ) {
+			return image->header_cache[i];
+		}
+	}
+	return NULL;
+}
+
+
+void image_cache_header_int(struct image *image,
+                            const char *header_name,
+                            int header_val)
+{
+	if ( image->n_cached_headers >= HEADER_CACHE_SIZE ) {
+		ERROR("Too many headers to copy.\n");
+	} else {
+
+		struct header_cache_entry *ce;
+		ce = malloc(sizeof(struct header_cache_entry));
+
+		if ( ce != NULL ) {
+			ce->header_name = strdup(header_name);
+			ce->val_int = header_val;
+			ce->type = 'i';
+			image->header_cache[image->n_cached_headers++] = ce;
+		} else {
+			ERROR("Failed to add header cache entry.\n");
+		}
+	}
+}
+
+
+void image_cache_header_float(struct image *image,
+                              const char *header_name,
+                              float header_val)
+{
+	if ( image->n_cached_headers >= HEADER_CACHE_SIZE ) {
+		ERROR("Too many headers to copy.\n");
+	} else {
+
+		struct header_cache_entry *ce;
+		ce = malloc(sizeof(struct header_cache_entry));
+
+		if ( ce != NULL ) {
+			ce->header_name = strdup(header_name);
+			ce->val_float = header_val;
+			ce->type = 'f';
+			image->header_cache[image->n_cached_headers++] = ce;
+		} else {
+			ERROR("Failed to add header cache entry.\n");
+		}
+	}
+}
+
+
 static double get_value(struct image *image, const char *from,
                         int *is_literal_number)
 {
 	double val;
 	char *rval;
+	struct header_cache_entry *ce;
+	char type;
 
 	if ( from == NULL ) return NAN;
 
@@ -302,10 +363,21 @@ static double get_value(struct image *image, const char *from,
 		return NAN;
 	}
 
-	if ( is_hdf5_file(image->filename) ) {
-		return image_hdf5_get_value(from,
-		                            image->filename,
-		                            image->ev);
+	ce = find_cache_entry(image, from);
+	if ( ce != NULL ) {
+		if ( ce->type == 'f' ) {
+			return ce->val_float;
+		} else if ( ce->type == 'i' ) {
+			return ce->val_int;
+		} else {
+			ERROR("Unrecognised header cache type '%c'\n",
+			      ce->type);
+			return NAN;
+		}
+
+	} else if ( is_hdf5_file(image->filename) ) {
+		val = image_hdf5_get_value(from, image->filename, image->ev,
+		                           &type);
 
 	} else if ( is_cbf_file(image->filename) ) {
 		/* FIXME: From headers */
@@ -319,6 +391,13 @@ static double get_value(struct image *image, const char *from,
 		ERROR("Unrecognised file type: %s\n", image->filename);
 		return NAN;
 	}
+
+	if ( type == 'f' ) {
+		image_cache_header_float(image, from, val);
+	} else if ( type == 'i' ) {
+		image_cache_header_int(image, from, val);
+	}
+	return val;
 }
 
 
@@ -954,6 +1033,7 @@ struct image *image_read(const DataTemplate *dtempl,
 {
 	struct image *image;
 	int r;
+	int i;
 
 	if ( dtempl == NULL ) {
 		ERROR("NULL data template!\n");
@@ -1002,6 +1082,10 @@ struct image *image_read(const DataTemplate *dtempl,
 		return NULL;
 	}
 
+	for ( i=0; i<dtempl->n_headers_to_copy; i++ ) {
+		get_value(image, dtempl->headers_to_copy[i], NULL);
+	}
+
 	return image;
 }
 
@@ -1030,6 +1114,10 @@ void image_free(struct image *image)
 		if ( image->bad != NULL ) free(image->bad[i]);
 	}
 
+	for ( i=0; i<image->n_cached_headers; i++ ) {
+		free(image->header_cache[i]);
+	}
+
 	free(image->dp);
 	free(image->sat);
 	free(image->bad);
@@ -1055,7 +1143,7 @@ struct image *image_new()
        image->detgeom = NULL;
        image->filename = NULL;
        image->ev = NULL;
-       image->copied_headers = NULL;
+       image->n_cached_headers = 0;
        image->id = 0;
        image->serial = 0;
        image->spectrum = NULL;
diff --git a/libcrystfel/src/image.h b/libcrystfel/src/image.h
index c3b55ba0..181d0433 100644
--- a/libcrystfel/src/image.h
+++ b/libcrystfel/src/image.h
@@ -7,7 +7,7 @@
  *                       a research centre of the Helmholtz Association.
  *
  * Authors:
- *   2009-2020 Thomas White <taw@physics.org>
+ *   2009-2021 Thomas White <taw@physics.org>
  *   2014      Valerio Mariani
  *
  *
@@ -76,6 +76,18 @@ struct imagefeature {
 typedef struct _imagefeaturelist ImageFeatureList;
 
 
+#define HEADER_CACHE_SIZE (128)
+
+struct header_cache_entry {
+	char *header_name;
+	char type;
+	union {
+		int val_int;
+		float val_float;
+	};
+};
+
+
 struct image
 {
 	/** The image data, by panel */
@@ -112,7 +124,8 @@ struct image
 	/** @} */
 
 	/** A list of metadata read from the stream */
-	char                    *copied_headers;
+	struct header_cache_entry *header_cache[HEADER_CACHE_SIZE];
+	int                        n_cached_headers;
 
 	/** ID number of the worker processing handling this image */
 	int                     id;
@@ -184,6 +197,14 @@ extern struct image *image_read(const DataTemplate *dtempl,
 extern struct image *image_create_for_simulation(const DataTemplate *dtempl);
 extern void image_free(struct image *image);
 
+extern void image_cache_header_float(struct image *image,
+                                     const char *header_name,
+                                     float header_val);
+
+extern void image_cache_header_int(struct image *image,
+                                   const char *header_name,
+                                   int header_val);
+
 extern ImageFeatureList *image_read_peaks(const DataTemplate *dtempl,
                                           const char *filename,
                                           const char *event,
diff --git a/libcrystfel/src/stream.c b/libcrystfel/src/stream.c
index 83756c7a..68f623dd 100644
--- a/libcrystfel/src/stream.c
+++ b/libcrystfel/src/stream.c
@@ -639,8 +639,19 @@ int stream_write_chunk(Stream *st, const struct image *i,
 	fprintf(st->fh, "beam_divergence = %.2e rad\n", i->div);
 	fprintf(st->fh, "beam_bandwidth = %.2e (fraction)\n", i->bw);
 
-	/* FIXME: Better way of doing this */
-	//imagefile_copy_fields(imfile, i->copyme, st->fh, ev);
+	for ( j=0; j<i->n_cached_headers; j++ ) {
+		struct header_cache_entry *ce = i->header_cache[j];
+		if ( ce->type == 'f' ) {
+			fprintf(st->fh, "header/float/%s = %f\n",
+			        ce->header_name, ce->val_float);
+		} else if ( ce->type == 'i' ) {
+			fprintf(st->fh, "header/int/%s = %i\n",
+			        ce->header_name, ce->val_int);
+		} else {
+			ERROR("Unrecognised header cache type '%c'\n",
+			      ce->type);
+		}
+	}
 
 	if ( i->detgeom != NULL ) {
 
@@ -891,6 +902,48 @@ static void read_crystal(Stream *st, struct image *image,
 }
 
 
+static void parse_header(const char *line_in, struct image *image, char type)
+{
+	char *line;
+	char *pos;
+
+	line = strdup(line_in);
+
+	pos = strchr(line, ' ');
+	if ( pos == NULL ) {
+		ERROR("Invalid header line '%s' (no space)\n", line);
+		return;
+	}
+	pos[0] = '\0';
+
+	if ( strlen(line_in) < strlen(line) + 3 ) {
+		ERROR("Invalid header line '%s' (too short)\n", line);
+		return;
+	}
+
+	if ( (pos[1] != '=') || (pos[2] != ' ') ) {
+		ERROR("Invalid header line '%s' (wrong separator)\n", line);
+		return;
+	}
+
+	if ( type == 'f' ) {
+		float v;
+		if ( sscanf(pos+3, "%f", &v) != 1 ) {
+			ERROR("Invalid header line '%s' (invalid value)\n", line);
+			return;
+		}
+		image_cache_header_float(image, line, v);
+	} else {
+		int v;
+		if ( sscanf(pos+3, "%i", &v) != 1 ) {
+			ERROR("Invalid header line '%s' (invalid value)\n", line);
+			return;
+		}
+		image_cache_header_int(image, line, v);
+	}
+}
+
+
 /**
  * Read the next chunk from a stream and return an image structure
  */
@@ -928,6 +981,18 @@ struct image *stream_read_chunk(Stream *st, StreamFlags srf)
 			image->ev = strdup(line+7);
 		}
 
+		if ( strncmp(line, "hdf5/", 5) == 0 ) {
+			parse_header(line+5, image, 'f');
+		}
+
+		if ( strncmp(line, "header/int/", 11) == 0 ) {
+			parse_header(line+11, image, 'i');
+		}
+
+		if ( strncmp(line, "header/float/", 13) == 0 ) {
+			parse_header(line+13, image, 'f');
+		}
+
 		if ( strncmp(line, "indexed_by = ", 13) == 0 ) {
 			int err = 0;
 			image->indexed_by = get_indm_from_string_2(line+13, &err);