From 00f88dac5a2b63ec1c5f7ad65efef3d4f9d93b77 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Tue, 30 Mar 2021 17:01:11 +0200 Subject: Add header caching layer This simultaneously implements --copy-header/--copy-hdf5-field and gives access to metadata (e.g. detector position or wavelength) when loading from a stream with header-specified values. --- libcrystfel/src/datatemplate.c | 13 ++++- libcrystfel/src/datatemplate_priv.h | 6 +++ libcrystfel/src/image-hdf5.c | 43 ++++++++++++---- libcrystfel/src/image-hdf5.h | 5 +- libcrystfel/src/image.c | 98 +++++++++++++++++++++++++++++++++++-- libcrystfel/src/image.h | 25 +++++++++- libcrystfel/src/stream.c | 69 +++++++++++++++++++++++++- src/partialator.c | 2 +- 8 files changed, 237 insertions(+), 24 deletions(-) diff --git a/libcrystfel/src/datatemplate.c b/libcrystfel/src/datatemplate.c index 290b9227..185847ea 100644 --- a/libcrystfel/src/datatemplate.c +++ b/libcrystfel/src/datatemplate.c @@ -1047,6 +1047,7 @@ DataTemplate *data_template_new_from_string(const char *string_in) dt->peak_list = NULL; dt->shift_x_from = NULL; dt->shift_y_from = NULL; + dt->n_headers_to_copy = 0; /* The default defaults... */ defaults.orig_min_fs = -1; @@ -1489,6 +1490,10 @@ void data_template_free(DataTemplate *dt) } } + for ( i=0; in_headers_to_copy; i++ ) { + free(dt->headers_to_copy[i]); + } + free(dt->wavelength_from); free(dt->peak_list); @@ -1574,8 +1579,12 @@ int data_template_panel_name_to_number(const DataTemplate *dt, void data_template_add_copy_header(DataTemplate *dt, const char *header) { - /* FIXME: Add "header" to list of things to copy */ - STATUS("Adding %s\n", header); + if ( dt->n_headers_to_copy >= MAX_COPY_HEADERS ) { + ERROR("Too many extra headers to copy\n"); + return; + } + + dt->headers_to_copy[dt->n_headers_to_copy++] = strdup(header); } diff --git a/libcrystfel/src/datatemplate_priv.h b/libcrystfel/src/datatemplate_priv.h index 71569273..9d9485bf 100644 --- a/libcrystfel/src/datatemplate_priv.h +++ b/libcrystfel/src/datatemplate_priv.h @@ -78,6 +78,9 @@ enum peak_layout #define DIM_UNDEFINED (-3) #define DIM_PLACEHOLDER (-4) +/* Maximum number of headers to cache, in addition to anything already + * referenced by the DataTemplate */ +#define MAX_COPY_HEADERS (32) /* Maximum number of masks per panel */ #define MAX_MASKS (8) @@ -232,6 +235,9 @@ struct _datatemplate /* Shift of whole detector, in m */ char *shift_x_from; char *shift_y_from; + + char *headers_to_copy[MAX_COPY_HEADERS]; + int n_headers_to_copy; }; extern double convert_to_m(double val, int units); diff --git a/libcrystfel/src/image-hdf5.c b/libcrystfel/src/image-hdf5.c index 16f20e8a..25e9d3f7 100644 --- a/libcrystfel/src/image-hdf5.c +++ b/libcrystfel/src/image-hdf5.c @@ -615,7 +615,7 @@ int image_hdf5_read_mask(struct panel_template *p, double image_hdf5_get_value(const char *name, const char *filename, - const char *event) + const char *event, char *ptype) { hid_t dh; hid_t type; @@ -667,7 +667,11 @@ double image_hdf5_get_value(const char *name, const char *filename, type = H5Dget_type(dh); class = H5Tget_class(type); - if ( (class != H5T_FLOAT) && (class != H5T_INTEGER) ) { + if ( class == H5T_FLOAT ) { + *ptype = 'f'; + } else if ( class == H5T_INTEGER ) { + *ptype = 'i'; + } else { ERROR("Not a floating point or integer value.\n"); close_hdf5(fh); return NAN; @@ -691,16 +695,35 @@ double image_hdf5_get_value(const char *name, const char *filename, ms = H5Screate_simple(1, msdims, NULL); if ( ndims == 0 ) { + /* Easy case, because value is a scalar */ - r = H5Dread(dh, H5T_NATIVE_DOUBLE, ms, sh, H5P_DEFAULT, &val); - if ( r < 0 ) { - ERROR("Couldn't read scalar value from %s.\n", - subst_name); - free(subst_name); - close_hdf5(fh); - return NAN; + + if ( class == H5T_FLOAT ) { + + r = H5Dread(dh, H5T_NATIVE_DOUBLE, ms, sh, H5P_DEFAULT, &val); + if ( r < 0 ) { + ERROR("Couldn't read scalar value from %s.\n", + subst_name); + free(subst_name); + close_hdf5(fh); + return NAN; + } + return val; + + } else { + + int vali; + r = H5Dread(dh, H5T_NATIVE_INT, ms, sh, H5P_DEFAULT, &vali); + if ( r < 0 ) { + ERROR("Couldn't read scalar value from %s.\n", + subst_name); + free(subst_name); + close_hdf5(fh); + return NAN; + } + return vali; + } - return val; } dim_vals = read_dim_parts(event, &n_dim_vals); diff --git a/libcrystfel/src/image-hdf5.h b/libcrystfel/src/image-hdf5.h index efb8a3b7..96b21558 100644 --- a/libcrystfel/src/image-hdf5.h +++ b/libcrystfel/src/image-hdf5.h @@ -7,7 +7,7 @@ * a research centre of the Helmholtz Association. * * Authors: - * 2020 Thomas White + * 2020-2021 Thomas White * * This file is part of CrystFEL. * @@ -36,7 +36,8 @@ extern double image_hdf5_get_value(const char *from, const char *filename, - const char *ev); + const char *ev, + char *type); extern int image_hdf5_read(struct image *image, const DataTemplate *dtempl, diff --git a/libcrystfel/src/image.c b/libcrystfel/src/image.c index 99b0891e..8d9f1640 100644 --- a/libcrystfel/src/image.c +++ b/libcrystfel/src/image.c @@ -281,11 +281,72 @@ void free_all_crystals(struct image *image) } +static struct header_cache_entry *find_cache_entry(struct image *image, + const char *name) +{ + int i; + for ( i=0; in_cached_headers; i++ ) { + if ( strcmp(name, image->header_cache[i]->header_name) == 0 ) { + return image->header_cache[i]; + } + } + return NULL; +} + + +void image_cache_header_int(struct image *image, + const char *header_name, + int header_val) +{ + if ( image->n_cached_headers >= HEADER_CACHE_SIZE ) { + ERROR("Too many headers to copy.\n"); + } else { + + struct header_cache_entry *ce; + ce = malloc(sizeof(struct header_cache_entry)); + + if ( ce != NULL ) { + ce->header_name = strdup(header_name); + ce->val_int = header_val; + ce->type = 'i'; + image->header_cache[image->n_cached_headers++] = ce; + } else { + ERROR("Failed to add header cache entry.\n"); + } + } +} + + +void image_cache_header_float(struct image *image, + const char *header_name, + float header_val) +{ + if ( image->n_cached_headers >= HEADER_CACHE_SIZE ) { + ERROR("Too many headers to copy.\n"); + } else { + + struct header_cache_entry *ce; + ce = malloc(sizeof(struct header_cache_entry)); + + if ( ce != NULL ) { + ce->header_name = strdup(header_name); + ce->val_float = header_val; + ce->type = 'f'; + image->header_cache[image->n_cached_headers++] = ce; + } else { + ERROR("Failed to add header cache entry.\n"); + } + } +} + + static double get_value(struct image *image, const char *from, int *is_literal_number) { double val; char *rval; + struct header_cache_entry *ce; + char type; if ( from == NULL ) return NAN; @@ -302,10 +363,21 @@ static double get_value(struct image *image, const char *from, return NAN; } - if ( is_hdf5_file(image->filename) ) { - return image_hdf5_get_value(from, - image->filename, - image->ev); + ce = find_cache_entry(image, from); + if ( ce != NULL ) { + if ( ce->type == 'f' ) { + return ce->val_float; + } else if ( ce->type == 'i' ) { + return ce->val_int; + } else { + ERROR("Unrecognised header cache type '%c'\n", + ce->type); + return NAN; + } + + } else if ( is_hdf5_file(image->filename) ) { + val = image_hdf5_get_value(from, image->filename, image->ev, + &type); } else if ( is_cbf_file(image->filename) ) { /* FIXME: From headers */ @@ -319,6 +391,13 @@ static double get_value(struct image *image, const char *from, ERROR("Unrecognised file type: %s\n", image->filename); return NAN; } + + if ( type == 'f' ) { + image_cache_header_float(image, from, val); + } else if ( type == 'i' ) { + image_cache_header_int(image, from, val); + } + return val; } @@ -954,6 +1033,7 @@ struct image *image_read(const DataTemplate *dtempl, { struct image *image; int r; + int i; if ( dtempl == NULL ) { ERROR("NULL data template!\n"); @@ -1002,6 +1082,10 @@ struct image *image_read(const DataTemplate *dtempl, return NULL; } + for ( i=0; in_headers_to_copy; i++ ) { + get_value(image, dtempl->headers_to_copy[i], NULL); + } + return image; } @@ -1030,6 +1114,10 @@ void image_free(struct image *image) if ( image->bad != NULL ) free(image->bad[i]); } + for ( i=0; in_cached_headers; i++ ) { + free(image->header_cache[i]); + } + free(image->dp); free(image->sat); free(image->bad); @@ -1055,7 +1143,7 @@ struct image *image_new() image->detgeom = NULL; image->filename = NULL; image->ev = NULL; - image->copied_headers = NULL; + image->n_cached_headers = 0; image->id = 0; image->serial = 0; image->spectrum = NULL; diff --git a/libcrystfel/src/image.h b/libcrystfel/src/image.h index c3b55ba0..181d0433 100644 --- a/libcrystfel/src/image.h +++ b/libcrystfel/src/image.h @@ -7,7 +7,7 @@ * a research centre of the Helmholtz Association. * * Authors: - * 2009-2020 Thomas White + * 2009-2021 Thomas White * 2014 Valerio Mariani * * @@ -76,6 +76,18 @@ struct imagefeature { typedef struct _imagefeaturelist ImageFeatureList; +#define HEADER_CACHE_SIZE (128) + +struct header_cache_entry { + char *header_name; + char type; + union { + int val_int; + float val_float; + }; +}; + + struct image { /** The image data, by panel */ @@ -112,7 +124,8 @@ struct image /** @} */ /** A list of metadata read from the stream */ - char *copied_headers; + struct header_cache_entry *header_cache[HEADER_CACHE_SIZE]; + int n_cached_headers; /** ID number of the worker processing handling this image */ int id; @@ -184,6 +197,14 @@ extern struct image *image_read(const DataTemplate *dtempl, extern struct image *image_create_for_simulation(const DataTemplate *dtempl); extern void image_free(struct image *image); +extern void image_cache_header_float(struct image *image, + const char *header_name, + float header_val); + +extern void image_cache_header_int(struct image *image, + const char *header_name, + int header_val); + extern ImageFeatureList *image_read_peaks(const DataTemplate *dtempl, const char *filename, const char *event, diff --git a/libcrystfel/src/stream.c b/libcrystfel/src/stream.c index 83756c7a..68f623dd 100644 --- a/libcrystfel/src/stream.c +++ b/libcrystfel/src/stream.c @@ -639,8 +639,19 @@ int stream_write_chunk(Stream *st, const struct image *i, fprintf(st->fh, "beam_divergence = %.2e rad\n", i->div); fprintf(st->fh, "beam_bandwidth = %.2e (fraction)\n", i->bw); - /* FIXME: Better way of doing this */ - //imagefile_copy_fields(imfile, i->copyme, st->fh, ev); + for ( j=0; jn_cached_headers; j++ ) { + struct header_cache_entry *ce = i->header_cache[j]; + if ( ce->type == 'f' ) { + fprintf(st->fh, "header/float/%s = %f\n", + ce->header_name, ce->val_float); + } else if ( ce->type == 'i' ) { + fprintf(st->fh, "header/int/%s = %i\n", + ce->header_name, ce->val_int); + } else { + ERROR("Unrecognised header cache type '%c'\n", + ce->type); + } + } if ( i->detgeom != NULL ) { @@ -891,6 +902,48 @@ static void read_crystal(Stream *st, struct image *image, } +static void parse_header(const char *line_in, struct image *image, char type) +{ + char *line; + char *pos; + + line = strdup(line_in); + + pos = strchr(line, ' '); + if ( pos == NULL ) { + ERROR("Invalid header line '%s' (no space)\n", line); + return; + } + pos[0] = '\0'; + + if ( strlen(line_in) < strlen(line) + 3 ) { + ERROR("Invalid header line '%s' (too short)\n", line); + return; + } + + if ( (pos[1] != '=') || (pos[2] != ' ') ) { + ERROR("Invalid header line '%s' (wrong separator)\n", line); + return; + } + + if ( type == 'f' ) { + float v; + if ( sscanf(pos+3, "%f", &v) != 1 ) { + ERROR("Invalid header line '%s' (invalid value)\n", line); + return; + } + image_cache_header_float(image, line, v); + } else { + int v; + if ( sscanf(pos+3, "%i", &v) != 1 ) { + ERROR("Invalid header line '%s' (invalid value)\n", line); + return; + } + image_cache_header_int(image, line, v); + } +} + + /** * Read the next chunk from a stream and return an image structure */ @@ -928,6 +981,18 @@ struct image *stream_read_chunk(Stream *st, StreamFlags srf) image->ev = strdup(line+7); } + if ( strncmp(line, "hdf5/", 5) == 0 ) { + parse_header(line+5, image, 'f'); + } + + if ( strncmp(line, "header/int/", 11) == 0 ) { + parse_header(line+11, image, 'i'); + } + + if ( strncmp(line, "header/float/", 13) == 0 ) { + parse_header(line+13, image, 'f'); + } + if ( strncmp(line, "indexed_by = ", 13) == 0 ) { int err = 0; image->indexed_by = get_indm_from_string_2(line+13, &err); diff --git a/src/partialator.c b/src/partialator.c index 13795c65..249eda93 100644 --- a/src/partialator.c +++ b/src/partialator.c @@ -1550,7 +1550,7 @@ int main(int argc, char *argv[]) image_for_crystal->detgeom = NULL; image_for_crystal->features = NULL; image_for_crystal->spectrum = NULL; - image_for_crystal->copied_headers = NULL; + image_for_crystal->n_cached_headers = 0; image_for_crystal->dp = NULL; image_for_crystal->bad = NULL; image_for_crystal->sat = NULL; -- cgit v1.2.3