From 45492b842c3af2af542256417a8bab5bbc7bd5f7 Mon Sep 17 00:00:00 2001 From: Valerio Mariani Date: Fri, 9 May 2014 11:02:17 +0200 Subject: Multi-event mode --- libcrystfel/src/beam-parameters.c | 11 +- libcrystfel/src/beam-parameters.h | 7 +- libcrystfel/src/detector.c | 156 ++++++- libcrystfel/src/detector.h | 11 +- libcrystfel/src/events.c | 696 +++++++++++++++++++++++++++++++ libcrystfel/src/events.h | 122 ++++++ libcrystfel/src/hdf5-file.c | 856 ++++++++++++++++++++++++++++++++++---- libcrystfel/src/hdf5-file.h | 32 +- libcrystfel/src/image.c | 2 +- libcrystfel/src/image.h | 4 + libcrystfel/src/index.c | 2 +- libcrystfel/src/stream.c | 9 +- libcrystfel/src/stream.h | 5 +- 13 files changed, 1810 insertions(+), 103 deletions(-) create mode 100644 libcrystfel/src/events.c create mode 100644 libcrystfel/src/events.h (limited to 'libcrystfel/src') diff --git a/libcrystfel/src/beam-parameters.c b/libcrystfel/src/beam-parameters.c index 4b9941ac..d5bb93a1 100644 --- a/libcrystfel/src/beam-parameters.c +++ b/libcrystfel/src/beam-parameters.c @@ -8,6 +8,7 @@ * * Authors: * 2010,2012,2014 Thomas White + * 2014 Valerio Mariani * 2012 Chunhong Yoon * * This file is part of CrystFEL. @@ -160,10 +161,16 @@ void free_beam_parameters(struct beam_params *beam) } -void fill_in_beam_parameters(struct beam_params *beam, struct hdfile *f) +void fill_in_beam_parameters(struct beam_params *beam, struct hdfile *f, + struct event* ev) { if ( beam->photon_energy_from != NULL ) { - beam->photon_energy = get_value(f, beam->photon_energy_from); + if ( ev != NULL ) { + beam->photon_energy = get_ev_based_value(f, + beam->photon_energy_from, ev); + } else { + beam->photon_energy = get_value(f, beam->photon_energy_from); + } beam->photon_energy *= beam->photon_energy_scale; } } diff --git a/libcrystfel/src/beam-parameters.h b/libcrystfel/src/beam-parameters.h index e4085a0b..245c7c16 100644 --- a/libcrystfel/src/beam-parameters.h +++ b/libcrystfel/src/beam-parameters.h @@ -8,6 +8,7 @@ * * Authors: * 2010,2012-2014 Thomas White + * 2014 Valerio Mariani * 2012 Chunhong Yoon * * This file is part of CrystFEL. @@ -35,7 +36,10 @@ #endif struct beam_params; +struct event; +struct hdfile; +#include "events.h" #include "hdf5-file.h" struct beam_params @@ -62,7 +66,8 @@ extern "C" { extern struct beam_params *get_beam_parameters(const char *filename); extern void free_beam_parameters(struct beam_params *beam); -extern void fill_in_beam_parameters(struct beam_params *beam, struct hdfile *f); +extern void fill_in_beam_parameters(struct beam_params *beam, struct hdfile *f, + struct event* ev); #ifdef __cplusplus } diff --git a/libcrystfel/src/detector.c b/libcrystfel/src/detector.c index 2ccf6a75..4cf6dac9 100644 --- a/libcrystfel/src/detector.c +++ b/libcrystfel/src/detector.c @@ -462,7 +462,7 @@ struct panel *find_panel(struct detector *det, double fs, double ss) } -void fill_in_values(struct detector *det, struct hdfile *f) +void fill_in_values(struct detector *det, struct hdfile *f, struct event* ev) { int i; @@ -471,7 +471,13 @@ void fill_in_values(struct detector *det, struct hdfile *f) struct panel *p = &det->panels[i]; if ( p->clen_from != NULL ) { - p->clen = get_value(f, p->clen_from) * 1.0e-3; + + + if (det->path_dim !=0 || det->dim_dim !=0 ){ + p->clen = get_ev_based_value(f, p->clen_from, ev) * 1.0e-3; + } else { + p->clen = get_value(f, p->clen_from) * 1.0e-3; + } } p->clen += p->coffset; @@ -720,6 +726,11 @@ static int parse_field_for_panel(struct panel *panel, const char *key, ERROR("Invalid slow scan direction '%s'\n", val); reject = 1; } + } else if ( strncmp(key, "dim", 3) == 0) { + if ( panel->dim_structure == NULL ) { + panel->dim_structure = initialize_dim_structure(); + } + set_dim_structure_entry(panel->dim_structure, key, val); } else { ERROR("Unrecognised field '%s'\n", key); } @@ -868,7 +879,11 @@ struct detector *get_detector_geometry(const char *filename) char **bits; int i; int reject = 0; + int path_dim; + int dim_dim; int x, y, max_fs, max_ss; + int dim_reject = 0; + int dim_dim_reject = 0; fh = fopen(filename, "r"); if ( fh == NULL ) return NULL; @@ -887,6 +902,8 @@ struct detector *get_detector_geometry(const char *filename) det->mask_bad = 0; det->n_rigid_groups = 0; det->rigid_groups = NULL; + det->path_dim = 0; + det->dim_dim = 0; /* The default defaults... */ det->defaults.min_fs = -1; @@ -913,6 +930,7 @@ struct detector *get_detector_geometry(const char *filename) det->defaults.max_adu = +INFINITY; det->defaults.mask = NULL; det->defaults.data = NULL; + det->defaults.dim_structure = NULL; strncpy(det->defaults.name, "", 1023); do { @@ -1001,9 +1019,133 @@ struct detector *get_detector_geometry(const char *filename) max_fs = 0; max_ss = 0; + + path_dim = -1; + dim_reject = 0; + + for ( i=0; in_panels; i++ ) { + + int panel_dim = 0; + char *next_instance; + + next_instance = det->panels[i].data; + + while(next_instance) + { + next_instance = strstr(next_instance, "%"); + if ( next_instance != NULL ) { + next_instance += 1*sizeof(char); + panel_dim += 1; + } + } + + if ( path_dim == -1 ) { + path_dim = panel_dim; + } else { + if ( panel_dim != path_dim ) { + dim_reject = 1; + } + } + + } + for ( i=0; in_panels; i++ ) { - if ( det->panels[i].min_fs < 0 ) { + int panel_mask_dim = 0; + char *next_instance; + + if ( det->panels[i].mask != NULL ) { + + next_instance = det->panels[i].mask; + + while(next_instance) + { + next_instance = strstr(next_instance, "%"); + if ( next_instance != NULL ) { + next_instance += 1*sizeof(char); + panel_mask_dim += 1; + } + } + + if ( panel_mask_dim != path_dim ) { + dim_reject = 1; + } + } + } + + if ( dim_reject == 1) { + ERROR("All panels' data and mask entries must have the same number "\ + "of placeholders\n"); + reject = 1; + } + + det->path_dim = path_dim; + + dim_dim_reject = 0; + dim_dim = -1; + + for ( i=0; in_panels; i++ ) { + + int di; + int found_ss = 0; + int found_fs = 0; + int panel_dim_dim = 0; + + if ( det->panels[i].dim_structure == NULL ) { + det->panels[i].dim_structure = default_dim_structure(); + } + + for ( di=0; dipanels[i].dim_structure->num_dims; di++ ) { + + if ( det->panels[i].dim_structure->dims[di] == HYSL_UNDEFINED ) { + dim_dim_reject = 1; + } + if ( det->panels[i].dim_structure->dims[di] == HYSL_PLACEHOLDER ) { + panel_dim_dim += 1; + } + if ( det->panels[i].dim_structure->dims[di] == HYSL_SS ) { + found_ss += 1; + } + if ( det->panels[i].dim_structure->dims[di] == HYSL_FS ) { + found_fs += 1; + } + + } + + if ( found_ss != 1 ) { + ERROR("Only one slow scan dim coordinate is allowed\n"); + dim_dim_reject = 1; + } + + if ( found_fs != 1 ) { + ERROR("Only one fast scan dim coordinate is allowed\n"); + dim_dim_reject = 1; + } + + if ( panel_dim_dim > 1 ) { + ERROR("Maximum one placeholder dim coordinate is allowed\n"); + dim_dim_reject = 1; + } + + if ( dim_dim == -1 ) { + dim_dim = panel_dim_dim; + } else { + if ( panel_dim_dim != dim_dim ) { + dim_dim_reject = 1; + } + } + + } + + if ( dim_dim_reject == 1) { + reject = 1; + } + + det->dim_dim = dim_dim; + + for ( i=0; in_panels; i++ ) { + + if ( det->panels[i ].min_fs < 0 ) { ERROR("Please specify the minimum FS coordinate for" " panel %s\n", det->panels[i].name); reject = 1; @@ -1049,6 +1191,7 @@ struct detector *get_detector_geometry(const char *filename) " panel %s\n", det->panels[i].name); reject = 1; } + /* It's OK if the badrow direction is '0' */ /* It's not a problem if "no_index" is still zero */ /* The default transformation matrix is at least valid */ @@ -1240,6 +1383,10 @@ struct detector *simple_geometry(const struct image *image) geom->panels[0].max_fs = image->width-1; geom->panels[0].min_ss = 0; geom->panels[0].max_ss = image->height-1; + geom->panels[0].orig_min_fs = 0; + geom->panels[0].orig_max_fs = image->width-1; + geom->panels[0].orig_min_ss = 0; + geom->panels[0].orig_max_ss = image->height-1; geom->panels[0].cnx = -image->width / 2.0; geom->panels[0].cny = -image->height / 2.0; geom->panels[0].rigid_group = NULL; @@ -1262,6 +1409,9 @@ struct detector *simple_geometry(const struct image *image) geom->panels[0].w = image->width; geom->panels[0].h = image->height; + geom->panels[0].mask = NULL; + geom->panels[0].data = NULL; + find_min_max_d(geom); return geom; diff --git a/libcrystfel/src/detector.h b/libcrystfel/src/detector.h index 65865da7..3fa3c141 100644 --- a/libcrystfel/src/detector.h +++ b/libcrystfel/src/detector.h @@ -10,8 +10,8 @@ * Authors: * 2009-2014 Thomas White * 2011-2012 Richard Kirian - * 2011 Andrew Aquila * 2014 Valerio Mariani + * 2011 Andrew Aquila * * This file is part of CrystFEL. * @@ -42,6 +42,7 @@ struct detector; struct panel; struct badregion; struct detector; +struct hdfile; #include "hdf5-file.h" #include "image.h" @@ -90,6 +91,8 @@ struct panel double max_adu; /* Treat pixel as unreliable if higher than this */ char *data; + struct dim_structure *dim_structure; + double fsx; double fsy; double ssx; @@ -158,6 +161,9 @@ struct detector double furthest_in_fs; double furthest_in_ss; + int path_dim; + int dim_dim; + struct panel defaults; }; @@ -189,7 +195,8 @@ extern void get_pixel_extents(struct detector *det, double *min_x, double *min_y, double *max_x, double *max_y); -extern void fill_in_values(struct detector *det, struct hdfile *f); +extern void fill_in_values(struct detector *det, struct hdfile *f, + struct event* ev); extern struct detector *copy_geom(const struct detector *in); diff --git a/libcrystfel/src/events.c b/libcrystfel/src/events.c new file mode 100644 index 00000000..87c0530d --- /dev/null +++ b/libcrystfel/src/events.c @@ -0,0 +1,696 @@ +/* + * events.c + * + * Event properties + * + * Copyright © 2012-2014 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2014 Valerio Mariani + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see . + * + */ + +#define _ISOC99_SOURCE +#define _GNU_SOURCE + +#include "events.h" + +#include +#include +#include + + +struct event *initialize_event() +{ + + struct event *ev; + + ev = malloc(sizeof(struct event)); + ev->path_entries = NULL; + ev->path_length = 0; + + ev->dim_entries = NULL; + ev->dim_length = 0; + + return ev; + +} + + +struct event_list *initialize_event_list() +{ + + struct event_list *ev_list; + + ev_list = malloc(sizeof(struct event_list)); + + ev_list->events = NULL; + ev_list->num_events = 0; + + return ev_list; + +} + +struct filename_plus_event *initialize_filename_plus_event() +{ + + struct filename_plus_event *fpe; + + fpe = malloc(sizeof(struct filename_plus_event)); + + fpe->filename = NULL; + fpe->ev = NULL; + + return fpe; +} + + +int event_cmp(struct event *ev1, struct event *ev2) +{ + + int pi; + int di; + + if ( ev1->path_length != ev2->path_length || + ev1->dim_length != ev2->dim_length ) { + return 1; + } + + for ( pi=0; pipath_length; pi++ ) { + if ( strcmp(ev1->path_entries[pi], ev2->path_entries[pi]) != 0 ) { + return 1; + } + } + + for ( di=0; didim_length; di++ ) { + if ( ev1->path_entries[di] != ev2->path_entries[di] ) { + return 1; + } + } + + return 0; + +} + + +int add_non_existing_event_to_event_list(struct event_list *ev_list, + struct event *ev) +{ + + int evi; + int found = 0; + + for ( evi=0; evinum_events; evi++ ) { + if (event_cmp(ev_list->events[evi], ev) == 0 ) { + found = 1; + break; + } + } + + if ( found == 0) { + return append_event_to_event_list(ev_list, ev); + } + + return 0; +} + + +int append_event_to_event_list(struct event_list *ev_list, struct event *ev) +{ + + struct event **new_el; + + new_el = realloc(ev_list->events, + (1+ev_list->num_events)*sizeof(struct event*)); + if ( new_el == NULL ) { + return 1; + } + ev_list->events = new_el; + ev_list->events[ev_list->num_events] = copy_event(ev); + ev_list->num_events +=1; + + return 0; +} + + +struct event* copy_event(struct event *ev) +{ + + struct event *new_ev; + int pi, di; + + if ( ev->dim_length == 0 && ev->path_length == 0) { + + new_ev = initialize_event(); + + } else { + + new_ev=malloc(sizeof(struct event)); + + new_ev->path_entries = malloc(ev->path_length*sizeof(char *)); + new_ev->path_length = ev->path_length; + + new_ev->dim_entries = malloc(ev->dim_length*sizeof(int *)); + new_ev->dim_length = ev->dim_length; + + for ( pi=0; pipath_length; pi++ ) { + new_ev->path_entries[pi] = strdup(ev->path_entries[pi]); + } + + for ( di=0; didim_length; di++ ) { + new_ev->dim_entries[di] = ev->dim_entries[di]; + } + + } + return new_ev; +} + + +extern struct event_list *copy_event_list(struct event_list *el) +{ + int ei; + struct event_list *el_copy; + struct event ** events_copy; + + el_copy = malloc(1); + if ( el_copy == NULL ) { + return NULL; + } + + events_copy = malloc(el->num_events); + if ( events_copy == NULL ) { + free (el_copy); + return NULL; + } + el_copy->events = events_copy; + + for ( ei=0; einum_events; ei++ ) { + el_copy->events[ei]=copy_event(el->events[ei]); + } + + el_copy->num_events = el->num_events; + + return el_copy; +} + + + +void free_event(struct event *ev) +{ + int pi; + + if ( ev->path_length != 0 ) { + for ( pi=0; pipath_length; pi++ ) { + free(ev->path_entries[pi]); + } + } + free(ev->dim_entries); + free(ev); +} + + +void free_event_list(struct event_list *el) +{ + int ei; + + for ( ei=0; einum_events; ei++ ) { + free_event(el->events[ei]); + } + free(el); +} + + +void free_filename_plus_event(struct filename_plus_event *fpe) +{ + + free(fpe->filename); + + if ( fpe->ev != NULL ) { + free_event(fpe->ev); + } +} + + +char *get_event_string(struct event *ev) +{ + char *ret_string; + char *new_ret_string; + int ret_string_len; + + if ( ev->path_length != 0) { + + int pi; + + ret_string = strdup(ev->path_entries[0]); + ret_string_len = strlen(ret_string); + + for ( pi=1; pipath_length; pi++ ) { + + new_ret_string = realloc(ret_string, + (ret_string_len+1+strlen(ev->path_entries[pi])) + *sizeof(char)); + if ( new_ret_string == NULL ) { + return NULL; + } + + ret_string=new_ret_string; + strncpy(&ret_string[ret_string_len],"/", 1); + strncpy(&ret_string[ret_string_len+1],ev->path_entries[pi], + strlen(ev->path_entries[pi])); + + ret_string_len += 1+strlen(ev->path_entries[pi]); + + } + + new_ret_string = realloc(ret_string, + (1+ret_string_len)*sizeof(char)); + if ( new_ret_string == NULL ) { + return NULL; + } + + ret_string = new_ret_string; + + strncpy(&ret_string[ret_string_len], "/", 1); + ret_string_len += 1; + + } else { + + ret_string = strdup("/"); + ret_string_len = strlen(ret_string); + + } + + if ( ev->dim_length !=0 ) { + + char num_buf[64]; + int di; + + for ( di=0; didim_length; di++ ) { + sprintf(num_buf, "%i", ev->dim_entries[di]); + + new_ret_string = realloc(ret_string, + (ret_string_len+1+strlen(num_buf)) + *sizeof(char)); + if ( new_ret_string == NULL ) { + return NULL; + } + + ret_string=new_ret_string; + + strncpy(&ret_string[ret_string_len],"/", 1); + strncpy(&ret_string[ret_string_len+1], num_buf, + strlen(num_buf)); + ret_string_len += 1+strlen(num_buf); + + } + + } else { + + new_ret_string = realloc(ret_string, + (1+ret_string_len)*sizeof(char)); + if ( new_ret_string == NULL ) { + return NULL; + } + + ret_string = new_ret_string; + + strncpy(&ret_string[ret_string_len], "/", 1); + ret_string_len += 1; + + } + + new_ret_string = realloc(ret_string, + (1+ret_string_len)*sizeof(char)); + if ( new_ret_string == NULL ) { + return NULL; + } + + ret_string = new_ret_string; + + strncpy(&ret_string[ret_string_len], "\0", 1); + + return ret_string; +} + + +struct event* get_event_from_event_string(char* ev_string) +{ + struct event* ev; + char *ev_sep; + char buf_path[1024]; + char buf_dim[1024]; + char *sep; + char *start; + + ev = initialize_event(); + if ( ev == NULL ) { + return NULL; + } + + ev_sep = strstr(ev_string, "//"); + if ( ev_sep == NULL ) { + return NULL; + } + + strncpy(buf_path, ev_string, ev_sep-ev_string); + buf_path[ev_sep-ev_string] = '\0'; + + strncpy(buf_dim, ev_sep+2, strlen(ev_sep)-2); + buf_dim[strlen(ev_sep)-2] = '\0'; + + if ( strlen(buf_path) !=0 ) { + + do { + + start = buf_path; + + char buf[2014]; + + sep = strstr(start, "/"); + if ( sep != NULL ) { + + strncpy(buf, start, sep-start); + buf[sep-start]='\0'; + push_path_entry_to_event(ev, buf); + start = sep + 1; + + } else { + + sprintf(buf,"%s",start); + push_path_entry_to_event(ev, buf); + + } + } while (sep); + + } + + + if ( strlen(buf_dim) !=0 ) { + + start = buf_dim; + + do { + + char buf[2014]; + int buf_int; + + sep = strstr(start, "/"); + if ( sep != NULL ) { + strncpy(buf, start, sep-start); + buf[sep-start]='\0'; + buf_int = atoi(buf); + push_dim_entry_to_event(ev, buf_int); + start = sep + 1; + + } else { + + sprintf(buf,"%s",start); + buf_int = atoi(buf); + push_dim_entry_to_event(ev, buf_int); + + } + } while (sep); + + } + + + return ev; +} + + +int push_path_entry_to_event(struct event *ev, const char * entry) +{ + char **new_path_entries; + + new_path_entries = realloc(ev->path_entries, + (1+ev->path_length)*sizeof(char *)); + if ( new_path_entries == NULL ) { + return 1; + } + + ev->path_entries = new_path_entries; + ev->path_entries[ev->path_length] = strdup(entry); + ev->path_length += 1; + + return 0; +} + + +int push_dim_entry_to_event(struct event *ev, int entry) +{ + int *new_dim_entries; + + new_dim_entries = realloc(ev->dim_entries, + (1+ev->dim_length)*sizeof(int)); + if ( new_dim_entries == NULL ) { + return 1; + } + + ev->dim_entries = new_dim_entries; + ev->dim_entries[ev->dim_length] = entry; + ev->dim_length += 1; + + return 0; +} + + +int pop_path_entry_from_event(struct event *ev) +{ + char **new_path_entries; + + if ( ev->path_length == 0 ) { + return 1; + } + + free(ev->path_entries[ev->path_length-1]); + + if ( ev->path_length == 1 ) { + ev->path_entries = NULL; + ev->path_length = 0; + return 0; + } + + new_path_entries = realloc(ev->path_entries, + (ev->path_length-1)*sizeof(char *)); + + if ( new_path_entries == NULL) { + return 1; + } + + ev->path_entries = new_path_entries; + ev->path_length = ev->path_length-1; + + return 0; +} + + +int pop_dim_entry_from_event(struct event *ev) +{ + int *new_dim_entries; + + if ( ev->dim_length == 0 ) { + return 1; + } + + if ( ev->dim_length == 1 ) { + ev->dim_entries = NULL; + ev->dim_length = 0; + return 0; + } + + new_dim_entries = realloc(ev->dim_entries, + (ev->dim_length-1)*sizeof(int)); + + if ( new_dim_entries == NULL) { + return 1; + } + + ev->dim_entries = new_dim_entries; + ev->dim_length = ev->dim_length-1; + + return 0; +} + + +char *event_path_placeholder_subst(const char * entry, + const char * data) +{ + + char *ph_loc; + char *full_path; + int len_head, len_tail; + + full_path = malloc(strlen(data) + strlen(entry)); + ph_loc = strstr(data, "%"); + len_head = ph_loc-data; + len_tail = strlen(ph_loc); + + strncpy(full_path, data, len_head); + strncpy(full_path+len_head, entry, strlen(entry)); + strncpy(full_path+len_head+strlen(entry), ph_loc+1, len_tail); + strncpy(&full_path[strlen(data) + strlen(entry)],"\0",1); + + return full_path; +} + + +char *retrieve_full_path(struct event *ev, const char *data) +{ + + int ei ; + char *return_value; + + return_value = strdup(data); + + for ( ei=0; eipath_length; ei++ ) { + + char *tmp_subst_data; + tmp_subst_data = event_path_placeholder_subst(ev->path_entries[ei], + return_value); + + free(return_value); + return_value = strdup(tmp_subst_data); + free(tmp_subst_data); + + } + + return return_value; + +} + + +extern char *partial_event_substitution(struct event *ev, const char *data) +{ + int ei ; + char *return_value; + char *pholder; + + return_value = strdup(data); + pholder = strstr(return_value,"%"); + ei = 0; + + while( pholder != NULL) { + + char *tmp_subst_data; + + tmp_subst_data = event_path_placeholder_subst(ev->path_entries[ei], + return_value); + free(return_value); + return_value = strdup(tmp_subst_data); + free(tmp_subst_data); + pholder = strstr(return_value, "%"); + ei += 1; + } + + return return_value; +} + + +struct dim_structure *initialize_dim_structure() +{ + struct dim_structure *hs; + hs = malloc(sizeof(struct dim_structure)); + if ( hs == NULL ) { + return NULL; + } + + hs->dims = NULL; + hs->num_dims = 0; + + return hs; +} + + +struct dim_structure *default_dim_structure() +{ + struct dim_structure *hsd; + + hsd = initialize_dim_structure(); + + set_dim_structure_entry(hsd, "dim0", "ss"); + set_dim_structure_entry(hsd, "dim1", "fs"); + + return hsd; +} + + +void free_dim_structure(struct dim_structure *hsd) +{ + int di; + + for ( di=0; dinum_dims; di++ ) { + free (hsd->dims); + free (hsd); + } +} + + +static int parse_dim_structure_val(const char *val) +{ + if ( strcmp(val,"%") == 0 ) { + return HYSL_PLACEHOLDER; + } else if ( strcmp(val,"ss") == 0 ) { + return HYSL_SS; + } else if ( strcmp(val,"fs") == 0 ) { + return HYSL_FS; + } + return atoi(val); + +} + + +int set_dim_structure_entry(struct dim_structure *hsd, const char *string_dim, + const char *val_string) +{ + int dim_entry; + + dim_entry = atoi(string_dim+3)+1; + + if ( dim_entry > hsd->num_dims ) { + + int di; + + int *new_dims = malloc(dim_entry*sizeof(int)); + if ( new_dims == NULL ) { + return 0; + } + + + for ( di=0; dinum_dims; di++ ) { + new_dims[di] = hsd->dims[di]; + } + + new_dims[dim_entry-1] = parse_dim_structure_val(val_string); + if ( hsd->dims == NULL ) { + free (hsd->dims); + } + hsd->dims = new_dims; + hsd->num_dims = dim_entry; + + return 1; + + } + + hsd->dims[dim_entry] = parse_dim_structure_val(val_string); + return 1; +} diff --git a/libcrystfel/src/events.h b/libcrystfel/src/events.h new file mode 100644 index 00000000..797df4d2 --- /dev/null +++ b/libcrystfel/src/events.h @@ -0,0 +1,122 @@ +/* + * events.h + * + * Event properties + * + * Copyright © 2012-2014 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2014 Valerio Mariani + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see . + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#ifndef EVENTS_H +#define EVENTS_H + +#include "detector.h" + +struct event +{ + char **path_entries; + int path_length; + int *dim_entries; + int dim_length; +}; + +struct event_list +{ + struct event **events; + int num_events; +}; + +struct filename_plus_event +{ + char *filename; + struct event *ev; +}; + +enum +{ + HYSL_UNDEFINED = -99, + HYSL_PLACEHOLDER = -98, + HYSL_FS = -1, + HYSL_SS = -2 +}; + +struct dim_structure +{ + int *dims; + int num_dims; +}; + +extern struct event *initialize_event(); +extern int push_path_entry_to_event(struct event *ev, const char * entry); +extern int pop_path_entry_from_event(struct event *ev); +extern int push_dim_entry_to_event(struct event *ev, int entry); +extern int pop_dim_entry_from_event(struct event *ev); +extern struct event* copy_event(struct event *ev); +extern void free_event(struct event *ev); +extern char *get_event_string(struct event * ev); +extern struct event* get_event_from_event_string(char* ev_string); +extern char *event_path_placeholder_subst(const char * ev_name, + const char * data); +extern char *partial_event_substitution(struct event *ev, const char *data); +extern char *retrieve_full_path(struct event *ev, const char *data); + + +extern struct filename_plus_event *initialize_filename_plus_event(); +extern void free_filename_plus_event(struct filename_plus_event *fpe); + + +extern struct event_list *initialize_event_list(); +extern int append_event_to_event_list(struct event_list *ev_list, + struct event *ev); +int add_non_existing_event_to_event_list(struct event_list *ev_list, + struct event *ev); +extern struct event_list *copy_event_list(struct event_list *el); +extern void free_event_list(struct event_list *el); + + +extern struct dim_structure *initialize_dim_structure(); +extern struct dim_structure *default_dim_structure(); +extern int set_dim_structure_entry(struct dim_structure *hsd, + const char *string_dim, + const char *val_string); +extern void free_dim_structure_entry(struct dim_structure *hsd); + + + + + + + + + + + + + + + + +#endif /* EVENTS_H */ diff --git a/libcrystfel/src/hdf5-file.c b/libcrystfel/src/hdf5-file.c index 297f78ac..df37e77f 100644 --- a/libcrystfel/src/hdf5-file.c +++ b/libcrystfel/src/hdf5-file.c @@ -37,6 +37,7 @@ #include #include +#include "events.h" #include "image.h" #include "hdf5-file.h" #include "utils.h" @@ -103,9 +104,6 @@ struct hdfile *hdfile_open(const char *filename) f = malloc(sizeof(struct hdfile)); if ( f == NULL ) return NULL; - /* Please stop spamming my terminal */ - H5Eset_auto2(H5E_DEFAULT, NULL, NULL); - f->fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); if ( f->fh < 0 ) { ERROR("Couldn't open file: %s\n", filename); @@ -118,11 +116,14 @@ struct hdfile *hdfile_open(const char *filename) } -int hdfile_set_image(struct hdfile *f, const char *path) +int hdfile_set_image(struct hdfile *f, const char *path, + struct panel *p) { hsize_t size[2]; hsize_t max_size[2]; hid_t sh; + int sh_dim; + int di; f->dh = H5Dopen2(f->fh, path, H5P_DEFAULT); if ( f->dh < 0 ) { @@ -130,17 +131,47 @@ int hdfile_set_image(struct hdfile *f, const char *path) return -1; } f->data_open = 1; - sh = H5Dget_space(f->dh); - if ( H5Sget_simple_extent_ndims(sh) != 2 ) { - ERROR("Dataset is not two-dimensional\n"); - return -1; + sh_dim = H5Sget_simple_extent_ndims(sh); + + if ( p == NULL ) { + + if ( sh_dim != 2 ) { + ERROR("Dataset is not two-dimensional\n"); + return -1; + } + + } else { + + if ( sh_dim != p->dim_structure->num_dims ) { + ERROR("Dataset dimensionality does not match geometry file\n"); + return -1; + } + } + H5Sget_simple_extent_dims(sh, size, max_size); H5Sclose(sh); - f->nx = size[0]; - f->ny = size[1]; + if ( p == NULL ) { + + f->nx = size[0]; + f->ny = size[1]; + + } else { + + for (di=0; didim_structure->num_dims; di++ ) { + + if (p->dim_structure->dims[di] == HYSL_SS ) { + f->ny = size[di]; + } + if (p->dim_structure->dims[di] == HYSL_FS ) { + f->nx = size[di]; + } + + } + + } return 0; } @@ -234,15 +265,17 @@ int get_peaks(struct image *image, struct hdfile *f, const char *p) static void cleanup(hid_t fh) { int n_ids, i; - hid_t ids[256]; + hid_t ids[2048]; + + n_ids = H5Fget_obj_ids(fh, H5F_OBJ_ALL, 2048, ids); - n_ids = H5Fget_obj_ids(fh, H5F_OBJ_ALL, 256, ids); for ( i=0; idata_open ) { H5Dclose(f->dh); } + cleanup(f->fh); H5Fclose(f->fh); + free(f); } @@ -401,8 +438,8 @@ int hdf5_write_image(const char *filename, struct image *image, char *element) new_panel_idxs = realloc(locations[li].panel_idxs, (locations[li].n_panels+1)*sizeof(int)); if ( new_panel_idxs == NULL ) { - ERROR("Error while managing write location list for file: %s\n", - filename); + ERROR("Error while managing write location list for" + "file: %s\n", filename); return 1; } locations[li].panel_idxs = new_panel_idxs; @@ -422,9 +459,11 @@ int hdf5_write_image(const char *filename, struct image *image, char *element) struct hdf5_write_location * new_locations; new_locations = realloc(locations, - (num_locations+1)*sizeof(struct hdf5_write_location)); + (num_locations+1)* + sizeof(struct hdf5_write_location)); if ( new_locations == NULL ) { - ERROR("Error while managing write location list for file: %s\n", + ERROR("Error while managing write location list for " + "file: %s\n", filename); return 1; } @@ -432,8 +471,8 @@ int hdf5_write_image(const char *filename, struct image *image, char *element) new_location = &locations[num_locations]; new_location = malloc(sizeof(struct hdf5_write_location)); if ( new_location == NULL ) { - ERROR("Error while managing write location list for file: %s\n", - filename); + ERROR("Error while managing write location list for " + "file: %s\n", filename); return 1; } locations[num_locations].max_ss = p.orig_max_ss; @@ -441,8 +480,8 @@ int hdf5_write_image(const char *filename, struct image *image, char *element) locations[num_locations].location = p_location; locations[num_locations].panel_idxs = malloc(sizeof(int)); if ( locations[num_locations].panel_idxs == NULL ) { - ERROR("Error while managing write location list for file: %s\n", - filename); + ERROR("Error while managing write location list for " + "file: %s\n", filename); return 1; } locations[num_locations].panel_idxs[0] = pi; @@ -845,13 +884,66 @@ static int unpack_panels(struct image *image, struct detector *det) } -int hdf5_read(const char *filename, struct image *image, const char* element, int satcorr) +int hdf5_read(struct hdfile *f, struct image *image, + const char* element, int satcorr) { - return hdf5_read2(filename, image, element, satcorr, 0); + herr_t r; + float *buf; + int fail; + + if ( element == NULL ) { + fail = hdfile_set_first_image(f, "/"); + } else { + fail = hdfile_set_image(f, element, NULL); + } + + if ( fail ) { + ERROR("Couldn't select path\n"); + return 1; + } + + /* Note the "swap" here, according to section 3.2.5, + * "C versus Fortran Dataspaces", of the HDF5 user's guide. */ + image->width = f->ny; + image->height = f->nx; + + buf = malloc(sizeof(float)*f->nx*f->ny); + + r = H5Dread(f->dh, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, buf); + if ( r < 0 ) { + ERROR("Couldn't read data\n"); + free(buf); + return 1; + } + image->data = buf; + + if ( satcorr ) debodge_saturation(f, image); + + if ( image->beam != NULL ) { + + fill_in_beam_parameters(image->beam, f, NULL); + image->lambda = ph_en_to_lambda(eV_to_J(image->beam->photon_energy)); + + if ( (image->beam->photon_energy < 0.0) + || (image->lambda > 1000) ) { + /* Error message covers a silly value in the beam file + * or in the HDF5 file. */ + ERROR("Nonsensical wavelength (%e m or %e eV) value " + "for %s.\n", + image->lambda, image->beam->photon_energy, + image->filename); + return 1; + } + + } + + return 0; } -int hdf5_read2(const char *filename, struct image *image, const char* element, int satcorr, int override_data_and_mask) +int hdf5_read2(struct hdfile *f, struct image *image, + struct event *ev, int satcorr) { herr_t r; float *buf; @@ -862,8 +954,7 @@ int hdf5_read2(const char *filename, struct image *image, const char* element, i int mask_is_present; int no_mask_loaded; int pi; - hid_t mask_dh = NULL; - struct hdfile *f; + hid_t mask_dh = 0; if ( image->det == NULL ) { ERROR("Geometry not available\n"); @@ -898,15 +989,12 @@ int hdf5_read2(const char *filename, struct image *image, const char* element, i curr_ss = 0; no_mask_loaded = 1; - f = hdfile_open(filename); - if ( f == NULL ) { - return 1; - } - for ( pi=0; pidet->n_panels; pi++ ) { int data_width, data_height; - hsize_t f_offset[2], f_count[2]; + hsize_t *f_offset, *f_count; + int hsi; + struct dim_structure *hsd; hsize_t m_offset[2], m_count[2]; hsize_t dimsm[2]; herr_t check; @@ -916,26 +1004,53 @@ int hdf5_read2(const char *filename, struct image *image, const char* element, i struct panel *p; p=&image->det->panels[pi]; - if ( override_data_and_mask ) { - fail = hdfile_set_image(f, element); + if ( ev != NULL ) { + + int exists; + char *panel_full_path; + + panel_full_path = retrieve_full_path + (ev, p->data); + + exists = check_path_existence(f->fh, panel_full_path); + if ( !exists ) { + ERROR("Cannot find data for panel %s\n", + p->name); + return 1; + } + + fail = hdfile_set_image(f, panel_full_path, p); + free(panel_full_path); + } else { - if ( p->data != NULL ) { - fail = hdfile_set_image(f, p->data); - } else if ( element != NULL ) { - fail = hdfile_set_image(f, element); + + if ( p->data == NULL ) { + + fail = hdfile_set_first_image(f, "/"); + } else { - fail = hdfile_set_first_image(f,"/"); + + int exists; + exists = check_path_existence(f->fh, p->data); + if ( !exists ) { + ERROR("Cannot find data for panel %s\n", + p->name); + return 1; + } + fail = hdfile_set_image(f, p->data, p); + } + } if ( fail ) { ERROR("Couldn't select path for panel %s\n", p->name); - hdfile_close(f); return 1; } data_width = f->ny; data_height = f->nx; + if ( (data_width < p->w ) || (data_height < p->h) ) { @@ -948,10 +1063,29 @@ int hdf5_read2(const char *filename, struct image *image, const char* element, i return 1; } - f_offset[0] = p->orig_min_ss; - f_offset[1] = p->orig_min_fs; - f_count[0] = p->orig_max_ss - p->orig_min_ss +1; - f_count[1] = p->orig_max_fs - p->orig_min_fs +1; + hsd = image->det->panels[pi].dim_structure; + + f_offset = malloc(hsd->num_dims*sizeof(hsize_t)); + f_count = malloc(hsd->num_dims*sizeof(hsize_t)); + + for ( hsi=0; hsinum_dims; hsi++ ) { + + if ( hsd->dims[hsi] == HYSL_FS ) { + f_offset[hsi] = p->orig_min_fs; + f_count[hsi] = p->orig_max_fs - p->orig_min_fs +1; + } else if ( hsd->dims[hsi] == HYSL_SS ) { + f_offset[hsi] = p->orig_min_ss; + f_count[hsi] = p->orig_max_ss - p->orig_min_ss +1; + } else if (hsd->dims[hsi] == HYSL_PLACEHOLDER ) { + f_offset[hsi] = ev->dim_entries[0]; + f_count[hsi] = 1; + } else { + f_offset[hsi] = hsd->dims[hsi]; + f_count[hsi] = 1; + } + + } + dataspace = H5Dget_space(f->dh); check = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, f_offset, NULL, f_count, NULL); @@ -959,7 +1093,6 @@ int hdf5_read2(const char *filename, struct image *image, const char* element, i ERROR("Error selecting file dataspace for panel %s\n", p->name); free(buf); - hdfile_close(f); return 1; } @@ -976,60 +1109,118 @@ int hdf5_read2(const char *filename, struct image *image, const char* element, i ERROR("Error selecting memory dataspace for panel %s\n", p->name); free(buf); - hdfile_close(f); return 1; } + r = H5Dread(f->dh, H5T_NATIVE_FLOAT, memspace, dataspace, H5P_DEFAULT, buf); if ( r < 0 ) { ERROR("Couldn't read data for panel %s\n", p->name); free(buf); - hdfile_close(f); return 1; } H5Dclose(f->dh); f->data_open = 0; H5Sclose(dataspace); + H5Sclose(memspace); if ( p->mask != NULL ) { - mask_dh = H5Dopen2(f->fh, p->mask, H5P_DEFAULT); - if ( mask_dh <= 0 ) { - ERROR("Couldn't open flags for panel %s\n", - p->name); - image->flags = NULL; - } else { - mask_dataspace = H5Dget_space(mask_dh); - check = H5Sselect_hyperslab(mask_dataspace, H5S_SELECT_SET, - f_offset, NULL, f_count, NULL); - if ( check < 0 ) { - ERROR("Error selecting mask dataspace for panel %s\n", + if ( ev != NULL ) { + + int exists; + char *mask_full_path; + + mask_full_path = retrieve_full_path (ev, p->mask); + + exists = check_path_existence(f->fh, mask_full_path); + if ( !exists ) { + ERROR("Cannot find flags for panel %s\n", p->name); + return 1; } - r = H5Dread(mask_dh, H5T_NATIVE_UINT16, memspace, mask_dataspace, - H5P_DEFAULT, flags); - if ( r < 0 ) { - ERROR("Couldn't read flags for panel %s\n", + + mask_dh = H5Dopen2(f->fh, mask_full_path, H5P_DEFAULT); + + if ( mask_dh <= 0 ) { + ERROR("Couldn't open flags for panel %s\n", p->name); + image->flags = NULL; } else { - no_mask_loaded = 0; + + mask_dataspace = H5Dget_space(mask_dh); + check = H5Sselect_hyperslab(mask_dataspace, H5S_SELECT_SET, + f_offset, NULL, f_count, NULL); + if ( check < 0 ) { + ERROR("Error selecting mask dataspace for panel %s\n", + p->name); + } + r = H5Dread(mask_dh, H5T_NATIVE_UINT16, memspace, + mask_dataspace, H5P_DEFAULT, flags); + if ( r < 0 ) { + ERROR("Couldn't read flags for panel %s\n", + p->name); + } else { + no_mask_loaded = 0; + } + + H5Sclose(mask_dataspace); + H5Dclose(mask_dh); + } - H5Sclose(mask_dataspace); - H5Dclose(mask_dh); + } else { + + int exists; + exists = check_path_existence(f->fh, p->mask); + if ( !exists ) { + ERROR("Cannot find flags for panel %s\n", + p->name); + return 1; + } + + mask_dh = H5Dopen2(f->fh, p->mask, H5P_DEFAULT); + if ( mask_dh <= 0 ) { + ERROR("Couldn't open flags for panel %s\n", + p->name); + image->flags = NULL; + } else { + + mask_dataspace = H5Dget_space(mask_dh); + check = H5Sselect_hyperslab(mask_dataspace, H5S_SELECT_SET, + f_offset, NULL, f_count, NULL); + if ( check < 0 ) { + ERROR("Error selecting mask dataspace for panel %s\n", + p->name); + } + r = H5Dread(mask_dh, H5T_NATIVE_UINT16, memspace, + mask_dataspace, H5P_DEFAULT, flags); + if ( r < 0 ) { + ERROR("Couldn't read flags for panel %s\n", + p->name); + } else { + no_mask_loaded = 0; + } + + H5Sclose(mask_dataspace); + H5Dclose(mask_dh); + + } } } - H5Sclose(memspace); - p->min_fs = m_min_fs; p->max_fs = m_max_fs; p->min_ss = curr_ss; p->max_ss = curr_ss + p->h-1; curr_ss += p->h; + + free(f_offset); + free(f_count); + } image->data = buf; @@ -1042,13 +1233,14 @@ int hdf5_read2(const char *filename, struct image *image, const char* element, i if ( satcorr ) debodge_saturation(f, image); - fill_in_values(image->det, f); + fill_in_values(image->det, f, ev); unpack_panels(image, image->det); if ( image->beam != NULL ) { - fill_in_beam_parameters(image->beam, f); + fill_in_beam_parameters(image->beam, f, ev); + image->lambda = ph_en_to_lambda(eV_to_J(image->beam->photon_energy)); if ( (image->beam->photon_energy < 0.0) @@ -1056,16 +1248,15 @@ int hdf5_read2(const char *filename, struct image *image, const char* element, i /* Error message covers a silly value in the beam file * or in the HDF5 file. */ ERROR("Nonsensical wavelength (%e m or %e eV) value " - "for %s.\n", + "for file: %s, event: %s.\n", image->lambda, image->beam->photon_energy, - image->filename); + image->filename, get_event_string(image->event)); hdfile_close(f); return 1; } } - hdfile_close(f); return 0; } @@ -1099,13 +1290,15 @@ int hdfile_is_scalar(struct hdfile *f, const char *name, int verbose) hid_t type; int ndims; int i; + int check; - dh = H5Dopen2(f->fh, name, H5P_DEFAULT); - if ( dh < 0 ) { + check = check_path_existence(f->fh, name); + if ( check == 0 ) { ERROR("No such field '%s'\n", name); return 0; } + dh = H5Dopen2(f->fh, name, H5P_DEFAULT); type = H5Dget_type(dh); /* Get the dimensionality. We have to cope with scalars expressed as @@ -1143,6 +1336,8 @@ int hdfile_is_scalar(struct hdfile *f, const char *name, int verbose) } + + static int get_f_value(struct hdfile *f, const char *name, double *val) { hid_t dh; @@ -1150,15 +1345,18 @@ static int get_f_value(struct hdfile *f, const char *name, double *val) hid_t class; herr_t r; double buf; + int check; if ( !hdfile_is_scalar(f, name, 1) ) return 1; - dh = H5Dopen2(f->fh, name, H5P_DEFAULT); - if ( dh < 0 ) { + check = check_path_existence(f->fh, name); + if ( check == 0 ) { ERROR("No such field '%s'\n", name); return 1; } + dh = H5Dopen2(f->fh, name, H5P_DEFAULT); + type = H5Dget_type(dh); class = H5Tget_class(type); @@ -1183,6 +1381,152 @@ static int get_f_value(struct hdfile *f, const char *name, double *val) } +static int get_ev_based_f_value(struct hdfile *f, const char *name, + struct event *ev, double *val) +{ + hid_t dh; + hid_t type; + hid_t class; + hid_t sh; + hid_t ms; + hsize_t *f_offset = NULL; + hsize_t *f_count = NULL; + hsize_t m_offset[1]; + hsize_t m_count[1]; + hsize_t msdims[1]; + hsize_t size[3]; + herr_t r; + herr_t check; + double buf; + int check_pe; + int dim_flag; + int ndims; + int i; + char* subst_name = NULL; + + if ( ev->path_length != 0 ) { + subst_name = partial_event_substitution(ev, name); + } else { + subst_name = strdup(name); + } + + check_pe = check_path_existence(f->fh, subst_name); + if ( check_pe == 0 ) { + ERROR("No such field '%s'\n", subst_name); + return 1; + } + + dh = H5Dopen2(f->fh, name, H5P_DEFAULT); + type = H5Dget_type(dh); + class = H5Tget_class(type); + + if ( class != H5T_FLOAT ) { + ERROR("Not a floating point value.\n"); + H5Tclose(type); + H5Dclose(dh); + return 1; + } + + /* Get the dimensionality. We have to cope with scalars expressed as + * arrays with all dimensions 1, as well as zero-d arrays. */ + sh = H5Dget_space(dh); + ndims = H5Sget_simple_extent_ndims(sh); + if ( ndims > 3 ) { + H5Tclose(type); + H5Dclose(dh); + return 1; + } + H5Sget_simple_extent_dims(sh, size, NULL); + + m_offset[0] = 0; + m_count[0] = 1; + msdims[0] = 1; + ms = H5Screate_simple(1,msdims,NULL); + + /* Check that the size in all dimensions is 1 */ + /* or that one of the dimensions has the same */ + /* size as the hyperplane events */ + + dim_flag = 0; + + for ( i=0; i ev->dim_entries[0] ) { + dim_flag = 1; + } else { + H5Tclose(type); + H5Dclose(dh); + return 1; + } + } + } + + if ( dim_flag == 0 ) { + + r = H5Dread(dh, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, + H5P_DEFAULT, &buf); + + if ( r < 0 ) { + ERROR("Couldn't read value.\n"); + H5Tclose(type); + H5Dclose(dh); + return 1; + } + + } else { + + f_offset = malloc(ndims*sizeof(hsize_t)); + f_count = malloc(ndims*sizeof(hsize_t)); + + for ( i=0; idim_entries[0]; + f_count[i] = 1; + } else { + f_offset[i] = 0; + f_count[i] = 0; + } + + } + + check = H5Sselect_hyperslab(sh, H5S_SELECT_SET, + f_offset, NULL, f_count, NULL); + if ( check <0 ) { + ERROR("Error selecting dataspace for float value"); + free(f_offset); + free(f_count); + return 1; + } + + ms = H5Screate_simple(1,msdims,NULL); + check = H5Sselect_hyperslab(ms, H5S_SELECT_SET, + m_offset, NULL, m_count, NULL); + if ( check < 0 ) { + ERROR("Error selecting memory dataspace for float value"); + free(f_offset); + free(f_count); + return 1; + } + + r = H5Dread(dh, H5T_NATIVE_DOUBLE, ms, sh, + H5P_DEFAULT, &buf); + if ( r < 0 ) { + ERROR("Couldn't read value.\n"); + H5Tclose(type); + H5Dclose(dh); + return 1; + } + + } + + free(subst_name); + *val = buf; + + return 0; +} + + static int get_i_value(struct hdfile *f, const char *name, int *val) { hid_t dh; @@ -1190,15 +1534,17 @@ static int get_i_value(struct hdfile *f, const char *name, int *val) hid_t class; herr_t r; int buf; + int check; if ( !hdfile_is_scalar(f, name, 1) ) return 1; - dh = H5Dopen2(f->fh, name, H5P_DEFAULT); - if ( dh < 0 ) { + check = check_path_existence(f->fh, name); + if ( check == 0 ) { ERROR("No such field '%s'\n", name); return 1; } + dh = H5Dopen2(f->fh, name, H5P_DEFAULT); type = H5Dget_type(dh); class = H5Tget_class(type); @@ -1230,6 +1576,14 @@ double get_value(struct hdfile *f, const char *name) return val; } +double get_ev_based_value(struct hdfile *f, const char *name, + struct event *ev) +{ + double val = -1; + get_ev_based_f_value(f, name, ev, &val); + return val; +} + struct copy_hdf5_field { @@ -1306,7 +1660,7 @@ void add_copy_hdf5_field(struct copy_hdf5_field *copyme, void copy_hdf5_fields(struct hdfile *f, const struct copy_hdf5_field *copyme, - FILE *fh) + FILE *fh, struct event *ev) { int i; @@ -1318,7 +1672,7 @@ void copy_hdf5_fields(struct hdfile *f, const struct copy_hdf5_field *copyme, char *field; field = copyme->fields[i]; - val = hdfile_get_string_value(f, field); + val = hdfile_get_string_value(f, field, ev); if ( field[0] == '/' ) { fprintf(fh, "hdf5%s = %s\n", field, val); @@ -1332,7 +1686,8 @@ void copy_hdf5_fields(struct hdfile *f, const struct copy_hdf5_field *copyme, } -char *hdfile_get_string_value(struct hdfile *f, const char *name) +char *hdfile_get_string_value(struct hdfile *f, const char *name, + struct event* ev) { hid_t dh; hsize_t size; @@ -1344,7 +1699,6 @@ char *hdfile_get_string_value(struct hdfile *f, const char *name) dh = H5Dopen2(f->fh, name, H5P_DEFAULT); if ( dh < 0 ) return NULL; - type = H5Dget_type(dh); class = H5Tget_class(type); @@ -1376,7 +1730,11 @@ char *hdfile_get_string_value(struct hdfile *f, const char *name) switch ( class ) { case H5T_FLOAT : - if ( get_f_value(f, name, &buf_f) ) goto fail; + if ( ev != NULL ) { + if ( get_ev_based_f_value(f, name, ev, &buf_f) ) goto fail; + } else { + if ( get_f_value(f, name, &buf_f) ) goto fail; + } tmp = malloc(256); snprintf(tmp, 255, "%f", buf_f); return tmp; @@ -1461,6 +1819,8 @@ char **hdfile_read_group(struct hdfile *f, int *n, const char *parent, } + H5Gclose(gh); + return res; } @@ -1478,7 +1838,7 @@ int hdfile_set_first_image(struct hdfile *f, const char *group) for ( i=0; itop_level ) { + + int fail_push; + + fail_push = push_path_entry_to_event(pp->curr_event, name); + if ( fail_push ) { + return -1; + } + + substituted_path = event_path_placeholder_subst(name, pp->path); + + } else { + substituted_path = strdup(pp->path); + } + + if ( pp->top_level == 1 ) { + pp->top_level = 0; + } + + truncated_path = strdup(substituted_path); + ph_loc = strstr(substituted_path,"%"); + if ( ph_loc != NULL) { + strncpy(&truncated_path[ph_loc-substituted_path],"\0",1); + } + + herrt_iterate = 0; + herrt_info = 0; + + check = check_path_existence(pp->hdfile->fh, truncated_path); + if ( check == 0 ) { + pop_path_entry_from_event(pp->curr_event); + return 0; + } else { + + herrt_info = H5Oget_info_by_name(pp->hdfile->fh, truncated_path, + &object_info, H5P_DEFAULT); + if ( herrt_info < 0 ) { + free(truncated_path); + free(substituted_path); + return -1; + } + + if ( pp->curr_event->path_length == pp->path_dim && + object_info.type == H5O_TYPE_DATASET ) { + + int fail_append; + + fail_append = append_event_to_event_list(pp->ev_list, + pp->curr_event); + if ( fail_append ) { + free(truncated_path); + free(substituted_path); + return -1; + } + + pop_path_entry_from_event(pp->curr_event); + return 0; + + } else { + + pp->path = substituted_path; + + if ( object_info.type == H5O_TYPE_GROUP ) { + + herrt_iterate = H5Literate_by_name(pp->hdfile->fh, + truncated_path, + H5_INDEX_NAME, + H5_ITER_NATIVE, NULL, + parse_file_event_structure, + (void *) pp, H5P_DEFAULT); + } + } + } + + pop_path_entry_from_event(pp->curr_event); + + free(truncated_path); + free(substituted_path); + + return herrt_iterate; + +} + + +struct event_list *fill_event_list(struct hdfile* hdfile, struct detector* det) +{ + int pi; + int evi; + herr_t check; + struct event_list *master_el; + struct event_list *master_el_with_dims; + + master_el = initialize_event_list(); + + if ( det->path_dim != 0 ) { + + for ( pi=0; pin_panels; pi++ ) { + + struct parse_params pparams; + struct event *empty_event; + struct event_list *panel_ev_list; + int ei; + + empty_event = initialize_event(); + panel_ev_list = initialize_event_list(); + + pparams.path = det->panels[pi].data; + pparams.hdfile = hdfile; + pparams.path_dim = det->path_dim; + pparams.curr_event = empty_event; + pparams.top_level = 1; + pparams.ev_list = panel_ev_list; + + check = parse_file_event_structure(hdfile->fh, NULL, + NULL, + (void *) &pparams); + + if ( check < 0 ) { + free_event(empty_event); + free_event_list(panel_ev_list); + return NULL; + } + + for ( ei=0; einum_events; ei++ ) { + + int fail_add; + + fail_add = add_non_existing_event_to_event_list(master_el, + panel_ev_list->events[ei]); + if ( fail_add ) { + + free_event(empty_event); + free_event_list(panel_ev_list); + return NULL; + } + } + + free_event(empty_event); + free_event_list(panel_ev_list); + } + + } + + if ( det->dim_dim > 0 ) { + + if ( master_el->num_events == 0 ) { + + struct event *empty_ev; + empty_ev = initialize_event(); + append_event_to_event_list(master_el, empty_ev); + free(empty_ev); + + } + + master_el_with_dims = initialize_event_list(); + + for (evi=0; evinum_events; evi++ ) { + + int global_path_dim = -1; + int pai; + int mlwd; + + for ( pai=0; pain_panels; pai++ ) { + + char *full_panel_path; + hid_t dh; + hid_t sh; + int dims; + hsize_t *size; + hsize_t *max_size; + int hsdi; + int panel_path_dim = 0; + + full_panel_path = retrieve_full_path(master_el->events[evi], + det->panels[pai].data); + + dh = H5Dopen2(hdfile->fh, full_panel_path, H5P_DEFAULT); + sh = H5Dget_space(dh); + dims = H5Sget_simple_extent_ndims(sh); + + size = malloc(dims*sizeof(hsize_t)); + max_size = malloc(dims*sizeof(hsize_t)); + + dims = H5Sget_simple_extent_dims(sh, size, max_size); + + for ( hsdi=0; hsdipanels[pai].dim_structure->num_dims; + hsdi++ ) { + if (det->panels[pai].dim_structure->dims[hsdi] == + HYSL_PLACEHOLDER ) { + panel_path_dim = size[hsdi]; + break; + } + } + + + if ( global_path_dim == -1 ) { + + global_path_dim = panel_path_dim; + + } else if ( panel_path_dim != global_path_dim ) { + + ERROR("Data blocks paths for panels must have the same " + "number of placeholders"); + free(size); + free(max_size); + return NULL; + } + + } + + for ( mlwd=0; mlwdevents[evi]); + push_dim_entry_to_event(mlwd_ev, mlwd); + append_event_to_event_list(master_el_with_dims, + mlwd_ev); + free(mlwd_ev); + } + + } + + free_event_list(master_el); + return master_el_with_dims; + + } + + return master_el; +} diff --git a/libcrystfel/src/hdf5-file.h b/libcrystfel/src/hdf5-file.h index 7aa76982..336684a5 100644 --- a/libcrystfel/src/hdf5-file.h +++ b/libcrystfel/src/hdf5-file.h @@ -9,6 +9,7 @@ * Authors: * 2009-2012 Thomas White * 2014 Valerio Mariani + * * This file is part of CrystFEL. * @@ -34,13 +35,18 @@ #ifndef HDF5_H #define HDF5_H +struct event_list; + #include #include +#include "image.h" +#include "events.h" struct hdfile; struct copy_hdf5_field; #include "image.h" +#include "events.h" #ifdef __cplusplus extern "C" { @@ -49,14 +55,20 @@ extern "C" { extern int hdf5_write(const char *filename, const void *data, int width, int height, int type); -extern int hdf5_write_image(const char *filename, struct image *image, char *element); +extern int hdf5_write_image(const char *filename, struct image *image, + char *element); -extern int hdf5_read(const char *filename, struct image *image, const char *element, int satcorr); +extern int hdf5_read(struct hdfile *f, struct image *image, + const char* element, int satcorr); -extern int hdf5_read2(const char *filename, struct image *image, const char* element, int satcorr, int override_data_and_mask); +extern int hdf5_read2(struct hdfile *f, struct image *image, + struct event *ev, int satcorr); + +extern int check_path_existence(hid_t fh, const char *path); extern struct hdfile *hdfile_open(const char *filename); -extern int hdfile_set_image(struct hdfile *f, const char *path); +int hdfile_set_image(struct hdfile *f, const char *path, + struct panel *p); extern int16_t *hdfile_get_image_binned(struct hdfile *hdfile, int binning, int16_t *maxp); extern char **hdfile_read_group(struct hdfile *f, int *n, const char *parent, @@ -65,16 +77,24 @@ extern int hdfile_set_first_image(struct hdfile *f, const char *group); extern void hdfile_close(struct hdfile *f); extern int hdfile_is_scalar(struct hdfile *f, const char *name, int verbose); -extern char *hdfile_get_string_value(struct hdfile *f, const char *name); +char *hdfile_get_string_value(struct hdfile *f, const char *name, + struct event* ev); extern int get_peaks(struct image *image, struct hdfile *f, const char *p); extern double get_value(struct hdfile *f, const char *name); +extern double get_ev_based_value(struct hdfile *f, const char *name, + struct event *ev); + extern struct copy_hdf5_field *new_copy_hdf5_field_list(void); extern void free_copy_hdf5_field_list(struct copy_hdf5_field *f); + extern void copy_hdf5_fields(struct hdfile *f, - const struct copy_hdf5_field *copyme, FILE *fh); + const struct copy_hdf5_field *copyme, + FILE *fh, struct event *ev); extern void add_copy_hdf5_field(struct copy_hdf5_field *copyme, const char *name); +extern struct event_list *fill_event_list(struct hdfile* hdfile, + struct detector* det); #ifdef __cplusplus } diff --git a/libcrystfel/src/image.c b/libcrystfel/src/image.c index 6a0495b1..6caeeb3c 100644 --- a/libcrystfel/src/image.c +++ b/libcrystfel/src/image.c @@ -7,8 +7,8 @@ * a research centre of the Helmholtz Association. * * Authors: - * 2011-2014 Thomas White * 2014 Kenneth Beyerlein + * 2011-2014 Thomas White * * This file is part of CrystFEL. * diff --git a/libcrystfel/src/image.h b/libcrystfel/src/image.h index 068affae..10e1eef7 100644 --- a/libcrystfel/src/image.h +++ b/libcrystfel/src/image.h @@ -10,6 +10,7 @@ * 2009-2013 Thomas White * 2014 Valerio Mariani * + * * This file is part of CrystFEL. * * CrystFEL is free software: you can redistribute it and/or modify @@ -34,6 +35,8 @@ #ifndef IMAGE_H #define IMAGE_H +struct detector; + #include #include #include @@ -166,6 +169,7 @@ struct image { struct detector *det; struct beam_params *beam; /* The nominal beam parameters */ char *filename; + struct event *event; const struct copy_hdf5_field *copyme; int id; /* ID number of the thread diff --git a/libcrystfel/src/index.c b/libcrystfel/src/index.c index 68b4c38e..241d9311 100644 --- a/libcrystfel/src/index.c +++ b/libcrystfel/src/index.c @@ -477,4 +477,4 @@ IndexingMethod *build_indexer_list(const char *str) list[++nmeth] = INDEXING_NONE; return list; -} +} \ No newline at end of file diff --git a/libcrystfel/src/stream.c b/libcrystfel/src/stream.c index 59525c04..a826dc02 100644 --- a/libcrystfel/src/stream.c +++ b/libcrystfel/src/stream.c @@ -9,10 +9,10 @@ * * Authors: * 2010-2014 Thomas White + * 2014 Valerio Mariani * 2011 Richard Kirian * 2011 Andrew Aquila * 2014 Takanori Nakane - * 2014 Valerio Mariani * * This file is part of CrystFEL. * @@ -612,7 +612,7 @@ static void write_crystal(Stream *st, Crystal *cr, int include_reflections) void write_chunk(Stream *st, struct image *i, struct hdfile *hdfile, - int include_peaks, int include_reflections) + int include_peaks, int include_reflections, struct event* ev) { int j; char *indexer; @@ -620,6 +620,9 @@ void write_chunk(Stream *st, struct image *i, struct hdfile *hdfile, fprintf(st->fh, CHUNK_START_MARKER"\n"); fprintf(st->fh, "Image filename: %s\n", i->filename); + if ( i->event != NULL ) { + fprintf(st->fh, "Event: %s\n", get_event_string(i->event)); + } indexer = indexer_str(i->indexed_by); fprintf(st->fh, "indexed_by = %s\n", indexer); @@ -631,7 +634,7 @@ void write_chunk(Stream *st, struct image *i, struct hdfile *hdfile, fprintf(st->fh, "beam_divergence = %.5f mrad\n", i->div*1e3); fprintf(st->fh, "beam_bandwidth = %.5f %%\n", i->bw*100.0); - copy_hdf5_fields(hdfile, i->copyme, st->fh); + copy_hdf5_fields(hdfile, i->copyme, st->fh, ev); if ( i->det != NULL ) { diff --git a/libcrystfel/src/stream.h b/libcrystfel/src/stream.h index 1c2e57c6..e98ee02d 100644 --- a/libcrystfel/src/stream.h +++ b/libcrystfel/src/stream.h @@ -8,6 +8,7 @@ * * Authors: * 2010-2014 Thomas White + * 2014 Valerio Mariani * 2011 Andrew Aquila * * This file is part of CrystFEL. @@ -37,6 +38,7 @@ struct image; struct hdfile; +struct event; #define GEOM_START_MARKER "----- Begin geometry file -----" #define GEOM_END_MARKER "----- End geometry file -----" @@ -91,7 +93,8 @@ extern void close_stream(Stream *st); extern int read_chunk(Stream *st, struct image *image); extern int read_chunk_2(Stream *st, struct image *image, StreamReadFlags srf); extern void write_chunk(Stream *st, struct image *image, struct hdfile *hdfile, - int include_peaks, int include_reflections); + int include_peaks, int include_reflections, + struct event *ev); extern void write_command(Stream *st, int argc, char *argv[]); extern void write_geometry_file(Stream *st, const char *geom_filename); -- cgit v1.2.3