diff options
Diffstat (limited to 'libcrystfel/src')
80 files changed, 10046 insertions, 9516 deletions
diff --git a/libcrystfel/src/cell-utils.c b/libcrystfel/src/cell-utils.c index d269e703..bb654bc5 100644 --- a/libcrystfel/src/cell-utils.c +++ b/libcrystfel/src/cell-utils.c @@ -3,12 +3,12 @@ * * Unit Cell utility functions * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Lorenzo Galli * * Authors: - * 2009-2019 Thomas White <taw@physics.org> + * 2009-2021 Thomas White <taw@physics.org> * 2012 Lorenzo Galli * * This file is part of CrystFEL. @@ -256,6 +256,48 @@ void cell_print(UnitCell *cell) } +void cell_print_oneline(UnitCell *cell) +{ + LatticeType lt; + char cen; + + if ( cell == NULL ) { + STATUS("(NULL cell)\n"); + return; + } + + lt = cell_get_lattice_type(cell); + cen = cell_get_centering(cell); + + STATUS("%s %c", str_lattice(lt), cen); + + if ( (lt==L_MONOCLINIC) || (lt==L_TETRAGONAL) || ( lt==L_HEXAGONAL) + || ( (lt==L_ORTHORHOMBIC) && (cen=='A') ) + || ( (lt==L_ORTHORHOMBIC) && (cen=='B') ) + || ( (lt==L_ORTHORHOMBIC) && (cen=='C') ) ) + { + STATUS(", unique axis %c", cell_get_unique_axis(cell)); + } + + if ( cell_has_parameters(cell) ) { + + double a, b, c, alpha, beta, gamma; + + if ( !right_handed(cell) ) { + STATUS(" (left handed)"); + } + + cell_get_parameters(cell, &a, &b, &c, &alpha, &beta, &gamma); + + STATUS(" %.2f %.2f %.2f A, %.2f %.2f %.2f deg\n", + a*1e10, b*1e10, c*1e10, + rad2deg(alpha), rad2deg(beta), rad2deg(gamma)); + } else { + STATUS(", no cell parameters.\n"); + } +} + + void cell_print_full(UnitCell *cell) { cell_print(cell); @@ -291,8 +333,6 @@ void cell_print_full(UnitCell *cell) rad2deg(angle_between(asx, asy, asz, csx, csy, csz)), rad2deg(angle_between(asx, asy, asz, bsx, bsy, bsz))); - STATUS("Cell representation is %s.\n", cell_rep(cell)); - } } @@ -577,28 +617,18 @@ UnitCell *uncenter_cell(UnitCell *in, IntegerMatrix **pC, RationalMatrix **pCi) /* Return sin(theta)/lambda = 1/2d. Multiply by two if you want 1/d */ double resolution(UnitCell *cell, signed int h, signed int k, signed int l) { - double a, b, c, alpha, beta, gamma; - - cell_get_parameters(cell, &a, &b, &c, &alpha, &beta, &gamma); - - const double Vsq = a*a*b*b*c*c*(1 - cos(alpha)*cos(alpha) - - cos(beta)*cos(beta) - - cos(gamma)*cos(gamma) - + 2*cos(alpha)*cos(beta)*cos(gamma) ); - - const double S11 = b*b*c*c*sin(alpha)*sin(alpha); - const double S22 = a*a*c*c*sin(beta)*sin(beta); - const double S33 = a*a*b*b*sin(gamma)*sin(gamma); - const double S12 = a*b*c*c*(cos(alpha)*cos(beta) - cos(gamma)); - const double S23 = a*a*b*c*(cos(beta)*cos(gamma) - cos(alpha)); - const double S13 = a*b*b*c*(cos(gamma)*cos(alpha) - cos(beta)); + double asx, asy, asz; + double bsx, bsy, bsz; + double csx, csy, csz; - const double brackets = S11*h*h + S22*k*k + S33*l*l - + 2*S12*h*k + 2*S23*k*l + 2*S13*h*l; - const double oneoverdsq = brackets / Vsq; - const double oneoverd = sqrt(oneoverdsq); + cell_get_reciprocal(cell, + &asx, &asy, &asz, + &bsx, &bsy, &bsz, + &csx, &csy, &csz); - return oneoverd / 2; + return modulus(h*asx + k*bsx + l*csx, + h*asy + k*bsy + l*csy, + h*asz + k*bsz + l*csz) / 2.0; } @@ -1271,6 +1301,49 @@ double cell_get_volume(UnitCell *cell) /** + * \param cell: A %UnitCell + * + * \returns the value of 1/d for the lowest order reflection + * that is not systematically absent according to the centering. + * + */ +double lowest_reflection(UnitCell *cell) +{ + signed int h, k, l; + double lowres = INFINITY; + + /* FIXME: Inelegant and nasty. Anyone want to work out + * all the possible cases? */ + for ( h=0; h<4; h++ ) { + for ( k=0; k<4; k++ ) { + for ( l=0; l<4; l++ ) { + if ( (h==0) && (k==0) && (l==0) ) continue; + if ( !forbidden_reflection(cell, h, k, l) ) { + double r = resolution(cell, h, k, l); + if ( r < lowres ) { + lowres = r; + } + } + } + } + } + return lowres; +} + + +/* Return true if the two centering symbols are identical, + * or if they are a pair of R/P, which should be considered the + * same for the purposes of cell comparison */ +static int centering_equivalent(char cen1, char cen2) +{ + if ( cen1 == cen2 ) return 1; + if ( (cen1=='P') && (cen2=='R') ) return 1; + if ( (cen1=='R') && (cen2=='P') ) return 1; + return 0; +} + + +/** * \param cell: A UnitCell * \param reference: Another UnitCell * \param tols: Pointer to tolerances for a,b,c (fractional), al,be,ga (radians) @@ -1294,11 +1367,13 @@ int compare_cell_parameters(UnitCell *cell, UnitCell *reference, /* Centering must match: we don't arbitrate primitive vs centered, * different cell choices etc */ - if ( cell_get_centering(cell) != cell_get_centering(reference) ) return 0; + if ( !centering_equivalent(cell_get_centering(cell), + cell_get_centering(reference)) ) return 0; cell_get_parameters(cell, &a1, &b1, &c1, &al1, &be1, &ga1); cell_get_parameters(reference, &a2, &b2, &c2, &al2, &be2, &ga2); + /* within_tolerance() takes a percentage */ if ( !within_tolerance(a1, a2, tols[0]*100.0) ) return 0; if ( !within_tolerance(b1, b2, tols[1]*100.0) ) return 0; if ( !within_tolerance(c1, c2, tols[2]*100.0) ) return 0; @@ -1344,6 +1419,7 @@ static double moduli_check(double ax, double ay, double az, * \returns non-zero if the cells match. * */ +/* 'tols' is in frac (not %) and radians */ int compare_cell_parameters_and_orientation(UnitCell *cell, UnitCell *reference, const double *tols) { @@ -1399,6 +1475,7 @@ int compare_cell_parameters_and_orientation(UnitCell *cell, UnitCell *reference, * \returns non-zero if the cells match. * */ +/* 'tols' is in frac (not %) and radians */ int compare_permuted_cell_parameters_and_orientation(UnitCell *cell, UnitCell *reference, const double *tols, diff --git a/libcrystfel/src/cell-utils.h b/libcrystfel/src/cell-utils.h index 1c356588..8eefe800 100644 --- a/libcrystfel/src/cell-utils.h +++ b/libcrystfel/src/cell-utils.h @@ -3,12 +3,12 @@ * * Unit Cell utility functions * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Lorenzo Galli * * Authors: - * 2009-2018 Thomas White <taw@physics.org> + * 2009-2020 Thomas White <taw@physics.org> * 2012 Lorenzo Galli * * This file is part of CrystFEL. @@ -31,10 +31,6 @@ #ifndef CELL_UTILS_H #define CELL_UTILS_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include <gsl/gsl_matrix.h> #include "cell.h" @@ -57,6 +53,7 @@ extern UnitCell *rotate_cell(UnitCell *in, double omega, double phi, double rot); extern void cell_print(UnitCell *cell); +extern void cell_print_oneline(UnitCell *cell); extern void cell_print_full(UnitCell *cell); extern UnitCell *load_cell_from_pdb(const char *filename); @@ -82,6 +79,8 @@ extern int forbidden_reflection(UnitCell *cell, extern double cell_get_volume(UnitCell *cell); +extern double lowest_reflection(UnitCell *cell); + extern int compare_cell_parameters(UnitCell *cell, UnitCell *reference, const double *tols); diff --git a/libcrystfel/src/cell.c b/libcrystfel/src/cell.c index f6ed412d..2e944ddb 100644 --- a/libcrystfel/src/cell.c +++ b/libcrystfel/src/cell.c @@ -3,15 +3,15 @@ * * A class representing a unit cell * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * Copyright © 2012 Lorenzo Galli * * Authors: - * 2009-2012,2014,2017 Thomas White <taw@physics.org> - * 2010 Richard Kirian - * 2012 Lorenzo Galli + * 2009-2021 Thomas White <taw@physics.org> + * 2010 Richard Kirian + * 2012 Lorenzo Galli * * This file is part of CrystFEL. * @@ -35,6 +35,7 @@ #endif #include <math.h> +#include <assert.h> #include <stdlib.h> #include <stdio.h> #include <string.h> @@ -54,19 +55,14 @@ */ -typedef enum { - CELL_REP_CRYST, - CELL_REP_CART, - CELL_REP_RECIP -} CellRepresentation; - struct _unitcell { - CellRepresentation rep; - - int have_parameters; + LatticeType lattice_type; + char centering; + char unique_axis; /* Crystallographic representation */ + int have_cryst; double a; /* m */ double b; /* m */ double c; /* m */ @@ -75,18 +71,16 @@ struct _unitcell { double gamma; /* Radians */ /* Cartesian representation */ + int have_cart; double ax; double bx; double cx; double ay; double by; double cy; double az; double bz; double cz; /* Cartesian representation of reciprocal axes */ + int have_recip; double axs; double bxs; double cxs; double ays; double bys; double cys; double azs; double bzs; double czs; - - LatticeType lattice_type; - char centering; - char unique_axis; }; typedef enum { @@ -127,12 +121,13 @@ UnitCell *cell_new() cell->beta = 0.0; cell->gamma = 0.0; - cell->rep = CELL_REP_CRYST; + cell->have_cryst = 0; + cell->have_cart = 0; + cell->have_recip = 0; cell->lattice_type = L_TRICLINIC; cell->centering = 'P'; cell->unique_axis = '?'; - cell->have_parameters = 0; return cell; } @@ -157,10 +152,12 @@ void cell_free(UnitCell *cell) * \returns True if cell has its parameters specified. * */ -int cell_has_parameters(UnitCell *cell) +int cell_has_parameters(const UnitCell *cell) { if ( cell == NULL ) return 0; - return cell->have_parameters; + return (cell->have_cryst > 0) + || (cell->have_cart > 0 ) + || (cell->have_recip > 0); } @@ -176,8 +173,9 @@ void cell_set_parameters(UnitCell *cell, double a, double b, double c, cell->beta = beta; cell->gamma = gamma; - cell->rep = CELL_REP_CRYST; - cell->have_parameters = 1; + cell->have_cryst = 1; + cell->have_cart = 0; + cell->have_recip = 0; } @@ -192,8 +190,9 @@ void cell_set_cartesian(UnitCell *cell, cell->bx = bx; cell->by = by; cell->bz = bz; cell->cx = cx; cell->cy = cy; cell->cz = cz; - cell->rep = CELL_REP_CART; - cell->have_parameters = 1; + cell->have_cryst = 0; + cell->have_cart = 1; + cell->have_recip = 0; } @@ -223,8 +222,9 @@ UnitCell *cell_new_from_reciprocal_axes(struct rvec as, struct rvec bs, cell->bxs = bs.u; cell->bys = bs.v; cell->bzs = bs.w; cell->cxs = cs.u; cell->cys = cs.v; cell->czs = cs.w; - cell->rep = CELL_REP_RECIP; - cell->have_parameters = 1; + cell->have_cryst = 0; + cell->have_cart = 0; + cell->have_recip = 1; return cell; } @@ -241,8 +241,9 @@ UnitCell *cell_new_from_direct_axes(struct rvec a, struct rvec b, struct rvec c) cell->bx = b.u; cell->by = b.v; cell->bz = b.w; cell->cx = c.u; cell->cy = c.v; cell->cz = c.w; - cell->rep = CELL_REP_CART; - cell->have_parameters = 1; + cell->have_cryst = 0; + cell->have_cart = 1; + cell->have_recip = 0; return cell; } @@ -268,8 +269,9 @@ void cell_set_reciprocal(UnitCell *cell, cell->bxs = bsx; cell->bys = bsy; cell->bzs = bsz; cell->cxs = csx; cell->cys = csy; cell->czs = csz; - cell->rep = CELL_REP_RECIP; - cell->have_parameters = 1; + cell->have_cryst = 0; + cell->have_cart = 0; + cell->have_recip = 1; } @@ -291,31 +293,25 @@ void cell_set_unique_axis(UnitCell *cell, char unique_axis) } -/************************* Getter helper functions ****************************/ +/************************* Conversion functions ****************************/ -static int cell_crystallographic_to_cartesian(const UnitCell *cell, - double *ax, double *ay, double *az, - double *bx, double *by, double *bz, - double *cx, double *cy, double *cz) +static void crystallographic_to_cartesian(UnitCell *cell) { double tmp, V, cosalphastar, cstar; - if ( !cell->have_parameters ) { - ERROR("Unit cell has unspecified parameters.\n"); - return 1; - } + assert(cell->have_cryst == 1); /* Firstly: Get a in terms of x, y and z * +a (cryst) is defined to lie along +x (cart) */ - *ax = cell->a; - *ay = 0.0; - *az = 0.0; + cell->ax = cell->a; + cell->ay = 0.0; + cell->az = 0.0; /* b in terms of x, y and z * b (cryst) is defined to lie in the xy (cart) plane */ - *bx = cell->b*cos(cell->gamma); - *by = cell->b*sin(cell->gamma); - *bz = 0.0; + cell->bx = cell->b*cos(cell->gamma); + cell->by = cell->b*sin(cell->gamma); + cell->bz = 0.0; tmp = cos(cell->alpha)*cos(cell->alpha) + cos(cell->beta)*cos(cell->beta) @@ -329,21 +325,41 @@ static int cell_crystallographic_to_cartesian(const UnitCell *cell, cstar = (cell->a * cell->b * sin(cell->gamma))/V; /* c in terms of x, y and z */ - *cx = cell->c*cos(cell->beta); - *cy = -cell->c*sin(cell->beta)*cosalphastar; - *cz = 1.0/cstar; + cell->cx = cell->c*cos(cell->beta); + cell->cy = -cell->c*sin(cell->beta)*cosalphastar; + cell->cz = 1.0/cstar; - return 0; + cell->have_cart = 1; +} + + +static void cartesian_to_crystallographic(UnitCell *cell) +{ + assert(cell->have_cart); + + /* Convert cartesian -> crystallographic */ + cell->a = modulus(cell->ax, cell->ay, cell->az); + cell->b = modulus(cell->bx, cell->by, cell->bz); + cell->c = modulus(cell->cx, cell->cy, cell->cz); + + cell->alpha = angle_between(cell->bx, cell->by, cell->bz, + cell->cx, cell->cy, cell->cz); + cell->beta = angle_between(cell->ax, cell->ay, cell->az, + cell->cx, cell->cy, cell->cz); + cell->gamma = angle_between(cell->ax, cell->ay, cell->az, + cell->bx, cell->by, cell->bz); + + cell->have_cryst = 1; } /* Why yes, I do enjoy long argument lists...! */ -static int cell_invert(double ax, double ay, double az, - double bx, double by, double bz, - double cx, double cy, double cz, - double *asx, double *asy, double *asz, - double *bsx, double *bsy, double *bsz, - double *csx, double *csy, double *csz) +static int invert(double ax, double ay, double az, + double bx, double by, double bz, + double cx, double cy, double cz, + double *asx, double *asy, double *asz, + double *bsx, double *bsy, double *bsz, + double *csx, double *csy, double *csz) { int s; gsl_matrix *m; @@ -413,167 +429,153 @@ static int cell_invert(double ax, double ay, double az, } +static int reciprocal_to_cartesian(UnitCell *cell) +{ + assert(cell->have_recip); + + if ( invert(cell->axs, cell->ays, cell->azs, + cell->bxs, cell->bys, cell->bzs, + cell->cxs, cell->cys, cell->czs, + &cell->ax, &cell->ay, &cell->az, + &cell->bx, &cell->by, &cell->bz, + &cell->cx, &cell->cy, &cell->cz) ) return 1; + + cell->have_cart = 1; + return 0; +} + + +static int cartesian_to_reciprocal(UnitCell *cell) +{ + assert(cell->have_cart); + + if ( invert(cell->ax, cell->ay, cell->az, + cell->bx, cell->by, cell->bz, + cell->cx, cell->cy, cell->cz, + &cell->axs, &cell->ays, &cell->azs, + &cell->bxs, &cell->bys, &cell->bzs, + &cell->cxs, &cell->cys, &cell->czs) ) return 1; + + cell->have_recip = 1; + return 0; +} + + /********************************** Getters ***********************************/ -int cell_get_parameters(const UnitCell *cell, double *a, double *b, double *c, +int cell_get_parameters(UnitCell *cell, + double *a, double *b, double *c, double *alpha, double *beta, double *gamma) { - double ax, ay, az, bx, by, bz, cx, cy, cz; - if ( cell == NULL ) return 1; - if ( !cell->have_parameters ) { + if ( cell->have_cryst ) { + + /* Nothing to do */ + + } else if ( cell->have_cart ) { + + cartesian_to_crystallographic(cell); + + } else if ( cell->have_recip ) { + + if ( reciprocal_to_cartesian(cell) ) return 1; + cartesian_to_crystallographic(cell); + + } else { + ERROR("Unit cell has unspecified parameters.\n"); return 1; - } - switch ( cell->rep ) { - - case CELL_REP_CRYST: - /* Direct response */ - *a = cell->a; - *b = cell->b; - *c = cell->c; - *alpha = cell->alpha; - *beta = cell->beta; - *gamma = cell->gamma; - return 0; - - case CELL_REP_CART: - /* Convert cartesian -> crystallographic */ - *a = modulus(cell->ax, cell->ay, cell->az); - *b = modulus(cell->bx, cell->by, cell->bz); - *c = modulus(cell->cx, cell->cy, cell->cz); - - *alpha = angle_between(cell->bx, cell->by, cell->bz, - cell->cx, cell->cy, cell->cz); - *beta = angle_between(cell->ax, cell->ay, cell->az, - cell->cx, cell->cy, cell->cz); - *gamma = angle_between(cell->ax, cell->ay, cell->az, - cell->bx, cell->by, cell->bz); - return 0; - - case CELL_REP_RECIP: - /* Convert reciprocal -> crystallographic. - * Start by converting reciprocal -> cartesian */ - if ( cell_invert(cell->axs, cell->ays, cell->azs, - cell->bxs, cell->bys, cell->bzs, - cell->cxs, cell->cys, cell->czs, - &ax, &ay, &az, - &bx, &by, &bz, - &cx, &cy, &cz) ) return 1; - - /* Now convert cartesian -> crystallographic */ - *a = modulus(ax, ay, az); - *b = modulus(bx, by, bz); - *c = modulus(cx, cy, cz); - - *alpha = angle_between(bx, by, bz, cx, cy, cz); - *beta = angle_between(ax, ay, az, cx, cy, cz); - *gamma = angle_between(ax, ay, az, bx, by, bz); - return 0; } - return 1; + *a = cell->a; + *b = cell->b; + *c = cell->c; + *alpha = cell->alpha; + *beta = cell->beta; + *gamma = cell->gamma; + return 0; } -int cell_get_cartesian(const UnitCell *cell, +int cell_get_cartesian(UnitCell *cell, double *ax, double *ay, double *az, double *bx, double *by, double *bz, double *cx, double *cy, double *cz) { if ( cell == NULL ) return 1; - if ( !cell->have_parameters ) { + if ( cell->have_cart ) { + + /* Nothing to do */ + + } else if ( cell->have_recip ) { + + /* NB recip->cart has priority over + * cryst->cart, to preserve orientation */ + reciprocal_to_cartesian(cell); + + } else if ( cell->have_cryst ) { + + crystallographic_to_cartesian(cell); + + } else { + ERROR("Unit cell has unspecified parameters.\n"); return 1; - } - - switch ( cell->rep ) { - - case CELL_REP_CRYST: - /* Convert crystallographic -> cartesian. */ - return cell_crystallographic_to_cartesian(cell, - ax, ay, az, - bx, by, bz, - cx, cy, cz); - - case CELL_REP_CART: - /* Direct response */ - *ax = cell->ax; *ay = cell->ay; *az = cell->az; - *bx = cell->bx; *by = cell->by; *bz = cell->bz; - *cx = cell->cx; *cy = cell->cy; *cz = cell->cz; - return 0; - - case CELL_REP_RECIP: - /* Convert reciprocal -> cartesian */ - return cell_invert(cell->axs, cell->ays, cell->azs, - cell->bxs, cell->bys, cell->bzs, - cell->cxs, cell->cys, cell->czs, - ax, ay, az, bx, by, bz, cx, cy, cz); } - return 1; + *ax = cell->ax; *ay = cell->ay; *az = cell->az; + *bx = cell->bx; *by = cell->by; *bz = cell->bz; + *cx = cell->cx; *cy = cell->cy; *cz = cell->cz; + return 0; } -int cell_get_reciprocal(const UnitCell *cell, +int cell_get_reciprocal(UnitCell *cell, double *asx, double *asy, double *asz, double *bsx, double *bsy, double *bsz, double *csx, double *csy, double *csz) { - int r; - double ax, ay, az, bx, by, bz, cx, cy, cz; - if ( cell == NULL ) return 1; - if ( !cell->have_parameters ) { + if ( cell->have_recip ) { + + /* Nothing to do */ + + } else if ( cell->have_cart ) { + + /* NB cart->recip has priority over cryst->recip */ + cartesian_to_reciprocal(cell); + + } else if ( cell->have_cryst ) { + + crystallographic_to_cartesian(cell); + cartesian_to_reciprocal(cell); + + } else { + ERROR("Unit cell has unspecified parameters.\n"); return 1; - } - - switch ( cell->rep ) { - - case CELL_REP_CRYST: - /* Convert crystallographic -> reciprocal */ - r = cell_crystallographic_to_cartesian(cell, - &ax, &ay, &az, - &bx, &by, &bz, - &cx, &cy, &cz); - if ( r ) return r; - return cell_invert(ax, ay, az,bx, by, bz, cx, cy, cz, - asx, asy, asz, bsx, bsy, bsz, csx, csy, csz); - - case CELL_REP_CART: - /* Convert cartesian -> reciprocal */ - cell_invert(cell->ax, cell->ay, cell->az, - cell->bx, cell->by, cell->bz, - cell->cx, cell->cy, cell->cz, - asx, asy, asz, bsx, bsy, bsz, csx, csy, csz); - return 0; - - case CELL_REP_RECIP: - /* Direct response */ - *asx = cell->axs; *asy = cell->ays; *asz = cell->azs; - *bsx = cell->bxs; *bsy = cell->bys; *bsz = cell->bzs; - *csx = cell->cxs; *csy = cell->cys; *csz = cell->czs; - return 0; } - return 1; + *asx = cell->axs; *asy = cell->ays; *asz = cell->azs; + *bsx = cell->bxs; *bsy = cell->bys; *bsz = cell->bzs; + *csx = cell->cxs; *csy = cell->cys; *csz = cell->czs; + return 0; } -char cell_get_centering(UnitCell *cell) +char cell_get_centering(const UnitCell *cell) { return cell->centering; } -LatticeType cell_get_lattice_type(UnitCell *cell) +LatticeType cell_get_lattice_type(const UnitCell *cell) { return cell->lattice_type; } @@ -594,31 +596,12 @@ struct g6 cell_get_G6(UnitCell *cell) } -char cell_get_unique_axis(UnitCell *cell) +char cell_get_unique_axis(const UnitCell *cell) { return cell->unique_axis; } -const char *cell_rep(UnitCell *cell) -{ - switch ( cell->rep ) { - - case CELL_REP_CRYST: - return "crystallographic, direct space"; - - case CELL_REP_CART: - return "cartesian, direct space"; - - case CELL_REP_RECIP: - return "cartesian, reciprocal space"; - - } - - return "unknown"; -} - - UnitCell *cell_transform_gsl_direct(UnitCell *in, gsl_matrix *m) { gsl_matrix *c; diff --git a/libcrystfel/src/cell.h b/libcrystfel/src/cell.h index 3deb92ff..ed82fff7 100644 --- a/libcrystfel/src/cell.h +++ b/libcrystfel/src/cell.h @@ -3,15 +3,15 @@ * * A class representing a unit cell * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * Copyright © 2012 Lorenzo Galli * * Authors: - * 2009-2012,2014,2017 Thomas White <taw@physics.org> - * 2010,2012 Richard Kirian - * 2012 Lorenzo Galli + * 2009-2021 Thomas White <taw@physics.org> + * 2010-2012 Richard Kirian + * 2012 Lorenzo Galli * * This file is part of CrystFEL. * @@ -33,10 +33,6 @@ #ifndef CELL_H #define CELL_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include "utils.h" #include "integer_matrix.h" @@ -101,7 +97,7 @@ extern UnitCell *cell_new_from_reciprocal_axes(struct rvec as, struct rvec bs, extern UnitCell *cell_new_from_direct_axes(struct rvec as, struct rvec bs, struct rvec cs); -extern int cell_has_parameters(UnitCell *cell); +extern int cell_has_parameters(const UnitCell *cell); extern void cell_set_cartesian(UnitCell *cell, double ax, double ay, double az, @@ -111,15 +107,15 @@ extern void cell_set_cartesian(UnitCell *cell, extern void cell_set_parameters(UnitCell *cell, double a, double b, double c, double alpha, double beta, double gamma); -extern int cell_get_parameters(const UnitCell *cell, double *a, double *b, double *c, +extern int cell_get_parameters(UnitCell *cell, double *a, double *b, double *c, double *alpha, double *beta, double *gamma); -extern int cell_get_cartesian(const UnitCell *cell, +extern int cell_get_cartesian(UnitCell *cell, double *ax, double *ay, double *az, double *bx, double *by, double *bz, double *cx, double *cy, double *cz); -extern int cell_get_reciprocal(const UnitCell *cell, +extern int cell_get_reciprocal(UnitCell *cell, double *asx, double *asy, double *asz, double *bsx, double *bsy, double *bsz, double *csx, double *csy, double *csz); @@ -129,7 +125,7 @@ extern void cell_set_reciprocal(UnitCell *cell, double bsx, double bsy, double bsz, double csx, double csy, double csz); -extern LatticeType cell_get_lattice_type(UnitCell *cell); +extern LatticeType cell_get_lattice_type(const UnitCell *cell); extern void cell_set_lattice_type(UnitCell *cell, LatticeType lattice_type); struct g6 @@ -144,14 +140,12 @@ struct g6 extern struct g6 cell_get_G6(UnitCell *cell); -extern char cell_get_centering(UnitCell *cell); +extern char cell_get_centering(const UnitCell *cell); extern void cell_set_centering(UnitCell *cell, char centering); -extern char cell_get_unique_axis(UnitCell *cell); +extern char cell_get_unique_axis(const UnitCell *cell); extern void cell_set_unique_axis(UnitCell *cell, char unique_axis); -extern const char *cell_rep(UnitCell *cell); - extern UnitCell *cell_transform_gsl_direct(UnitCell *in, gsl_matrix *m); extern UnitCell *cell_transform_rational(UnitCell *cell, RationalMatrix *m); diff --git a/libcrystfel/src/render.c b/libcrystfel/src/colscale.c index 284f4c46..4a39de8d 100644 --- a/libcrystfel/src/render.c +++ b/libcrystfel/src/colscale.c @@ -1,13 +1,13 @@ /* - * render.c + * colscale.c * - * Render a high dynamic range buffer in some sensible way + * Colour scales * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2009-2012,2014 Thomas White <taw@physics.org> + * 2009-2020 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -35,14 +35,9 @@ #include <math.h> #include <stdint.h> +#include "colscale.h" -#include "hdf5-file.h" -#include "render.h" -#include "peaks.h" -#include "filters.h" -#include "utils.h" - -/** \file render.h */ +/** \file colscale.h */ static void render_rgb(double val, double max, double *rp, double *gp, double *bp) @@ -223,8 +218,8 @@ static void render_invmono(double val, double max, } -void render_scale(double val, double max, int scale, - double *rp, double *gp, double *bp) +void colscale_lookup(double val, double max, int scale, + double *rp, double *gp, double *bp) { switch ( scale ) { diff --git a/libcrystfel/src/render.h b/libcrystfel/src/colscale.h index a3292515..972b659b 100644 --- a/libcrystfel/src/render.h +++ b/libcrystfel/src/colscale.h @@ -1,13 +1,13 @@ /* - * render.h + * colscale.h * - * Render a high dynamic range buffer in some sensible way + * Colour scales * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2009-2012 Thomas White <taw@physics.org> + * 2009-2020 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -26,16 +26,12 @@ * */ -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - -#ifndef RENDER_H -#define RENDER_H +#ifndef COLSCALE_H +#define COLSCALE_H /** - * \file render.h - * Colour scale for rendering + * \file colscale.h + * Colour scales for rendering */ enum { @@ -51,12 +47,12 @@ extern "C" { #endif /* Colour scale lookup */ -extern void render_scale(double val, double max, int scale, - double *rp, double *gp, double *bp); +extern void colscale_lookup(double val, double max, int scale, + double *rp, double *gp, double *bp); #ifdef __cplusplus } #endif -#endif /* RENDER_H */ +#endif /* COLSCALE_H */ diff --git a/libcrystfel/src/crystal.c b/libcrystfel/src/crystal.c index 9202c668..c9f59bb7 100644 --- a/libcrystfel/src/crystal.c +++ b/libcrystfel/src/crystal.c @@ -3,11 +3,11 @@ * * A class representing a single crystal * - * Copyright © 2013-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2013-2016 Thomas White <taw@physics.org> + * 2013-2020 Thomas White <taw@physics.org> * 2016 Valerio Mariani * * This file is part of CrystFEL. @@ -33,6 +33,7 @@ #include "crystal.h" #include "utils.h" +#include "reflist-utils.h" /** @@ -66,7 +67,7 @@ struct _crystal /* Text notes, which go in the stream */ char *notes; - /* Detector shift */ + /* Detector shift in metres */ double det_shift_x; double det_shift_y; }; @@ -127,6 +128,44 @@ Crystal *crystal_copy(const Crystal *cryst) /** + * \param cryst: A \ref Crystal to copy. + * + * Creates a new \ref Crystal which is a copy of \p cryst. The copy is a "deep + * copy", which means that copies ARE made of the data structures which + * \p cryst contains references to, for example its \ref RefList. + * + * \returns A (deep) copy of \p cryst, or NULL on failure. + * + */ +Crystal *crystal_copy_deep(const Crystal *cryst) +{ + Crystal *c; + + c = crystal_new(); + if ( c == NULL ) return NULL; + + memcpy(c, cryst, sizeof(Crystal)); + if ( c->notes != NULL ) c->notes = strdup(c->notes); + + if ( cryst->cell != NULL ) { + UnitCell *cell; + cell = cell_new_from_cell(cryst->cell); + if ( cell == NULL ) return NULL; + c->cell = cell; + } + + if ( cryst->reflections != NULL ) { + RefList *refls; + refls = copy_reflist(cryst->reflections); + if ( refls == NULL ) return NULL; + c->reflections = refls; + } + + return c; +} + + +/** * \param cryst: A \ref Crystal to free. * * Frees a \ref Crystal, and all internal resources concerning that crystal. diff --git a/libcrystfel/src/crystal.h b/libcrystfel/src/crystal.h index 669c173f..5a4ca3f6 100644 --- a/libcrystfel/src/crystal.h +++ b/libcrystfel/src/crystal.h @@ -3,11 +3,11 @@ * * A class representing a single crystal * - * Copyright © 2013-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2013-2016 Thomas White <taw@physics.org> + * 2013-2020 Thomas White <taw@physics.org> * 2016 Valerio Mariani * * This file is part of CrystFEL. @@ -30,11 +30,6 @@ #ifndef CRYSTAL_H #define CRYSTAL_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - - #include "cell.h" /** @@ -56,6 +51,7 @@ extern "C" { extern Crystal *crystal_new(void); extern Crystal *crystal_copy(const Crystal *cryst); +extern Crystal *crystal_copy_deep(const Crystal *cryst); extern void crystal_free(Crystal *cryst); extern UnitCell *crystal_get_cell(Crystal *cryst); diff --git a/libcrystfel/src/datatemplate.c b/libcrystfel/src/datatemplate.c new file mode 100644 index 00000000..290b9227 --- /dev/null +++ b/libcrystfel/src/datatemplate.c @@ -0,0 +1,1683 @@ +/* + * datatemplate.c + * + * Data template structure + * + * Copyright © 2019-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2019-2021 Thomas White <taw@physics.org> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include <stdlib.h> +#include <math.h> +#include <string.h> +#include <assert.h> +#include <ctype.h> + +#include "utils.h" +#include "datatemplate.h" + +#include "datatemplate_priv.h" + + +/** + * \file datatemplate.h + */ + +struct rg_definition { + char *name; + char *pns; +}; + + +struct rgc_definition { + char *name; + char *rgs; +}; + + +static struct panel_template *new_panel(DataTemplate *det, + const char *name, + struct panel_template *defaults) +{ + struct panel_template *new; + int i; + + det->n_panels++; + det->panels = realloc(det->panels, + det->n_panels*sizeof(struct panel_template)); + + new = &det->panels[det->n_panels-1]; + memcpy(new, defaults, sizeof(struct panel_template)); + + /* Set name */ + new->name = strdup(name); + + /* Copy strings */ + new->cnz_from = safe_strdup(defaults->cnz_from); + new->data = safe_strdup(defaults->data); + new->satmap = safe_strdup(defaults->satmap); + new->satmap_file = safe_strdup(defaults->satmap_file); + for ( i=0; i<MAX_MASKS; i++ ) { + new->masks[i].data_location = safe_strdup(defaults->masks[i].data_location); + new->masks[i].filename = safe_strdup(defaults->masks[i].filename); + } + + return new; +} + + +static struct dt_badregion *new_bad_region(DataTemplate *det, const char *name) +{ + struct dt_badregion *new; + + det->n_bad++; + det->bad = realloc(det->bad, det->n_bad*sizeof(struct dt_badregion)); + + new = &det->bad[det->n_bad-1]; + new->min_x = NAN; + new->max_x = NAN; + new->min_y = NAN; + new->max_y = NAN; + new->min_fs = 0; + new->max_fs = 0; + new->min_ss = 0; + new->max_ss = 0; + new->is_fsss = 99; /* Slightly nasty: means "unassigned" */ + new->panel_name = NULL; + new->panel_number = 0; /* Needs to be set after loading */ + strcpy(new->name, name); + + return new; +} + + +static struct panel_template *find_panel_by_name(DataTemplate *det, + const char *name) +{ + int i; + + for ( i=0; i<det->n_panels; i++ ) { + if ( strcmp(det->panels[i].name, name) == 0 ) { + return &det->panels[i]; + } + } + + return NULL; +} + + +static struct dt_badregion *find_bad_region_by_name(DataTemplate *det, + const char *name) +{ + int i; + + for ( i=0; i<det->n_bad; i++ ) { + if ( strcmp(det->bad[i].name, name) == 0 ) { + return &det->bad[i]; + } + } + + return NULL; +} + + +static struct rigid_group *find_or_add_rg(DataTemplate *det, + const char *name) +{ + int i; + struct rigid_group **new; + struct rigid_group *rg; + + for ( i=0; i<det->n_rigid_groups; i++ ) { + + if ( strcmp(det->rigid_groups[i]->name, name) == 0 ) { + return det->rigid_groups[i]; + } + + } + + new = realloc(det->rigid_groups, + (1+det->n_rigid_groups)*sizeof(struct rigid_group *)); + if ( new == NULL ) return NULL; + + det->rigid_groups = new; + + rg = malloc(sizeof(struct rigid_group)); + if ( rg == NULL ) return NULL; + + det->rigid_groups[det->n_rigid_groups++] = rg; + + rg->name = strdup(name); + rg->panel_numbers = NULL; + rg->n_panels = 0; + + return rg; +} + + +static struct rg_collection *find_or_add_rg_coll(DataTemplate *det, + const char *name) +{ + int i; + struct rg_collection **new; + struct rg_collection *rgc; + + for ( i=0; i<det->n_rg_collections; i++ ) { + if ( strcmp(det->rigid_group_collections[i]->name, name) == 0 ) + { + return det->rigid_group_collections[i]; + } + } + + new = realloc(det->rigid_group_collections, + (1+det->n_rg_collections)*sizeof(struct rg_collection *)); + if ( new == NULL ) return NULL; + + det->rigid_group_collections = new; + + rgc = malloc(sizeof(struct rg_collection)); + if ( rgc == NULL ) return NULL; + + det->rigid_group_collections[det->n_rg_collections++] = rgc; + + rgc->name = strdup(name); + rgc->rigid_groups = NULL; + rgc->n_rigid_groups = 0; + + return rgc; +} + + +static void add_to_rigid_group(struct rigid_group *rg, int panel_number) +{ + int *pn; + + pn = realloc(rg->panel_numbers, (1+rg->n_panels)*sizeof(int)); + if ( pn == NULL ) { + ERROR("Couldn't add panel to rigid group.\n"); + return; + } + + rg->panel_numbers = pn; + rg->panel_numbers[rg->n_panels++] = panel_number; +} + + +static void add_to_rigid_group_coll(struct rg_collection *rgc, + struct rigid_group *rg) +{ + struct rigid_group **r; + + r = realloc(rgc->rigid_groups, (1+rgc->n_rigid_groups)* + sizeof(struct rigid_group *)); + if ( r == NULL ) { + ERROR("Couldn't add rigid group to collection.\n"); + return; + } + + rgc->rigid_groups = r; + rgc->rigid_groups[rgc->n_rigid_groups++] = rg; +} + + +/* Free all rigid groups in detector */ +static void free_all_rigid_groups(DataTemplate *det) +{ + int i; + + if ( det->rigid_groups == NULL ) return; + for ( i=0; i<det->n_rigid_groups; i++ ) { + free(det->rigid_groups[i]->name); + free(det->rigid_groups[i]->panel_numbers); + free(det->rigid_groups[i]); + } + free(det->rigid_groups); +} + + +/* Free all rigid groups in detector */ +static void free_all_rigid_group_collections(DataTemplate *det) +{ + int i; + + if ( det->rigid_group_collections == NULL ) return; + for ( i=0; i<det->n_rg_collections; i++ ) { + free(det->rigid_group_collections[i]->name); + free(det->rigid_group_collections[i]->rigid_groups); + free(det->rigid_group_collections[i]); + } + free(det->rigid_group_collections); +} + + +static struct rigid_group *find_rigid_group_by_name(DataTemplate *det, + char *name) +{ + int i; + + for ( i=0; i<det->n_rigid_groups; i++ ) { + if ( strcmp(det->rigid_groups[i]->name, name) == 0 ) { + return det->rigid_groups[i]; + } + } + + return NULL; +} + + +static int atob(const char *a) +{ + if ( strcasecmp(a, "true") == 0 ) return 1; + if ( strcasecmp(a, "false") == 0 ) return 0; + return atoi(a); +} + + +static int assplode_algebraic(const char *a_orig, char ***pbits) +{ + int len, i; + int nexp; + char **bits; + char *a; + int idx, istr; + + len = strlen(a_orig); + + /* Add plus at start if no sign there already */ + if ( (a_orig[0] != '+') && (a_orig[0] != '-') ) { + len += 1; + a = malloc(len+1); + snprintf(a, len+1, "+%s", a_orig); + a[len] = '\0'; + + } else { + a = strdup(a_orig); + } + + /* Count the expressions */ + nexp = 0; + for ( i=0; i<len; i++ ) { + if ( (a[i] == '+') || (a[i] == '-') ) nexp++; + } + + bits = calloc(nexp, sizeof(char *)); + + /* Break the string up */ + idx = -1; + istr = 0; + assert((a[0] == '+') || (a[0] == '-')); + for ( i=0; i<len; i++ ) { + + char ch; + + ch = a[i]; + + if ( ch == ' ' ) continue; + + if ( (ch == '+') || (ch == '-') ) { + if ( idx >= 0 ) bits[idx][istr] = '\0'; + idx++; + bits[idx] = malloc(len+1); + istr = 0; + } + + if ( !isdigit(ch) && (ch != '.') && (ch != '+') && (ch != '-') + && (ch != 'x') && (ch != 'y') && (ch != 'z') ) + { + ERROR("Invalid character '%c' found.\n", ch); + return 0; + } + + assert(idx >= 0); + bits[idx][istr++] = ch; + + } + if ( idx >= 0 ) bits[idx][istr] = '\0'; + + *pbits = bits; + free(a); + + return nexp; +} + + +/* Parses the scan directions (accounting for possible rotation) + * Assumes all white spaces have been already removed */ +static int dir_conv(const char *a, double *sx, double *sy, double *sz) +{ + int n; + char **bits; + int i; + + *sx = 0.0; *sy = 0.0; *sz = 0.0; + + n = assplode_algebraic(a, &bits); + + if ( n == 0 ) { + ERROR("Invalid direction '%s'\n", a); + return 1; + } + + for ( i=0; i<n; i++ ) { + + int len; + double val; + char axis; + int j; + + len = strlen(bits[i]); + assert(len != 0); + axis = bits[i][len-1]; + if ( (axis != 'x') && (axis != 'y') && (axis != 'z') ) { + ERROR("Invalid symbol '%c' - must be x, y or z.\n", + axis); + return 1; + } + + /* Chop off the symbol now it's dealt with */ + bits[i][len-1] = '\0'; + + /* Check for anything that isn't part of a number */ + for ( j=0; j<strlen(bits[i]); j++ ) { + if ( isdigit(bits[i][j]) ) continue; + if ( bits[i][j] == '+' ) continue; + if ( bits[i][j] == '-' ) continue; + if ( bits[i][j] == '.' ) continue; + ERROR("Invalid coefficient '%s'\n", bits[i]); + } + + if ( strlen(bits[i]) == 0 ) { + val = 1.0; + } else { + val = atof(bits[i]); + } + if ( strlen(bits[i]) == 1 ) { + if ( bits[i][0] == '+' ) val = 1.0; + if ( bits[i][0] == '-' ) val = -1.0; + } + switch ( axis ) { + + case 'x' : + *sx += val; + break; + + case 'y' : + *sy += val; + break; + + case 'z' : + *sz += val; + break; + } + + free(bits[i]); + + } + free(bits); + + return 0; +} + + +int set_dim(struct panel_template *panel, int dimension, + const char *val) +{ + if ( dimension >= MAX_DIMS ) { + ERROR("Too many dimensions!\n"); + return 1; + } + + if ( strcmp(val, "fs") == 0 ) { + panel->dims[dimension] = DIM_FS; + } else if ( strcmp(val, "ss") == 0 ) { + panel->dims[dimension] = DIM_SS; + } else if ( strcmp(val, "%") == 0 ) { + panel->dims[dimension] = DIM_PLACEHOLDER; + } else { + char *endptr; + unsigned long int fix_val = strtoul(val, &endptr, 10); + if ( endptr[0] != '\0' ) { + ERROR("Invalid dimension value '%s'\n", val); + return 1; + } else { + panel->dims[dimension] = fix_val; + } + } + return 0; +} + + +static int add_flag_value(struct panel_template *p, + float val, + enum flag_value_type type) +{ + int i; + + for ( i=0; i<MAX_FLAG_VALUES; i++ ) { + if ( p->flag_types[i] == FLAG_NOTHING ) { + p->flag_types[i] = type; + p->flag_values[i] = val; + return 0; + } + } + + ERROR("Too many flag values.\n"); + return 1; +} + + +static int parse_mask(struct panel_template *panel, + const char *key_orig, + const char *val) +{ + int n; + char *key; + + if ( sscanf(key_orig, "mask%d_", &n) != 1 ) { + ERROR("Invalid mask directive '%s'\n", key_orig); + return 1; + } + + key = strdup(key_orig); + if ( key == NULL ) return 1; + + key[4] = '_'; + + if ( strcmp(key, "mask__file") == 0 ) { + + panel->masks[n].filename = strdup(val); + + } else if ( strcmp(key, "mask__data") == 0 ) { + + if ( strncmp(val, "/", 1) != 0 ) { + ERROR("Invalid mask location '%s'\n", val); + free(key); + return 1; + } + panel->masks[n].data_location = strdup(val); + + } else if ( strcmp(key, "mask__goodbits") == 0 ) { + + char *end; + double v = strtod(val, &end); + + if ( end != val ) { + panel->masks[n].good_bits = v; + } else { + free(key); + return 1; + } + + } else if ( strcmp(key, "mask__badbits") == 0 ) { + + char *end; + double v = strtod(val, &end); + + if ( end != val ) { + panel->masks[n].bad_bits = v; + } else { + free(key); + return 1; + } + + } + + free(key); + return 0; +} + + +static int parse_field_for_panel(struct panel_template *panel, const char *key, + const char *val, DataTemplate *det) +{ + int reject = 0; + + if ( strcmp(key, "min_fs") == 0 ) { + panel->orig_min_fs = atof(val); + } else if ( strcmp(key, "max_fs") == 0 ) { + panel->orig_max_fs = atof(val); + } else if ( strcmp(key, "min_ss") == 0 ) { + panel->orig_min_ss = atof(val); + } else if ( strcmp(key, "max_ss") == 0 ) { + panel->orig_max_ss = atof(val); + } else if ( strcmp(key, "corner_x") == 0 ) { + panel->cnx = atof(val); + } else if ( strcmp(key, "corner_y") == 0 ) { + panel->cny = atof(val); + } else if ( strcmp(key, "rail_direction") == 0 ) { + if ( dir_conv(val, &panel->rail_x, + &panel->rail_y, + &panel->rail_z) ) + { + ERROR("Invalid rail direction '%s'\n", val); + reject = 1; + } + } else if ( strcmp(key, "clen_for_centering") == 0 ) { + panel->clen_for_centering = atof(val); + } else if ( strcmp(key, "adu_per_eV") == 0 ) { + panel->adu_scale = atof(val); + panel->adu_scale_unit = ADU_PER_EV; + } else if ( strcmp(key, "adu_per_photon") == 0 ) { + panel->adu_scale = atof(val); + panel->adu_scale_unit = ADU_PER_PHOTON; + } else if ( strcmp(key, "clen") == 0 ) { + /* Gets expanded when image is loaded */ + panel->cnz_from = strdup(val); + + } else if ( strcmp(key, "data") == 0 ) { + if ( strncmp(val,"/",1) != 0 ) { + ERROR("Invalid data location '%s'\n", val); + reject = -1; + } + free(panel->data); + panel->data = strdup(val); + + } else if ( strcmp(key, "mask_bad") == 0 ) { + parse_field_for_panel(panel, "mask0_badbits", val, det); + } else if ( strcmp(key, "mask_good") == 0 ) { + parse_field_for_panel(panel, "mask0_goodbits", val, det); + } else if ( strcmp(key, "mask") == 0 ) { + parse_field_for_panel(panel, "mask0_data", val, det); + } else if ( strcmp(key, "mask_file") == 0 ) { + parse_field_for_panel(panel, "mask0_file", val, det); + + } else if ( strncmp(key, "mask", 4) == 0 ) { + reject = parse_mask(panel, key, val); + + } else if ( strcmp(key, "saturation_map") == 0 ) { + panel->satmap = strdup(val); + } else if ( strcmp(key, "saturation_map_file") == 0 ) { + panel->satmap_file = strdup(val); + + } else if ( strcmp(key, "coffset") == 0) { + panel->cnz_offset = atof(val); + } else if ( strcmp(key, "res") == 0 ) { + panel->pixel_pitch = 1.0/atof(val); + } else if ( strcmp(key, "max_adu") == 0 ) { + panel->max_adu = atof(val); + ERROR("WARNING: It's usually better not to set max_adu " + "in the geometry file. Use --max-adu during " + "merging instead.\n"); + + } else if ( strcmp(key, "flag_equal") == 0 ) { + if ( add_flag_value(panel, atof(val), FLAG_EQUAL) ) { + reject = -1; + } + } else if ( strcmp(key, "flag_lessthan") == 0 ) { + if ( add_flag_value(panel, atof(val), FLAG_LESSTHAN) ) { + reject = -1; + } + } else if ( strcmp(key, "flag_morethan") == 0 ) { + if ( add_flag_value(panel, atof(val), FLAG_MORETHAN) ) { + reject = -1; + } + + } else if ( strcmp(key, "badrow_direction") == 0 ) { + ERROR("WARNING 'badrow_direction' is ignored in this version.\n"); + } else if ( strcmp(key, "no_index") == 0 ) { + panel->bad = atob(val); + } else if ( strcmp(key, "fs") == 0 ) { + if ( dir_conv(val, &panel->fsx, &panel->fsy, &panel->fsz) != 0 ) + { + ERROR("Invalid fast scan direction '%s'\n", val); + reject = 1; + } + } else if ( strcmp(key, "ss") == 0 ) { + if ( dir_conv(val, &panel->ssx, &panel->ssy, &panel->ssz) != 0 ) + { + ERROR("Invalid slow scan direction '%s'\n", val); + reject = 1; + } + } else if ( strncmp(key, "dim", 3) == 0) { + int dim_entry; + char *endptr; + if ( key[3] != '\0' ) { + dim_entry = strtoul(key+3, &endptr, 10); + if ( endptr[0] != '\0' ) { + ERROR("Invalid dimension number %s\n", + key+3); + } else { + if ( set_dim(panel, dim_entry, val) ) { + ERROR("Failed to set dim structure entry\n"); + } + } + } else { + ERROR("'dim' must be followed by a number, e.g. 'dim0'\n"); + } + } else { + ERROR("Unrecognised field '%s'\n", key); + } + + return reject; +} + + +static int check_badr_fsss(struct dt_badregion *badr, int is_fsss) +{ + /* First assignment? */ + if ( badr->is_fsss == 99 ) { + badr->is_fsss = is_fsss; + return 0; + } + + if ( is_fsss != badr->is_fsss ) { + ERROR("You can't mix x/y and fs/ss in a bad region.\n"); + return 1; + } + + return 0; +} + + +static int parse_field_bad(struct dt_badregion *badr, const char *key, + const char *val) +{ + int reject = 0; + + if ( strcmp(key, "min_x") == 0 ) { + badr->min_x = atof(val); + reject = check_badr_fsss(badr, 0); + } else if ( strcmp(key, "max_x") == 0 ) { + badr->max_x = atof(val); + reject = check_badr_fsss(badr, 0); + } else if ( strcmp(key, "min_y") == 0 ) { + badr->min_y = atof(val); + reject = check_badr_fsss(badr, 0); + } else if ( strcmp(key, "max_y") == 0 ) { + badr->max_y = atof(val); + reject = check_badr_fsss(badr, 0); + } else if ( strcmp(key, "min_fs") == 0 ) { + badr->min_fs = atof(val); + reject = check_badr_fsss(badr, 1); + } else if ( strcmp(key, "max_fs") == 0 ) { + badr->max_fs = atof(val); + reject = check_badr_fsss(badr, 1); + } else if ( strcmp(key, "min_ss") == 0 ) { + badr->min_ss = atof(val); + reject = check_badr_fsss(badr, 1); + } else if ( strcmp(key, "max_ss") == 0 ) { + badr->max_ss = atof(val); + reject = check_badr_fsss(badr, 1); + } else if ( strcmp(key, "panel") == 0 ) { + badr->panel_name = strdup(val); + } else { + ERROR("Unrecognised field '%s'\n", key); + } + + return reject; +} + + +static int parse_electron_voltage(const char *val, + char **p_from, + enum wavelength_unit *punit) +{ + char *valcpy; + char *sp; + + valcpy = strdup(val); + if ( valcpy == NULL ) return 1; + + /* "electron_voltage" directive must have explicit units */ + sp = strchr(valcpy, ' '); + if ( sp == NULL ) { + free(valcpy); + return 1; + } + + if ( strcmp(sp+1, "V") == 0 ) { + *punit = WAVELENGTH_ELECTRON_V; + } else if ( strcmp(sp+1, "kV") == 0 ) { + *punit = WAVELENGTH_ELECTRON_KV; + } else { + free(valcpy); + return 1; + } + + sp[0] = '\0'; + *p_from = valcpy; + return 0; +} + + +static int parse_wavelength(const char *val, + char **p_from, + enum wavelength_unit *punit) +{ + char *valcpy; + char *sp; + + valcpy = strdup(val); + if ( valcpy == NULL ) return 1; + + /* "wavelength" directive must have explicit units */ + sp = strchr(valcpy, ' '); + if ( sp == NULL ) { + free(valcpy); + return 1; + } + + if ( strcmp(sp+1, "m") == 0 ) { + *punit = WAVELENGTH_M; + } else if ( strcmp(sp+1, "A") == 0 ) { + *punit = WAVELENGTH_A; + } else { + free(valcpy); + return 1; + } + + sp[0] = '\0'; + *p_from = valcpy; + return 0; +} + + +static int parse_photon_energy(const char *val, + char **p_from, + enum wavelength_unit *punit) +{ + char *valcpy; + char *sp; + + valcpy = strdup(val); + if ( valcpy == NULL ) return 1; + + /* "photon_energy" is the only one of the wavelength + * directives which is allowed to not have units */ + sp = strchr(valcpy, ' '); + if ( sp == NULL ) { + *punit = WAVELENGTH_PHOTON_EV; + } else if ( strcmp(sp+1, "eV") == 0 ) { + *punit = WAVELENGTH_PHOTON_EV; + sp[0] = '\0'; + } else if ( strcmp(sp+1, "keV") == 0 ) { + *punit = WAVELENGTH_PHOTON_KEV; + sp[0] = '\0'; + } else { + /* Unit specified, but unrecognised */ + free(valcpy); + return 1; + } + + *p_from = valcpy; + return 0; +} + + +static int parse_peak_layout(const char *val, + enum peak_layout *layout) +{ + if ( strcmp(val, "auto") == 0 ) { + *layout = PEAK_LIST_AUTO; + return 0; + } + + if ( strcmp(val, "cxi") == 0 ) { + *layout = PEAK_LIST_CXI; + return 0; + } + + if ( (strcmp(val, "list3") == 0) ) { + *layout = PEAK_LIST_LIST3; + return 0; + } + + return 1; +} + + +static int parse_toplevel(DataTemplate *dt, + const char *key, + const char *val, + struct rg_definition ***rg_defl, + struct rgc_definition ***rgc_defl, + int *n_rg_defs, + int *n_rgc_defs, + struct panel_template *defaults) +{ + if ( strcmp(key, "detector_shift_x") == 0 ) { + dt->shift_x_from = strdup(val); + + } else if ( strcmp(key, "detector_shift_y") == 0 ) { + dt->shift_y_from = strdup(val); + + } else if ( strcmp(key, "photon_energy") == 0 ) { + return parse_photon_energy(val, + &dt->wavelength_from, + &dt->wavelength_unit); + + } else if ( strcmp(key, "electron_voltage") == 0 ) { + return parse_electron_voltage(val, + &dt->wavelength_from, + &dt->wavelength_unit); + + } else if ( strcmp(key, "wavelength") == 0 ) { + return parse_wavelength(val, + &dt->wavelength_from, + &dt->wavelength_unit); + + } else if ( strcmp(key, "peak_list") == 0 ) { + dt->peak_list = strdup(val); + + } else if ( strcmp(key, "peak_list_type") == 0 ) { + return parse_peak_layout(val, &dt->peak_list_type); + + } else if ( strcmp(key, "bandwidth") == 0 ) { + double v; + char *end; + v = strtod(val, &end); + if ( (val[0] != '\0') && (end[0] == '\0') ) { + dt->bandwidth = v; + } else { + ERROR("Invalid value for bandwidth\n"); + } + + } else if (strncmp(key, "rigid_group", 11) == 0 + && strncmp(key, "rigid_group_collection", 22) != 0 ) { + + struct rg_definition **new; + + new = realloc(*rg_defl, + ((*n_rg_defs)+1)*sizeof(struct rg_definition*)); + *rg_defl = new; + + (*rg_defl)[*n_rg_defs] = malloc(sizeof(struct rg_definition)); + (*rg_defl)[*n_rg_defs]->name = strdup(key+12); + (*rg_defl)[*n_rg_defs]->pns = strdup(val); + *n_rg_defs = *n_rg_defs+1; + + } else if ( strncmp(key, "rigid_group_collection", 22) == 0 ) { + + struct rgc_definition **new; + + new = realloc(*rgc_defl, ((*n_rgc_defs)+1)* + sizeof(struct rgc_definition*)); + *rgc_defl = new; + + (*rgc_defl)[*n_rgc_defs] = + malloc(sizeof(struct rgc_definition)); + (*rgc_defl)[*n_rgc_defs]->name = strdup(key+23); + (*rgc_defl)[*n_rgc_defs]->rgs = strdup(val); + *n_rgc_defs = *n_rgc_defs+1; + + } else if ( parse_field_for_panel(defaults, key, val, dt) ) { + return 1; + } + + return 0; +} + + +static int dt_num_path_placeholders(const char *str) +{ + size_t i, len; + int n_pl = 0; + + if ( str == NULL ) return 0; + + len = strlen(str); + for ( i=0; i<len; i++ ) { + if ( str[i] == '%' ) n_pl++; + } + + return n_pl; +} + + +signed int find_dim(signed int *dims, int which) +{ + int i; + + for ( i=0; i<MAX_DIMS; i++ ) { + if ( dims[i] == DIM_UNDEFINED ) break; + if ( dims[i] == which ) return i; + } + + return -1; +} + + +static int lookup_panel(const char *panel_name, + const DataTemplate *dt, + int *res) +{ + int i; + + /* If there is exactly one panel, you can get away without + * specifying the panel name */ + if ( (panel_name == NULL) && (dt->n_panels == 1) ) { + *res = 0; + return 0; + } + + for ( i=0; i<dt->n_panels; i++ ) { + if ( strcmp(dt->panels[i].name, panel_name) == 0 ) { + *res = i; + return 0; + } + } + + return 1; +} + + +static int check_mask_and_satmap_placeholders(const DataTemplate *dt) +{ + int i; + + for ( i=0; i<dt->n_panels; i++ ) { + + int num_data_pl; + int num_satmap_pl; + int j; + + num_data_pl = dt_num_path_placeholders(dt->panels[i].data); + num_satmap_pl = dt_num_path_placeholders(dt->panels[i].satmap); + + if ( num_satmap_pl > num_data_pl ) return 1; + + for ( j=0; j<MAX_MASKS; j++ ) { + + int num_mask_pl; + + /* Unused slot? */ + if ( dt->panels[i].masks[j].data_location == NULL ) continue; + + num_mask_pl = dt_num_path_placeholders(dt->panels[i].masks[j].data_location); + if ( num_mask_pl > num_data_pl ) return 1; + } + } + + return 0; +} + + +DataTemplate *data_template_new_from_string(const char *string_in) +{ + DataTemplate *dt; + char **bits; + int done = 0; + int i; + int rgi, rgci; + int reject = 0; + struct rg_definition **rg_defl = NULL; + struct rgc_definition **rgc_defl = NULL; + int n_rg_definitions = 0; + int n_rgc_definitions = 0; + char *string; + char *string_orig; + size_t len; + struct panel_template defaults; + + dt = calloc(1, sizeof(DataTemplate)); + if ( dt == NULL ) return NULL; + + dt->n_panels = 0; + dt->panels = NULL; + dt->n_bad = 0; + dt->bad = NULL; + dt->n_rigid_groups = 0; + dt->rigid_groups = NULL; + dt->n_rg_collections = 0; + dt->rigid_group_collections = NULL; + dt->bandwidth = 0.00000001; + dt->peak_list = NULL; + dt->shift_x_from = NULL; + dt->shift_y_from = NULL; + + /* The default defaults... */ + defaults.orig_min_fs = -1; + defaults.orig_min_ss = -1; + defaults.orig_max_fs = -1; + defaults.orig_max_ss = -1; + defaults.cnx = NAN; + defaults.cny = NAN; + defaults.cnz_from = NULL; + defaults.cnz_offset = 0.0; + defaults.pixel_pitch = -1.0; + defaults.bad = 0; + defaults.fsx = 1.0; + defaults.fsy = 0.0; + defaults.fsz = 0.0; + defaults.ssx = 0.0; + defaults.ssy = 1.0; + defaults.ssz = 0.0; + defaults.rail_x = NAN; /* The actual default rail direction */ + defaults.rail_y = NAN; /* is below */ + defaults.rail_z = NAN; + defaults.clen_for_centering = NAN; + defaults.adu_scale = NAN; + defaults.adu_scale_unit = ADU_PER_PHOTON; + for ( i=0; i<MAX_FLAG_VALUES; i++ ) defaults.flag_values[i] = 0; + for ( i=0; i<MAX_FLAG_VALUES; i++ ) defaults.flag_types[i] = FLAG_NOTHING; + for ( i=0; i<MAX_MASKS; i++ ) { + defaults.masks[i].data_location = NULL; + defaults.masks[i].filename = NULL; + defaults.masks[i].good_bits = 0; + defaults.masks[i].bad_bits = 0; + } + defaults.max_adu = +INFINITY; + defaults.satmap = NULL; + defaults.satmap_file = NULL; + defaults.data = strdup("/data/data"); + defaults.name = NULL; + defaults.dims[0] = DIM_SS; + defaults.dims[1] = DIM_FS; + for ( i=2; i<MAX_DIMS; i++ ) defaults.dims[i] = DIM_UNDEFINED; + + string = strdup(string_in); + if ( string == NULL ) return NULL; + len = strlen(string); + for ( i=0; i<len; i++ ) { + if ( string_in[i] == '\r' ) string[i] = '\n'; + } + + /* Becaue 'string' will get modified */ + string_orig = string; + + do { + + char *line; + struct dt_badregion *badregion = NULL; + struct panel_template *panel = NULL; + + /* Copy the next line from the big string */ + const char *nl = strchr(string, '\n'); + if ( nl != NULL ) { + size_t len = nl - string; + line = strndup(string, nl-string); + line[len] = '\0'; + string += len+1; + } else { + line = strdup(string); + done = 1; + } + + /* Trim leading spaces */ + i = 0; + char *line_orig = line; + while ( (line_orig[i] == ' ') + || (line_orig[i] == '\t') ) i++; + line = strdup(line+i); + free(line_orig); + + /* Stop at comment symbol */ + char *comm = strchr(line, ';'); + if ( comm != NULL ) comm[0] = '\0'; + + /* Nothing left? Entire line was commented out, + * and can be silently ignored */ + if ( line[0] == '\0' ) { + free(line); + continue; + } + + /* Find the equals sign */ + char *eq = strchr(line, '='); + if ( eq == NULL ) { + ERROR("Bad line in geometry file: '%s'\n", line); + free(line); + reject = 1; + continue; + } + + /* Split into two strings */ + eq[0] = '\0'; + char *val = eq+1; + + /* Trim leading and trailing spaces in value */ + while ( (val[0] == ' ') || (val[0] == '\t') ) val++; + notrail(val); + + /* Trim trailing spaces in key + * (leading spaces already done above) */ + notrail(line); + + /* Find slash after panel name */ + char *slash = strchr(line, '/'); + if ( slash == NULL ) { + + /* Top-level option */ + if ( parse_toplevel(dt, line, val, + &rg_defl, + &rgc_defl, + &n_rg_definitions, + &n_rgc_definitions, + &defaults) ) + { + ERROR("Invalid top-level line '%s'\n", + line); + reject = 1; + } + free(line); + continue; + } + + slash[0] = '\0'; + char *key = slash+1; + /* No space trimming this time - must be "panel/key" */ + + /* Find either panel or bad region */ + if ( strncmp(line, "bad", 3) == 0 ) { + badregion = find_bad_region_by_name(dt, line); + if ( badregion == NULL ) { + badregion = new_bad_region(dt, line); + } + } else { + panel = find_panel_by_name(dt, line); + if ( panel == NULL ) { + panel = new_panel(dt, line, &defaults); + } + } + + if ( panel != NULL ) { + if ( parse_field_for_panel(panel, key, val, + dt) ) reject = 1; + } else { + if ( parse_field_bad(badregion, key, + val) ) reject = 1; + } + + free(line); + + } while ( !done ); + + if ( dt->n_panels == 0 ) { + ERROR("No panel descriptions in geometry file.\n"); + free(dt); + return NULL; + } + + if ( check_mask_and_satmap_placeholders(dt) ) { + ERROR("Mask and saturation map paths must have fewer " + "placeholders than image data path.\n"); + reject = 1; + } + + for ( i=0; i<dt->n_panels; i++ ) { + + int j; + struct panel_template *p = &dt->panels[i]; + signed int dim_fs = find_dim(p->dims, DIM_FS); + signed int dim_ss = find_dim(p->dims, DIM_SS); + + if ( (dim_fs<0) || (dim_ss<0) ) { + ERROR("Panel %s does not have dimensions " + "assigned to both fs and ss.\n", + p->name); + reject = 1; + } + + if ( dim_ss > dim_fs ) { + ERROR("Fast scan dimension must be lower than " + "slow scan (panel %s)\n", p->name); + reject = 1; + } + + if ( p->orig_min_fs < 0 ) { + ERROR("Please specify the minimum FS coordinate for" + " panel %s\n", dt->panels[i].name); + reject = 1; + } + if ( p->orig_max_fs < 0 ) { + ERROR("Please specify the maximum FS coordinate for" + " panel %s\n", dt->panels[i].name); + reject = 1; + } + if ( p->orig_min_ss < 0 ) { + ERROR("Please specify the minimum SS coordinate for" + " panel %s\n", dt->panels[i].name); + reject = 1; + } + if ( p->orig_max_ss < 0 ) { + ERROR("Please specify the maximum SS coordinate for" + " panel %s\n", dt->panels[i].name); + reject = 1; + } + if ( isnan(p->cnx) ) { + ERROR("Please specify the corner X coordinate for" + " panel %s\n", dt->panels[i].name); + reject = 1; + } + if ( isnan(p->cny) ) { + ERROR("Please specify the corner Y coordinate for" + " panel %s\n", dt->panels[i].name); + reject = 1; + } + if ( p->cnz_from == NULL ) { + ERROR("Please specify the camera length for panel %s\n", + dt->panels[i].name); + reject = 1; + } + if ( p->pixel_pitch < 0 ) { + ERROR("Please specify the pixel size for" + " panel %s\n", dt->panels[i].name); + reject = 1; + } + if ( p->data == NULL ) { + ERROR("Please specify the data location for panel %s\n", + p->name); + reject = 1; + } + if ( isnan(p->adu_scale) ) { + ERROR("Please specify either adu_per_eV or " + "adu_per_photon for panel %s\n", + dt->panels[i].name); + reject = 1; + } + + if ( isnan(p->clen_for_centering) && !isnan(p->rail_x) ) + { + ERROR("You must specify clen_for_centering if you " + "specify the rail direction (panel %s)\n", + p->name); + reject = 1; + } + + for ( j=0; j<MAX_MASKS; j++ ) { + if ( (p->masks[j].filename != NULL) + && (p->masks[j].data_location == NULL) ) + { + ERROR("You have specified filename but not data" + " location for mask %i of panel %s\n", + j, p->name); + reject = 1; + } + } + + /* The default rail direction */ + if ( isnan(p->rail_x) ) { + p->rail_x = 0.0; + p->rail_y = 0.0; + p->rail_z = 1.0; + } + if ( isnan(p->clen_for_centering) ) p->clen_for_centering = 0.0; + + } + + for ( i=0; i<dt->n_bad; i++ ) { + + if ( dt->bad[i].is_fsss == 99 ) { + ERROR("Please specify the coordinate ranges for" + " bad region %s\n", dt->bad[i].name); + reject = 1; + } + + if ( dt->bad[i].is_fsss ) { + if ( dt->bad[i].panel_name == NULL ) { + + ERROR("Panel not specified for bad region '%s'\n", + dt->bad[i].name); + reject = 1; + + } else if ( lookup_panel(dt->bad[i].panel_name, dt, + &dt->bad[i].panel_number) ) + { + ERROR("No such panel '%s' for bad region %s\n", + dt->bad[i].panel_name, dt->bad[i].name); + reject = 1; + + } else { + struct panel_template *p; + struct dt_badregion *bad; + int r = 0; + p = &dt->panels[dt->bad[i].panel_number]; + bad = &dt->bad[i]; + if ( bad->min_fs < p->orig_min_fs ) r = 1; + if ( bad->min_ss < p->orig_min_ss ) r = 1; + if ( bad->max_fs > p->orig_max_fs ) r = 1; + if ( bad->max_ss > p->orig_max_ss ) r = 1; + if ( r ) { + ERROR("Bad region '%s' is outside the " + "panel bounds (%s) as presented " + "in data (%i %i, %i %i inclusive): " + "Bad region %i,%i to %i, %i " + "inclusive\n", + bad->name, p->name, + p->orig_min_fs, p->orig_min_ss, + p->orig_max_fs, p->orig_max_ss, + bad->min_fs, bad->min_ss, + bad->max_fs, bad->max_ss); + reject = 1; + } + bad->min_fs -= p->orig_min_fs; + bad->max_fs -= p->orig_min_fs; + bad->min_ss -= p->orig_min_ss; + bad->max_ss -= p->orig_min_ss; + } + } + } + + free(defaults.cnz_from); + free(defaults.data); + for ( i=0; i<MAX_MASKS; i++ ) { + free(defaults.masks[i].data_location); + free(defaults.masks[i].filename); + } + + for ( rgi=0; rgi<n_rg_definitions; rgi++) { + + int pi, n1; + struct rigid_group *rigidgroup = NULL; + + rigidgroup = find_or_add_rg(dt, rg_defl[rgi]->name); + + n1 = assplode(rg_defl[rgi]->pns, ",", &bits, ASSPLODE_NONE); + + for ( pi=0; pi<n1; pi++ ) { + + int panel_number; + if ( data_template_panel_name_to_number(dt, + bits[pi], + &panel_number) ) + { + ERROR("Cannot add panel to rigid group\n"); + ERROR("Panel not found: %s\n", bits[pi]); + return NULL; + } + add_to_rigid_group(rigidgroup, panel_number); + free(bits[pi]); + + } + free(bits); + free(rg_defl[rgi]->name); + free(rg_defl[rgi]->pns); + free(rg_defl[rgi]); + } + free(rg_defl); + + for ( rgci=0; rgci<n_rgc_definitions; rgci++ ) { + + int rgi, n2; + struct rg_collection *rgcollection = NULL; + + rgcollection = find_or_add_rg_coll(dt, rgc_defl[rgci]->name); + + n2 = assplode(rgc_defl[rgci]->rgs, ",", &bits, ASSPLODE_NONE); + + for ( rgi=0; rgi<n2; rgi++ ) { + + struct rigid_group *r; + + r = find_rigid_group_by_name(dt, bits[rgi]); + if ( r == NULL ) { + ERROR("Cannot add rigid group to collection\n"); + ERROR("Rigid group not found: %s\n", bits[rgi]); + return NULL; + } + add_to_rigid_group_coll(rgcollection, r); + free(bits[rgi]); + } + free(bits); + free(rgc_defl[rgci]->name); + free(rgc_defl[rgci]->rgs); + free(rgc_defl[rgci]); + + } + free(rgc_defl); + + free(string_orig); + + if ( reject ) return NULL; + + return dt; +} + + +DataTemplate *data_template_new_from_file(const char *filename) +{ + char *contents; + DataTemplate *dt; + + contents = load_entire_file(filename); + if ( contents == NULL ) { + ERROR("Failed to load geometry file '%s'\n", filename); + return NULL; + } + + dt = data_template_new_from_string(contents); + free(contents); + return dt; +} + + +void data_template_free(DataTemplate *dt) +{ + int i; + + if ( dt == NULL ) return; + + free_all_rigid_groups(dt); + free_all_rigid_group_collections(dt); + + for ( i=0; i<dt->n_panels; i++ ) { + + int j; + + free(dt->panels[i].name); + free(dt->panels[i].data); + free(dt->panels[i].satmap); + free(dt->panels[i].satmap_file); + free(dt->panels[i].cnz_from); + + for ( j=0; j<MAX_MASKS; j++ ) { + free(dt->panels[i].masks[j].filename); + free(dt->panels[i].masks[j].data_location); + } + } + + free(dt->wavelength_from); + free(dt->peak_list); + + free(dt->panels); + free(dt->bad); + free(dt); +} + + +static int data_template_find_panel(const DataTemplate *dt, + int fs, int ss, int *ppn) +{ + int p; + + for ( p=0; p<dt->n_panels; p++ ) { + if ( (fs >= dt->panels[p].orig_min_fs) + && (fs < dt->panels[p].orig_max_fs+1) + && (ss >= dt->panels[p].orig_min_ss) + && (ss < dt->panels[p].orig_max_ss+1) ) { + *ppn = p; + return 0; + } + } + + return 1; +} + + +int data_template_file_to_panel_coords(const DataTemplate *dt, + float *pfs, float *pss, + int *ppn) +{ + int pn; + + if ( data_template_find_panel(dt, *pfs, *pss, &pn) ) { + return 1; + } + + *ppn = pn; + *pfs = *pfs - dt->panels[pn].orig_min_fs; + *pss = *pss - dt->panels[pn].orig_min_ss; + return 0; +} + + +int data_template_panel_to_file_coords(const DataTemplate *dt, + int pn, float *pfs, float *pss) +{ + if ( pn >= dt->n_panels ) return 1; + *pfs = *pfs + dt->panels[pn].orig_min_fs; + *pss = *pss + dt->panels[pn].orig_min_ss; + return 0; +} + + +const char *data_template_panel_number_to_name(const DataTemplate *dt, + int pn) +{ + if ( pn >= dt->n_panels ) return NULL; + return dt->panels[pn].name; +} + + +int data_template_panel_name_to_number(const DataTemplate *dt, + const char *panel_name, + int *pn) +{ + int i; + + if ( panel_name == NULL ) return 1; + + for ( i=0; i<dt->n_panels; i++ ) { + if ( strcmp(panel_name, dt->panels[i].name) == 0 ) { + *pn = i; + return 0; + } + } + + return 1; +} + + +void data_template_add_copy_header(DataTemplate *dt, + const char *header) +{ + /* FIXME: Add "header" to list of things to copy */ + STATUS("Adding %s\n", header); +} + + +static int dt_num_placeholders(const struct panel_template *p) +{ + int i; + int n_pl = 0; + for ( i=0; i<MAX_DIMS; i++ ) { + if ( p->dims[i] == DIM_PLACEHOLDER ) n_pl++; + } + return n_pl; +} + + +int data_template_get_slab_extents(const DataTemplate *dt, + int *pw, int *ph) +{ + int w, h; + char *data_from; + int i; + + data_from = dt->panels[0].data; + + w = 0; h = 0; + for ( i=0; i<dt->n_panels; i++ ) { + + struct panel_template *p = &dt->panels[i]; + + if ( strcmp(data_from, p->data) != 0 ) { + /* Not slabby */ + return 1; + } + + if ( dt_num_placeholders(p) > 0 ) { + /* Not slabby */ + return 1; + } + + if ( p->orig_max_fs > w ) { + w = p->orig_max_fs; + } + if ( p->orig_max_ss > h ) { + h = p->orig_max_ss; + } + + } + + /* Inclusive -> exclusive */ + *pw = w + 1; + *ph = h + 1; + return 0; +} + + +double convert_to_m(double val, int units) +{ + switch ( units ) { + + case WAVELENGTH_M : + return val; + + case WAVELENGTH_A : + return val * 1e-10; + + case WAVELENGTH_PHOTON_EV : + return ph_eV_to_lambda(val); + + case WAVELENGTH_PHOTON_KEV : + return ph_eV_to_lambda(val*1e3); + + case WAVELENGTH_ELECTRON_V : + return el_V_to_lambda(val); + + case WAVELENGTH_ELECTRON_KV : + return el_V_to_lambda(val*1e3); + + } + + return NAN; +} + + +/** + * Get the wavelength from a DataTemplate, if possible. + * + * WARNING: This is probably not the routine you are looking for! + * See the disclaimer for image_create_for_simulation(), which applies + * equally to this routine. + * + * \returns the wavelength, in metres, or NAN if impossible. + */ +double data_template_get_wavelength_if_possible(const DataTemplate *dt) +{ + float val; + char *rval; + + if ( dt->wavelength_from == NULL ) return NAN; + + val = strtod(dt->wavelength_from, &rval); + if ( (*rval == '\0') && (rval != dt->wavelength_from) ) { + return convert_to_m(val, dt->wavelength_unit); + } else { + return NAN; + } +} diff --git a/libcrystfel/src/datatemplate.h b/libcrystfel/src/datatemplate.h new file mode 100644 index 00000000..3d457f80 --- /dev/null +++ b/libcrystfel/src/datatemplate.h @@ -0,0 +1,99 @@ +/* + * datatemplate.h + * + * Template for loading data + * + * Copyright © 2019-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2019-2020 Thomas White <taw@physics.org> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifndef DATATEMPLATE_H +#define DATATEMPLATE_H + +#include "detgeom.h" + +/** + * \file datatemplate.h + * Template for loading data. + */ + +/** + * This data structure is opaque. You must use the available accessor functions + * to read and write its contents. + **/ +typedef struct _datatemplate DataTemplate; + + +#ifdef __cplusplus +extern "C" { +#endif + +struct rigid_group +{ + char *name; + int *panel_numbers; + int n_panels; +}; + + +struct rg_collection +{ + char *name; + struct rigid_group **rigid_groups; + int n_rigid_groups; +}; + + +extern DataTemplate *data_template_new_from_file(const char *filename); +extern DataTemplate *data_template_new_from_string(const char *string_in); +extern void data_template_free(DataTemplate *dt); + +extern const char *data_template_panel_number_to_name(const DataTemplate *dt, + int pn); + +extern int data_template_panel_name_to_number(const DataTemplate *dt, + const char *panel_name, + int *pn); + +extern int data_template_file_to_panel_coords(const DataTemplate *dt, + float *pfs, float *pss, + int *pn); + +extern int data_template_panel_to_file_coords(const DataTemplate *dt, + int pn, + float *pfs, float *pss); + +extern void data_template_add_copy_header(DataTemplate *dt, + const char *header); + +extern int data_template_get_slab_extents(const DataTemplate *dt, int *pw, int *ph); + +extern struct rg_collection *data_template_get_rigid_groups(const DataTemplate *dtempl, + const char *collection_name); + +extern double data_template_get_wavelength_if_possible(const DataTemplate *dt); + +#ifdef __cplusplus +} +#endif + +#endif /* DATATEMPLATE_H */ diff --git a/libcrystfel/src/datatemplate_priv.h b/libcrystfel/src/datatemplate_priv.h new file mode 100644 index 00000000..71569273 --- /dev/null +++ b/libcrystfel/src/datatemplate_priv.h @@ -0,0 +1,239 @@ +/* + * datatemplate_priv.h + * + * Data template structure (private parts) + * + * Copyright © 2019-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2019-2021 Thomas White <taw@physics.org> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +/* NB This file is NOT part of the public API, and should NOT + * be installed, but rather stays in the libcrystfel source folder. */ + +#ifndef DATATEMPLATE_PRIV_H +#define DATATEMPLATE_PRIV_H + +/* Maximum number of dimensions expected in data files */ +#define MAX_DIMS (16) + +/* Maximum number of placeholders expected in path structure */ +#define MAX_PATH_PARTS (16) + +enum adu_per_unit +{ + ADU_PER_PHOTON, + ADU_PER_EV +}; + + +enum wavelength_unit +{ + WAVELENGTH_M, + WAVELENGTH_A, + WAVELENGTH_ELECTRON_KV, + WAVELENGTH_ELECTRON_V, + WAVELENGTH_PHOTON_KEV, + WAVELENGTH_PHOTON_EV +}; + +#define MAX_FLAG_VALUES (16) + +enum flag_value_type +{ + FLAG_NOTHING, + FLAG_EQUAL, + FLAG_MORETHAN, + FLAG_LESSTHAN +}; + +enum peak_layout +{ + PEAK_LIST_AUTO, + PEAK_LIST_CXI, + PEAK_LIST_LIST3 +}; + +/* Special values for dimension IDs */ +#define DIM_FS (-1) +#define DIM_SS (-2) +#define DIM_UNDEFINED (-3) +#define DIM_PLACEHOLDER (-4) + + +/* Maximum number of masks per panel */ +#define MAX_MASKS (8) + +struct mask_template +{ + /** Location of mask data */ + char *data_location; + + /** Filename for mask data */ + char *filename; + + /** Bit mask for bad pixels + * (pixel is bad if any of these are set) */ + unsigned int bad_bits; + + /** Bit mask for good pixels + * (pixel cannot be good unless all of these are set) */ + unsigned int good_bits; +}; + + +/** + * Represents one panel of a detector + */ +struct panel_template +{ + /** Text name for panel */ + char *name; + + /** \name Location of corner in units of the pixel size of this panel */ + /**@{*/ + double cnx; + double cny; + /**@}*/ + + /** Location to get \ref cnz from, e.g. from HDF5 file */ + char *cnz_from; + + /** The offset to be applied from \ref clen */ + double cnz_offset; + + /** Mask definitions */ + struct mask_template masks[MAX_MASKS]; + + /** Location of per-pixel saturation map */ + char *satmap; + + /** Filename for saturation map */ + char *satmap_file; + + /** Mark entire panel as bad if set */ + int bad; + + /** Resolution in pixels per metre */ + double pixel_pitch; + + /** Number of detector intensity units per photon, or eV */ + double adu_scale; + enum adu_per_unit adu_scale_unit; + + /** Treat pixel as unreliable if higher than this */ + double max_adu; + + /** Pixels with exactly this value will be marked as bad */ + enum flag_value_type flag_types[MAX_FLAG_VALUES]; + signed int flag_values[MAX_FLAG_VALUES]; + + /** Location of data in file (possibly with placeholders) */ + char *data; + + /** Dimensions (see definitions for DIM_FS etc above) */ + signed int dims[MAX_DIMS]; + + /** \name Transformation matrix from pixel coordinates to lab frame */ + /*@{*/ + double fsx; + double fsy; + double fsz; + double ssx; + double ssy; + double ssz; + /*@}*/ + + /** \name Rail direction */ + /*@{*/ + double rail_x; + double rail_y; + double rail_z; + /*@}*/ + + /* Value of clen (without coffset) at which beam is centered */ + double clen_for_centering; + + /** \name Position of the panel in the data block in the file. */ + /*@{*/ + int orig_min_fs; + int orig_max_fs; + int orig_min_ss; + int orig_max_ss; + /*@}*/ +}; + + +#define PANEL_WIDTH(p) ((p)->orig_max_fs - (p)->orig_min_fs + 1) +#define PANEL_HEIGHT(p) ((p)->orig_max_ss - (p)->orig_min_ss + 1) + + +struct dt_badregion +{ + char name[1024]; + int is_fsss; + + double min_x; + double max_x; + double min_y; + double max_y; + + /* Coordinates are specified INCLUSIVELY */ + int panel_number; + char *panel_name; + int min_fs; + int max_fs; + int min_ss; + int max_ss; + +}; + + +struct _datatemplate +{ + struct panel_template *panels; + int n_panels; + + struct dt_badregion *bad; + int n_bad; + + char *wavelength_from; + enum wavelength_unit wavelength_unit; + + double bandwidth; + + struct rigid_group **rigid_groups; + int n_rigid_groups; + + struct rg_collection **rigid_group_collections; + int n_rg_collections; + + char *peak_list; + enum peak_layout peak_list_type; + + /* Shift of whole detector, in m */ + char *shift_x_from; + char *shift_y_from; +}; + +extern double convert_to_m(double val, int units); + +#endif /* DATATEMPLATE_PRIV_H */ diff --git a/libcrystfel/src/detector.c b/libcrystfel/src/detector.c deleted file mode 100644 index 629e13f0..00000000 --- a/libcrystfel/src/detector.c +++ /dev/null @@ -1,2401 +0,0 @@ -/* - * detector.c - * - * Detector properties - * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, - * a research centre of the Helmholtz Association. - * Copyright © 2012 Richard Kirian - * - * Authors: - * 2009-2019 Thomas White <taw@physics.org> - * 2014 Valerio Mariani - * 2014 Kenneth Beyerlein <kenneth.beyerlein@desy.de> - * 2011 Andrew Aquila - * 2011 Richard Kirian <rkirian@asu.edu> - * - * This file is part of CrystFEL. - * - * CrystFEL is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * CrystFEL is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. - * - */ - -#include <stdlib.h> -#include <math.h> -#include <stdio.h> -#include <string.h> -#include <assert.h> -#include <ctype.h> -#include <sys/stat.h> - -#include "image.h" -#include "utils.h" -#include "detector.h" -#include "hdf5-file.h" - - -/** - * \file detector.h - */ - - -struct rg_definition { - char *name; - char *pns; -}; - - -struct rgc_definition { - char *name; - char *rgs; -}; - - -static int atob(const char *a) -{ - if ( strcasecmp(a, "true") == 0 ) return 1; - if ( strcasecmp(a, "false") == 0 ) return 0; - return atoi(a); -} - - -static int assplode_algebraic(const char *a_orig, char ***pbits) -{ - int len, i; - int nexp; - char **bits; - char *a; - int idx, istr; - - len = strlen(a_orig); - - /* Add plus at start if no sign there already */ - if ( (a_orig[0] != '+') && (a_orig[0] != '-') ) { - len += 1; - a = malloc(len+1); - snprintf(a, len+1, "+%s", a_orig); - a[len] = '\0'; - - } else { - a = strdup(a_orig); - } - - /* Count the expressions */ - nexp = 0; - for ( i=0; i<len; i++ ) { - if ( (a[i] == '+') || (a[i] == '-') ) nexp++; - } - - bits = calloc(nexp, sizeof(char *)); - - /* Break the string up */ - idx = -1; - istr = 0; - assert((a[0] == '+') || (a[0] == '-')); - for ( i=0; i<len; i++ ) { - - char ch; - - ch = a[i]; - - if ( (ch == '+') || (ch == '-') ) { - if ( idx >= 0 ) bits[idx][istr] = '\0'; - idx++; - bits[idx] = malloc(len+1); - istr = 0; - } - - if ( !isdigit(ch) && (ch != '.') && (ch != '+') && (ch != '-') - && (ch != 'x') && (ch != 'y') && (ch != 'z') ) - { - ERROR("Invalid character '%c' found.\n", ch); - return 0; - } - - assert(idx >= 0); - bits[idx][istr++] = ch; - - } - if ( idx >= 0 ) bits[idx][istr] = '\0'; - - *pbits = bits; - free(a); - - return nexp; -} - - -/* Parses the scan directions (accounting for possible rotation) - * Assumes all white spaces have been already removed */ -static int dir_conv(const char *a, double *sx, double *sy, double *sz) -{ - int n; - char **bits; - int i; - - *sx = 0.0; *sy = 0.0; *sz = 0.0; - - n = assplode_algebraic(a, &bits); - - if ( n == 0 ) { - ERROR("Invalid direction '%s'\n", a); - return 1; - } - - for ( i=0; i<n; i++ ) { - - int len; - double val; - char axis; - int j; - - len = strlen(bits[i]); - assert(len != 0); - axis = bits[i][len-1]; - if ( (axis != 'x') && (axis != 'y') && (axis != 'z') ) { - ERROR("Invalid symbol '%c' - must be x, y or z.\n", - axis); - return 1; - } - - /* Chop off the symbol now it's dealt with */ - bits[i][len-1] = '\0'; - - /* Check for anything that isn't part of a number */ - for ( j=0; j<strlen(bits[i]); j++ ) { - if ( isdigit(bits[i][j]) ) continue; - if ( bits[i][j] == '+' ) continue; - if ( bits[i][j] == '-' ) continue; - if ( bits[i][j] == '.' ) continue; - ERROR("Invalid coefficient '%s'\n", bits[i]); - } - - if ( strlen(bits[i]) == 0 ) { - val = 1.0; - } else { - val = atof(bits[i]); - } - if ( strlen(bits[i]) == 1 ) { - if ( bits[i][0] == '+' ) val = 1.0; - if ( bits[i][0] == '-' ) val = -1.0; - } - switch ( axis ) { - - case 'x' : - *sx += val; - break; - - case 'y' : - *sy += val; - break; - - case 'z' : - *sz += val; - break; - } - - free(bits[i]); - - } - free(bits); - - return 0; -} - - -static int count_trailing_spaces(const char *string) { - - int i; - - for ( i=0; i<strlen(string); i++ ) { - if ( !isspace(string[i]) ) { - return i; - } - } - - return -1; -} - - -static void build_output_line(const char *line, char *new_line, - const char *string_to_write) -{ - int nsp, i, w, ww; - int trailsp; - int n_bits; - char **bits; - - n_bits = assplode(line, "=", &bits, ASSPLODE_NONE); - trailsp = count_trailing_spaces(bits[1]); - - strcat(new_line, bits[0]); - strcat(new_line, "="); - - for ( nsp=0; nsp < trailsp; nsp++ ) { - strcat(new_line, " "); - } - strcat(new_line, string_to_write); - - for ( w=0; w<strlen(line); w++ ) { - if ( strncmp(&line[w],";",1) == 0 ) { - for ( ww=w-1; ww>=0; ww-- ) { - if ( !isspace(line[ww])) { - strcat(new_line, &line[ww]); - break; - } - } - break; - } - } - for ( i=0; i<n_bits; i++) free(bits[i]); - free(bits); -} - - -struct rvec get_q_for_panel(struct panel *p, double fs, double ss, - double *ttp, double k) -{ - struct rvec q; - double ctt, twotheta; - double xs, ys, zs; - double az; - - /* Calculate 3D position of given position, in m */ - xs = (p->cnx + fs*p->fsx + ss*p->ssx) / p->res; - ys = (p->cny + fs*p->fsy + ss*p->ssy) / p->res; - zs = p->clen + (fs*p->fsz + ss*p->ssz) / p->res; - - ctt = zs/sqrt(xs*xs + ys*ys + zs*zs); - twotheta = acos(ctt); - az = atan2(ys, xs); - if ( ttp != NULL ) *ttp = twotheta; - - q.u = k * sin(twotheta)*cos(az); - q.v = k * sin(twotheta)*sin(az); - q.w = k * (ctt - 1.0); - - return q; -} - - -int in_bad_region(struct detector *det, struct panel *p, double fs, double ss) -{ - double rx, ry; - double xs, ys; - int i; - - /* No panel found -> definitely bad! */ - if ( p == NULL ) return 1; - - /* Convert xs and ys, which are in fast scan/slow scan coordinates, - * to x and y */ - xs = fs*p->fsx + ss*p->ssx; - ys = fs*p->fsy + ss*p->ssy; - - rx = xs + p->cnx; - ry = ys + p->cny; - - for ( i=0; i<det->n_bad; i++ ) { - - struct badregion *b = &det->bad[i]; - - if ( (b->panel != NULL) - && (strcmp(b->panel, p->name) != 0) ) continue; - - if ( b->is_fsss ) { - - int nfs, nss; - - /* fs/ss bad regions are specified according to the - * original coordinates */ - nfs = fs + p->orig_min_fs; - nss = ss + p->orig_min_ss; - - if ( nfs < b->min_fs ) continue; - if ( nfs > b->max_fs ) continue; - if ( nss < b->min_ss ) continue; - if ( nss > b->max_ss ) continue; - - } else { - - if ( rx < b->min_x ) continue; - if ( rx > b->max_x ) continue; - if ( ry < b->min_y ) continue; - if ( ry > b->max_y ) continue; - - } - - return 1; - } - - return 0; -} - - -int detector_has_clen_references(struct detector *det) -{ - int i; - - for ( i=0; i<det->n_panels; i++ ) { - if ( det->panels[i].clen_from != NULL ) return 1; - } - - return 0; -} - - -static void record_panel(struct panel *p, float *dp, int do_poisson, - gsl_rng *rng, double ph_per_e, double background, - double lambda, - int *n_neg1, int *n_inf1, int *n_nan1, - int *n_neg2, int *n_inf2, int *n_nan2, - double *max_tt) -{ - int fs, ss; - - for ( ss=0; ss<p->h; ss++ ) { - for ( fs=0; fs<p->w; fs++ ) { - - double counts; - double cf; - double intensity, sa; - double pix_area, Lsq; - double xs, ys, rx, ry; - double dsq, proj_area; - float dval; - double twotheta; - - intensity = (double)dp[fs + p->w*ss]; - if ( isinf(intensity) ) (*n_inf1)++; - if ( intensity < 0.0 ) (*n_neg1)++; - if ( isnan(intensity) ) (*n_nan1)++; - - /* Area of one pixel */ - pix_area = pow(1.0/p->res, 2.0); - Lsq = pow(p->clen, 2.0); - - /* Calculate distance from crystal to pixel */ - xs = fs*p->fsx + ss*p->ssx; - ys = ss*p->fsy + ss*p->ssy; - rx = (xs + p->cnx) / p->res; - ry = (ys + p->cny) / p->res; - dsq = pow(rx, 2.0) + pow(ry, 2.0); - twotheta = atan2(sqrt(dsq), p->clen); - - /* Area of pixel as seen from crystal (approximate) */ - proj_area = pix_area * cos(twotheta); - - /* Projected area of pixel divided by distance squared */ - sa = proj_area / (dsq + Lsq); - - if ( do_poisson ) { - counts = poisson_noise(rng, intensity * ph_per_e * sa); - } else { - cf = intensity * ph_per_e * sa; - counts = cf; - } - - /* Number of photons in pixel */ - dval = counts + poisson_noise(rng, background); - - /* Convert to ADU */ - dval *= p->adu_per_photon; - - /* Saturation */ - if ( dval > p->max_adu ) dval = p->max_adu; - - dp[fs + p->w*ss] = dval; - - /* Sanity checks */ - if ( isinf(dp[fs + p->w*ss]) ) n_inf2++; - if ( isnan(dp[fs + p->w*ss]) ) n_nan2++; - if ( dp[fs + p->w*ss] < 0.0 ) n_neg2++; - - if ( twotheta > *max_tt ) *max_tt = twotheta; - - } - } -} - - -void record_image(struct image *image, int do_poisson, double background, - gsl_rng *rng, double beam_radius, double nphotons) -{ - double total_energy, energy_density; - double ph_per_e; - double area; - double max_tt = 0.0; - int n_inf1 = 0; - int n_neg1 = 0; - int n_nan1 = 0; - int n_inf2 = 0; - int n_neg2 = 0; - int n_nan2 = 0; - int pn; - - /* How many photons are scattered per electron? */ - area = M_PI*pow(beam_radius, 2.0); - total_energy = nphotons * ph_lambda_to_en(image->lambda); - energy_density = total_energy / area; - ph_per_e = (nphotons /area) * pow(THOMSON_LENGTH, 2.0); - STATUS("Fluence = %8.2e photons, " - "Energy density = %5.3f kJ/cm^2, " - "Total energy = %5.3f microJ\n", - nphotons, energy_density/1e7, total_energy*1e6); - - fill_in_adu(image); - - for ( pn=0; pn<image->det->n_panels; pn++ ) { - - record_panel(&image->det->panels[pn], image->dp[pn], - do_poisson, rng, ph_per_e, background, - image->lambda, - &n_neg1, &n_inf1, &n_nan1, - &n_neg2, &n_inf2, &n_nan2, &max_tt); - } - - STATUS("Max 2theta = %.2f deg, min d = %.2f nm\n", - rad2deg(max_tt), (image->lambda/(2.0*sin(max_tt/2.0)))/1e-9); - - STATUS("Halve the d values to get the voxel size for a synthesis.\n"); - - if ( n_neg1 + n_inf1 + n_nan1 + n_neg2 + n_inf2 + n_nan2 ) { - ERROR("WARNING: The raw calculation produced %i negative" - " values, %i infinities and %i NaNs.\n", - n_neg1, n_inf1, n_nan1); - ERROR("WARNING: After processing, there were %i negative" - " values, %i infinities and %i NaNs.\n", - n_neg2, n_inf2, n_nan2); - } -} - - -signed int find_orig_panel_number(struct detector *det, double fs, double ss) -{ - int p; - - for ( p=0; p<det->n_panels; p++ ) { - if ( (fs >= det->panels[p].orig_min_fs) - && (fs < det->panels[p].orig_max_fs+1) - && (ss >= det->panels[p].orig_min_ss) - && (ss < det->panels[p].orig_max_ss+1) ) return p; - } - - return -1; -} - - -/* Like find_panel(), but uses the original panel bounds, i.e. referring to - * what's in the HDF5 file */ -struct panel *find_orig_panel(struct detector *det, double fs, double ss) -{ - signed int pn = find_orig_panel_number(det, fs, ss); - if ( pn == -1 ) return NULL; - return &det->panels[pn]; -} - - -int panel_number(const struct detector *det, const struct panel *p) -{ - int pn; - - for ( pn=0; pn<det->n_panels; pn++ ) { - if ( &det->panels[pn] == p ) return pn; - } - - return det->n_panels; -} - - -void adjust_centering_for_rail(struct panel *p) -{ - double offs; - - /* Offset in +z direction from calibrated clen to actual */ - offs = p->clen - p->clen_for_centering; - p->cnx += p->rail_x * offs; - p->cny += p->rail_y * offs; - p->clen = p->clen_for_centering + p->coffset + p->rail_z * offs; -} - - -void fill_in_adu(struct image *image) -{ - int i; - - if ( image->det == NULL ) return; - - for ( i=0; i<image->det->n_panels; i++ ) { - - struct panel *p = &image->det->panels[i]; - - /* Already have ADU per photon? */ - if ( !isnan(p->adu_per_photon) ) continue; - - if ( isnan(p->adu_per_eV) ) { - ERROR("Neither adu_per_eV nor adu_per_photon set for " - "panel %s\n", p->name); - continue; - } - - /* Convert ADU per eV to ADU per photon */ - p->adu_per_photon = ph_lambda_to_eV(image->lambda) - * p->adu_per_eV; - } -} - - -int panel_is_in_rigid_group(const struct rigid_group *rg, struct panel *p) -{ - int i; - - for ( i=0; i<rg->n_panels; i++ ) { - if ( rg->panels[i] == p ) { - return 1; - } - } - - return 0; -} - - -int rigid_group_is_in_collection(struct rg_collection *c, - struct rigid_group *rg) -{ - int i; - - for ( i=0; i<c->n_rigid_groups; i++ ) { - if ( c->rigid_groups[i] == rg ) { - return 1; - } - } - - return 0; -} - - -struct rg_collection *find_rigid_group_collection_by_name(struct detector *det, - const char *name) -{ - int i; - - for ( i=0; i<det->n_rg_collections; i++ ) { - if ( strcmp(det->rigid_group_collections[i]->name, - name) == 0 ) { - return det->rigid_group_collections[i]; - } - } - - return NULL; -} - - -static struct panel *new_panel(struct detector *det, const char *name) -{ - struct panel *new; - - det->n_panels++; - det->panels = realloc(det->panels, det->n_panels*sizeof(struct panel)); - - new = &det->panels[det->n_panels-1]; - memcpy(new, &det->defaults, sizeof(struct panel)); - - strcpy(new->name, name); - - /* Create a new copy of the camera length location if needed */ - if ( new->clen_from != NULL ) { - new->clen_from = strdup(new->clen_from); - } - - /* Create a new copy of the data location if needed */ - if ( new->data != NULL ) { - new->data = strdup(new->data); - } - - /* Create a new copy of the dim_structure if needed */ - if ( new->dim_structure != NULL ) { - - struct dim_structure *dim_copy; - int di; - - dim_copy = initialize_dim_structure(); - dim_copy->num_dims = new->dim_structure->num_dims; - dim_copy->dims = malloc(dim_copy->num_dims*sizeof(int)); - for ( di=0; di<dim_copy->num_dims; di++ ) { - dim_copy->dims[di] = new->dim_structure->dims[di]; - } - - new->dim_structure = dim_copy; - } - - /* Create a new copy of the bad pixel mask location */ - if ( new->mask != NULL ) { - new->mask = strdup(new->mask); - } - - return new; -} - - -static struct badregion *new_bad_region(struct detector *det, const char *name) -{ - struct badregion *new; - - det->n_bad++; - det->bad = realloc(det->bad, det->n_bad*sizeof(struct badregion)); - - new = &det->bad[det->n_bad-1]; - new->min_x = NAN; - new->max_x = NAN; - new->min_y = NAN; - new->max_y = NAN; - new->min_fs = 0; - new->max_fs = 0; - new->min_ss = 0; - new->max_ss = 0; - new->is_fsss = 99; /* Slightly nasty: means "unassigned" */ - new->panel = NULL; - strcpy(new->name, name); - - return new; -} - - -struct panel *find_panel_by_name(struct detector *det, const char *name) -{ - int i; - - for ( i=0; i<det->n_panels; i++ ) { - if ( strcmp(det->panels[i].name, name) == 0 ) { - return &det->panels[i]; - } - } - - return NULL; -} - - -static struct badregion *find_bad_region_by_name(struct detector *det, - const char *name) -{ - int i; - - for ( i=0; i<det->n_bad; i++ ) { - if ( strcmp(det->bad[i].name, name) == 0 ) { - return &det->bad[i]; - } - } - - return NULL; -} - - -static struct rigid_group *find_or_add_rg(struct detector *det, - const char *name) -{ - int i; - struct rigid_group **new; - struct rigid_group *rg; - - for ( i=0; i<det->n_rigid_groups; i++ ) { - - if ( strcmp(det->rigid_groups[i]->name, name) == 0 ) { - return det->rigid_groups[i]; - } - - } - - new = realloc(det->rigid_groups, - (1+det->n_rigid_groups)*sizeof(struct rigid_group *)); - if ( new == NULL ) return NULL; - - det->rigid_groups = new; - - rg = malloc(sizeof(struct rigid_group)); - if ( rg == NULL ) return NULL; - - det->rigid_groups[det->n_rigid_groups++] = rg; - - rg->name = strdup(name); - rg->panels = NULL; - rg->n_panels = 0; - - return rg; -} - - -static struct rg_collection *find_or_add_rg_coll(struct detector *det, - const char *name) -{ - int i; - struct rg_collection **new; - struct rg_collection *rgc; - - for ( i=0; i<det->n_rg_collections; i++ ) { - if ( strcmp(det->rigid_group_collections[i]->name, name) == 0 ) - { - return det->rigid_group_collections[i]; - } - } - - new = realloc(det->rigid_group_collections, - (1+det->n_rg_collections)*sizeof(struct rg_collection *)); - if ( new == NULL ) return NULL; - - det->rigid_group_collections = new; - - rgc = malloc(sizeof(struct rg_collection)); - if ( rgc == NULL ) return NULL; - - det->rigid_group_collections[det->n_rg_collections++] = rgc; - - rgc->name = strdup(name); - rgc->rigid_groups = NULL; - rgc->n_rigid_groups = 0; - - return rgc; -} - - -static void add_to_rigid_group(struct rigid_group *rg, struct panel *p) -{ - struct panel **pn; - - pn = realloc(rg->panels, (1+rg->n_panels)*sizeof(struct panel *)); - if ( pn == NULL ) { - ERROR("Couldn't add panel to rigid group.\n"); - return; - } - - rg->panels = pn; - rg->panels[rg->n_panels++] = p; -} - - -static void add_to_rigid_group_coll(struct rg_collection *rgc, - struct rigid_group *rg) -{ - struct rigid_group **r; - - r = realloc(rgc->rigid_groups, (1+rgc->n_rigid_groups)* - sizeof(struct rigid_group *)); - if ( r == NULL ) { - ERROR("Couldn't add rigid group to collection.\n"); - return; - } - - rgc->rigid_groups = r; - rgc->rigid_groups[rgc->n_rigid_groups++] = rg; -} - - -/* Free all rigid groups in detector */ -static void free_all_rigid_groups(struct detector *det) -{ - int i; - - if ( det->rigid_groups == NULL ) return; - for ( i=0; i<det->n_rigid_groups; i++ ) { - free(det->rigid_groups[i]->name); - free(det->rigid_groups[i]->panels); - free(det->rigid_groups[i]); - } - free(det->rigid_groups); -} - - -/* Free all rigid groups in detector */ -static void free_all_rigid_group_collections(struct detector *det) -{ - int i; - - if ( det->rigid_group_collections == NULL ) return; - for ( i=0; i<det->n_rg_collections; i++ ) { - free(det->rigid_group_collections[i]->name); - free(det->rigid_group_collections[i]->rigid_groups); - free(det->rigid_group_collections[i]); - } - free(det->rigid_group_collections); -} - - -static struct rigid_group *find_rigid_group_by_name(struct detector *det, - char *name) -{ - int i; - - for ( i=0; i<det->n_rigid_groups; i++ ) { - if ( strcmp(det->rigid_groups[i]->name, name) == 0 ) { - return det->rigid_groups[i]; - } - } - - return NULL; -} - - -static int parse_field_for_panel(struct panel *panel, const char *key, - const char *val, struct detector *det) -{ - int reject = 0; - - if ( strcmp(key, "min_fs") == 0 ) { - panel->orig_min_fs = atof(val); - } else if ( strcmp(key, "max_fs") == 0 ) { - panel->orig_max_fs = atof(val); - } else if ( strcmp(key, "min_ss") == 0 ) { - panel->orig_min_ss = atof(val); - } else if ( strcmp(key, "max_ss") == 0 ) { - panel->orig_max_ss = atof(val); - } else if ( strcmp(key, "corner_x") == 0 ) { - panel->cnx = atof(val); - } else if ( strcmp(key, "corner_y") == 0 ) { - panel->cny = atof(val); - } else if ( strcmp(key, "rail_direction") == 0 ) { - if ( dir_conv(val, &panel->rail_x, - &panel->rail_y, - &panel->rail_z) ) - { - ERROR("Invalid rail direction '%s'\n", val); - reject = 1; - } - } else if ( strcmp(key, "clen_for_centering") == 0 ) { - panel->clen_for_centering = atof(val); - } else if ( strcmp(key, "adu_per_eV") == 0 ) { - panel->adu_per_eV = atof(val); - } else if ( strcmp(key, "adu_per_photon") == 0 ) { - panel->adu_per_photon = atof(val); - } else if ( strcmp(key, "rigid_group") == 0 ) { - add_to_rigid_group(find_or_add_rg(det, val), panel); - } else if ( strcmp(key, "clen") == 0 ) { - - char *end; - double v = strtod(val, &end); - if ( end == val ) { - /* This means "fill in later" */ - panel->clen = -1.0; - panel->clen_from = strdup(val); - } else { - panel->clen = v; - panel->clen_from = NULL; - } - - } else if ( strcmp(key, "data") == 0 ) { - if ( strncmp(val,"/",1) != 0 ) { - ERROR("Invalid data location '%s'\n", val); - reject = -1; - } - panel->data = strdup(val); - - } else if ( strcmp(key, "mask") == 0 ) { - if ( strncmp(val,"/",1) != 0 ) { - ERROR("Invalid mask location '%s'\n", val); - reject = -1; - } - panel->mask = strdup(val); - - } else if ( strcmp(key, "mask_file") == 0 ) { - panel->mask_file = strdup(val); - - } else if ( strcmp(key, "saturation_map") == 0 ) { - panel->satmap = strdup(val); - } else if ( strcmp(key, "saturation_map_file") == 0 ) { - panel->satmap_file = strdup(val); - - } else if ( strcmp(key, "coffset") == 0) { - panel->coffset = atof(val); - } else if ( strcmp(key, "res") == 0 ) { - panel->res = atof(val); - } else if ( strcmp(key, "max_adu") == 0 ) { - panel->max_adu = atof(val); - } else if ( strcmp(key, "badrow_direction") == 0 ) { - panel->badrow = val[0]; /* First character only */ - if ( (panel->badrow != 'x') && (panel->badrow != 'y') - && (panel->badrow != 'f') && (panel->badrow != 's') - && (panel->badrow != '-') ) { - ERROR("badrow_direction must be x, y, f, s or '-'\n"); - ERROR("Assuming '-'\n."); - panel->badrow = '-'; - } - if ( panel->badrow == 'x' ) panel->badrow = 'f'; - if ( panel->badrow == 'y' ) panel->badrow = 's'; - } else if ( strcmp(key, "no_index") == 0 ) { - panel->no_index = atob(val); - } else if ( strcmp(key, "fs") == 0 ) { - if ( dir_conv(val, &panel->fsx, &panel->fsy, &panel->fsz) != 0 ) - { - ERROR("Invalid fast scan direction '%s'\n", val); - reject = 1; - } - } else if ( strcmp(key, "ss") == 0 ) { - if ( dir_conv(val, &panel->ssx, &panel->ssy, &panel->ssz) != 0 ) - { - ERROR("Invalid slow scan direction '%s'\n", val); - reject = 1; - } - } else if ( strncmp(key, "dim", 3) == 0) { - int dim_entry; - char *endptr; - if ( key[3] != '\0' ) { - if ( panel->dim_structure == NULL ) { - panel->dim_structure = initialize_dim_structure(); - } - dim_entry = strtoul(key+3, &endptr, 10); - if ( endptr[0] != '\0' ) { - ERROR("Invalid dimension number %s\n", key+3); - } else { - if ( set_dim_structure_entry(panel->dim_structure, - dim_entry, val) ) - { - ERROR("Failed to set dim structure entry\n"); - } - } - } else { - ERROR("'dim' must be followed by a number, e.g. 'dim0'\n"); - } - } else { - ERROR("Unrecognised field '%s'\n", key); - } - - return reject; -} - - -static int check_badr_fsss(struct badregion *badr, int is_fsss) -{ - /* First assignment? */ - if ( badr->is_fsss == 99 ) { - badr->is_fsss = is_fsss; - return 0; - } - - if ( is_fsss != badr->is_fsss ) { - ERROR("You can't mix x/y and fs/ss in a bad region.\n"); - return 1; - } - - return 0; -} - - -static int parse_field_bad(struct badregion *badr, const char *key, - const char *val) -{ - int reject = 0; - - if ( strcmp(key, "min_x") == 0 ) { - badr->min_x = atof(val); - reject = check_badr_fsss(badr, 0); - } else if ( strcmp(key, "max_x") == 0 ) { - badr->max_x = atof(val); - reject = check_badr_fsss(badr, 0); - } else if ( strcmp(key, "min_y") == 0 ) { - badr->min_y = atof(val); - reject = check_badr_fsss(badr, 0); - } else if ( strcmp(key, "max_y") == 0 ) { - badr->max_y = atof(val); - reject = check_badr_fsss(badr, 0); - } else if ( strcmp(key, "min_fs") == 0 ) { - badr->min_fs = atof(val); - reject = check_badr_fsss(badr, 1); - } else if ( strcmp(key, "max_fs") == 0 ) { - badr->max_fs = atof(val); - reject = check_badr_fsss(badr, 1); - } else if ( strcmp(key, "min_ss") == 0 ) { - badr->min_ss = atof(val); - reject = check_badr_fsss(badr, 1); - } else if ( strcmp(key, "max_ss") == 0 ) { - badr->max_ss = atof(val); - reject = check_badr_fsss(badr, 1); - } else if ( strcmp(key, "panel") == 0 ) { - badr->panel = strdup(val); - } else { - ERROR("Unrecognised field '%s'\n", key); - } - - return reject; -} - - -static void parse_toplevel(struct detector *det, struct beam_params *beam, - const char *key, const char *val, - struct rg_definition ***rg_defl, - struct rgc_definition ***rgc_defl, int *n_rg_defs, - int *n_rgc_defs, char **hdf5_peak_path) -{ - - if ( strcmp(key, "mask_bad") == 0 ) { - - char *end; - double v = strtod(val, &end); - - if ( end != val ) { - det->mask_bad = v; - } - - } else if ( strcmp(key, "mask_good") == 0 ) { - - char *end; - double v = strtod(val, &end); - - if ( end != val ) { - det->mask_good = v; - } - - } else if ( strcmp(key, "coffset") == 0 ) { - det->defaults.coffset = atof(val); - - } else if ( strcmp(key, "photon_energy") == 0 ) { - if ( beam != NULL ) { - double v; - char *end; - v = strtod(val, &end); - if ( (val[0] != '\0') && (end[0] == '\0') ) { - beam->photon_energy = v; - beam->photon_energy_from = NULL; - } else { - beam->photon_energy = 0.0; - beam->photon_energy_from = strdup(val); - } - } - - } else if ( strcmp(key, "photon_energy_bandwidth") == 0 ) { - if ( beam != NULL ) { - double v; - char *end; - v = strtod(val, &end); - if ( (val[0] != '\0') && (end[0] == '\0') ) { - beam->bandwidth = v; - } else { - ERROR("Invalid value for " - "photon_energy_bandwidth\n"); - } - } - - } else if ( strcmp(key, "photon_energy_scale") == 0 ) { - if ( beam != NULL ) { - beam->photon_energy_scale = atof(val); - } - - } else if ( strcmp(key, "peak_info_location") == 0 ) { - if ( hdf5_peak_path != NULL ) { - *hdf5_peak_path = strdup(val); - } - - } else if (strncmp(key, "rigid_group", 11) == 0 - && strncmp(key, "rigid_group_collection", 22) != 0 ) { - - struct rg_definition **new; - - new = realloc(*rg_defl, - ((*n_rg_defs)+1)*sizeof(struct rg_definition*)); - *rg_defl = new; - - (*rg_defl)[*n_rg_defs] = malloc(sizeof(struct rg_definition)); - (*rg_defl)[*n_rg_defs]->name = strdup(key+12); - (*rg_defl)[*n_rg_defs]->pns = strdup(val); - *n_rg_defs = *n_rg_defs+1; - - } else if ( strncmp(key, "rigid_group_collection", 22) == 0 ) { - - struct rgc_definition **new; - - new = realloc(*rgc_defl, ((*n_rgc_defs)+1)* - sizeof(struct rgc_definition*)); - *rgc_defl = new; - - (*rgc_defl)[*n_rgc_defs] = - malloc(sizeof(struct rgc_definition)); - (*rgc_defl)[*n_rgc_defs]->name = strdup(key+23); - (*rgc_defl)[*n_rgc_defs]->rgs = strdup(val); - *n_rgc_defs = *n_rgc_defs+1; - - } else if ( parse_field_for_panel(&det->defaults, key, val, det) ) { - ERROR("Unrecognised top level field '%s'\n", key); - } -} - - -/* Test if fs,ss in panel "p" is further {out,in} than {*p_max_d,*p_min_d}, and - * if so update det->furthest_{out,in}_{panel,fs,ss}. */ -static void check_point(struct panel *p, double fs, double ss, - double *p_min_d, double *p_max_d, struct detector *det) -{ - double xs, ys, rx, ry, d; - - xs = fs*p->fsx + ss*p->ssx; - ys = fs*p->fsy + ss*p->ssy; - - rx = (xs + p->cnx) / p->res; - ry = (ys + p->cny) / p->res; - - d = sqrt(pow(rx, 2.0) + pow(ry, 2.0)); - - if ( d > *p_max_d ) { - - det->furthest_out_panel = p; - det->furthest_out_fs = fs; - det->furthest_out_ss = ss; - *p_max_d = d; - - } else if ( d < *p_min_d ) { - - det->furthest_in_panel = p; - det->furthest_in_fs = fs; - det->furthest_in_ss = ss; - *p_min_d = d; - - } -} - - -static void find_min_max_d(struct detector *det) -{ - double max_d, min_d; - int i; - - min_d = +INFINITY; - max_d = 0.0; - for ( i=0; i<det->n_panels; i++ ) { - - struct panel *p; - - p = &det->panels[i]; - - check_point(p, 0, 0, &min_d, &max_d, det); - check_point(p, p->w, 0, &min_d, &max_d, det); - check_point(p, 0, p->h, &min_d, &max_d, det); - check_point(p, p->w, p->h, &min_d, &max_d, det); - - } -} - - -struct detector *get_detector_geometry(const char *filename, - struct beam_params *beam) -{ - return get_detector_geometry_2(filename, beam, NULL); -} - - -struct detector *get_detector_geometry_from_string(const char *string_in, - struct beam_params *beam, - char **hdf5_peak_path) -{ - struct detector *det; - char **bits; - int done = 0; - int i; - int rgi, rgci; - int reject = 0; - int path_dim, mask_path_dim; - int dim_dim; - int dim_reject = 0; - int dim_dim_reject = 0; - struct rg_definition **rg_defl = NULL; - struct rgc_definition **rgc_defl = NULL; - int n_rg_definitions = 0; - int n_rgc_definitions = 0; - char *string; - char *string_orig; - size_t len; - - det = calloc(1, sizeof(struct detector)); - if ( det == NULL ) return NULL; - - if ( beam != NULL ) { - beam->photon_energy = 0.0; - beam->photon_energy_from = NULL; - beam->photon_energy_scale = 1.0; - beam->bandwidth = 0.00000001; - } - - det->n_panels = 0; - det->panels = NULL; - det->n_bad = 0; - det->bad = NULL; - det->mask_good = 0; - det->mask_bad = 0; - det->n_rigid_groups = 0; - det->rigid_groups = NULL; - det->path_dim = 0; - det->dim_dim = 0; - det->n_rg_collections = 0; - det->rigid_group_collections = NULL; - - /* The default defaults... */ - det->defaults.orig_min_fs = -1; - det->defaults.orig_min_ss = -1; - det->defaults.orig_max_fs = -1; - det->defaults.orig_max_ss = -1; - det->defaults.cnx = NAN; - det->defaults.cny = NAN; - det->defaults.clen = NAN; - det->defaults.coffset = 0.0; - det->defaults.res = -1.0; - det->defaults.badrow = '-'; - det->defaults.no_index = 0; - det->defaults.fsx = 1.0; - det->defaults.fsy = 0.0; - det->defaults.fsz = 0.0; - det->defaults.ssx = 0.0; - det->defaults.ssy = 1.0; - det->defaults.ssz = 0.0; - det->defaults.rail_x = NAN; /* The actual default rail direction */ - det->defaults.rail_y = NAN; /* is below */ - det->defaults.rail_z = NAN; - det->defaults.clen_for_centering = NAN; - det->defaults.adu_per_eV = NAN; - det->defaults.adu_per_photon = NAN; - det->defaults.max_adu = +INFINITY; - det->defaults.mask = NULL; - det->defaults.mask_file = NULL; - det->defaults.satmap = NULL; - det->defaults.satmap_file = NULL; - det->defaults.data = NULL; - det->defaults.dim_structure = NULL; - strncpy(det->defaults.name, "", 1023); - - string = strdup(string_in); - if ( string == NULL ) return NULL; - len = strlen(string); - for ( i=0; i<len; i++ ) { - if ( string_in[i] == '\r' ) string[i] = '\n'; - } - - /* Because 'string' will get modified */ - string_orig = string; - - do { - - int n1, n2; - char **path; - char *line; - struct badregion *badregion = NULL; - struct panel *panel = NULL; - char wholeval[1024]; - - const char *nl = strchr(string, '\n'); - if ( nl != NULL ) { - size_t len = nl - string; - line = strndup(string, nl-string); - line[len] = '\0'; - string += len+1; - } else { - line = strdup(string); - done = 1; - } - - if ( line[0] == ';' ) { - free(line); - continue; - } - - n1 = assplode(line, " \t", &bits, ASSPLODE_NONE); - if ( n1 < 3 ) { - for ( i=0; i<n1; i++ ) free(bits[i]); - free(bits); - free(line); - continue; - } - - /* Stitch the pieces of the "value" back together */ - wholeval[0] = '\0'; /* Empty string */ - for ( i=2; i<n1; i++ ) { - if ( bits[i][0] == ';' ) break; /* Stop on comment */ - strncat(wholeval, bits[i], 1023); - } - - if ( bits[1][0] != '=' ) { - for ( i=0; i<n1; i++ ) free(bits[i]); - free(bits); - free(line); - continue; - } - - n2 = assplode(bits[0], "/\\.", &path, ASSPLODE_NONE); - if ( n2 < 2 ) { - - /* This was a top-level option, not handled above. */ - parse_toplevel(det, beam, bits[0], wholeval, &rg_defl, - &rgc_defl, &n_rg_definitions, - &n_rgc_definitions, hdf5_peak_path); - for ( i=0; i<n1; i++ ) free(bits[i]); - free(bits); - for ( i=0; i<n2; i++ ) free(path[i]); - free(path); - free(line); - continue; - } - - if ( strncmp(path[0], "bad", 3) == 0 ) { - badregion = find_bad_region_by_name(det, path[0]); - if ( badregion == NULL ) { - badregion = new_bad_region(det, path[0]); - } - } else { - panel = find_panel_by_name(det, path[0]); - if ( panel == NULL ) { - panel = new_panel(det, path[0]); - } - } - - if ( panel != NULL ) { - if ( parse_field_for_panel(panel, path[1], - wholeval, det) ) - { - reject = 1; - } - } else { - if ( parse_field_bad(badregion, path[1], wholeval) ) { - reject = 1; - } - } - - for ( i=0; i<n1; i++ ) free(bits[i]); - for ( i=0; i<n2; i++ ) free(path[i]); - free(bits); - free(path); - free(line); - - } while ( !done ); - - if ( det->n_panels == -1 ) { - ERROR("No panel descriptions in geometry file.\n"); - free(det); - return NULL; - } - - path_dim = -1; - dim_reject = 0; - - for ( i=0; i<det->n_panels; i++ ) { - - int panel_dim = 0; - char *next_instance; - - next_instance = det->panels[i].data; - - while ( next_instance ) { - next_instance = strstr(next_instance, "%"); - if ( next_instance != NULL ) { - next_instance += 1*sizeof(char); - panel_dim += 1; - } - } - - if ( path_dim == -1 ) { - path_dim = panel_dim; - } else { - if ( panel_dim != path_dim ) { - dim_reject = 1; - } - } - - } - - mask_path_dim = -1; - for ( i=0; i<det->n_panels; i++ ) { - - int panel_mask_dim = 0; - char *next_instance; - - if ( det->panels[i].mask != NULL ) { - - next_instance = det->panels[i].mask; - - while ( next_instance ) { - next_instance = strstr(next_instance, "%"); - if ( next_instance != NULL ) { - next_instance += 1*sizeof(char); - panel_mask_dim += 1; - } - } - - if ( mask_path_dim == -1 ) { - mask_path_dim = panel_mask_dim; - } else { - if ( panel_mask_dim != mask_path_dim ) { - dim_reject = 1; - } - } - - } - } - - if ( dim_reject == 1 ) { - ERROR("All panels' data and mask entries must have the same " - "number of placeholders\n"); - reject = 1; - } - - if ( mask_path_dim > path_dim ) { - ERROR("Number of placeholders in mask cannot be larger than " - "for data\n"); - reject = 1; - } - - det->path_dim = path_dim; - - dim_dim_reject = 0; - dim_dim = -1; - - for ( i=0; i<det->n_panels; i++ ) { - - int di; - int found_ss = 0; - int found_fs = 0; - int panel_dim_dim = 0; - - if ( det->panels[i].dim_structure == NULL ) { - det->panels[i].dim_structure = default_dim_structure(); - } - - for ( di=0; di<det->panels[i].dim_structure->num_dims; di++ ) { - - if ( det->panels[i].dim_structure->dims[di] == - HYSL_UNDEFINED ) { - dim_dim_reject = 1; - ERROR("Dimension %i for panel %s is undefined.\n", - di, det->panels[i].name); - } - if ( det->panels[i].dim_structure->dims[di] == - HYSL_PLACEHOLDER ) { - panel_dim_dim += 1; - } - if ( det->panels[i].dim_structure->dims[di] == - HYSL_SS ) { - found_ss += 1; - } - if ( det->panels[i].dim_structure->dims[di] == - HYSL_FS ) { - found_fs += 1; - } - - } - - if ( found_ss != 1 ) { - ERROR("Exactly one slow scan dim coordinate is needed " - "(found %i for panel %s)\n", found_ss, - det->panels[i].name); - dim_dim_reject = 1; - } - - if ( found_fs != 1 ) { - ERROR("Exactly one fast scan dim coordinate is needed " - "(found %i for panel %s)\n", found_fs, - det->panels[i].name); - dim_dim_reject = 1; - } - - if ( panel_dim_dim > 1 ) { - ERROR("Maximum one placeholder dim coordinate is " - "allowed (found %i for panel %s)\n", - panel_dim_dim, det->panels[i].name); - dim_dim_reject = 1; - } - - if ( dim_dim == -1 ) { - dim_dim = panel_dim_dim; - } else { - if ( panel_dim_dim != dim_dim ) { - dim_dim_reject = 1; - } - } - - } - - if ( dim_dim_reject == 1) { - reject = 1; - } - - det->dim_dim = dim_dim; - - for ( i=0; i<det->n_panels; i++ ) { - - struct panel *p = &det->panels[i]; - - if ( p->orig_min_fs < 0 ) { - ERROR("Please specify the minimum FS coordinate for" - " panel %s\n", det->panels[i].name); - reject = 1; - } - if ( p->orig_max_fs < 0 ) { - ERROR("Please specify the maximum FS coordinate for" - " panel %s\n", det->panels[i].name); - reject = 1; - } - if ( p->orig_min_ss < 0 ) { - ERROR("Please specify the minimum SS coordinate for" - " panel %s\n", det->panels[i].name); - reject = 1; - } - if ( p->orig_max_ss < 0 ) { - ERROR("Please specify the maximum SS coordinate for" - " panel %s\n", det->panels[i].name); - reject = 1; - } - if ( isnan(p->cnx) ) { - ERROR("Please specify the corner X coordinate for" - " panel %s\n", det->panels[i].name); - reject = 1; - } - if ( isnan(p->cny) ) { - ERROR("Please specify the corner Y coordinate for" - " panel %s\n", det->panels[i].name); - reject = 1; - } - if ( isnan(p->clen) && (p->clen_from == NULL) ) { - ERROR("Please specify the camera length for" - " panel %s\n", det->panels[i].name); - reject = 1; - } - if ( p->res < 0 ) { - ERROR("Please specify the resolution for" - " panel %s\n", det->panels[i].name); - reject = 1; - } - if ( isnan(p->adu_per_eV) && isnan(p->adu_per_photon) ) { - ERROR("Please specify either adu_per_eV or " - "adu_per_photon for panel %s\n", - det->panels[i].name); - reject = 1; - } - - if ( isnan(p->clen_for_centering) && !isnan(p->rail_x) ) - { - ERROR("You must specify clen_for_centering if you " - "specify the rail direction (panel %s)\n", - p->name); - reject = 1; - } - - if ( (p->mask_file != NULL) && (p->mask == NULL) ) { - ERROR("You have specified 'mask_file' but not 'mask'. " - "'mask_file' will therefore have no effect. " - "(panel %s)\n", p->name); - reject = 1; - } - - /* The default rail direction */ - if ( isnan(p->rail_x) ) { - p->rail_x = 0.0; - p->rail_y = 0.0; - p->rail_z = 1.0; - } - if ( isnan(p->clen_for_centering) ) p->clen_for_centering = 0.0; - - /* It's OK if the badrow direction is '0' */ - /* It's not a problem if "no_index" is still zero */ - /* The default transformation matrix is at least valid */ - - det->panels[i].w = det->panels[i].orig_max_fs - - det->panels[i].orig_min_fs+1; - det->panels[i].h = det->panels[i].orig_max_ss - - det->panels[i].orig_min_ss+1; - - } - - for ( i=0; i<det->n_bad; i++ ) { - if ( det->bad[i].is_fsss == 99 ) { - ERROR("Please specify the coordinate ranges for" - " bad region %s\n", det->bad[i].name); - reject = 1; - } - } - - free(det->defaults.clen_from); - free(det->defaults.data); - free(det->defaults.mask); - - for ( rgi=0; rgi<n_rg_definitions; rgi++) { - - int pi, n1; - struct rigid_group *rigidgroup = NULL; - - rigidgroup = find_or_add_rg(det, rg_defl[rgi]->name); - - n1 = assplode(rg_defl[rgi]->pns, ",", &bits, ASSPLODE_NONE); - - for ( pi=0; pi<n1; pi++ ) { - - struct panel *p; - - p = find_panel_by_name(det, bits[pi]); - if ( p == NULL ) { - ERROR("Cannot add panel to rigid group\n"); - ERROR("Panel not found: %s\n", bits[pi]); - return NULL; - } - add_to_rigid_group(rigidgroup, p); - free(bits[pi]); - } - free(bits); - free(rg_defl[rgi]->name); - free(rg_defl[rgi]->pns); - free(rg_defl[rgi]); - } - free(rg_defl); - - for ( rgci=0; rgci<n_rgc_definitions; rgci++ ) { - - int rgi, n2; - struct rg_collection *rgcollection = NULL; - - rgcollection = find_or_add_rg_coll(det, rgc_defl[rgci]->name); - - n2 = assplode(rgc_defl[rgci]->rgs, ",", &bits, ASSPLODE_NONE); - - for ( rgi=0; rgi<n2; rgi++ ) { - - struct rigid_group *r; - - r = find_rigid_group_by_name(det, bits[rgi]); - if ( r == NULL ) { - ERROR("Cannot add rigid group to collection\n"); - ERROR("Rigid group not found: %s\n", bits[rgi]); - return NULL; - } - add_to_rigid_group_coll(rgcollection, r); - free(bits[rgi]); - } - free(bits); - free(rgc_defl[rgci]->name); - free(rgc_defl[rgci]->rgs); - free(rgc_defl[rgci]); - - } - free(rgc_defl); - - if ( n_rg_definitions == 0 ) { - - int pi; - - for ( pi=0; pi<det->n_panels; pi++ ) { - - struct rigid_group *rigidgroup = NULL; - - rigidgroup = find_or_add_rg(det, det->panels[pi].name); - add_to_rigid_group(rigidgroup, &det->panels[pi]); - - } - } - - if ( n_rgc_definitions == 0 ) { - - int rgi; - struct rg_collection *rgcollection = NULL; - - rgcollection = find_or_add_rg_coll(det, "default"); - - for ( rgi=0; rgi<det->n_rigid_groups; rgi++ ) { - - add_to_rigid_group_coll(rgcollection, - det->rigid_groups[rgi]); - - } - } - - /* Calculate matrix inverses and other stuff */ - for ( i=0; i<det->n_panels; i++ ) { - - struct panel *p; - double d; - - p = &det->panels[i]; - - if ( p->fsx*p->ssy == p->ssx*p->fsy ) { - ERROR("Panel %i transformation singular.\n", i); - } - - d = (double)p->fsx*p->ssy - p->ssx*p->fsy; - p->xfs = p->ssy / d; - p->yfs = -p->ssx / d; - p->xss = -p->fsy / d; - p->yss = p->fsx / d; - - p->w = p->orig_max_fs - p->orig_min_fs + 1; - p->h = p->orig_max_ss - p->orig_min_ss + 1; - - } - - find_min_max_d(det); - free(string_orig); - - if ( reject ) return NULL; - - return det; -} - - -char *load_entire_file(const char *filename) -{ - struct stat statbuf; - int r; - char *contents; - FILE *fh; - - r = stat(filename, &statbuf); - if ( r != 0 ) { - ERROR("File '%s' not found\n", filename); - return NULL; - } - - contents = malloc(statbuf.st_size+1); - if ( contents == NULL ) { - ERROR("Failed to allocate memory for file\n"); - return NULL; - } - - fh = fopen(filename, "r"); - if ( fh == NULL ) { - ERROR("Failed to open file '%s'\n", filename); - free(contents); - return NULL; - } - - if ( fread(contents, 1, statbuf.st_size, fh) != statbuf.st_size ) { - ERROR("Failed to read file '%s'\n", filename); - free(contents); - return NULL; - } - contents[statbuf.st_size] = '\0'; - - fclose(fh); - - return contents; -} - - -struct detector *get_detector_geometry_2(const char *filename, - struct beam_params *beam, - char **hdf5_peak_path) -{ - char *contents; - struct detector *det; - - contents = load_entire_file(filename); - if ( contents == NULL ) { - ERROR("Failed to load geometry file '%s'\n", filename); - return NULL; - } - - det = get_detector_geometry_from_string(contents, beam, hdf5_peak_path); - free(contents); - return det; -} - - -void free_detector_geometry(struct detector *det) -{ - int i; - - free_all_rigid_groups(det); - free_all_rigid_group_collections(det); - - for ( i=0; i<det->n_panels; i++ ) { - free(det->panels[i].clen_from); - free_dim_structure(det->panels[i].dim_structure); - } - - free(det->panels); - free(det->bad); - free(det); -} - - -static int rg_number(const struct detector *det, const struct rigid_group *rg) -{ - int i; - for ( i=0; i<det->n_rigid_groups; i++ ) { - if ( det->rigid_groups[i] == rg ) return i; - } - return det->n_rigid_groups; -} - - -struct detector *copy_geom(const struct detector *in) -{ - struct detector *out; - int i; - - if ( in == NULL ) return NULL; - - out = malloc(sizeof(struct detector)); - if ( out == NULL ) return NULL; - - /* Copy everything */ - memcpy(out, in, sizeof(struct detector)); - - out->panels = malloc(out->n_panels * sizeof(struct panel)); - memcpy(out->panels, in->panels, out->n_panels * sizeof(struct panel)); - - out->bad = malloc(out->n_bad * sizeof(struct badregion)); - memcpy(out->bad, in->bad, out->n_bad * sizeof(struct badregion)); - - /* Copy the panels */ - for ( i=0; i<out->n_panels; i++ ) { - - struct panel *p; - - /* Copy all fields */ - p = &out->panels[i]; - - /* Now fix up everything involving pointers... */ - - if ( p->clen_from != NULL ) { - /* Make a copy of the clen_from fields unique to this - * copy of the structure. */ - p->clen_from = strdup(p->clen_from); - } - - if ( p->data != NULL ) { - /* Make a copy of the data fields unique to this - * copy of the structure. */ - p->data = strdup(p->data); - } - - if ( p->dim_structure != NULL ) { - /* Make a copy of the dim_structure fields unique to this - * copy of the structure. */ - - struct dim_structure *dim_new; - int di; - - dim_new = initialize_dim_structure(); - dim_new->num_dims = p->dim_structure->num_dims; - dim_new->dims = malloc(dim_new->num_dims*sizeof(int)); - for ( di=0; di<dim_new->num_dims; di++ ) { - dim_new->dims[di] = p->dim_structure->dims[di]; - } - - p->dim_structure = dim_new; - - } - - if ( &in->panels[i] == in->furthest_out_panel ) { - out->furthest_out_panel = &out->panels[i]; - } - if ( &in->panels[i] == in->furthest_in_panel ) { - out->furthest_in_panel = &out->panels[i]; - } - - } - - /* Copy all the rigid groups */ - out->rigid_groups = malloc(out->n_rigid_groups*sizeof(struct rigid_group)); - if ( out->rigid_groups == NULL ) return NULL; - for ( i=0; i<out->n_rigid_groups; i++ ) { - - struct rigid_group *inrg; - struct rigid_group *rg; - int j; - - rg = malloc(sizeof(struct rigid_group)); - if ( rg == NULL ) return NULL; - - out->rigid_groups[i] = rg; - - inrg = in->rigid_groups[i]; - - rg->name = strdup(inrg->name); - if ( rg->name == NULL ) return NULL; - - rg->n_panels = inrg->n_panels; - rg->panels = malloc(inrg->n_panels*sizeof(struct panel *)); - if ( rg->panels == NULL ) return NULL; - - for ( j=0; j<rg->n_panels; j++ ) { - int k = panel_number(in, inrg->panels[j]); - rg->panels[j] = &out->panels[k]; - } - - } - - /* Copy all the rigid group collections */ - out->rigid_group_collections = malloc(out->n_rg_collections*sizeof(struct rg_collection)); - if ( out->rigid_group_collections == NULL ) return NULL; - for ( i=0; i<out->n_rg_collections; i++ ) { - - struct rg_collection *inrgc; - struct rg_collection *rgc; - int j; - - rgc = malloc(sizeof(struct rg_collection)); - if ( rgc == NULL ) return NULL; - - out->rigid_group_collections[i] = rgc; - - inrgc = in->rigid_group_collections[i]; - - rgc->name = strdup(inrgc->name); - if ( rgc->name == NULL ) return NULL; - - rgc->n_rigid_groups = inrgc->n_rigid_groups; - rgc->rigid_groups = malloc(rgc->n_rigid_groups*sizeof(struct rg_collection)); - if ( rgc->rigid_groups == NULL ) return NULL; - - for ( j=0; j<rgc->n_rigid_groups; j++ ) { - int k = rg_number(in, inrgc->rigid_groups[j]); - if ( k == in->n_rigid_groups ) return NULL; - rgc->rigid_groups[j] = out->rigid_groups[k]; - } - - } - - return out; -} - - -struct detector *simple_geometry(const struct image *image, int w, int h) -{ - struct detector *geom; - - geom = calloc(1, sizeof(struct detector)); - - geom->n_panels = 1; - geom->panels = calloc(1, sizeof(struct panel)); - - geom->panels[0].orig_min_fs = 0; - geom->panels[0].orig_max_fs = w-1; - geom->panels[0].orig_min_ss = 0; - geom->panels[0].orig_max_ss = h-1; - geom->panels[0].cnx = -w / 2.0; - geom->panels[0].cny = -h / 2.0; - geom->panels[0].max_adu = INFINITY; - geom->panels[0].orig_min_fs = -1; - geom->panels[0].orig_max_fs = -1; - geom->panels[0].orig_min_ss = -1; - geom->panels[0].orig_max_ss = -1; - - geom->panels[0].fsx = 1; - geom->panels[0].fsy = 0; - geom->panels[0].fsz = 0; - geom->panels[0].ssx = 0; - geom->panels[0].ssy = 1; - geom->panels[0].ssz = 0; - - geom->panels[0].xfs = 1; - geom->panels[0].xss = 0; - geom->panels[0].yfs = 0; - geom->panels[0].yss = 1; - - geom->panels[0].w = w; - geom->panels[0].h = h; - - geom->panels[0].mask = NULL; - geom->panels[0].data = NULL; - - find_min_max_d(geom); - - return geom; -} - - -int reverse_2d_mapping(double x, double y, struct detector *det, - struct panel **pp, double *pfs, double *pss) -{ - int i; - - for ( i=0; i<det->n_panels; i++ ) { - - struct panel *p = &det->panels[i]; - double cx, cy, fs, ss; - - /* Get position relative to corner */ - cx = x - p->cnx; - cy = y - p->cny; - - /* Reverse the transformation matrix */ - fs = cx*p->xfs + cy*p->yfs; - ss = cx*p->xss + cy*p->yss; - - /* In range? */ - if ( fs < 0 ) continue; - if ( ss < 0 ) continue; - if ( fs > p->w ) continue; - if ( ss > p->h ) continue; - - *pfs = fs; - *pss = ss; - *pp = p; - return 0; - - } - - return 1; -} - - -static void check_extents(struct panel p, double *min_x, double *min_y, - double *max_x, double *max_y, double fs, double ss) -{ - double xs, ys, rx, ry; - - xs = fs*p.fsx + ss*p.ssx; - ys = fs*p.fsy + ss*p.ssy; - - rx = xs + p.cnx; - ry = ys + p.cny; - - if ( rx > *max_x ) *max_x = rx; - if ( ry > *max_y ) *max_y = ry; - if ( rx < *min_x ) *min_x = rx; - if ( ry < *min_y ) *min_y = ry; -} - - -static void rewrite_panel_fields(const struct panel *p, char *line, - FILE *fh, char **bits, - int write_panel_coffset) -{ - char new_line[1024]; - char string_to_write[512]; - - strcpy(new_line,"\0"); - strcpy(string_to_write,"\0"); - - if(strstr(bits[1], "fs") != NULL && - strstr(bits[1], "min_fs") == NULL && - strstr(bits[1], "max_fs") == NULL && - strstr(bits[1], "offset") == NULL ) { - - sprintf(string_to_write, "%+fx %+fy", - p->fsx, p->fsy); - build_output_line(line, new_line, - string_to_write); - fputs(new_line, fh); - fputs("\n", fh); - return; - - } else if ( strstr(bits[1], "ss") != NULL && - strstr(bits[1], "min_ss") == NULL && - strstr(bits[1], "max_ss") == NULL) { - - sprintf(string_to_write, "%+fx %+fy", - p->ssx, p->ssy); - build_output_line(line, new_line, - string_to_write); - fputs(new_line, fh); - fputs("\n", fh); - return; - - } else if ( strstr(bits[1], "corner_x") != NULL) { - - sprintf(string_to_write, "%g", - p->cnx); - build_output_line(line, new_line, - string_to_write); - fputs(new_line, fh); - fputs("\n", fh); - return; - - } else if ( strstr(bits[1], "corner_y") != NULL) { - - sprintf(string_to_write, "%g", - p->cny); - build_output_line(line, new_line, - string_to_write); - fputs(new_line, fh); - fputs("\n", fh); - return; - - } else if ( strstr(bits[1], "coffset") != NULL) { - - if ( write_panel_coffset ) { - return; - } else { - fputs(line, fh); - fputs("\n", fh); - return; - } - - } else { - fputs(line, fh); - fputs("\n", fh); - } -} - - -double largest_q(struct image *image) -{ - struct rvec q; - double tt; - - if ( image->det == NULL ) { - ERROR("No detector geometry. assuming detector is infinite!\n"); - return INFINITY; - } - - q = get_q_for_panel(image->det->furthest_out_panel, - image->det->furthest_out_fs, - image->det->furthest_out_ss, - &tt, 1.0/image->lambda); - - return modulus(q.u, q.v, q.w); -} - - -double smallest_q(struct image *image) -{ - struct rvec q; - double tt; - - q = get_q_for_panel(image->det->furthest_in_panel, - image->det->furthest_in_fs, - image->det->furthest_in_ss, - &tt, 1.0/image->lambda); - - return modulus(q.u, q.v, q.w); -} - - -void get_pixel_extents(struct detector *det, - double *min_x, double *min_y, - double *max_x, double *max_y) -{ - int i; - - *min_x = 0.0; - *max_x = 0.0; - *min_y = 0.0; - *max_y = 0.0; - - /* To determine the maximum extents of the detector, put all four - * corners of each panel through the transformations and watch for the - * biggest */ - - for ( i=0; i<det->n_panels; i++ ) { - - check_extents(det->panels[i], min_x, min_y, max_x, max_y, - 0.0, 0.0); - - check_extents(det->panels[i], min_x, min_y, max_x, max_y, - 0.0, det->panels[i].h+1); - - check_extents(det->panels[i], min_x, min_y, max_x, max_y, - det->panels[i].w+1, 0.0); - - check_extents(det->panels[i], min_x, min_y, max_x, max_y, - det->panels[i].w+1, det->panels[i].h+1); - - - } -} - - -int write_detector_geometry_3(const char *geometry_data, - const char *output_filename, struct detector *det, - const char *additional_comment, - int write_panel_coffset) -{ - FILE *fh; - int done = 0; - - if ( geometry_data == NULL ) return 2; - if ( output_filename == NULL ) return 2; - if ( det->n_panels < 1 ) return 3; - - fh = fopen(output_filename, "w"); - if ( fh == NULL ) return 1; - - if ( additional_comment != NULL ) { - fputs("; ", fh); - fputs(additional_comment, fh); - fputs("\n", fh); - } - - if ( write_panel_coffset ) { - fputs("; Optimized panel offsets can be found at the " - "end of the file\n", fh); - } - - do { - - int n_bits; - char **bits; - int i; - struct panel *p; - char *line; - const char *nl; - - /* Get the next line */ - nl = strchr(geometry_data, '\n'); - if ( nl != NULL ) { - size_t len = nl - geometry_data; - line = strndup(geometry_data, nl-geometry_data); - line[len] = '\0'; - geometry_data += len+1; - } else { - /* Last line might now have newline at end */ - line = strdup(geometry_data); - done = 1; - } - - n_bits = assplode(line, "/=", &bits, ASSPLODE_NONE); - - if ( n_bits != 3 ) { - if ( write_panel_coffset && (bits != NULL) - && (strstr(bits[0], "coffset" ) != NULL) ) continue; - fputs(line, fh); - fputs("\n", fh); - } else { - - p = find_panel_by_name(det, bits[0]); - - if ( p != NULL ) { - rewrite_panel_fields(p, line, fh, bits, - write_panel_coffset); - } else { - fputs(line, fh); - fputs("\n", fh); - } - } - - for ( i=0; i<n_bits; i++ ) free(bits[i]); - - } while ( !done ); - - if ( write_panel_coffset ) { - - int pi; - - fputs("\n", fh); - - for ( pi=0; pi<det->n_panels; pi++ ) { - fprintf(fh, "%s/coffset = %f\n", - det->panels[pi].name, det->panels[pi].coffset); - } - } - - return 0; -} - - -int write_detector_geometry_2(const char *geometry_filename, - const char *output_filename, struct detector *det, - const char *additional_comment, - int write_panel_coffset) -{ - int r; - char *geometry_data = load_entire_file(geometry_filename); - r = write_detector_geometry_3(geometry_data, output_filename, det, - additional_comment, write_panel_coffset); - free(geometry_data); - return r; -} - - -int write_detector_geometry(const char *geometry_filename, - const char *output_filename, struct detector *det) -{ - return write_detector_geometry_2(geometry_filename, output_filename, - det, NULL, 0); -} - - -/** - * \param image An image structure - * \param min Minimum value of 1/d to be marked as bad - * \param max Maximum value of 1/d to be marked as bad - * - * Flags, in the bad pixel mask for \p image, every pixel whose resolution is - * between \p min and \p max. - * - */ - -void mark_resolution_range_as_bad(struct image *image, - double min, double max) -{ - int i; - - if ( isinf(min) && isinf(max) ) return; /* nothing to do */ - - for ( i=0; i<image->det->n_panels; i++ ) { - - int fs, ss; - struct panel *p = &image->det->panels[i]; - - for ( ss=0; ss<p->h; ss++ ) { - for ( fs=0; fs<p->w; fs++ ) { - struct rvec q; - double r; - q = get_q_for_panel(p, fs, ss, NULL, 1.0/image->lambda); - r = modulus(q.u, q.v, q.w); - if ( (r >= min) && (r <= max) ) { - image->bad[i][fs+p->w*ss] = 1; - } - } - } - - } -} - - -static int safe_strcmp(const char *a, const char *b) -{ - /* If both are NULL, they count as equal */ - if ( (a == NULL) && (b == NULL) ) return 0; - - /* Otherwise, if either is NULL then they're different */ - if ( a == NULL ) return 1; - if ( b == NULL ) return 1; - - /* Otherwise, normal string comparison */ - return strcmp(a, b); -} - - -/** - * \param det A detector structure - * \param element If manually selected by the user, the HDF5 element being used. - * Otherwise NULL. - * - * \returns Non-zero if the combination of \p det and \p element mean that all the - * data comes from a single block. - */ -int single_panel_data_source(struct detector *det, const char *element) -{ - int pi; - const char *first_datafrom = NULL; - const char *curr_datafrom = NULL; - - if ( det->panels[0].data == NULL ) { - first_datafrom = element; /* Might be NULL */ - } else { - first_datafrom = det->panels[0].data; - } - - for ( pi=1; pi<det->n_panels; pi++ ) { - - if ( det->panels[pi].data == NULL ) { - curr_datafrom = element; /* Might be NULL */ - } else { - curr_datafrom = det->panels[pi].data; - } - - if ( safe_strcmp(curr_datafrom, first_datafrom) != 0 ) { - return 0; - } - - } - - return 1; -} - - -int multi_event_geometry(struct detector *det) -{ - return (det->path_dim != 0) || (det->dim_dim != 0); -} diff --git a/libcrystfel/src/detector.h b/libcrystfel/src/detector.h deleted file mode 100644 index 2176861f..00000000 --- a/libcrystfel/src/detector.h +++ /dev/null @@ -1,333 +0,0 @@ -/* - * detector.h - * - * Detector properties - * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, - * a research centre of the Helmholtz Association. - * Copyright © 2012 Richard Kirian - * - * Authors: - * 2009-2019 Thomas White <taw@physics.org> - * 2011-2012 Richard Kirian <rkirian@asu.edu> - * 2014 Valerio Mariani - * 2011 Andrew Aquila - * - * This file is part of CrystFEL. - * - * CrystFEL is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * CrystFEL is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. - * - */ - -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - -#ifndef DETECTOR_H -#define DETECTOR_H - -struct rigid_group; -struct rg_collection; -struct detector; -struct panel; -struct badregion; -struct beam_params; -struct hdfile; -struct event; - -#include "hdf5-file.h" -#include "image.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \file detector.h - * Detector geometry structure and related functions. - */ - - -struct rigid_group -{ - char *name; - struct panel **panels; - int n_panels; -}; - - -struct rg_collection -{ - char *name; - struct rigid_group **rigid_groups; - int n_rigid_groups; -}; - - -/** - * Represents one panel of a detector - */ -struct panel -{ - /** Text name for panel (fixed length array) */ - char name[1024]; - - /** \name Location of corner in units of the pixel size of this panel */ - /**@{*/ - double cnx; - double cny; - /**@}*/ - - /** The offset to be applied from \ref clen */ - double coffset; - - /** The distance from the interaction point to the corner of the - * first pixel */ - double clen; - - /** Location to get \ref clen from, e.g. from HDF5 file */ - char *clen_from; - - /** Location of mask data */ - char *mask; - - /** Filename for mask data */ - char *mask_file; - - /** Location of per-pixel saturation map */ - char *satmap; - - /** Filename for saturation map */ - char *satmap_file; - - /** Resolution in pixels per metre */ - double res; - - /** Readout direction (for filtering out clusters of peaks) - * ('x' or 'y') */ - char badrow; - - /** Non-zero if panel should be considered entirely bad */ - int no_index; - - /** Number of detector intensity units per photon */ - double adu_per_photon; - - /** Treat pixel as unreliable if higher than this */ - double max_adu; - - /** Location of data in file */ - char *data; - - /** Number of detector intensity units per eV of photon energy */ - double adu_per_eV; - - /** Dimension structure */ - struct dim_structure *dim_structure; - - /** \name Transformation matrix from pixel coordinates to lab frame */ - /*@{*/ - double fsx; - double fsy; - double fsz; - double ssx; - double ssy; - double ssz; - /*@}*/ - - /** \name Rail direction */ - /*@{*/ - double rail_x; - double rail_y; - double rail_z; - /*@}*/ - - /* Value of clen (without coffset) at which beam is centered */ - double clen_for_centering; - - /** \name Inverse of 2D part of transformation matrix */ - /*@{*/ - double xfs; - double yfs; - double xss; - double yss; - /*@}*/ - - /** \name Position of the panel in the data block in the file. - * The panels may get moved around when the file is loaded (see - * hdf5_read2()), especially if the panels come from different HDF5 - * elements. */ - /*@{*/ - int orig_min_fs; - int orig_max_fs; - int orig_min_ss; - int orig_max_ss; - /*@}*/ - - /** Width, calculated as max_fs-min_fs+1 */ - int w; - - /*** Height, calculated as max_ss-min_ss+1 */ - int h; -}; - - -struct badregion -{ - char name[1024]; - int is_fsss; - char *panel; - - double min_x; - double max_x; - double min_y; - double max_y; - - /* Specified INCLUSIVELY */ - int min_fs; - int max_fs; - int min_ss; - int max_ss; - -}; - - -struct detector -{ - struct panel *panels; - int n_panels; - - struct badregion *bad; - int n_bad; - - unsigned int mask_bad; - unsigned int mask_good; - - struct rigid_group **rigid_groups; - int n_rigid_groups; - - struct rg_collection **rigid_group_collections; - int n_rg_collections; - - /* Location of the pixel furthest away from the beam position, which - * will have the largest value of 2theta regardless of camera length - * and wavelength */ - struct panel *furthest_out_panel; - double furthest_out_fs; - double furthest_out_ss; - - /* As above, but for the smallest 2theta */ - struct panel *furthest_in_panel; - double furthest_in_fs; - double furthest_in_ss; - - int path_dim; - int dim_dim; - - struct panel defaults; -}; - - -extern struct rvec get_q_for_panel(struct panel *p, double fs, double ss, - double *ttp, double k); - -extern double get_tt(struct image *image, double xs, double ys, int *err); - -extern int in_bad_region(struct detector *det, struct panel *p, - double fs, double ss); - -extern void record_image(struct image *image, int do_poisson, double background, - gsl_rng *rng, double beam_radius, double nphotons); - -extern struct panel *find_orig_panel(struct detector *det, - double fs, double ss); - -extern signed int find_orig_panel_number(struct detector *det, - double fs, double ss); - -extern int panel_number(const struct detector *det, const struct panel *p); - -extern struct detector *get_detector_geometry(const char *filename, - struct beam_params *beam); - -extern struct detector *get_detector_geometry_2(const char *filename, - struct beam_params *beam, - char **hdf5_peak_path); - -extern struct detector *get_detector_geometry_from_string(const char *string, - struct beam_params *beam, - char **hdf5_peak_path); - -extern void free_detector_geometry(struct detector *det); - -extern struct detector *simple_geometry(const struct image *image, int w, int h); - -extern void get_pixel_extents(struct detector *det, - double *min_x, double *min_y, - double *max_x, double *max_y); - -extern void fill_in_adu(struct image *image); -extern void adjust_centering_for_rail(struct panel *p); - -extern int panel_is_in_rigid_group(const struct rigid_group *rg, - struct panel *p); - -extern int rigid_group_is_in_collection(struct rg_collection *c, - struct rigid_group *rg); - -extern struct detector *copy_geom(const struct detector *in); - -extern int reverse_2d_mapping(double x, double y, struct detector *det, - struct panel **pp, double *pfs, double *pss); - -extern double largest_q(struct image *image); - -extern double smallest_q(struct image *image); - -extern struct panel *find_panel_by_name(struct detector *det, const char *name); - -extern int write_detector_geometry_2(const char *geometry_filename, - const char *output_filename, - struct detector *det, - const char *additional_comment, - int write_panel_coffset); - -extern int write_detector_geometry_3(const char *geometry_data, - const char *output_filename, - struct detector *det, - const char *additional_comment, - int write_panel_coffset); - -extern int write_detector_geometry(const char *geometry_filename, - const char *output_filename, - struct detector *det); - -extern void mark_resolution_range_as_bad(struct image *image, - double min, double max); - - -extern int single_panel_data_source(struct detector *det, const char *element); - -struct rg_collection *find_rigid_group_collection_by_name(struct detector *det, - const char *name); - -extern int detector_has_clen_references(struct detector *det); - -extern int multi_event_geometry(struct detector *det); - -#ifdef __cplusplus -} -#endif - -#endif /* DETECTOR_H */ diff --git a/libcrystfel/src/detgeom.c b/libcrystfel/src/detgeom.c new file mode 100644 index 00000000..5612a225 --- /dev/null +++ b/libcrystfel/src/detgeom.c @@ -0,0 +1,141 @@ +/* + * detgeom.c + * + * Utility functions for detgeom structure + * + * Copyright © 2019-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2020-2021 Thomas White <taw@physics.org> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include <math.h> +#include <stdlib.h> + +#include "detgeom.h" +#include "utils.h" + + +/** + * \file detgeom.h + */ + + +void detgeom_transform_coords(struct detgeom_panel *p, + double fs, double ss, + double wavelength, + double dx, double dy, + double *r) +{ + double xs, ys, zs; + double fac; + + /* Calculate 3D position of given position, in pixels */ + xs = p->cnx + fs*p->fsx + ss*p->ssx + dx*p->pixel_pitch; + ys = p->cny + fs*p->fsy + ss*p->ssy + dy*p->pixel_pitch; + zs = p->cnz + fs*p->fsz + ss*p->ssz; + + fac = wavelength * sqrt(xs*xs + ys*ys + zs*zs); + + r[0] = xs / fac; + r[1] = ys / fac; + r[2] = zs / fac - 1.0/wavelength; +} + + +void detgeom_free(struct detgeom *detgeom) +{ + int i; + + for ( i=0; i<detgeom->n_panels; i++ ) { + free(detgeom->panels[i].name); + } + + free(detgeom->panels); + free(detgeom); +} + + +static double panel_max_res(struct detgeom_panel *p, + double wavelength) +{ + double r[3]; + double max_res = 0.0; + + detgeom_transform_coords(p, 0, 0, wavelength, 0.0, 0.0, r); + max_res = biggest(max_res, modulus(r[0], r[1], r[2])); + + detgeom_transform_coords(p, 0, p->h, wavelength, 0.0, 0.0, r); + max_res = biggest(max_res, modulus(r[0], r[1], r[2])); + + detgeom_transform_coords(p, p->w, 0, wavelength, 0.0, 0.0, r); + max_res = biggest(max_res, modulus(r[0], r[1], r[2])); + + detgeom_transform_coords(p, p->w, p->h, wavelength, 0.0, 0.0, r); + max_res = biggest(max_res, modulus(r[0], r[1], r[2])); + + return max_res; +} + + +double detgeom_max_resolution(struct detgeom *detgeom, + double wavelength) +{ + int i; + double max_res = 0.0; + + for ( i=0; i<detgeom->n_panels; i++ ) { + + double panel_maxres; + + panel_maxres = panel_max_res(&detgeom->panels[i], + wavelength); + if ( panel_maxres > max_res ) { + max_res = panel_maxres; + } + } + + return max_res; +} + + +void show_panel(struct detgeom_panel *p) +{ + STATUS("Panel '%s':\n", p->name); + STATUS(" Size %i x %i px\n", p->w, p->h); + STATUS(" Transformation [cnx] + [%6.2f %6.2f] [fs] = [x]\n", + p->fsx, p->ssx); + STATUS(" [cny] + [%6.2f %6.2f] [ss] = [y]\n", + p->fsy, p->ssy); + STATUS(" [cnz] + [%6.2f %6.2f] = [z]\n", + p->fsz, p->ssz); + STATUS(" corner x,y,z = %f, %f, %f px\n", + p->cnx, p->cny, p->cnz); + STATUS(" = %f, %f, %f mm\n", + p->cnx*p->pixel_pitch*1e3, + p->cny*p->pixel_pitch*1e3, + p->cnz*p->pixel_pitch*1e3); + STATUS(" %f adu/photon, max %f adu\n", + p->adu_per_photon, p->max_adu); +} diff --git a/libcrystfel/src/detgeom.h b/libcrystfel/src/detgeom.h new file mode 100644 index 00000000..5e3815ac --- /dev/null +++ b/libcrystfel/src/detgeom.h @@ -0,0 +1,112 @@ +/* + * detgeom.h + * + * Detector geometry structure + * + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * Copyright © 2012 Richard Kirian + * + * Authors: + * 2009-2020 Thomas White <taw@physics.org> + * 2011-2012 Richard Kirian <rkirian@asu.edu> + * 2014 Valerio Mariani + * 2011 Andrew Aquila + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifndef DETGEOM_H +#define DETGEOM_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \file detgeom.h + * Detector geometry structure and related functions. + */ + + +/** + * Represents one panel of a detector + */ +struct detgeom_panel +{ + /** Text name for panel */ + char *name; + + /** \name Location of corner in units of the pixel size of this panel, \ + * measured from the interaction point. */ + /**@{*/ + double cnx; + double cny; + double cnz; + /**@}*/ + + /** Pixel size in metres */ + double pixel_pitch; + + /** Number of detector intensity units per photon (or electron, etc) */ + double adu_per_photon; + + /** Treat pixel as unreliable if higher than this */ + double max_adu; + + /** \name Transformation matrix from pixel coordinates to lab frame */ + /*@{*/ + double fsx; + double fsy; + double fsz; + double ssx; + double ssy; + double ssz; + /*@}*/ + + /** \name Width and height of panel */ + /*@{*/ + int w; + int h; + /*@}*/ +}; + + +struct detgeom +{ + struct detgeom_panel *panels; + int n_panels; +}; + +extern void detgeom_transform_coords(struct detgeom_panel *p, + double fs, double ss, + double wavelength, + double dx, double dy, + double *r); + +extern void detgeom_free(struct detgeom *detgeom); + +extern double detgeom_max_resolution(struct detgeom *detgeom, + double wavelength); + +extern void show_panel(struct detgeom_panel *p); + +#ifdef __cplusplus +} +#endif + +#endif /* DETGEOM_H */ diff --git a/libcrystfel/src/events.c b/libcrystfel/src/events.c deleted file mode 100644 index 491f7811..00000000 --- a/libcrystfel/src/events.c +++ /dev/null @@ -1,636 +0,0 @@ -/* - * events.c - * - * Event properties - * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, - * a research centre of the Helmholtz Association. - * - * Authors: - * 2017 Thomas White - * 2014 Valerio Mariani - * - * This file is part of CrystFEL. - * - * CrystFEL is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * CrystFEL is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. - * - */ - -#include "events.h" -#include "utils.h" - -#include <hdf5.h> -#include <string.h> -#include <stdlib.h> -#include <assert.h> - -/** \file events.h */ - -struct event *initialize_event() -{ - - struct event *ev; - - ev = malloc(sizeof(struct event)); - ev->path_entries = NULL; - ev->path_length = 0; - - ev->dim_entries = NULL; - ev->dim_length = 0; - - return ev; - -} - - -struct event_list *initialize_event_list() -{ - - struct event_list *ev_list; - - ev_list = malloc(sizeof(struct event_list)); - - ev_list->events = NULL; - ev_list->num_events = 0; - - return ev_list; - -} - -struct filename_plus_event *initialize_filename_plus_event() -{ - - struct filename_plus_event *fpe; - - fpe = malloc(sizeof(struct filename_plus_event)); - - fpe->filename = NULL; - fpe->ev = NULL; - - return fpe; -} - - -int event_cmp(struct event *ev1, struct event *ev2) -{ - int pi; - int di; - - if ( ev1->path_length != ev2->path_length || - ev1->dim_length != ev2->dim_length ) { - return 1; - } - - for ( pi=0; pi<ev1->path_length; pi++ ) { - if ( strcmp(ev1->path_entries[pi], ev2->path_entries[pi]) != 0 ) { - return 1; - } - } - - for ( di=0; di<ev1->dim_length; di++ ) { - if ( ev1->path_entries[di] != ev2->path_entries[di] ) { - return 1; - } - } - - return 0; - -} - - -int add_non_existing_event_to_event_list(struct event_list *ev_list, - struct event *ev) -{ - int evi; - int found = 0; - - for ( evi=0; evi<ev_list->num_events; evi++ ) { - if (event_cmp(ev_list->events[evi], ev) == 0 ) { - found = 1; - break; - } - } - - if ( found == 0) { - return append_event_to_event_list(ev_list, ev); - } - - return 0; -} - - -int append_event_to_event_list(struct event_list *ev_list, struct event *ev) -{ - struct event **new_el; - - new_el = realloc(ev_list->events, - (1+ev_list->num_events)*sizeof(struct event*)); - if ( new_el == NULL ) return 1; - - ev_list->events = new_el; - ev_list->events[ev_list->num_events] = copy_event(ev); - ev_list->num_events +=1; - - return 0; -} - - -struct event *copy_event(struct event *ev) -{ - struct event *new_ev; - int pi, di; - - if ( ev == NULL ) return NULL; - - if ( ev->dim_length == 0 && ev->path_length == 0) { - - new_ev = initialize_event(); - - } else { - - new_ev=malloc(sizeof(struct event)); - - new_ev->path_entries = malloc(ev->path_length*sizeof(char *)); - new_ev->path_length = ev->path_length; - - new_ev->dim_entries = malloc(ev->dim_length*sizeof(int *)); - new_ev->dim_length = ev->dim_length; - - for ( pi=0; pi<new_ev->path_length; pi++ ) { - new_ev->path_entries[pi] = strdup(ev->path_entries[pi]); - } - - for ( di=0; di<new_ev->dim_length; di++ ) { - new_ev->dim_entries[di] = ev->dim_entries[di]; - } - - } - return new_ev; -} - - -struct event_list *copy_event_list(struct event_list *el) -{ - int ei; - struct event_list *el_copy; - struct event **events_copy; - - el_copy = malloc(1); - if ( el_copy == NULL ) { - return NULL; - } - - events_copy = malloc(el->num_events); - if ( events_copy == NULL ) { - free (el_copy); - return NULL; - } - el_copy->events = events_copy; - - for ( ei=0; ei<el->num_events; ei++ ) { - el_copy->events[ei]=copy_event(el->events[ei]); - } - - el_copy->num_events = el->num_events; - - return el_copy; -} - - -static int events_equal(struct event *ev1, struct event *ev2) -{ - int i; - - if ( ev1->path_length != ev2->path_length ) return 0; - if ( ev1->dim_length != ev2->dim_length ) return 0; - - for ( i=0; i<ev1->path_length; i++ ) { - if ( strcmp(ev1->path_entries[i], ev2->path_entries[i]) != 0 ) { - return 0; - } - } - - for ( i=0; i<ev1->dim_length; i++ ) { - if ( ev1->dim_entries[i] != ev2->dim_entries[i] ) return 0; - } - - return 1; -} - - -/** - * \param ev: An event structure - * \param el: An event list - * - * \returns The indexing into \p el of the event matching \p ev, of el->num_events - * if no such event is found. - **/ -int find_event(struct event *ev, struct event_list *el) -{ - int i; - - if ( ev == NULL ) return el->num_events; - - for ( i=0; i<el->num_events; i++ ) { - if ( events_equal(ev, el->events[i]) ) return i; - } - - return i; -} - - -void free_event(struct event *ev) -{ - int pi; - - if ( ev == NULL ) return; - - if ( ev->path_length != 0 ) { - for ( pi=0; pi<ev->path_length; pi++ ) { - free(ev->path_entries[pi]); - } - } - free(ev->dim_entries); - free(ev); -} - - -void free_event_list(struct event_list *el) -{ - int ei; - - for ( ei=0; ei<el->num_events; ei++ ) { - free_event(el->events[ei]); - } - free(el); -} - - -void free_filename_plus_event(struct filename_plus_event *fpe) -{ - free(fpe->filename); - - if ( fpe->ev != NULL ) { - free_event(fpe->ev); - } - - free(fpe); -} - - -char *get_event_string(struct event *ev) -{ - char *evstr; - int i; - size_t ev_len; - - if ( ev == NULL ) return strdup("(none)"); - - ev_len = 1; /* Zero terminator */ - for ( i=0; i<ev->path_length; i++ ) { - ev_len += strlen(ev->path_entries[i]); - ev_len += 1; /* Slash afterwards */ - } - ev_len += 16*ev->dim_length; /* Max length of number plus slash */ - ev_len += 2; /* Double slash in middle */ - - evstr = malloc(ev_len); - if ( evstr == NULL ) return NULL; - evstr[0] = '\0'; - - for ( i=0; i<ev->path_length; i++ ) { - if ( i > 0 ) strcat(evstr, "/"); - strcat(evstr, ev->path_entries[i]); - } - - strcat(evstr, "//"); - - for ( i=0; i<ev->dim_length; i++ ) { - char num_buf[16]; - snprintf(num_buf, 16, "%i", ev->dim_entries[i]); - if ( i > 0 ) strcat(evstr, "/"); - strcat(evstr, num_buf); - } - - return evstr; -} - - -struct event *get_event_from_event_string(const char *ev_string) -{ - struct event *ev; - char *ev_sep; - char buf_path[1024]; - char buf_dim[1024]; - char *sep; - char *start; - - ev_sep = strstr(ev_string, "//"); - if ( ev_sep == NULL ) return NULL; - - strncpy(buf_path, ev_string, ev_sep-ev_string); - buf_path[ev_sep-ev_string] = '\0'; - - strncpy(buf_dim, ev_sep+2, strlen(ev_sep)-2); - buf_dim[strlen(ev_sep)-2] = '\0'; - - ev = initialize_event(); - if ( ev == NULL ) return NULL; - - if ( strlen(buf_path) !=0 ) { - start = buf_path; - do { - char buf[2014]; - - sep = strstr(start, "/"); - - if ( sep != NULL ) { - strncpy(buf, start, sep-start); - buf[sep-start]='\0'; - push_path_entry_to_event(ev, buf); - start = sep + 1; - - } else { - - sprintf(buf,"%s",start); - push_path_entry_to_event(ev, buf); - - } - - } while (sep); - - } - - if ( strlen(buf_dim) !=0 ) { - - start = buf_dim; - - do { - - char buf[2014]; - int buf_int; - - sep = strstr(start, "/"); - if ( sep != NULL ) { - strncpy(buf, start, sep-start); - buf[sep-start]='\0'; - buf_int = atoi(buf); - push_dim_entry_to_event(ev, buf_int); - start = sep + 1; - - } else { - - sprintf(buf,"%s",start); - buf_int = atoi(buf); - push_dim_entry_to_event(ev, buf_int); - - } - - } while (sep); - - } - - - return ev; -} - - -int push_path_entry_to_event(struct event *ev, const char *entry) -{ - char **new_path_entries; - - new_path_entries = realloc(ev->path_entries, - (1+ev->path_length)*sizeof(char *)); - if ( new_path_entries == NULL ) return 1; - - ev->path_entries = new_path_entries; - ev->path_entries[ev->path_length] = strdup(entry); - ev->path_length += 1; - - return 0; -} - - -int push_dim_entry_to_event(struct event *ev, int entry) -{ - int *new_dim_entries; - - new_dim_entries = realloc(ev->dim_entries, - (1+ev->dim_length)*sizeof(int)); - if ( new_dim_entries == NULL ) return 1; - - ev->dim_entries = new_dim_entries; - ev->dim_entries[ev->dim_length] = entry; - ev->dim_length += 1; - - return 0; -} - - -int pop_path_entry_from_event(struct event *ev) -{ - char **new_path_entries; - - if ( ev->path_length == 0 ) return 1; - - free(ev->path_entries[ev->path_length-1]); - - if ( ev->path_length == 1 ) { - ev->path_entries = NULL; - ev->path_length = 0; - return 0; - } - - new_path_entries = realloc(ev->path_entries, - (ev->path_length-1)*sizeof(char *)); - - if ( new_path_entries == NULL ) return 1; - - ev->path_entries = new_path_entries; - ev->path_length = ev->path_length-1; - - return 0; -} - - -int pop_dim_entry_from_event(struct event *ev) -{ - int *new_dim_entries; - - if ( ev->dim_length == 0 ) { - return 1; - } - - if ( ev->dim_length == 1 ) { - ev->dim_entries = NULL; - ev->dim_length = 0; - return 0; - } - - new_dim_entries = realloc(ev->dim_entries, - (ev->dim_length-1)*sizeof(int)); - - if ( new_dim_entries == NULL) { - return 1; - } - - ev->dim_entries = new_dim_entries; - ev->dim_length = ev->dim_length-1; - - return 0; -} - - -char *event_path_placeholder_subst(const char *entry, const char *data) -{ - - char *ph_loc; - char *full_path; - ptrdiff_t len_head; - size_t len_entry, len_data; - - len_entry = strlen(entry); - len_data = strlen(data); - full_path = malloc(len_data + len_entry + 1); - if ( full_path == NULL ) return NULL; - - ph_loc = strchr(data, '%'); - len_head = ph_loc - data; - assert(len_head >= 0); - - strncpy(full_path, data, len_head); - full_path[len_head] = '\0'; - strcat(full_path, entry); - strcat(full_path, ph_loc+1); - - return full_path; -} - - -char *retrieve_full_path(struct event *ev, const char *data) -{ - int ei ; - char *return_value; - char *pholder; - - return_value = strdup(data); - pholder = strstr(return_value,"%"); - ei = 0; - - while ( pholder != NULL ) { - - char *tmp; - - /* Check we have enough things to put in the placeholders */ - if ( ei >= ev->path_length ) { - ERROR("Too many placeholders ('%%') in location.\n"); - return NULL; - } - - /* Substitute one placeholder */ - tmp = event_path_placeholder_subst(ev->path_entries[ei++], - return_value); - - if ( tmp == NULL ) { - ERROR("Couldn't substitute placeholder\n"); - return NULL; - } - - /* Next time, we will substitute the next part of the path into - * the partially substituted string */ - free(return_value); - return_value = tmp; - - pholder = strstr(return_value, "%"); - - } - - return return_value; -} - - -struct dim_structure *initialize_dim_structure() -{ - struct dim_structure *hs; - - hs = malloc(sizeof(struct dim_structure)); - if ( hs == NULL ) return NULL; - - hs->dims = NULL; - hs->num_dims = 0; - - return hs; -} - - -struct dim_structure *default_dim_structure() -{ - struct dim_structure *hsd; - - hsd = initialize_dim_structure(); - - set_dim_structure_entry(hsd, 0, "ss"); - set_dim_structure_entry(hsd, 1, "fs"); - - return hsd; -} - - -void free_dim_structure(struct dim_structure *hsd) -{ - if ( hsd == NULL ) return; - free(hsd->dims); - free(hsd); -} - - -static int parse_dim_structure_val(const char *val) -{ - if ( strcmp(val,"%") == 0 ) { - return HYSL_PLACEHOLDER; - } else if ( strcmp(val,"ss") == 0 ) { - return HYSL_SS; - } else if ( strcmp(val,"fs") == 0 ) { - return HYSL_FS; - } - return atoi(val); - -} - - -int set_dim_structure_entry(struct dim_structure *hsd, int dim_entry, - const char *val_string) -{ - /* "dims" array needs element with zero index */ - if ( dim_entry >= hsd->num_dims ) { - - int di; - - int *new_dims = realloc(hsd->dims, (dim_entry+1)*sizeof(int)); - if ( new_dims == NULL ) return 1; - - /* Initialise the elements just allocated */ - for ( di=hsd->num_dims; di<=dim_entry; di++ ) { - new_dims[di] = HYSL_UNDEFINED; - } - - hsd->dims = new_dims; - hsd->num_dims = dim_entry+1; - - } - - hsd->dims[dim_entry] = parse_dim_structure_val(val_string); - - return 0; -} diff --git a/libcrystfel/src/events.h b/libcrystfel/src/events.h deleted file mode 100644 index b743f827..00000000 --- a/libcrystfel/src/events.h +++ /dev/null @@ -1,110 +0,0 @@ -/* - * events.h - * - * Event properties - * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, - * a research centre of the Helmholtz Association. - * - * Authors: - * 2014 Valerio Mariani - * - * This file is part of CrystFEL. - * - * CrystFEL is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * CrystFEL is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. - * - */ - -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - -#ifndef EVENTS_H -#define EVENTS_H - -/** - * \file events.h - * Event description structures - */ - -struct event -{ - char **path_entries; - int path_length; - int *dim_entries; - int dim_length; -}; - -struct event_list -{ - struct event **events; - int num_events; -}; - -struct filename_plus_event -{ - char *filename; - struct event *ev; -}; - -enum dimension_id -{ - HYSL_UNDEFINED = -99, - HYSL_PLACEHOLDER = -98, - HYSL_FS = -1, - HYSL_SS = -2 -}; - -struct dim_structure -{ - int *dims; - int num_dims; -}; - -extern struct event *initialize_event(void); -extern int push_path_entry_to_event(struct event *ev, const char *entry); -extern int pop_path_entry_from_event(struct event *ev); -extern int push_dim_entry_to_event(struct event *ev, int entry); -extern int pop_dim_entry_from_event(struct event *ev); -extern struct event *copy_event(struct event *ev); -extern void free_event(struct event *ev); -extern char *get_event_string(struct event *ev); -extern struct event *get_event_from_event_string(const char *ev_string); -extern char *event_path_placeholder_subst(const char *ev_name, - const char *data); -extern char *retrieve_full_path(struct event *ev, const char *data); - - -extern struct filename_plus_event *initialize_filename_plus_event(void); -extern void free_filename_plus_event(struct filename_plus_event *fpe); - - -extern struct event_list *initialize_event_list(void); -extern int append_event_to_event_list(struct event_list *ev_list, - struct event *ev); -extern int add_non_existing_event_to_event_list(struct event_list *ev_list, - struct event *ev); -extern struct event_list *copy_event_list(struct event_list *el); -extern int find_event(struct event *ev, struct event_list *el); -extern void free_event_list(struct event_list *el); - - -extern struct dim_structure *initialize_dim_structure(void); -extern struct dim_structure *default_dim_structure(void); -extern int set_dim_structure_entry(struct dim_structure *hsd, - int dim_entry, const char *val_string); -extern void free_dim_structure_entry(struct dim_structure *hsd); -extern void free_dim_structure(struct dim_structure *hsd); - -#endif /* EVENTS_H */ diff --git a/libcrystfel/src/filters.c b/libcrystfel/src/filters.c index 352046aa..9d01bac3 100644 --- a/libcrystfel/src/filters.c +++ b/libcrystfel/src/filters.c @@ -3,11 +3,11 @@ * * Image filtering * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2016 Thomas White <taw@physics.org> + * 2010-2020 Thomas White <taw@physics.org> * 2013 Anton Barty <anton.barty@desy.de> * * This file is part of CrystFEL. @@ -84,8 +84,8 @@ void filter_noise(struct image *image) { int i; - for ( i=0; i<image->det->n_panels; i++ ) { - struct panel *p = &image->det->panels[i]; + for ( i=0; i<image->detgeom->n_panels; i++ ) { + struct detgeom_panel *p = &image->detgeom->panels[i]; filter_noise_in_panel(image->dp[i], p->w, p->h); } } @@ -149,14 +149,14 @@ void filter_median(struct image *image, int size) /* Determine local background * (median over window width either side of current pixel) */ - for ( pn=0; pn<image->det->n_panels; pn++ ) { + for ( pn=0; pn<image->detgeom->n_panels; pn++ ) { int fs, ss; int i; - struct panel *p; + struct detgeom_panel *p; float *localBg; - p = &image->det->panels[pn]; + p = &image->detgeom->panels[pn]; localBg = calloc(p->w*p->h, sizeof(float)); if ( localBg == NULL ) { diff --git a/libcrystfel/src/filters.h b/libcrystfel/src/filters.h index 17c235b1..83b76b9b 100644 --- a/libcrystfel/src/filters.h +++ b/libcrystfel/src/filters.h @@ -1,13 +1,13 @@ /* - * peaks.h + * filters.h * * Image filtering * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010,2012-2013 Thomas White <taw@physics.org> + * 2010-2019 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -29,10 +29,6 @@ #ifndef FILTERS_H #define FILTERS_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #ifdef __cplusplus extern "C" { #endif diff --git a/libcrystfel/src/fom.c b/libcrystfel/src/fom.c new file mode 100644 index 00000000..af1ba8fe --- /dev/null +++ b/libcrystfel/src/fom.c @@ -0,0 +1,1465 @@ +/* + * fom.c + * + * Figure of merit calculation + * + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2010-2021 Thomas White <taw@physics.org> + * 2013 Lorenzo Galli <lorenzo.galli@desy.de> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include <gsl/gsl_errno.h> +#include <gsl/gsl_statistics.h> +#include <gsl/gsl_fit.h> +#include <assert.h> + +#include "utils.h" +#include "fom.h" +#include "cell.h" +#include "cell-utils.h" +#include "reflist.h" +#include "reflist-utils.h" + +/** + * \file fom.h + */ + +struct fom_context +{ + enum fom_type fom; + int nshells; + int *cts; + + /* For R-factors */ + double *num; + double *den; + + /* For "double" R-factors */ + double *num2; + double *den2; + + /* For CCs */ + double **vec1; + double **vec2; + int *n; + int nmax; + + /* For "counting" things e.g. d1sig or d2sig */ + int *n_within; + + long int *n_meas; + long int *possible; +}; + + +static struct fom_context *init_fom(enum fom_type fom, int nmax, int nshells) +{ + struct fom_context *fctx; + int i; + + fctx = malloc(sizeof(struct fom_context)); + if ( fctx == NULL ) return NULL; + + fctx->fom = fom; + fctx->nshells = nshells; + fctx->cts = malloc(nshells*sizeof(int)); + for ( i=0; i<nshells; i++ ) { + fctx->cts[i] = 0; + } + + fctx->num = NULL; + fctx->den = NULL; + fctx->num2 = NULL; + fctx->den2 = NULL; + fctx->possible = NULL; + + switch ( fctx->fom ) { + + case FOM_RANORSPLIT : + fctx->num2 = malloc(nshells*sizeof(double)); + fctx->den2 = malloc(nshells*sizeof(double)); + if ( (fctx->num2 == NULL) || (fctx->den2 == NULL) ) return NULL; + for ( i=0; i<nshells; i++ ) { + fctx->num2[i] = 0.0; + fctx->den2[i] = 0.0; + } + /* Intentional fall-through (no break) */ + + case FOM_R1I : + case FOM_R1F : + case FOM_R2 : + case FOM_RSPLIT : + case FOM_RANO : + case FOM_MEAN_INTENSITY : + case FOM_SNR : + case FOM_REDUNDANCY : + fctx->num = malloc(nshells*sizeof(double)); + fctx->den = malloc(nshells*sizeof(double)); + if ( (fctx->num == NULL) || (fctx->den == NULL) ) return NULL; + for ( i=0; i<nshells; i++ ) { + fctx->num[i] = 0.0; + fctx->den[i] = 0.0; + } + break; + + case FOM_COMPLETENESS : + /* Uses 'cts' and 'possible' only */ + break; + + case FOM_NUM_MEASUREMENTS : + fctx->n_meas = calloc(nshells, sizeof(long int)); + if ( fctx->n_meas == NULL ) return NULL; + break; + + case FOM_CC : + case FOM_CCSTAR : + case FOM_CCANO : + case FOM_CRDANO : + fctx->vec1 = malloc(nshells*sizeof(double *)); + fctx->vec2 = malloc(nshells*sizeof(double *)); + if ( (fctx->vec1 == NULL) || (fctx->vec2 == NULL) ) return NULL; + for ( i=0; i<nshells; i++ ) { + fctx->vec1[i] = malloc(nmax*sizeof(double)); + if ( fctx->vec1[i] == NULL ) return NULL; + fctx->vec2[i] = malloc(nmax*sizeof(double)); + if ( fctx->vec2[i] == NULL ) return NULL; + fctx->n = malloc(nshells*sizeof(int)); + if ( fctx->n == NULL ) return NULL; + } + for ( i=0; i<nshells; i++ ) { + fctx->n[i] = 0; + } + fctx->nmax = nmax; + break; + + case FOM_D1SIG : + case FOM_D2SIG : + fctx->n_within = malloc(nshells*sizeof(int)); + if ( fctx->n_within == NULL ) return NULL; + for ( i=0; i<nshells; i++ ) { + fctx->n_within[i] = 0; + } + break; + + } + + return fctx; +} + + +static int add_to_fom(struct fom_context *fctx, + Reflection *refl1, + Reflection *refl2, + Reflection *refl1bij, + Reflection *refl2bij, + int bin) +{ + double i1, i2, i1bij, i2bij, sig1, sig2; + double im, imbij; + int bad = 0; + + fctx->cts[bin]++; + + switch ( fctx->fom ) { + + case FOM_R1I : + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + fctx->num[bin] += fabs(i1 - i2); + fctx->den[bin] += i1; + break; + + case FOM_R1F : + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + fctx->num[bin] += fabs(sqrt(i1) - sqrt(i2)); + fctx->den[bin] += sqrt(i1); + break; + + case FOM_R2 : + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + fctx->num[bin] += pow(i1 - i2, 2.0); + fctx->den[bin] += pow(i1, 2.0); + break; + + case FOM_RSPLIT : + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + fctx->num[bin] += fabs(i1 - i2); + fctx->den[bin] += i1 + i2; + break; + + case FOM_CC : + case FOM_CCSTAR : + assert(fctx->n[bin] < fctx->nmax); + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + fctx->vec1[bin][fctx->n[bin]] = i1; + fctx->vec2[bin][fctx->n[bin]] = i2; + fctx->n[bin]++; + break; + + case FOM_CCANO : + case FOM_CRDANO : + assert(fctx->n[bin] < fctx->nmax); + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + i1bij = get_intensity(refl1bij); + i2bij = get_intensity(refl2bij); + fctx->vec1[bin][fctx->n[bin]] = i1 - i1bij; + fctx->vec2[bin][fctx->n[bin]] = i2 - i2bij; + fctx->n[bin]++; + break; + + case FOM_RANORSPLIT : + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + fctx->num2[bin] += fabs(i1 - i2); + fctx->den2[bin] += i1 + i2; + /* Intentional fall-through (no break) */ + + case FOM_RANO : + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + i1bij = get_intensity(refl1bij); + i2bij = get_intensity(refl2bij); + im = (i1 + i2)/2.0; + imbij = (i1bij + i2bij)/2.0; + fctx->num[bin] += fabs(im - imbij); + fctx->den[bin] += im + imbij; + break; + + case FOM_D1SIG : + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + sig1 = get_esd_intensity(refl1); + sig2 = get_esd_intensity(refl2); + if ( fabs(i1-i2) < sqrt(sig1*sig1 + sig2*sig2) ) { + fctx->n_within[bin]++; + } + break; + + case FOM_D2SIG : + i1 = get_intensity(refl1); + i2 = get_intensity(refl2); + sig1 = get_esd_intensity(refl1); + sig2 = get_esd_intensity(refl2); + if ( fabs(i1-i2) < 2.0*sqrt(sig1*sig1 + sig2*sig2) ) { + fctx->n_within[bin]++; + } + break; + + case FOM_NUM_MEASUREMENTS : + fctx->n_meas[bin] += get_redundancy(refl1); + break; + + case FOM_REDUNDANCY : + fctx->num[bin] += get_redundancy(refl1); + fctx->den[bin] += 1.0; + break; + + case FOM_SNR : + i1 = get_intensity(refl1); + sig1 = get_esd_intensity(refl1); + if ( isfinite(i1/sig1) ) { + fctx->num[bin] += i1/sig1; + fctx->den[bin] += 1.0; + } else { + bad = 1; + } + break; + + case FOM_MEAN_INTENSITY : + i1 = get_intensity(refl1); + fctx->num[bin] += i1; + fctx->den[bin] += 1.0; + break; + + case FOM_COMPLETENESS : + /* fctx->cts already incremented, as needed. + * Will calculate possible reflections later */ + break; + + } + + return bad; +} + + +/** + * Calculates the overall value for the %fom_context + * + * You must have previously called fom_calculate() + */ +double fom_overall_value(struct fom_context *fctx) +{ + double overall_num = INFINITY; + double overall_den = 0.0; + double overall_num2 = INFINITY; + double overall_den2 = 0.0; + int i; + double *overall_vec1; + double *overall_vec2; + int overall_n; + double *overall_along_diagonal; + double *overall_perpend_diagonal; + double variance_signal; + double variance_error; + double cc = INFINITY; + long int total_meas = 0; + long int overall_cts = 0; + long int overall_possible = 0; + + switch ( fctx->fom ) { + + case FOM_R1I : + case FOM_R1F : + case FOM_R2 : + case FOM_RSPLIT : + case FOM_RANO : + case FOM_REDUNDANCY : + case FOM_SNR : + case FOM_MEAN_INTENSITY : + overall_num = 0.0; + overall_den = 0.0; + for ( i=0; i<fctx->nshells; i++ ) { + overall_num += fctx->num[i]; + overall_den += fctx->den[i]; + } + break; + + case FOM_RANORSPLIT : + overall_num = 0.0; + overall_den = 0.0; + for ( i=0; i<fctx->nshells; i++ ) { + overall_num += fctx->num[i]; + overall_den += fctx->den[i]; + } + overall_num2 = 0.0; + overall_den2 = 0.0; + for ( i=0; i<fctx->nshells; i++ ) { + overall_num2 += fctx->num2[i]; + overall_den2 += fctx->den2[i]; + } + break; + + case FOM_CC : + case FOM_CCSTAR : + case FOM_CCANO : + overall_vec1 = malloc(fctx->nmax*sizeof(double)); + overall_vec2 = malloc(fctx->nmax*sizeof(double)); + overall_n = 0; + for ( i=0; i<fctx->nshells; i++ ) { + int j; + for ( j=0; j<fctx->n[i]; j++ ) { + overall_vec1[overall_n] = fctx->vec1[i][j]; + overall_vec2[overall_n] = fctx->vec2[i][j]; + overall_n++; + } + } + cc = gsl_stats_correlation(overall_vec1, 1, overall_vec2, 1, + overall_n); + free(overall_vec1); + free(overall_vec2); + break; + + case FOM_CRDANO : + overall_along_diagonal = malloc(fctx->nmax*sizeof(double)); + overall_perpend_diagonal = malloc(fctx->nmax*sizeof(double)); + overall_n = 0; + for ( i=0; i<fctx->nshells; i++ ) { + int j; + for ( j=0; j<fctx->n[i]; j++ ) { + overall_along_diagonal[overall_n] = + ( fctx->vec1[i][j] + fctx->vec2[i][j] ) + / sqrt(2.0); + overall_perpend_diagonal[overall_n] = + ( fctx->vec1[i][j] - fctx->vec2[i][j] ) + / sqrt(2.0); + overall_n++; + } + } + variance_signal = gsl_stats_variance_m(overall_along_diagonal, + 1, overall_n, 0.0); + variance_error = gsl_stats_variance_m(overall_perpend_diagonal, + 1, overall_n, 0.0); + cc = sqrt(variance_signal / variance_error ); + + free(overall_along_diagonal); + free(overall_perpend_diagonal); + break; + + case FOM_D1SIG : + case FOM_D2SIG : + overall_num = 0.0; + overall_den = 0.0; + for ( i=0; i<fctx->nshells; i++ ) { + overall_num += fctx->n_within[i]; + overall_den += fctx->cts[i]; + } + break; + + case FOM_NUM_MEASUREMENTS : + total_meas = 0; + for ( i=0; i<fctx->nshells; i++ ) { + total_meas += fctx->n_meas[i]; + } + break; + + case FOM_COMPLETENESS : + for ( i=0; i<fctx->nshells; i++ ) { + overall_cts += fctx->cts[i]; + overall_possible += fctx->possible[i]; + } + break; + + } + + switch ( fctx->fom ) { + + case FOM_R1I : + case FOM_R1F : + case FOM_REDUNDANCY : + case FOM_SNR : + case FOM_MEAN_INTENSITY : + return overall_num/overall_den; + + case FOM_COMPLETENESS : + return (double)overall_cts / overall_possible; + + case FOM_NUM_MEASUREMENTS : + return total_meas; + + case FOM_R2 : + return sqrt(overall_num/overall_den); + + case FOM_RSPLIT : + return 2.0*(overall_num/overall_den) / sqrt(2.0); + + case FOM_CC : + case FOM_CCANO : + case FOM_CRDANO : + return cc; + + case FOM_CCSTAR : + return sqrt((2.0*cc)/(1.0+cc)); + + case FOM_RANO : + return 2.0*(overall_num/overall_den); + + case FOM_RANORSPLIT : + return (2.0*(overall_num/overall_den)) / + (2.0*(overall_num2/overall_den2) / sqrt(2.0)); + + case FOM_D1SIG : + case FOM_D2SIG : + return overall_num/overall_den; + + } + + ERROR("This point is never reached.\n"); + abort(); +} + + +/** + * Calculates the figure of merit for the specified shell number. + * You must have previously called fom_calculate() + */ +double fom_shell_value(struct fom_context *fctx, int i) +{ + double cc; + int j; + double variance_signal; + double variance_error; + double *along_diagonal; + double *perpend_diagonal; + + switch ( fctx->fom ) { + + case FOM_R1I : + case FOM_R1F : + case FOM_REDUNDANCY : + case FOM_SNR : + case FOM_MEAN_INTENSITY : + return fctx->num[i]/fctx->den[i]; + + case FOM_R2 : + return sqrt(fctx->num[i]/fctx->den[i]); + + case FOM_RSPLIT : + return 2.0*(fctx->num[i]/fctx->den[i]) / sqrt(2.0); + + case FOM_CC : + case FOM_CCANO : + return gsl_stats_correlation(fctx->vec1[i], 1, fctx->vec2[i], 1, + fctx->n[i]); + + case FOM_CCSTAR : + cc = gsl_stats_correlation(fctx->vec1[i], 1, fctx->vec2[i], 1, + fctx->n[i]); + return sqrt((2.0*cc)/(1.0+cc)); + + case FOM_RANO : + return 2.0 * fctx->num[i]/fctx->den[i]; + + case FOM_RANORSPLIT : + return (2.0*fctx->num[i]/fctx->den[i]) / + (2.0*(fctx->num2[i]/fctx->den2[i]) / sqrt(2.0)); + + case FOM_CRDANO : + along_diagonal = malloc(fctx->n[i] * sizeof(double)); + perpend_diagonal = malloc(fctx->n[i] * sizeof(double)); + for ( j=0; j<fctx->n[i]; j++ ) { + along_diagonal[j] = ( fctx->vec1[i][j] + + fctx->vec2[i][j] ) / sqrt(2.0); + perpend_diagonal[j] = ( fctx->vec1[i][j] - + fctx->vec2[i][j] ) / sqrt(2.0); + } + variance_signal = gsl_stats_variance_m(along_diagonal, 1, + fctx->n[i], 0.0); + variance_error = gsl_stats_variance_m(perpend_diagonal, 1, + fctx->n[i], 0.0); + free(along_diagonal); + free(perpend_diagonal); + return sqrt(variance_signal / variance_error); + + case FOM_D1SIG : + case FOM_D2SIG : + return (double)fctx->n_within[i] / fctx->cts[i]; + + case FOM_NUM_MEASUREMENTS : + return fctx->n_meas[i]; + + case FOM_COMPLETENESS : + return (double)fctx->cts[i] / fctx->possible[i]; + + } + + ERROR("This point is never reached.\n"); + abort(); +} + + +/** + * \param rmin: The minimum value of 1/d, in m^-1 + * \param rmax: The maximum value of 1/d, in m^-1 + * \param nshells: The number of shells to use + * + * Create a %fom_shells structure for the specified minimum and maximum + * resolution limits + * + * Returns the %fom_shells structure, or NULL on error. + */ +struct fom_shells *fom_make_resolution_shells(double rmin, double rmax, + int nshells) +{ + struct fom_shells *s; + double total_vol, vol_per_shell; + int i; + + s = malloc(sizeof(struct fom_shells)); + if ( s == NULL ) return NULL; + + s->rmins = malloc(nshells*sizeof(double)); + s->rmaxs = malloc(nshells*sizeof(double)); + + if ( (s->rmins==NULL) || (s->rmaxs==NULL) ) { + ERROR("Couldn't allocate memory for resolution shells.\n"); + free(s); + return NULL; + } + + s->nshells = nshells; + + total_vol = pow(rmax, 3.0) - pow(rmin, 3.0); + vol_per_shell = total_vol / nshells; + s->rmins[0] = rmin; + for ( i=1; i<nshells; i++ ) { + + double r; + + r = vol_per_shell + pow(s->rmins[i-1], 3.0); + r = pow(r, 1.0/3.0); + + /* Shells of constant volume */ + s->rmaxs[i-1] = r; + s->rmins[i] = r; + + } + s->rmaxs[nshells-1] = rmax; + + return s; +} + + +/** + * \param s: A %fom_shells structure + * \param i: The shell number + * + * Returns the value of 1/d at the middle of the shell, + * i.e. the mean of the minimum and maximum 1/d values for the shell + */ +double fom_shell_centre(struct fom_shells *s, int i) +{ + return s->rmins[i] + (s->rmaxs[i] - s->rmins[i])/2.0; +} + + +static int get_bin(struct fom_shells *s, Reflection *refl, UnitCell *cell) +{ + double d; + int bin, j; + signed int h, k, l; + + get_indices(refl, &h, &k, &l); + d = 2.0 * resolution(cell, h, k, l); + + bin = -1; + for ( j=0; j<s->nshells; j++ ) { + if ( (d>s->rmins[j]) && (d<=s->rmaxs[j]) ) { + bin = j; + break; + } + } + + /* Allow for slight rounding errors */ + if ( (bin == -1) && (d <= s->rmins[0]) ) bin = 0; + if ( (bin == -1) && (d >= s->rmaxs[s->nshells-1]) ) bin = 0; + assert(bin != -1); + + return bin; +} + + +static int wilson_scale(RefList *list1, RefList *list2, UnitCell *cell) +{ + Reflection *refl1; + Reflection *refl2; + RefListIterator *iter; + int max_n = 256; + int n = 0; + double *x; + double *y; + int r; + double G, B; + double c0, c1, cov00, cov01, cov11, chisq; + + x = malloc(max_n*sizeof(double)); + y = malloc(max_n*sizeof(double)); + if ( (x==NULL) || (y==NULL) ) { + ERROR("Failed to allocate memory for scaling.\n"); + return 1; + } + + for ( refl1 = first_refl(list1, &iter); + refl1 != NULL; + refl1 = next_refl(refl1, iter) ) + { + signed int h, k, l; + double Ih1, Ih2; + double res; + + get_indices(refl1, &h, &k, &l); + res = resolution(cell, h, k, l); + + refl2 = find_refl(list2, h, k, l); + assert(refl2 != NULL); + + Ih1 = get_intensity(refl1); + Ih2 = get_intensity(refl2); + + if ( (Ih1 <= 0.0) || (Ih2 <= 0.0) ) continue; + if ( isnan(Ih1) || isinf(Ih1) ) continue; + if ( isnan(Ih2) || isinf(Ih2) ) continue; + + if ( n == max_n ) { + max_n *= 2; + x = realloc(x, max_n*sizeof(double)); + y = realloc(y, max_n*sizeof(double)); + if ( (x==NULL) || (y==NULL) ) { + ERROR("Failed to allocate memory for scaling.\n"); + return 1; + } + } + + x[n] = res*res; + y[n] = log(Ih1/Ih2); + n++; + + } + + if ( n < 2 ) { + ERROR("Not enough reflections for scaling\n"); + return 1; + } + + r = gsl_fit_linear(x, 1, y, 1, n, &c0, &c1, + &cov00, &cov01, &cov11, &chisq); + + if ( r ) { + ERROR("Scaling failed.\n"); + return 1; + } + + G = exp(c0); + B = c1/2.0; + + STATUS("Relative scale factor = %f, relative B factor = %f A^2\n", + G, B*1e20); + STATUS("A scale factor greater than 1 means that the second reflection " + "list is weaker than the first.\n"); + STATUS("A positive relative B factor means that the second reflection " + "list falls off with resolution more quickly than the first.\n"); + + free(x); + free(y); + + /* Apply the scaling factor */ + for ( refl2 = first_refl(list2, &iter); + refl2 != NULL; + refl2 = next_refl(refl2, iter) ) + { + signed int h, k, l; + double res; + double corr; + + get_indices(refl2, &h, &k, &l); + res = resolution(cell, h, k, l); + + corr = G * exp(2.0*B*res*res); + set_intensity(refl2, get_intensity(refl2)*corr); + set_esd_intensity(refl2, get_esd_intensity(refl2)*corr); + + } + return 0; +} + + +static int calculate_possible(struct fom_context *fctx, + struct fom_shells *shells, + UnitCell *cell, + const SymOpList *sym) +{ + RefList *counted; + int hmax, kmax, lmax; + double ax, ay, az; + double bx, by, bz; + double cx, cy, cz; + signed int h, k, l; + + fctx->possible = calloc(fctx->nshells, sizeof(long int)); + if ( fctx->possible == NULL ) return 1; + + counted = reflist_new(); + if ( counted == NULL ) { + free(fctx->possible); + return 1; + } + + cell_get_cartesian(cell, &ax, &ay, &az, + &bx, &by, &bz, + &cx, &cy, &cz); + hmax = shells->rmaxs[fctx->nshells-1] * modulus(ax, ay, az); + kmax = shells->rmaxs[fctx->nshells-1] * modulus(bx, by, bz); + lmax = shells->rmaxs[fctx->nshells-1] * modulus(cx, cy, cz); + for ( h=-hmax; h<=hmax; h++ ) { + for ( k=-kmax; k<=kmax; k++ ) { + for ( l=-lmax; l<=lmax; l++ ) { + + double d; + signed int hs, ks, ls; + int bin; + int i; + + get_asymm(sym, h, k, l, &hs, &ks, &ls); + d = 2.0 * resolution(cell, hs, ks, ls); + + if ( forbidden_reflection(cell, h, k, l) ) continue; + + bin = -1; + for ( i=0; i<fctx->nshells; i++ ) { + if ( (d>shells->rmins[i]) && (d<=shells->rmaxs[i]) ) { + bin = i; + break; + } + } + if ( bin == -1 ) continue; + + if ( find_refl(counted, hs, ks, ls) != NULL ) continue; + add_refl(counted, hs, ks, ls); + + fctx->possible[bin]++; + + } + } + } + reflist_free(counted); + + return 0; +} + + +int fom_is_anomalous(enum fom_type fom) +{ + switch ( fom ) { + + case FOM_CCANO: + case FOM_RANO: + case FOM_CRDANO: + case FOM_RANORSPLIT: + return 1; + + case FOM_R1I: + case FOM_R1F: + case FOM_R2: + case FOM_RSPLIT: + case FOM_CC: + case FOM_CCSTAR: + case FOM_D1SIG: + case FOM_D2SIG: + case FOM_NUM_MEASUREMENTS: + case FOM_REDUNDANCY: + case FOM_SNR: + case FOM_MEAN_INTENSITY: + case FOM_COMPLETENESS: + return 0; + } + + ERROR("This point never reached\n"); + abort(); +} + + +int fom_is_comparison(enum fom_type fom) +{ + switch ( fom ) { + + case FOM_CCANO: + case FOM_RANO: + case FOM_CRDANO: + case FOM_RANORSPLIT: + case FOM_R1I: + case FOM_R1F: + case FOM_R2: + case FOM_RSPLIT: + case FOM_CC: + case FOM_CCSTAR: + case FOM_D1SIG: + case FOM_D2SIG: + return 1; + + case FOM_NUM_MEASUREMENTS: + case FOM_REDUNDANCY: + case FOM_SNR: + case FOM_MEAN_INTENSITY: + case FOM_COMPLETENESS: + return 0; + } + + ERROR("This point never reached\n"); + abort(); +} + + +static int is_single_list(enum fom_type fom) +{ + switch ( fom ) { + + case FOM_CCANO: + case FOM_RANO: + case FOM_CRDANO: + case FOM_RANORSPLIT: + case FOM_R1I: + case FOM_R1F: + case FOM_R2: + case FOM_RSPLIT: + case FOM_CC: + case FOM_CCSTAR: + case FOM_D1SIG: + case FOM_D2SIG: + return 0; + + case FOM_NUM_MEASUREMENTS: + case FOM_REDUNDANCY: + case FOM_SNR: + case FOM_MEAN_INTENSITY: + case FOM_COMPLETENESS: + return 1; + } + + ERROR("This point never reached\n"); + abort(); +} + + +/** + * \param list1: A %RefList + * \param list2: A %RefList + * \param cell: A %UnitCell + * \param shells: A %fom_shells structure + * \param fom: The figure of merit to calculate + * \param noscale: Non-zero to disable scaline of reflection lists + * \param sym: The symmetry of \p list1 and \p list2. + * + * Calculates the specified figure of merit, comparing the two reflection lists. + * + * The \p cell and \p sym must match both reflection lists. You should also have + * called fom_select_reflection_pairs() to pre-process the lists. + * + * If the figure of merit does not involve comparison (e.g. %FOM_SNR), + * then \p list1 will be used. In this case, \p list2 and \p noscale will be + * ignored. Use fom_select_reflections() instead of fom_select_reflection_pairs() + * in this case. + * + * \returns a %fom_context structure. Use fom_shell_value() et al., to + * extract the actual figure of merit values. + */ +struct fom_context *fom_calculate(RefList *list1, RefList *list2, UnitCell *cell, + struct fom_shells *shells, enum fom_type fom, + int noscale, const SymOpList *sym) +{ + Reflection *refl1; + RefListIterator *iter; + struct fom_context *fctx; + long int n_out = 0; + long int n_rej = 0; + + fctx = init_fom(fom, num_reflections(list1), shells->nshells); + + if ( fctx == NULL ) { + ERROR("Couldn't allocate memory for resolution shells.\n"); + return NULL; + } + + if ( !is_single_list(fom) ) { + if ( !noscale && wilson_scale(list1, list2, cell) ) { + ERROR("Error with scaling.\n"); + return NULL; + } + + for ( refl1 = first_refl(list1, &iter); + refl1 != NULL; + refl1 = next_refl(refl1, iter) ) + { + Reflection *refl2; + signed int h, k, l; + set_flag(refl1, 0); + get_indices(refl1, &h, &k, &l); + refl2 = find_refl(list2, h, k, l); + assert(refl2 != NULL); + set_flag(refl2, 0); + } + } + + for ( refl1 = first_refl(list1, &iter); + refl1 != NULL; + refl1 = next_refl(refl1, iter) ) + { + signed int h, k, l; + int bin; + Reflection *refl2; + Reflection *refl1_bij = NULL; + Reflection *refl2_bij = NULL; + + get_indices(refl1, &h, &k, &l); + + if ( is_single_list(fom) ) { + refl2 = NULL; + } else { + refl2 = find_refl(list2, h, k, l); + if ( refl2 == NULL ) continue; + } + + bin = get_bin(shells, refl1, cell); + if ( bin == -1 ) { + n_out++; + continue; + } + + if ( fom_is_anomalous(fom) ) { + + signed int hb, kb, lb; + + if ( find_equiv_in_list(list1, -h, -k, -l, sym, + &hb, &kb, &lb) ) + { + refl1_bij = find_refl(list1, hb, kb, lb); + } + + if ( find_equiv_in_list(list2, -h, -k, -l, sym, + &hb, &kb, &lb) ) + { + refl2_bij = find_refl(list2, hb, kb, lb); + } + + /* Each reflection must only be counted once, whether + * we are visiting it now as "normal" or "bij" */ + if ( get_flag(refl1) ) continue; + assert(!get_flag(refl2)); + set_flag(refl1, 1); + set_flag(refl1_bij, 1); + set_flag(refl2, 1); + set_flag(refl2_bij, 1); + + assert(refl1_bij != NULL); + assert(refl2_bij != NULL); + + } + + n_rej += add_to_fom(fctx, refl1, refl2, refl1_bij, refl2_bij, bin); + + } + if ( n_out ) { + ERROR("WARNING: %i reflection pairs outside range.\n", n_out); + } + if ( n_rej ) { + if ( fom == FOM_SNR ) { + ERROR("WARNING: %li reflections had infinite or " + "invalid values of I/sigma(I).\n", n_rej); + } else { + ERROR("WARNING: %li reflections rejected by add_to_fom\n", + n_rej); + } + } + + if ( fom == FOM_COMPLETENESS ) { + calculate_possible(fctx, shells, cell, sym); + } + + return fctx; +} + + +/** + * \param list1: The first input %RefList + * \param list2: The second input %RefList + * \param plist1_acc: Pointer to location for accepted list + * \param plist2_acc: Pointer to location for accepted list + * \param cell: A %UnitCell + * \param sym: The symmetry of \p raw_list + * \param anom: Non-zero if you will calculate a FoM for anomalous signal + * \param rmin_fix: If positive, minimum resolution to use + * \param rmax_fix: If positive, maximum resolution to use + * \param sigma_cutoff: Minimum I/sigI value + * \param ignore_negs: Non-zero to filter out negative intensities + * \param zero_negs: Non-zero to set negative intensities to zero + * \param mul_cutoff: Minimum number of measurements per reflection + * + * Selects reflections suitable for use with fom_calculate(). + * + * Use -INFINITY for \p sigma_cutoff to disable the check. + * Set \p mul_cutoff to zero to disable the check. + * + * \returns a %fom_rejections structure with the counts of reflections. + */ +struct fom_rejections fom_select_reflection_pairs(RefList *list1, RefList *list2, + RefList **plist1_acc, + RefList **plist2_acc, + UnitCell *cell, SymOpList *sym, + int anom, double rmin_fix, double rmax_fix, + double sigma_cutoff, int ignore_negs, + int zero_negs, int mul_cutoff) +{ + Reflection *refl1; + RefListIterator *iter; + struct fom_rejections rej; + RefList *list1_acc; + RefList *list2_acc; + + rej.common = 0; + rej.low_snr = 0; + rej.negative_deleted = 0; + rej.negative_zeroed = 0; + rej.few_measurements = 0; + rej.outside_resolution_range = 0; + rej.no_bijvoet = 0; + rej.centric = 0; + rej.nan_inf_value = 0; + + list1_acc = reflist_new(); + list2_acc = reflist_new(); + + for ( refl1 = first_refl(list1, &iter); + refl1 != NULL; + refl1 = next_refl(refl1, iter) ) + { + signed int h, k, l; + double val1, val2; + double esd1, esd2; + int mul1, mul2; + Reflection *refl2; + Reflection *refl1_acc; + Reflection *refl2_acc; + + get_indices(refl1, &h, &k, &l); + + refl2 = find_refl(list2, h, k, l); + if ( refl2 == NULL ) continue; + + val1 = get_intensity(refl1); + val2 = get_intensity(refl2); + + esd1 = get_esd_intensity(refl1); + esd2 = get_esd_intensity(refl2); + + mul1 = get_redundancy(refl1); + mul2 = get_redundancy(refl2); + + if ( !isfinite(val1) || !isfinite(val2) + || !isfinite(esd1) || !isfinite(esd2) ) + { + rej.nan_inf_value++; + continue; + } + + if ( (val1 < sigma_cutoff * esd1) + || (val2 < sigma_cutoff * esd2) ) + { + rej.low_snr++; + continue; + } + + if ( ignore_negs && ((val1 < 0.0) || (val2 < 0.0)) ) { + rej.negative_deleted++; + continue; + } + + if ( (mul1 < mul_cutoff) || (mul2 < mul_cutoff) ) { + rej.few_measurements++; + continue; + } + + if ( zero_negs ) { + int d = 0; + if ( val1 < 0.0 ) { + val1 = 0.0; + d = 1; + } + if ( val2 < 0.0 ) { + val2 = 0.0; + d = 1; + } + if ( d ) rej.negative_zeroed++; + continue; + } + + if ( rmin_fix > 0.0 ) { + double res = 2.0*resolution(cell, h, k, l); + if ( res < rmin_fix ) { + rej.outside_resolution_range++; + continue; + } + } + + if ( rmax_fix > 0.0 ) { + double res = 2.0*resolution(cell, h, k, l); + if ( res > rmax_fix ) { + rej.outside_resolution_range++; + continue; + } + } + + refl1_acc = add_refl(list1_acc, h, k, l); + copy_data(refl1_acc, refl1); + set_intensity(refl1_acc, val1); + + refl2_acc = add_refl(list2_acc, h, k, l); + copy_data(refl2_acc, refl2); + set_intensity(refl2_acc, val2); + + rej.common++; + + } + + /* For anomalous figures of merit, we additionally require that we have + * all the Bijvoet pairs after the above rejection tests */ + if ( anom ) { + + list1 = list1_acc; + list2 = list2_acc; + list1_acc = reflist_new(); + list2_acc = reflist_new(); + + rej.common = 0; + + for ( refl1 = first_refl(list1, &iter); + refl1 != NULL; + refl1 = next_refl(refl1, iter) ) + { + Reflection *refl1_bij = NULL; + Reflection *refl2_bij = NULL; + signed int h, k, l; + signed int hb, kb, lb; + Reflection *refl1_acc; + Reflection *refl2_acc; + Reflection *refl2; + double val1, val2; + + get_indices(refl1, &h, &k, &l); + + refl2 = find_refl(list2, h, k, l); + assert(refl2 != NULL); + + val1 = get_intensity(refl1); + val2 = get_intensity(refl2); + + if ( is_centric(h, k, l, sym) ) { + rej.centric++; + continue; + } + + if ( find_equiv_in_list(list1, -h, -k, -l, sym, + &hb, &kb, &lb) ) + { + refl1_bij = find_refl(list1, hb, kb, lb); + } + + if ( find_equiv_in_list(list2, -h, -k, -l, sym, + &hb, &kb, &lb) ) + { + refl2_bij = find_refl(list2, hb, kb, lb); + } + + if ( (refl1_bij == NULL) || (refl2_bij == NULL) ) { + rej.no_bijvoet++; + continue; + } + + refl1_acc = add_refl(list1_acc, h, k, l); + copy_data(refl1_acc, refl1); + set_intensity(refl1_acc, val1); + + refl2_acc = add_refl(list2_acc, h, k, l); + copy_data(refl2_acc, refl2); + set_intensity(refl2_acc, val2); + + rej.common++; + } + } + + *plist1_acc = list1_acc; + *plist2_acc = list2_acc; + return rej; +} + + +/** + * \param raw_list: The input %RefList + * \param plist_acc: Pointer to location for accepted list + * \param cell: A %UnitCell + * \param sym: The symmetry of \p raw_list + * \param rmin_fix: If positive, minimum resolution to use + * \param rmax_fix: If positive, maximum resolution to use + * \param sigma_cutoff: Minimum I/sigI value + * \param ignore_negs: Non-zero to filter out negative intensities + * \param zero_negs: Non-zero to set negative intensities to zero + * \param mul_cutoff: Minimum number of measurements per reflection + * + * Use -INFINITY for \p sigma_cutoff to disable the check. + * Set \p mul_cutoff to zero to disable the check. + * + * \returns a %fom_rejections structure with the counts of reflections. + */ +struct fom_rejections fom_select_reflections(RefList *raw_list, + RefList **plist_acc, + UnitCell *cell, SymOpList *sym, + double rmin_fix, double rmax_fix, + double sigma_cutoff, int ignore_negs, + int zero_negs, int mul_cutoff) +{ + RefList *list; + Reflection *refl; + RefListIterator *iter; + struct fom_rejections rej; + + *plist_acc = NULL; + + rej.common = 0; + rej.low_snr = 0; + rej.negative_deleted = 0; + rej.negative_zeroed = 0; + rej.few_measurements = 0; + rej.outside_resolution_range = 0; + rej.no_bijvoet = 0; + rej.centric = 0; + rej.nan_inf_value = 0; + + list = reflist_new(); + if ( list == NULL ) return rej; + + for ( refl = first_refl(raw_list, &iter); + refl != NULL; + refl = next_refl(refl, iter) ) { + + signed int h, k, l; + double val, sig; + int ig = 0; + Reflection *new; + + get_indices(refl, &h, &k, &l); + + val = get_intensity(refl); + sig = get_esd_intensity(refl); + + if ( !isfinite(val) || !isfinite(sig) ) { + rej.nan_inf_value++; + continue; + } + + if ( val < sigma_cutoff * sig ) { + rej.low_snr++; + ig = 1; + } + + if ( ignore_negs && (val < 0.0) ) { + rej.negative_deleted++; + ig = 1; + } + + if ( zero_negs && (val < 0.0) ) { + set_intensity(refl, 0.0); + rej.negative_zeroed++; + } + + if ( rmin_fix > 0.0 ) { + double res = 2.0*resolution(cell, h, k, l); + if ( res < rmin_fix ) { + rej.outside_resolution_range++; + continue; + } + } + + if ( rmax_fix > 0.0 ) { + double res = 2.0*resolution(cell, h, k, l); + if ( res > rmax_fix ) { + rej.outside_resolution_range++; + continue; + } + } + + if ( ig ) continue; + + new = add_refl(list, h, k, l); + copy_data(new, refl); + } + + *plist_acc = list; + return rej; +} + + +/** + * \param fctx: A %fom_context structure + * + * \returns the total number of unique reflections + */ +int fom_overall_num_reflections(struct fom_context *fctx) +{ + int i; + long int n = 0; + + for ( i=0; i<fctx->nshells; i++ ) { + n += fctx->cts[i]; + } + return n; +} + + +/** + * \param fctx: A %fom_context structure + * \param i: Shell number + * + * \returns the number of unique reflections in the shell + */ +int fom_shell_num_reflections(struct fom_context *fctx, int i) +{ + return fctx->cts[i]; +} + + +/** + * \param fctx: A %fom_context structure + * + * This must only be called on a %fom_context for %FOM_COMPLETENESS. + * + * \returns the total number of reflections possible in all shells, taking into + * account symmetry and lattice absences, but not screw axis/glide place absences. + */ +int fom_overall_num_possible(struct fom_context *fctx) +{ + int i; + long int n = 0; + + assert(fctx->fom == FOM_COMPLETENESS); + + for ( i=0; i<fctx->nshells; i++ ) { + n += fctx->possible[i]; + } + return n; +} + + +/** + * \param fctx: A %fom_context structure + * \param i: Shell number + * + * This must only be called on a %fom_context for %FOM_COMPLETENESS. + * + * \returns the number of reflections possible in the shell, taking into account + * symmetry and lattice absences, but not screw axis/glide place absences. + */ +int fom_shell_num_possible(struct fom_context *fctx, int i) +{ + assert(fctx->fom == FOM_COMPLETENESS); + return fctx->possible[i]; +} + + +const char *fom_name(enum fom_type f) +{ + switch ( f ) { + case FOM_R1I : return "R1(I)"; + case FOM_R1F : return "R1(F)"; + case FOM_R2 : return "R2"; + case FOM_RSPLIT : return "Rsplit"; + case FOM_CC : return "CC"; + case FOM_CCSTAR : return "CC*"; + case FOM_CCANO : return "CCano"; + case FOM_CRDANO : return "CRDano"; + case FOM_RANO : return "Rano"; + case FOM_RANORSPLIT : return "Rano/Rsplit"; + case FOM_D1SIG : return "D<1sigma"; + case FOM_D2SIG : return "D<2sigma"; + case FOM_NUM_MEASUREMENTS : return "nMeas"; + case FOM_REDUNDANCY : return "Redundancy"; + case FOM_SNR : return "I/sigI"; + case FOM_MEAN_INTENSITY : return "mean I"; + case FOM_COMPLETENESS : return "Completeness"; + default : return "unknown FoM"; + } +} diff --git a/libcrystfel/src/fom.h b/libcrystfel/src/fom.h new file mode 100644 index 00000000..08bc655b --- /dev/null +++ b/libcrystfel/src/fom.h @@ -0,0 +1,141 @@ +/* + * fom.h + * + * Figure of merit calculation + * + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2010-2021 Thomas White <taw@physics.org> + * 2013 Lorenzo Galli <lorenzo.galli@desy.de> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifndef FOM_H +#define FOM_H + +/** + * \file fom.h + * Figure of merit calculation + */ + +#include <reflist.h> +#include <symmetry.h> + +/** + * Contains counts of rejected reflections + */ +struct fom_rejections +{ + int common; /**< Number of common reflection pairs accepted */ + int low_snr; /**< Reflections with I/sigI too low */ + int negative_deleted; /**< Negative intensities which were deleted */ + int negative_zeroed; /**< Negative intensities which were set to zero */ + int few_measurements; /**< Reflections with too few measurements */ + int outside_resolution_range; /**< Reflections outside resolution range */ + int no_bijvoet; /**< Reflections with no Bijvoet partner */ + int centric; /**< Reflections which are centric */ + int nan_inf_value; /**< Reflections with NaN or infinite intensity */ +}; + +/** + * An enumeration of possible figures of merit to calculate + */ +enum fom_type +{ + FOM_R1I, + FOM_R1F, + FOM_R2, + FOM_RSPLIT, + FOM_CC, + FOM_CCSTAR, + FOM_CCANO, + FOM_CRDANO, + FOM_RANO, + FOM_RANORSPLIT, + FOM_D1SIG, + FOM_D2SIG, + FOM_NUM_MEASUREMENTS, + FOM_REDUNDANCY, + FOM_SNR, + FOM_MEAN_INTENSITY, + FOM_COMPLETENESS, +}; + +struct fom_shells +{ + int nshells; + double *rmins; + double *rmaxs; +}; + +struct fom_context; + +extern struct fom_rejections fom_select_reflection_pairs(RefList *list1, + RefList *list2, + RefList **plist1_acc, + RefList **plist2_acc, + UnitCell *cell, + SymOpList *sym, + int anom, + double rmin_fix, + double rmax_fix, + double sigma_cutoff, + int ignore_negs, + int zero_negs, + int mul_cutoff); + +extern struct fom_rejections fom_select_reflections(RefList *list, + RefList **plist_acc, + UnitCell *cell, + SymOpList *sym, + double rmin_fix, + double rmax_fix, + double sigma_cutoff, + int ignore_negs, + int zero_negs, + int mul_cutoff); + + +extern struct fom_context *fom_calculate(RefList *list1, RefList *list2, + UnitCell *cell, + struct fom_shells *shells, + enum fom_type fom, int noscale, + const SymOpList *sym); + +extern struct fom_shells *fom_make_resolution_shells(double rmin, double rmax, + int nshells); + +extern double fom_shell_centre(struct fom_shells *s, int i); + +extern double fom_overall_value(struct fom_context *fctx); +extern double fom_shell_value(struct fom_context *fctx, int i); + +extern int fom_overall_num_reflections(struct fom_context *fctx); +extern int fom_shell_num_reflections(struct fom_context *fctx, int i); + +extern int fom_overall_num_possible(struct fom_context *fctx); +extern int fom_shell_num_possible(struct fom_context *fctx, int i); + +extern int fom_is_anomalous(enum fom_type f); +extern int fom_is_comparison(enum fom_type f); + +extern const char *fom_name(enum fom_type f); + +#endif /* FOM */ diff --git a/libcrystfel/src/geometry.c b/libcrystfel/src/geometry.c index 8f8a9672..adb95e69 100644 --- a/libcrystfel/src/geometry.c +++ b/libcrystfel/src/geometry.c @@ -3,11 +3,11 @@ * * Geometry of diffraction * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2016 Thomas White <taw@physics.org> + * 2010-2021 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -52,7 +52,7 @@ /** \file geometry.h */ static int locate_peak_on_panel(double x, double y, double z, double k, - struct panel *p, + struct detgeom_panel *p, double *pfs, double *pss) { double ctt, tta, phi; @@ -85,7 +85,7 @@ static int locate_peak_on_panel(double x, double y, double z, double k, gsl_matrix_set(M, 1, 0, p->cny); gsl_matrix_set(M, 1, 1, p->fsy); gsl_matrix_set(M, 1, 2, p->ssy); - gsl_matrix_set(M, 2, 0, p->clen*p->res); + gsl_matrix_set(M, 2, 0, p->cnz); gsl_matrix_set(M, 2, 1, p->fsz); gsl_matrix_set(M, 2, 2, p->ssz); @@ -103,6 +103,10 @@ static int locate_peak_on_panel(double x, double y, double z, double k, *pfs = fs; *pss = ss; + /* If "mu" is negative, then the reflection is in the + * wrong direction */ + if ( one_over_mu < 0.0 ) return 0; + /* Now, is this on this panel? */ if ( fs < 0.0 ) return 0; if ( fs >= p->w ) return 0; @@ -113,7 +117,8 @@ static int locate_peak_on_panel(double x, double y, double z, double k, } static signed int locate_peak(double x, double y, double z, double k, - struct detector *det, double *pfs, double *pss) + struct detgeom *det, + double *pfs, double *pss) { int i; @@ -121,7 +126,7 @@ static signed int locate_peak(double x, double y, double z, double k, for ( i=0; i<det->n_panels; i++ ) { - struct panel *p; + struct detgeom_panel *p; p = &det->panels[i]; @@ -379,29 +384,31 @@ static Reflection *check_reflection(struct image *image, Crystal *cryst, /* If we are updating a previous reflection, assume it stays * on the same panel and calculate the new position even if it's * fallen off the edge of the panel. */ - if ( (image->det != NULL) && (updateme != NULL) ) { + if ( (image->detgeom != NULL) && (updateme != NULL) ) { double fs, ss; + assert(get_panel_number(updateme) <= image->detgeom->n_panels); locate_peak_on_panel(xl, yl, zl, mean_kpred, - get_panel(updateme), &fs, &ss); + &image->detgeom->panels[get_panel_number(updateme)], + &fs, &ss); set_detector_pos(refl, fs, ss); } - /* Otherwise, calculate position if we have a detector structure, and + /* otherwise, calculate position if we have a detector structure, and * if we don't then just make do with partiality calculation */ - if ( (image->det != NULL) && (updateme == NULL) ) { + if ( (image->detgeom != NULL) && (updateme == NULL) ) { - double fs, ss; /* Position on detector */ - signed int p; /* Panel number */ + double fs, ss; /* position on detector */ + signed int p; /* panel number */ p = locate_peak(xl, yl, zl, mean_kpred, - image->det, &fs, &ss); + image->detgeom, &fs, &ss); if ( p == -1 ) { reflection_free(refl); return NULL; } set_detector_pos(refl, fs, ss); - set_panel(refl, &image->det->panels[p]); + set_panel_number(refl, p); } @@ -503,6 +510,7 @@ RefList *predict_to_res(Crystal *cryst, double max_res) double mres; signed int h, k, l; UnitCell *cell; + struct image *image; cell = crystal_get_cell(cryst); if ( cell == NULL ) return NULL; @@ -512,14 +520,15 @@ RefList *predict_to_res(Crystal *cryst, double max_res) /* Cell angle check from Foadi and Evans (2011) */ if ( !cell_is_sensible(cell) ) { ERROR("Invalid unit cell parameters given to" - " find_intersections()\n"); + " predict_to_res()\n"); cell_print(cell); return NULL; } cell_get_cartesian(cell, &ax, &ay, &az, &bx, &by, &bz, &cx, &cy, &cz); - mres = largest_q(crystal_get_image(cryst)); + image = crystal_get_image(cryst); + mres = detgeom_max_resolution(image->detgeom, image->lambda); if ( mres > max_res ) mres = max_res; hmax = mres * modulus(ax, ay, az); @@ -1076,7 +1085,8 @@ void polarisation_correction(RefList *list, UnitCell *cell, /* Returns dx_h/dP, where P = any parameter */ -double x_gradient(int param, Reflection *refl, UnitCell *cell, struct panel *p) +double x_gradient(int param, Reflection *refl, UnitCell *cell, + struct detgeom_panel *p) { signed int h, k, l; double xl, zl, kpred; @@ -1093,13 +1103,13 @@ double x_gradient(int param, Reflection *refl, UnitCell *cell, struct panel *p) switch ( param ) { case GPARAM_ASX : - return h * p->clen / (kpred + zl); + return h * p->cnz * p->pixel_pitch / (kpred + zl); case GPARAM_BSX : - return k * p->clen / (kpred + zl); + return k * p->cnz * p->pixel_pitch / (kpred + zl); case GPARAM_CSX : - return l * p->clen / (kpred + zl); + return l * p->cnz * p->pixel_pitch / (kpred + zl); case GPARAM_ASY : return 0.0; @@ -1111,13 +1121,13 @@ double x_gradient(int param, Reflection *refl, UnitCell *cell, struct panel *p) return 0.0; case GPARAM_ASZ : - return -h * xl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); + return -h * xl * p->cnz * p->pixel_pitch / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_BSZ : - return -k * xl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); + return -k * xl * p->cnz * p->pixel_pitch / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_CSZ : - return -l * xl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); + return -l * xl * p->cnz * p->pixel_pitch / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_DETX : return -1; @@ -1136,7 +1146,8 @@ double x_gradient(int param, Reflection *refl, UnitCell *cell, struct panel *p) /* Returns dy_h/dP, where P = any parameter */ -double y_gradient(int param, Reflection *refl, UnitCell *cell, struct panel *p) +double y_gradient(int param, Reflection *refl, UnitCell *cell, + struct detgeom_panel *p) { signed int h, k, l; double yl, zl, kpred; @@ -1162,22 +1173,22 @@ double y_gradient(int param, Reflection *refl, UnitCell *cell, struct panel *p) return 0.0; case GPARAM_ASY : - return h * p->clen / (kpred + zl); + return h * p->cnz * p->pixel_pitch / (kpred + zl); case GPARAM_BSY : - return k * p->clen / (kpred + zl); + return k * p->cnz * p->pixel_pitch / (kpred + zl); case GPARAM_CSY : - return l * p->clen / (kpred + zl); + return l * p->cnz * p->pixel_pitch / (kpred + zl); case GPARAM_ASZ : - return -h * yl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); + return -h * yl * p->cnz * p->pixel_pitch / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_BSZ : - return -k * yl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); + return -k * yl * p->cnz * p->pixel_pitch / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_CSZ : - return -l * yl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); + return -l * yl * p->cnz * p->pixel_pitch / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_DETX : return 0; diff --git a/libcrystfel/src/geometry.h b/libcrystfel/src/geometry.h index afaa9d6c..19c6a23a 100644 --- a/libcrystfel/src/geometry.h +++ b/libcrystfel/src/geometry.h @@ -3,12 +3,12 @@ * * Geometry of diffraction * - * Copyright © 2013-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * * Authors: - * 2010-2016 Thomas White <taw@physics.org> + * 2010-2020 Thomas White <taw@physics.org> * 2012 Richard Kirian * * This file is part of CrystFEL. @@ -31,14 +31,10 @@ #ifndef GEOMETRY_H #define GEOMETRY_H - -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include "reflist.h" #include "cell.h" #include "crystal.h" +#include "detgeom.h" #ifdef __cplusplus extern "C" { @@ -122,9 +118,9 @@ extern double sphere_fraction(double rlow, double rhigh, double pr); extern double gaussian_fraction(double rlow, double rhigh, double pr); extern double x_gradient(int param, Reflection *refl, UnitCell *cell, - struct panel *p); + struct detgeom_panel *p); extern double y_gradient(int param, Reflection *refl, UnitCell *cell, - struct panel *p); + struct detgeom_panel *p); #ifdef __cplusplus } diff --git a/libcrystfel/src/hdf5-file.c b/libcrystfel/src/hdf5-file.c deleted file mode 100644 index e2738a8b..00000000 --- a/libcrystfel/src/hdf5-file.c +++ /dev/null @@ -1,2785 +0,0 @@ -/* - * hdf5-file.c - * - * Read/write HDF5 data files - * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, - * a research centre of the Helmholtz Association. - * - * Authors: - * 2009-2016 Thomas White <taw@physics.org> - * 2014 Valerio Mariani - * - * This file is part of CrystFEL. - * - * CrystFEL is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * CrystFEL is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. - * - */ - -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - -#include <stdlib.h> -#include <stdio.h> -#include <stdint.h> -#include <hdf5.h> -#include <assert.h> -#include <unistd.h> - -#include "events.h" -#include "image.h" -#include "hdf5-file.h" -#include "utils.h" - -/** \file hdf5-file.h */ - -struct hdf5_write_location { - - const char *location; - int n_panels; - int *panel_idxs; - - int max_ss; - int max_fs; - -}; - - -int split_group_and_object(const char *path, char **group, char **object) -{ - const char *sep; - const char *store; - - sep = path; - store = sep; - sep = strpbrk(sep + 1, "/"); - if ( sep != NULL ) { - while ( 1 ) { - store = sep; - sep = strpbrk(sep + 1, "/"); - if ( sep == NULL ) { - break; - } - } - } - if ( store == path ) { - *group = NULL; - *object = strdup(path); - } else { - *group = strndup(path, store - path); - *object = strdup(store+1); - } - return 0; -}; - - -struct hdfile { - - const char *path; /* Current data path */ - - hid_t fh; /* HDF file handle */ - hid_t dh; /* Dataset handle */ - - int data_open; /* True if dh is initialised */ -}; - - -struct hdfile *hdfile_open(const char *filename) -{ - struct hdfile *f; - - f = malloc(sizeof(struct hdfile)); - if ( f == NULL ) return NULL; - - if ( access( filename, R_OK ) == -1 ) { - ERROR("File does not exist or cannot be read: %s\n", - filename); - free(f); - return NULL; - } - - f->fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); - if ( f->fh < 0 ) { - ERROR("Couldn't open file: %s\n", filename); - free(f); - return NULL; - } - - f->data_open = 0; - return f; -} - - -int hdfile_set_image(struct hdfile *f, const char *path) -{ - f->dh = H5Dopen2(f->fh, path, H5P_DEFAULT); - if ( f->dh < 0 ) { - ERROR("Couldn't open dataset\n"); - return -1; - } - f->data_open = 1; - return 0; -} - - -static int read_peak_count(struct hdfile *f, char *path, int line, - int *num_peaks) -{ - - hid_t dh, sh, mh; - hsize_t size[1]; - hsize_t max_size[1]; - hsize_t offset[1], count[1]; - hsize_t m_offset[1], m_count[1], dimmh[1]; - - - int tw, r; - - dh = H5Dopen2(f->fh, path, H5P_DEFAULT); - if ( dh < 0 ) { - ERROR("Data block %s not found.\n", path); - return 1; - } - - sh = H5Dget_space(dh); - if ( sh < 0 ) { - H5Dclose(dh); - ERROR("Couldn't get dataspace for data.\n"); - return 1; - } - - if ( H5Sget_simple_extent_ndims(sh) != 1 ) { - ERROR("Data block %s has the wrong dimensionality (%i).\n", - path, H5Sget_simple_extent_ndims(sh)); - H5Sclose(sh); - H5Dclose(dh); - return 1; - } - - H5Sget_simple_extent_dims(sh, size, max_size); - - tw = size[0]; - - if ( line > tw-1 ) { - H5Sclose(sh); - H5Dclose(dh); - ERROR("Data block %s does not contain data for required event.\n", - path); - return 1; - } - - offset[0] = line; - count[0] = 1; - - r = H5Sselect_hyperslab(sh, H5S_SELECT_SET, - offset, NULL, count, NULL); - if ( r < 0 ) { - ERROR("Error selecting file dataspace " - "for data block %s\n", path); - H5Dclose(dh); - H5Sclose(sh); - return 1; - } - - m_offset[0] = 0; - m_count[0] = 1; - dimmh[0] = 1; - mh = H5Screate_simple(1, dimmh, NULL); - r = H5Sselect_hyperslab(mh, H5S_SELECT_SET, - m_offset, NULL, m_count, NULL); - if ( r < 0 ) { - ERROR("Error selecting memory dataspace " - "for data block %s\n", path); - H5Dclose(dh); - H5Sclose(sh); - H5Sclose(mh); - return 1; - } - - r = H5Dread(dh, H5T_NATIVE_INT, mh, - sh, H5P_DEFAULT, num_peaks); - if ( r < 0 ) { - ERROR("Couldn't read data for block %s, line %i\n", path, line); - H5Dclose(dh); - H5Sclose(sh); - H5Sclose(mh); - return 1; - } - - H5Dclose(dh); - H5Sclose(sh); - H5Sclose(mh); - return 0; -} - - - -static float *read_hdf5_data(struct hdfile *f, char *path, int line) -{ - - hid_t dh, sh, mh; - hsize_t size[2]; - hsize_t max_size[2]; - hsize_t offset[2], count[2]; - hsize_t m_offset[2], m_count[2], dimmh[2]; - float *buf; - int tw, r; - - dh = H5Dopen2(f->fh, path, H5P_DEFAULT); - if ( dh < 0 ) { - ERROR("Data block (%s) not found.\n", path); - return NULL; - } - - sh = H5Dget_space(dh); - if ( sh < 0 ) { - H5Dclose(dh); - ERROR("Couldn't get dataspace for data.\n"); - return NULL; - } - - if ( H5Sget_simple_extent_ndims(sh) != 2 ) { - ERROR("Data block %s has the wrong dimensionality (%i).\n", - path, H5Sget_simple_extent_ndims(sh)); - H5Sclose(sh); - H5Dclose(dh); - return NULL; - } - - H5Sget_simple_extent_dims(sh, size, max_size); - - tw = size[0]; - if ( line> tw-1 ) { - H5Sclose(sh); - H5Dclose(dh); - ERROR("Data block %s does not contain data for required event.\n", - path); - return NULL; - } - - offset[0] = line; - offset[1] = 0; - count[0] = 1; - count[1] = size[1]; - - r = H5Sselect_hyperslab(sh, H5S_SELECT_SET, offset, NULL, count, NULL); - if ( r < 0 ) { - ERROR("Error selecting file dataspace " - "for data block %s\n", path); - H5Dclose(dh); - H5Sclose(sh); - return NULL; - } - - m_offset[0] = 0; - m_offset[1] = 0; - m_count[0] = 1; - m_count[1] = size[1]; - dimmh[0] = 1; - dimmh[1] = size[1]; - - mh = H5Screate_simple(2, dimmh, NULL); - r = H5Sselect_hyperslab(mh, H5S_SELECT_SET, - m_offset, NULL, m_count, NULL); - if ( r < 0 ) { - ERROR("Error selecting memory dataspace " - "for data block %s\n", path); - H5Dclose(dh); - H5Sclose(sh); - H5Sclose(mh); - return NULL; - } - - buf = malloc(size[1]*sizeof(float)); - if ( buf == NULL ) return NULL; - r = H5Dread(dh, H5T_NATIVE_FLOAT, mh, sh, H5P_DEFAULT, buf); - if ( r < 0 ) { - ERROR("Couldn't read data for block %s, line %i\n", path, line); - H5Dclose(dh); - H5Sclose(sh); - H5Sclose(mh); - return NULL; - } - - H5Dclose(dh); - H5Sclose(sh); - H5Sclose(mh); - return buf; -} - - -/** - * \param image: An \ref image structure - * \param f: An \ref hdfile structure - * \param p: The HDF5 path to the peak data - * \param fpe: A \ref filename_plus_event structure specifying the event - * \param half_pixel_shift: Non-zero if 0.5 should be added to all peak coordinates - * - * Get peaks from HDF5, in "CXI format" (as in "CXIDB"). The data should be in - * a set of arrays under \p p. The number of peaks should be in a 1D array at - * \p p/nPeaks. The fast-scan and slow-scan coordinates should be in 2D arrays at - * \p p/peakXPosRaw and \p p/peakYPosRaw respectively (sorry about the naming). The - * first dimension of these arrays should be the event number (as given by - * \p fpe). The intensities are expected to be at \p p/peakTotalIntensity in a - * similar 2D array. - * - * CrystFEL considers all peak locations to be distances from the corner of the - * detector panel, in pixel units, consistent with its description of detector - * geometry (see 'man crystfel_geometry'). The software which generates the - * CXI files, including Cheetah, may instead consider the peak locations to be - * pixel indices in the data array. In this case, the peak coordinates should - * have 0.5 added to them. This will be done if \p half_pixel_shift is non-zero. - * - * \returns Non-zero on error, zero otherwise. - * - */ -int get_peaks_cxi_2(struct image *image, struct hdfile *f, const char *p, - struct filename_plus_event *fpe, int half_pixel_shift) -{ - char path_n[1024]; - char path_x[1024]; - char path_y[1024]; - char path_i[1024]; - int r; - int pk; - - int line = 0; - int num_peaks; - - float *buf_x; - float *buf_y; - float *buf_i; - - double peak_offset = half_pixel_shift ? 0.5 : 0.0; - - if ( (fpe != NULL) && (fpe->ev != NULL) - && (fpe->ev->dim_entries != NULL) ) - { - line = fpe->ev->dim_entries[0]; - } else { - ERROR("CXI format peak list format selected," - "but file has no event structure"); - return 1; - } - - snprintf(path_n, 1024, "%s/nPeaks", p); - snprintf(path_x, 1024, "%s/peakXPosRaw", p); - snprintf(path_y, 1024, "%s/peakYPosRaw", p); - snprintf(path_i, 1024, "%s/peakTotalIntensity", p); - - r = read_peak_count(f, path_n, line, &num_peaks); - if ( r != 0 ) return 1; - - buf_x = read_hdf5_data(f, path_x, line); - if ( r != 0 ) return 1; - - buf_y = read_hdf5_data(f, path_y, line); - if ( r != 0 ) return 1; - - buf_i = read_hdf5_data(f, path_i, line); - if ( r != 0 ) return 1; - - if ( image->features != NULL ) { - image_feature_list_free(image->features); - } - image->features = image_feature_list_new(); - - for ( pk=0; pk<num_peaks; pk++ ) { - - float fs, ss, val; - struct panel *p; - - fs = buf_x[pk] + peak_offset; - ss = buf_y[pk] + peak_offset; - val = buf_i[pk]; - - p = find_orig_panel(image->det, fs, ss); - if ( p == NULL ) continue; - if ( p->no_index ) continue; - - /* Convert coordinates to panel-relative */ - fs = fs - p->orig_min_fs; - ss = ss - p->orig_min_ss; - - image_add_feature(image->features, fs, ss, p, image, val, NULL); - - } - - return 0; -} - - -/** - * \param image: An \ref image structure - * \param f: An \ref hdfile structure - * \param p: The HDF5 path to the peak data - * \param fpe: A \ref filename_plus_event structure specifying the event - * - * This is a wrapper function to preserve API compatibility with older CrystFEL - * versions. Use \ref get_peaks_cxi_2 instead. - * - * This function is equivalent to get_peaks_cxi_2(\p image, \p f, \p p, \p fpe, 1). - * - * \returns Non-zero on error, zero otherwise. - * - */ -int get_peaks_cxi(struct image *image, struct hdfile *f, const char *p, - struct filename_plus_event *fpe) -{ - return get_peaks_cxi_2(image, f, p, fpe, 1); -} - - -/** - * \param image: An \ref image structure - * \param f: An \ref hdfile structure - * \param p: The HDF5 path to the peak data - * \param half_pixel_shift: Non-zero if 0.5 should be added to all peak coordinates - * - * Get peaks from HDF5. The peak list should be located at \p p in the HDF5 file, - * a 2D array where the first dimension equals the number of peaks and second - * dimension is three. The first two columns contain the fast scan and slow - * scan coordinates, respectively, of the peaks. The third column contains the - * intensities. - * - * CrystFEL considers all peak locations to be distances from the corner of the - * detector panel, in pixel units, consistent with its description of detector - * geometry (see 'man crystfel_geometry'). The software which generates the - * CXI files, including Cheetah, may instead consider the peak locations to be - * pixel indices in the data array. In this case, the peak coordinates should - * have 0.5 added to them. This will be done if \p half_pixel_shift is non-zero. - * - * \returns Non-zero on error, zero otherwise. - * - */ -int get_peaks_2(struct image *image, struct hdfile *f, const char *p, - int half_pixel_shift) -{ - hid_t dh, sh; - hsize_t size[2]; - hsize_t max_size[2]; - int i; - float *buf; - herr_t r; - int tw; - char *np; - double peak_offset = half_pixel_shift ? 0.5 : 0.0; - - if ( image->event != NULL ) { - np = retrieve_full_path(image->event, p); - } else { - np = strdup(p); - } - - dh = H5Dopen2(f->fh, np, H5P_DEFAULT); - if ( dh < 0 ) { - ERROR("Peak list (%s) not found.\n", np); - return 1; - } - - sh = H5Dget_space(dh); - if ( sh < 0 ) { - H5Dclose(dh); - ERROR("Couldn't get dataspace for peak list.\n"); - free(np); - return 1; - } - - if ( H5Sget_simple_extent_ndims(sh) != 2 ) { - ERROR("Peak list has the wrong dimensionality (%i).\n", - H5Sget_simple_extent_ndims(sh)); - H5Sclose(sh); - H5Dclose(dh); - free(np); - return 1; - } - - H5Sget_simple_extent_dims(sh, size, max_size); - - tw = size[1]; - if ( (tw != 3) && (tw != 4) ) { - H5Sclose(sh); - H5Dclose(dh); - ERROR("Peak list has the wrong dimensions.\n"); - free(np); - return 1; - } - - buf = malloc(sizeof(float)*size[0]*size[1]); - if ( buf == NULL ) { - H5Sclose(sh); - H5Dclose(dh); - ERROR("Couldn't reserve memory for the peak list.\n"); - free(np); - return 1; - } - r = H5Dread(dh, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, buf); - if ( r < 0 ) { - ERROR("Couldn't read peak list.\n"); - free(buf); - free(np); - return 1; - } - - if ( image->features != NULL ) { - image_feature_list_free(image->features); - } - image->features = image_feature_list_new(); - - for ( i=0; i<size[0]; i++ ) { - - float fs, ss, val; - struct panel *p; - - fs = buf[tw*i+0] + peak_offset; - ss = buf[tw*i+1] + peak_offset; - val = buf[tw*i+2]; - - p = find_orig_panel(image->det, fs, ss); - if ( p == NULL ) continue; - if ( p->no_index ) continue; - - /* Convert coordinates to panel-relative */ - fs = fs - p->orig_min_fs; - ss = ss - p->orig_min_ss; - - image_add_feature(image->features, fs, ss, p, image, val, - NULL); - - } - - free(buf); - free(np); - H5Sclose(sh); - H5Dclose(dh); - - return 0; -} - - -/** - * \param image: An \ref image structure - * \param f: An \ref hdfile structure - * \param p: The HDF5 path to the peak data - * - * This is a wrapper function to preserve API compatibility with older CrystFEL - * versions. Use \ref get_peaks_2 instead. - * - * This function is equivalent to \ref get_peaks_2(\p image, \p f, \p p, 1). - * - * \returns Non-zero on error, zero otherwise. - * - */ -int get_peaks(struct image *image, struct hdfile *f, const char *p) -{ - return get_peaks_2(image, f, p, 1); -} - - -static void cleanup(hid_t fh) -{ - int n_ids, i; - hid_t ids[2048]; - - n_ids = H5Fget_obj_ids(fh, H5F_OBJ_ALL, 2048, ids); - - for ( i=0; i<n_ids; i++ ) { - - hid_t id; - H5I_type_t type; - - id = ids[i]; - - type = H5Iget_type(id); - - if ( type == H5I_GROUP ) H5Gclose(id); - if ( type == H5I_DATASET ) H5Dclose(id); - if ( type == H5I_DATATYPE ) H5Tclose(id); - if ( type == H5I_DATASPACE ) H5Sclose(id); - if ( type == H5I_ATTR ) H5Aclose(id); - - } - -} - - -void hdfile_close(struct hdfile *f) -{ - - if ( f->data_open ) { - H5Dclose(f->dh); - } - - cleanup(f->fh); - - H5Fclose(f->fh); - - free(f); -} - - -/* Deprecated */ -int hdf5_write(const char *filename, const void *data, - int width, int height, int type) -{ - hid_t fh, gh, sh, dh; /* File, group, dataspace and data handles */ - herr_t r; - hsize_t size[2]; - - fh = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - if ( fh < 0 ) { - ERROR("Couldn't create file: %s\n", filename); - return 1; - } - - gh = H5Gcreate2(fh, "data", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if ( gh < 0 ) { - ERROR("Couldn't create group\n"); - H5Fclose(fh); - return 1; - } - - /* Note the "swap" here, according to section 3.2.5, - * "C versus Fortran Dataspaces", of the HDF5 user's guide. */ - size[0] = height; - size[1] = width; - sh = H5Screate_simple(2, size, NULL); - - dh = H5Dcreate2(gh, "data", type, sh, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if ( dh < 0 ) { - ERROR("Couldn't create dataset\n"); - H5Fclose(fh); - return 1; - } - - /* Muppet check */ - H5Sget_simple_extent_dims(sh, size, NULL); - - r = H5Dwrite(dh, type, H5S_ALL, - H5S_ALL, H5P_DEFAULT, data); - if ( r < 0 ) { - ERROR("Couldn't write data\n"); - H5Dclose(dh); - H5Fclose(fh); - return 1; - } - H5Dclose(dh); - H5Gclose(gh); - H5Fclose(fh); - - return 0; -} - - -static void add_panel_to_location(struct hdf5_write_location *loc, - struct panel *p, int pi) -{ - int *new_panel_idxs; - - new_panel_idxs = realloc(loc->panel_idxs, - (loc->n_panels+1)*sizeof(int)); - if ( new_panel_idxs == NULL ) { - ERROR("Error while managing write location list.\n"); - return; - } - loc->panel_idxs = new_panel_idxs; - loc->panel_idxs[loc->n_panels] = pi; - loc->n_panels += 1; - if ( p->orig_max_fs > loc->max_fs ) { - loc->max_fs = p->orig_max_fs; - } - if ( p->orig_max_ss > loc->max_ss ) { - loc->max_ss = p->orig_max_ss; - } -} - - -static void add_panel_location(struct panel *p, const char *p_location, int pi, - struct hdf5_write_location **plocations, - int *pnum_locations) -{ - int li; - int num_locations = *pnum_locations; - struct hdf5_write_location *locations = *plocations; - int done = 0; - - /* Does this HDF5 path already exist in the location list? - * If so, add the new panel to it (with a unique index, we hope) */ - for ( li=0; li<num_locations; li++ ) { - if ( strcmp(p_location, locations[li].location) == 0 ) { - add_panel_to_location(&locations[li], p, pi); - done = 1; - } - } - - /* If not, add a new location to ths list */ - if ( !done ) { - - struct hdf5_write_location *new_locations; - size_t nsz; - - nsz = (num_locations+1)*sizeof(struct hdf5_write_location); - new_locations = realloc(locations, nsz); - if ( new_locations == NULL ) { - ERROR("Failed to grow location list.\n"); - return; - } - locations = new_locations; - - locations[num_locations].max_ss = p->orig_max_ss; - locations[num_locations].max_fs = p->orig_max_fs; - locations[num_locations].location = p_location; - locations[num_locations].panel_idxs = malloc(sizeof(int)); - if ( locations[num_locations].panel_idxs == NULL ) { - ERROR("Failed to allocate single idx (!)\n"); - return; - } - locations[num_locations].panel_idxs[0] = pi; - locations[num_locations].n_panels = 1; - - num_locations += 1; - - } - - *plocations = locations; - *pnum_locations = num_locations; -} - - -static struct hdf5_write_location *make_location_list(struct detector *det, - const char *def_location, - int *pnum_locations) -{ - int pi; - struct hdf5_write_location *locations = NULL; - int num_locations = 0; - - for ( pi=0; pi<det->n_panels; pi++ ) { - - struct panel *p; - const char *p_location; - - p = &det->panels[pi]; - - if ( p->data == NULL ) { - p_location = def_location; - } else { - p_location = p->data; - } - - add_panel_location(p, p_location, pi, - &locations, &num_locations); - - } - - *pnum_locations = num_locations; - return locations; -} - - -static void write_location(hid_t fh, struct detector *det, float **dp, - struct hdf5_write_location *loc) -{ - hid_t sh, dh, ph; - hid_t dh_dataspace; - hsize_t size[2]; - int pi; - - /* Note the "swap" here, according to section 3.2.5, - * "C versus Fortran Dataspaces", of the HDF5 user's guide. */ - size[0] = loc->max_ss+1; - size[1] = loc->max_fs+1; - sh = H5Screate_simple(2, size, NULL); - - ph = H5Pcreate(H5P_LINK_CREATE); - H5Pset_create_intermediate_group(ph, 1); - - dh = H5Dcreate2(fh, loc->location, H5T_NATIVE_FLOAT, sh, - ph, H5P_DEFAULT, H5P_DEFAULT); - if ( dh < 0 ) { - ERROR("Couldn't create dataset\n"); - H5Fclose(fh); - return; - } - - H5Sget_simple_extent_dims(sh, size, NULL); - - for ( pi=0; pi<loc->n_panels; pi++ ) { - - hsize_t f_offset[2], f_count[2], dims[2]; - hid_t memspace; - struct panel p; - int r; - - p = det->panels[loc->panel_idxs[pi]]; - - f_offset[0] = p.orig_min_ss; - f_offset[1] = p.orig_min_fs; - f_count[0] = p.orig_max_ss - p.orig_min_ss +1; - f_count[1] = p.orig_max_fs - p.orig_min_fs +1; - - dh_dataspace = H5Dget_space(dh); - r = H5Sselect_hyperslab(dh_dataspace, H5S_SELECT_SET, - f_offset, NULL, f_count, NULL); - if ( r < 0 ) { - ERROR("Error selecting file dataspace " - "for panel %s\n", p.name); - H5Pclose(ph); - H5Dclose(dh); - H5Sclose(dh_dataspace); - H5Sclose(sh); - H5Fclose(fh); - return; - } - - dims[0] = p.h; - dims[1] = p.w; - memspace = H5Screate_simple(2, dims, NULL); - - r = H5Dwrite(dh, H5T_NATIVE_FLOAT, memspace, dh_dataspace, - H5P_DEFAULT, dp[loc->panel_idxs[pi]]); - if ( r < 0 ) { - ERROR("Couldn't write data\n"); - H5Pclose(ph); - H5Dclose(dh); - H5Sclose(dh_dataspace); - H5Sclose(memspace); - H5Sclose(sh); - H5Fclose(fh); - return; - } - - H5Sclose(dh_dataspace); - H5Sclose(memspace); - } - H5Pclose(ph); - H5Sclose(sh); - H5Dclose(dh); -} - - -static void write_photon_energy(hid_t fh, double eV, const char *ph_en_loc) -{ - hid_t ph, sh, dh; - hsize_t size1d[1]; - int r; - - ph = H5Pcreate(H5P_LINK_CREATE); - H5Pset_create_intermediate_group(ph, 1); - - size1d[0] = 1; - sh = H5Screate_simple(1, size1d, NULL); - - dh = H5Dcreate2(fh, ph_en_loc, H5T_NATIVE_DOUBLE, sh, - ph, H5S_ALL, H5P_DEFAULT); - if ( dh < 0 ) { - ERROR("Couldn't create dataset for photon energy.\n"); - return; - } - r = H5Dwrite(dh, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &eV); - if ( r < 0 ) { - ERROR("Couldn't write photon energy.\n"); - /* carry on */ - } - - H5Pclose(ph); - H5Dclose(dh); -} - - -static void write_spectrum(hid_t fh, Spectrum *s) -{ - herr_t r; - double *arr; - int i; - hid_t sh, dh, ph; - double kmin, kmax, step; - const hsize_t n = 1024; - - ph = H5Pcreate(H5P_LINK_CREATE); - H5Pset_create_intermediate_group(ph, 1); - - arr = malloc(n*sizeof(double)); - if ( arr == NULL ) { - ERROR("Failed to allocate memory for spectrum.\n"); - return; - } - - /* Save the wavelength values */ - spectrum_get_range(s, &kmin, &kmax); - step = (kmax-kmin)/n; - for ( i=0; i<n; i++ ) { - arr[i] = 1.0e10/(kmin+i*step); - } - - sh = H5Screate_simple(1, &n, NULL); - - dh = H5Dcreate2(fh, "/spectrum/wavelengths_A", H5T_NATIVE_DOUBLE, - sh, ph, H5S_ALL, H5P_DEFAULT); - if ( dh < 0 ) { - ERROR("Failed to create dataset for spectrum wavelengths.\n"); - return; - } - r = H5Dwrite(dh, H5T_NATIVE_DOUBLE, H5S_ALL, - H5S_ALL, H5P_DEFAULT, arr); - if ( r < 0 ) { - ERROR("Failed to write spectrum wavelengths.\n"); - return; - } - H5Dclose(dh); - - /* Save the probability density values */ - for ( i=0; i<n; i++ ) { - arr[i] = spectrum_get_density_at_k(s, kmin+i*step); - } - - dh = H5Dcreate2(fh, "/spectrum/pdf", H5T_NATIVE_DOUBLE, sh, - H5P_DEFAULT, H5S_ALL, H5P_DEFAULT); - if ( dh < 0 ) { - ERROR("Failed to create dataset for spectrum p.d.f.\n"); - return; - } - r = H5Dwrite(dh, H5T_NATIVE_DOUBLE, H5S_ALL, - H5S_ALL, H5P_DEFAULT, arr); - if ( r < 0 ) { - ERROR("Failed to write spectrum p.d.f.\n"); - return; - } - - H5Dclose(dh); - H5Pclose(ph); - free(arr); -} - - -int hdf5_write_image(const char *filename, const struct image *image, - char *element) -{ - hid_t fh; - int li; - char *default_location; - struct hdf5_write_location *locations; - int num_locations; - const char *ph_en_loc; - - if ( image->det == NULL ) { - ERROR("Geometry not available\n"); - return 1; - } - - fh = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - if ( fh < 0 ) { - ERROR("Couldn't create file: %s\n", filename); - return 1; - } - - if ( element != NULL ) { - default_location = strdup(element); - } else { - default_location = strdup("/data/data"); - } - - locations = make_location_list(image->det, default_location, - &num_locations); - - for ( li=0; li<num_locations; li++ ) { - write_location(fh, image->det, image->dp, &locations[li]); - } - - if ( image->beam == NULL - || (image->beam != NULL && image->beam->photon_energy_from == NULL) ) { - ph_en_loc = "photon_energy_eV"; - } else { - ph_en_loc = image->beam->photon_energy_from; - } - - write_photon_energy(fh, ph_lambda_to_eV(image->lambda), ph_en_loc); - - if ( image->spectrum != NULL ) { - write_spectrum(fh, image->spectrum); - } - - H5Fclose(fh); - free(default_location); - for ( li=0; li<num_locations; li ++ ) { - free(locations[li].panel_idxs); - } - free(locations); - return 0; -} - - -static void debodge_saturation(struct hdfile *f, struct image *image) -{ - hid_t dh, sh; - hsize_t size[2]; - hsize_t max_size[2]; - int i; - float *buf; - herr_t r; - - dh = H5Dopen2(f->fh, "/processing/hitfinder/peakinfo_saturated", - H5P_DEFAULT); - - if ( dh < 0 ) { - /* This isn't an error */ - return; - } - - sh = H5Dget_space(dh); - if ( sh < 0 ) { - H5Dclose(dh); - ERROR("Couldn't get dataspace for saturation table.\n"); - return; - } - - if ( H5Sget_simple_extent_ndims(sh) != 2 ) { - H5Sclose(sh); - H5Dclose(dh); - return; - } - - H5Sget_simple_extent_dims(sh, size, max_size); - - if ( size[1] != 3 ) { - H5Sclose(sh); - H5Dclose(dh); - ERROR("Saturation table has the wrong dimensions.\n"); - return; - } - - buf = malloc(sizeof(float)*size[0]*size[1]); - if ( buf == NULL ) { - H5Sclose(sh); - H5Dclose(dh); - ERROR("Couldn't reserve memory for saturation table.\n"); - return; - } - r = H5Dread(dh, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); - if ( r < 0 ) { - ERROR("Couldn't read saturation table.\n"); - free(buf); - return; - } - - for ( i=0; i<size[0]; i++ ) { - - unsigned int fs, ss; - float val; - struct panel *p; - signed int pn; - - fs = buf[3*i+0]; - ss = buf[3*i+1]; - val = buf[3*i+2]; - - /* Turn "original" position into "panel" position */ - pn = find_orig_panel_number(image->det, fs, ss); - if ( pn == -1 ) { - ERROR("Failed to find panel!\n"); - continue; - } - p = &image->det->panels[pn]; - - image->dp[pn][fs+p->w*ss] = val/5.0; - image->dp[pn][fs+1+p->w*ss] = val/5.0; - image->dp[pn][fs-1+p->w*ss] = val/5.0; - image->dp[pn][fs+p->w*(ss-1)] = val/5.0; - image->dp[pn][fs+p->w*(ss+1)] = val/5.0; - - } - - free(buf); - H5Sclose(sh); - H5Dclose(dh); -} - - -static int *make_badmask(int *flags, struct detector *det, float *data, - struct panel *p) -{ - int *badmap; - int fs, ss; - - badmap = malloc(p->w*p->h*sizeof(int)); - if ( badmap == NULL ) { - ERROR("Failed to allocate bad mask for panel %s\n", - p->name); - return NULL; - } - - /* Defaults, then bad pixels arising from bad regions or panels */ - for ( ss=0; ss<p->h; ss++ ) { - for ( fs=0; fs<p->w; fs++ ) { - - int bad = 0; - - if ( p->no_index ) bad = 1; - - if ( in_bad_region(det, p, fs, ss) ) { - bad = 1; - } - - badmap[fs+p->w*ss] = bad; - } - } - - /* Bad pixels from mask */ - if ( flags != NULL ) { - for ( ss=0; ss<p->h; ss++ ) { - for ( fs=0; fs<p->w; fs++ ) { - - int f = flags[fs+p->w*ss]; - int bad = badmap[fs+p->w*ss]; - float val = data[fs+p->w*ss]; - - /* Bad if it's missing any of the "good" bits */ - if ( (f & det->mask_good) != det->mask_good ) bad = 1; - - /* Bad if it has any of the "bad" bits. */ - if ( f & det->mask_bad ) bad = 1; - - /* Bad if pixel value is NaN or inf */ - if ( isnan(val) || isinf(val) ) bad = 1; - - badmap[fs+p->w*ss] = bad; - - } - } - } - - return badmap; -} - - -int hdfile_get_value(struct hdfile *f, const char *name, struct event *ev, - void *val, hid_t memtype) -{ - hid_t dh; - hid_t type; - hid_t class; - hid_t sh; - hid_t ms; - hsize_t *f_offset = NULL; - hsize_t *f_count = NULL; - hsize_t m_offset[1]; - hsize_t m_count[1]; - hsize_t msdims[1]; - hsize_t size[64]; - herr_t r; - herr_t check; - int check_pe; - int dim_flag; - int ndims; - int i; - char *subst_name = NULL; - - if ( (ev != NULL) && (ev->path_length != 0) ) { - subst_name = retrieve_full_path(ev, name); - } else { - subst_name = strdup(name); - } - - check_pe = check_path_existence(f->fh, subst_name); - if ( check_pe == 0 ) { - ERROR("No such event-based float field '%s'\n", subst_name); - return 1; - } - - dh = H5Dopen2(f->fh, subst_name, H5P_DEFAULT); - type = H5Dget_type(dh); - class = H5Tget_class(type); - - if ( (class != H5T_FLOAT) && (class != H5T_INTEGER) ) { - ERROR("Not a floating point or integer value.\n"); - H5Tclose(type); - H5Dclose(dh); - return 1; - } - - /* Get the dimensionality. We have to cope with scalars expressed as - * arrays with all dimensions 1, as well as zero-d arrays. */ - sh = H5Dget_space(dh); - ndims = H5Sget_simple_extent_ndims(sh); - if ( ndims > 64 ) { - ERROR("Too many dimensions for hdfile_get_value\n"); - H5Tclose(type); - H5Dclose(dh); - return 1; - } - H5Sget_simple_extent_dims(sh, size, NULL); - - m_offset[0] = 0; - m_count[0] = 1; - msdims[0] = 1; - ms = H5Screate_simple(1,msdims,NULL); - - /* Check that the size in all dimensions is 1 - * or that one of the dimensions has the same - * size as the hyperplane events */ - - dim_flag = 0; - - for ( i=0; i<ndims; i++ ) { - if ( size[i] == 1 ) continue; - if ( ( i==0 ) && (ev != NULL) && (ev->dim_length > 0) - && (size[i] > ev->dim_entries[0]) ) - { - dim_flag = 1; - } else { - H5Tclose(type); - H5Dclose(dh); - return 1; - } - } - - if ( dim_flag == 0 ) { - - if ( H5Dread(dh, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, val) < 0 ) { - ERROR("Couldn't read value.\n"); - H5Tclose(type); - H5Dclose(dh); - return 1; - } - - } else { - - f_offset = malloc(ndims*sizeof(hsize_t)); - f_count = malloc(ndims*sizeof(hsize_t)); - - for ( i=0; i<ndims; i++ ) { - - if ( i == 0 ) { - f_offset[i] = ev->dim_entries[0]; - f_count[i] = 1; - } else { - f_offset[i] = 0; - f_count[i] = 0; - } - - } - - check = H5Sselect_hyperslab(sh, H5S_SELECT_SET, - f_offset, NULL, f_count, NULL); - if ( check <0 ) { - ERROR("Error selecting dataspace for float value"); - free(f_offset); - free(f_count); - return 1; - } - - ms = H5Screate_simple(1,msdims,NULL); - check = H5Sselect_hyperslab(ms, H5S_SELECT_SET, - m_offset, NULL, m_count, NULL); - if ( check < 0 ) { - ERROR("Error selecting memory dataspace for float value"); - free(f_offset); - free(f_count); - return 1; - } - - r = H5Dread(dh, memtype, ms, sh, H5P_DEFAULT, val); - if ( r < 0 ) { - ERROR("Couldn't read value.\n"); - H5Tclose(type); - H5Dclose(dh); - return 1; - } - - } - - free(subst_name); - - return 0; -} - - -static void hdfile_fill_in_beam_parameters(struct beam_params *beam, - struct hdfile *f, - struct event *ev, - struct image *image) -{ - double eV; - - if ( beam->photon_energy_from == NULL ) { - - /* Explicit value given */ - eV = beam->photon_energy; - - } else { - - int r; - - r = hdfile_get_value(f, beam->photon_energy_from, - ev, &eV, H5T_NATIVE_DOUBLE); - if ( r ) { - ERROR("Failed to read '%s'\n", - beam->photon_energy_from); - } - - } - - image->lambda = ph_en_to_lambda(eV_to_J(eV))*beam->photon_energy_scale; -} - - -static void hdfile_fill_in_clen(struct detector *det, struct hdfile *f, - struct event *ev) -{ - int i; - - for ( i=0; i<det->n_panels; i++ ) { - - struct panel *p = &det->panels[i]; - - if ( p->clen_from != NULL ) { - - double val; - int r; - - r = hdfile_get_value(f, p->clen_from, ev, &val, - H5T_NATIVE_DOUBLE); - if ( r ) { - ERROR("Failed to read '%s'\n", p->clen_from); - } else { - p->clen = val * 1.0e-3; - } - - } - - adjust_centering_for_rail(p); - - } -} - - -int hdf5_read(struct hdfile *f, struct image *image, const char *element, - int satcorr) -{ - herr_t r; - float *buf; - int fail; - hsize_t *size; - hsize_t *max_size; - hid_t sh; - int sh_dim; - int w, h; - - if ( element == NULL ) { - fail = hdfile_set_first_image(f, "/"); - } else { - fail = hdfile_set_image(f, element); - } - - if ( fail ) { - ERROR("Couldn't select path\n"); - return 1; - } - - sh = H5Dget_space(f->dh); - sh_dim = H5Sget_simple_extent_ndims(sh); - - if ( sh_dim != 2 ) { - ERROR("Dataset is not two-dimensional\n"); - return -1; - } - - size = malloc(sh_dim*sizeof(hsize_t)); - max_size = malloc(sh_dim*sizeof(hsize_t)); - H5Sget_simple_extent_dims(sh, size, max_size); - H5Sclose(sh); - w = size[1]; - h = size[0]; - free(size); - free(max_size); - - buf = malloc(sizeof(float)*w*h); - r = H5Dread(f->dh, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, buf); - if ( r < 0 ) { - ERROR("Couldn't read data\n"); - free(buf); - return 1; - } - - if ( image->det != NULL ) { - ERROR("WARNING: hdf5_read() called with geometry structure.\n"); - } - image->det = simple_geometry(image, w, h); - image->dp = malloc(sizeof(double *)); - if ( image->dp == NULL ) { - ERROR("Failed to allocate memory for image data!\n"); - return 1; - } - image->dp[0] = buf; - - if ( satcorr ) debodge_saturation(f, image); - - if ( image->beam != NULL ) { - - hdfile_fill_in_beam_parameters(image->beam, f, NULL, image); - - if ( image->lambda > 1000 ) { - /* Error message covers a silly value in the beam file - * or in the HDF5 file. */ - ERROR("WARNING: Missing or nonsensical wavelength " - "(%e m) for %s.\n", - image->lambda, image->filename); - } - - } - - fill_in_adu(image); - - return 0; -} - - -static hsize_t *first_two_dims(hsize_t *in, struct dim_structure *ds) -{ - int i, j; - hsize_t *out = malloc(2*sizeof(hsize_t)); - - if ( out == NULL ) return NULL; - - j = 0; - for ( i=0; i<ds->num_dims; i++ ) { - if ( (ds->dims[i] == HYSL_FS) || (ds->dims[i] == HYSL_SS) ) { - out[j++] = in[i]; - } - } - return out; -} - - -static int load_satmap(struct hdfile *f, struct event *ev, struct panel *p, - hsize_t *in_f_offset, hsize_t *in_f_count, - struct dim_structure *dim_struct, - float *satmap) -{ - char *loc; /* Sat map location after possible substitution */ - hid_t satmap_dataspace, satmap_dh; - int exists; - int check, r; - hid_t memspace; - hsize_t dimsm[2]; - hid_t fh; - hsize_t *f_offset, *f_count; - - if ( p->satmap_file != NULL ) { - - fh = H5Fopen(p->satmap_file, H5F_ACC_RDONLY, H5P_DEFAULT); - if ( fh < 0 ) { - ERROR("Couldn't open satmap file '%s'\n", p->satmap_file); - return 1; - } - - /* If we have an external map file, we assume it to be a simple - * 2D job */ - f_offset = first_two_dims(in_f_offset, dim_struct); - f_count = first_two_dims(in_f_count, dim_struct); - - } else { - - /* Otherwise, we assume it has the same dimensions as the - * image data itself */ - fh = f->fh; - f_offset = in_f_offset; - f_count = in_f_count; - } - - if ( ev != NULL ) { - loc = retrieve_full_path(ev, p->satmap); - } else { - loc = strdup(p->satmap); - } - - exists = check_path_existence(fh, loc); - if ( !exists ) { - ERROR("Cannot find satmap for panel %s\n", p->name); - goto err; - } - - satmap_dh = H5Dopen2(fh, loc, H5P_DEFAULT); - if ( satmap_dh <= 0 ) { - ERROR("Couldn't open satmap for panel %s\n", p->name); - goto err; - } - - satmap_dataspace = H5Dget_space(satmap_dh); - check = H5Sselect_hyperslab(satmap_dataspace, H5S_SELECT_SET, - f_offset, NULL, f_count, NULL); - if ( check < 0 ) { - ERROR("Error selecting satmap dataspace for panel %s\n", - p->name); - goto err; - } - - dimsm[0] = p->h; - dimsm[1] = p->w; - memspace = H5Screate_simple(2, dimsm, NULL); - if ( check < 0 ) { - ERROR("Error selecting satmap memory dataspace for panel %s\n", - p->name); - goto err; - } - - r = H5Dread(satmap_dh, H5T_NATIVE_FLOAT, memspace, - satmap_dataspace, H5P_DEFAULT, satmap); - if ( r < 0 ) { - ERROR("Couldn't read satmap for panel %s\n", p->name); - goto err; - } - - H5Sclose(satmap_dataspace); - H5Dclose(satmap_dh); - free(loc); - - return 0; - -err: - if ( p->satmap_file != NULL ) H5Fclose(fh); - free(loc); - return 1; -} - - - -static int load_mask(struct hdfile *f, struct event *ev, struct panel *p, - int *flags, - hsize_t *in_f_offset, hsize_t *in_f_count, - struct dim_structure *dim_struct) -{ - char *mask; /* Mask location after possible substitution */ - hid_t mask_dataspace, mask_dh; - int exists; - int check, r; - hid_t memspace; - hsize_t dimsm[2]; - hid_t fh; - hsize_t *f_offset, *f_count; - - if ( p->mask_file != NULL ) { - - fh = H5Fopen(p->mask_file, H5F_ACC_RDONLY, H5P_DEFAULT); - if ( fh < 0 ) { - ERROR("Couldn't open mask file '%s'\n", p->mask_file); - return 1; - } - - /* If we have an external map file, we assume it to be a simple - * 2D job */ - f_offset = first_two_dims(in_f_offset, dim_struct); - f_count = first_two_dims(in_f_count, dim_struct); - - } else { - fh = f->fh; - f_offset = in_f_offset; - f_count = in_f_count; - } - - if ( ev != NULL ) { - mask = retrieve_full_path(ev, p->mask); - } else { - mask = strdup(p->mask); - } - - exists = check_path_existence(fh, mask); - if ( !exists ) { - ERROR("Cannot find flags for panel %s\n", p->name); - goto err; - } - - mask_dh = H5Dopen2(fh, mask, H5P_DEFAULT); - if ( mask_dh <= 0 ) { - ERROR("Couldn't open flags for panel %s\n", p->name); - goto err; - } - - mask_dataspace = H5Dget_space(mask_dh); - check = H5Sselect_hyperslab(mask_dataspace, H5S_SELECT_SET, - f_offset, NULL, f_count, NULL); - if ( check < 0 ) { - ERROR("Error selecting mask dataspace for panel %s\n", p->name); - goto err; - } - - dimsm[0] = p->h; - dimsm[1] = p->w; - memspace = H5Screate_simple(2, dimsm, NULL); - if ( check < 0 ) { - ERROR("Error selecting memory dataspace for panel %s\n", p->name); - goto err; - } - - r = H5Dread(mask_dh, H5T_NATIVE_INT, memspace, - mask_dataspace, H5P_DEFAULT, flags); - if ( r < 0 ) { - ERROR("Couldn't read flags for panel %s\n", p->name); - goto err; - } - - H5Sclose(mask_dataspace); - H5Dclose(mask_dh); - free(mask); - - return 0; - -err: - if ( p->mask_file != NULL ) H5Fclose(fh); - free(mask); - return 1; -} - - -int hdf5_read2(struct hdfile *f, struct image *image, struct event *ev, - int satcorr) -{ - herr_t r; - int pi; - int i; - - if ( image->det == NULL ) { - ERROR("Geometry not available\n"); - return 1; - } - - image->dp = malloc(image->det->n_panels*sizeof(float *)); - image->bad = malloc(image->det->n_panels*sizeof(int *)); - image->sat = malloc(image->det->n_panels*sizeof(float *)); - if ( (image->dp==NULL) || (image->bad==NULL) || (image->sat==NULL) ) { - ERROR("Failed to allocate data arrays.\n"); - return 1; - } - - for ( pi=0; pi<image->det->n_panels; pi++ ) { - - hsize_t *f_offset, *f_count; - int hsi; - struct dim_structure *hsd; - herr_t check; - hid_t dataspace, memspace; - int fail; - struct panel *p; - hsize_t dims[2]; - - p = &image->det->panels[pi]; - - if ( ev != NULL ) { - - int exists; - char *panel_full_path; - - panel_full_path = retrieve_full_path(ev, p->data); - - exists = check_path_existence(f->fh, panel_full_path); - if ( !exists ) { - ERROR("Cannot find data for panel %s\n", - p->name); - free(image->dp); - free(image->bad); - free(image->sat); - return 1; - } - - fail = hdfile_set_image(f, panel_full_path); - - free(panel_full_path); - - } else { - - if ( p->data == NULL ) { - - fail = hdfile_set_first_image(f, "/"); - - } else { - - int exists; - exists = check_path_existence(f->fh, p->data); - if ( !exists ) { - ERROR("Cannot find data for panel %s\n", - p->name); - free(image->dp); - free(image->bad); - free(image->sat); - return 1; - } - fail = hdfile_set_image(f, p->data); - - } - - } - if ( fail ) { - ERROR("Couldn't select path for panel %s\n", - p->name); - free(image->dp); - free(image->bad); - free(image->sat); - return 1; - } - - /* Determine where to read the data from in the file */ - hsd = image->det->panels[pi].dim_structure; - f_offset = malloc(hsd->num_dims*sizeof(hsize_t)); - f_count = malloc(hsd->num_dims*sizeof(hsize_t)); - if ( (f_offset == NULL) || (f_count == NULL ) ) { - ERROR("Failed to allocate offset or count.\n"); - free(image->dp); - free(image->bad); - free(image->sat); - return 1; - } - for ( hsi=0; hsi<hsd->num_dims; hsi++ ) { - - if ( hsd->dims[hsi] == HYSL_FS ) { - f_offset[hsi] = p->orig_min_fs; - f_count[hsi] = p->orig_max_fs - p->orig_min_fs+1; - } else if ( hsd->dims[hsi] == HYSL_SS ) { - f_offset[hsi] = p->orig_min_ss; - f_count[hsi] = p->orig_max_ss - p->orig_min_ss+1; - } else if (hsd->dims[hsi] == HYSL_PLACEHOLDER ) { - f_offset[hsi] = ev->dim_entries[0]; - f_count[hsi] = 1; - } else { - f_offset[hsi] = hsd->dims[hsi]; - f_count[hsi] = 1; - } - - } - - /* Set up dataspace for file */ - dataspace = H5Dget_space(f->dh); - check = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, - f_offset, NULL, f_count, NULL); - if ( check < 0 ) { - ERROR("Error selecting file dataspace for panel %s\n", - p->name); - free(image->dp); - free(image->bad); - free(image->sat); - return 1; - } - - dims[0] = p->h; - dims[1] = p->w; - memspace = H5Screate_simple(2, dims, NULL); - - image->dp[pi] = malloc(p->w*p->h*sizeof(float)); - image->sat[pi] = malloc(p->w*p->h*sizeof(float)); - if ( (image->dp[pi] == NULL) || (image->sat[pi] == NULL) ) { - ERROR("Failed to allocate panel %s\n", p->name); - free(f_offset); - free(f_count); - free(image->dp); - free(image->bad); - free(image->sat); - return 1; - } - for ( i=0; i<p->w*p->h; i++ ) image->sat[pi][i] = INFINITY; - - r = H5Dread(f->dh, H5T_NATIVE_FLOAT, memspace, dataspace, - H5P_DEFAULT, image->dp[pi]); - if ( r < 0 ) { - ERROR("Couldn't read data for panel %s\n", - p->name); - free(f_offset); - free(f_count); - for ( i=0; i<=pi; i++ ) { - free(image->dp[i]); - free(image->sat[i]); - } - free(image->dp); - free(image->bad); - free(image->sat); - return 1; - } - - if ( p->mask != NULL ) { - int *flags = malloc(p->w*p->h*sizeof(int)); - if ( !load_mask(f, ev, p, flags, f_offset, f_count, hsd) ) { - image->bad[pi] = make_badmask(flags, image->det, - image->dp[pi], p); - } else { - image->bad[pi] = make_badmask(NULL, image->det, - image->dp[pi], p); - } - free(flags); - } else { - image->bad[pi] = make_badmask(NULL, image->det, - image->dp[pi], p); - } - - if ( p->satmap != NULL ) { - if ( load_satmap(f, ev, p, f_offset, f_count, hsd, - image->sat[pi]) ) - { - ERROR("Failed to load sat map for panel %s\n", - p->name); - } - } - - H5Sclose(dataspace); - free(f_offset); - free(f_count); - - } - - H5Dclose(f->dh); - f->data_open = 0; - hdfile_fill_in_clen(image->det, f, ev); - - if ( satcorr ) debodge_saturation(f, image); - - if ( image->beam != NULL ) { - - hdfile_fill_in_beam_parameters(image->beam, f, ev, image); - - if ( (image->lambda > 1.0) || (image->lambda < 1e-20) ) { - - ERROR("WARNING: Nonsensical wavelength (%e m) value " - "for file: %s, event: %s.\n", - image->lambda, image->filename, - get_event_string(image->event)); - } - - } - - fill_in_adu(image); - - return 0; -} - - -static int looks_like_image(hid_t h) -{ - hid_t sh; - hsize_t size[2]; - hsize_t max_size[2]; - - sh = H5Dget_space(h); - if ( sh < 0 ) return 0; - - if ( H5Sget_simple_extent_ndims(sh) != 2 ) { - H5Sclose(sh); - return 0; - } - - H5Sget_simple_extent_dims(sh, size, max_size); - H5Sclose(sh); - - if ( ( size[0] > 64 ) && ( size[1] > 64 ) ) return 1; - - return 0; -} - - -int hdfile_is_scalar(struct hdfile *f, const char *name, int verbose) -{ - hid_t dh; - hid_t sh; - hsize_t size[3]; - hid_t type; - int ndims; - int i; - int check; - - check = check_path_existence(f->fh, name); - if ( check == 0 ) { - ERROR("No such scalar field '%s'\n", name); - return 0; - } - - dh = H5Dopen2(f->fh, name, H5P_DEFAULT); - type = H5Dget_type(dh); - - /* Get the dimensionality. We have to cope with scalars expressed as - * arrays with all dimensions 1, as well as zero-d arrays. */ - sh = H5Dget_space(dh); - ndims = H5Sget_simple_extent_ndims(sh); - if ( ndims > 3 ) { - if ( verbose ) { - ERROR("Too many dimensions (%i).\n", ndims); - } - H5Sclose(sh); - H5Tclose(type); - H5Dclose(dh); - return 0; - } - - /* Check that the size in all dimensions is 1 */ - H5Sget_simple_extent_dims(sh, size, NULL); - H5Sclose(sh); - H5Tclose(type); - H5Dclose(dh); - for ( i=0; i<ndims; i++ ) { - if ( size[i] != 1 ) { - if ( verbose ) { - ERROR("%s not a scalar value (ndims=%i," - "size[%i]=%i)\n", - name, ndims, i, (int)size[i]); - } - return 0; - } - } - - return 1; -} - - -struct copy_hdf5_field -{ - char **fields; - int n_fields; - int max_fields; -}; - - -struct copy_hdf5_field *new_copy_hdf5_field_list() -{ - struct copy_hdf5_field *n; - - n = calloc(1, sizeof(struct copy_hdf5_field)); - if ( n == NULL ) return NULL; - - n->max_fields = 32; - n->fields = malloc(n->max_fields*sizeof(char *)); - if ( n->fields == NULL ) { - free(n); - return NULL; - } - - return n; -} - - -void free_copy_hdf5_field_list(struct copy_hdf5_field *n) -{ - int i; - for ( i=0; i<n->n_fields; i++ ) { - free(n->fields[i]); - } - free(n->fields); - free(n); -} - - -void add_copy_hdf5_field(struct copy_hdf5_field *copyme, - const char *name) -{ - int i; - - /* Already on the list? Don't re-add if so. */ - for ( i=0; i<copyme->n_fields; i++ ) { - if ( strcmp(copyme->fields[i], name) == 0 ) return; - } - - /* Need more space? */ - if ( copyme->n_fields == copyme->max_fields ) { - - char **nfields; - int nmax = copyme->max_fields + 32; - - nfields = realloc(copyme->fields, nmax*sizeof(char *)); - if ( nfields == NULL ) { - ERROR("Failed to allocate space for new HDF5 field.\n"); - return; - } - - copyme->max_fields = nmax; - copyme->fields = nfields; - - } - - copyme->fields[copyme->n_fields] = strdup(name); - if ( copyme->fields[copyme->n_fields] == NULL ) { - ERROR("Failed to add field for copying '%s'\n", name); - return; - } - - copyme->n_fields++; -} - - -void copy_hdf5_fields(struct hdfile *f, const struct copy_hdf5_field *copyme, - FILE *fh, struct event *ev) -{ - int i; - - if ( copyme == NULL ) return; - - for ( i=0; i<copyme->n_fields; i++ ) { - - char *val; - char *field; - - field = copyme->fields[i]; - val = hdfile_get_string_value(f, field, ev); - - if ( field[0] == '/' ) { - fprintf(fh, "hdf5%s = %s\n", field, val); - } else { - fprintf(fh, "hdf5/%s = %s\n", field, val); - } - - free(val); - - } -} - - -static int make_dataspaces(hid_t dh, struct event *ev, hid_t *memspace, - hid_t *filespace) -{ - hsize_t *f_offset, *f_count; - hsize_t *m_offset, *m_count; - hid_t sh, mh; - int ndims; - int i; - - if ( ev == NULL ) { - ERROR("Can't make dataspaces: no event ID\n"); - return 1; - } - - /* Check that there are at least as many dim entries as dimensions */ - sh = H5Dget_space(dh); - ndims = H5Sget_simple_extent_ndims(sh); - if ( ndims > ev->dim_length ) { - return 1; - } - - /* Now set up arrays of offsets and counts in files and memory */ - f_offset = malloc(sizeof(hsize_t)*ndims); - f_count = malloc(sizeof(hsize_t)*ndims); - m_offset = malloc(sizeof(hsize_t)*ndims); - m_count = malloc(sizeof(hsize_t)*ndims); - if ( (f_offset == NULL) || (f_count == NULL) - || (m_offset == NULL) || (m_count == NULL) ) return 1; - - for ( i=0; i<ev->dim_length; i++ ) { - f_offset[i] = ev->dim_entries[i]; - f_count[i] = 1; - m_offset[i] = 0; - m_count[i] = 1; - } - - if ( H5Sselect_hyperslab(sh, H5S_SELECT_SET, - f_offset, NULL, f_count, NULL) ) return 1; - - free(f_offset); - free(f_count); - - mh = H5Screate_simple(ndims, m_count, NULL); - if ( H5Sselect_hyperslab(mh, H5S_SELECT_SET, - m_offset, NULL, m_count, NULL) ) return 1; - free(m_offset); - free(m_count); - - *memspace = mh; - *filespace = sh; - - return 0; -} - - -static char *read_vlen_string(hid_t dh, struct event *ev) -{ - hid_t memspace, filespace; - herr_t r; - char *tmp; - hid_t type; - - if ( make_dataspaces(dh, ev, &memspace, &filespace) ) { - return strdup("[couldn't make dataspaces - variable len]"); - } - - type = H5Dget_type(dh); - r = H5Dread(dh, type, memspace, filespace, H5P_DEFAULT, &tmp); - H5Tclose(type); - if ( r < 0 ) { - return strdup("[couldn't read vlen string]"); - } - - H5Sclose(memspace); - H5Sclose(filespace); - - /* Variable strings are 0-terminated */ - chomp(tmp); - return tmp; -} - - -static char *read_fixed_string(hid_t dh, struct event *ev) -{ - hid_t memspace, filespace; - herr_t r; - hid_t sh, type; - size_t size; - char *tmp; - - type = H5Dget_type(dh); - size = H5Tget_size(type); - tmp = malloc(size+1); - if ( tmp == NULL ) { - H5Tclose(type); - return strdup("[couldn't allocate string]"); - } - - if ( ev == NULL ) { - - /* Try a simple fixed-length string */ - sh = H5Dget_space(dh); - if ( H5Sget_simple_extent_ndims(sh) ) { - H5Tclose(type); - return strdup("[non-scalar string]"); - } - - sh = H5Screate(H5S_SCALAR); - r = H5Dread(dh, type, sh, H5S_ALL, H5P_DEFAULT, tmp); - H5Sclose(sh); - if ( r < 0 ) { - free(tmp); - H5Tclose(type); - return strdup("[couldn't read scalar string]"); - } else { - H5Tclose(type); - tmp[size] = '\0'; - chomp(tmp); - return tmp; - } - } - - if ( make_dataspaces(dh, ev, &memspace, &filespace) ) { - H5Tclose(type); - return strdup("[couldn't make dataspaces - fixed len]"); - } - - r = H5Dread(dh, type, memspace, filespace, H5P_DEFAULT, tmp); - if ( r < 0 ) { - H5Tclose(type); - return strdup("[couldn't read string]"); - } - - H5Tclose(type); - H5Sclose(memspace); - H5Sclose(filespace); - - tmp[size] = '\0'; - chomp(tmp); - return tmp; -} - - -static char *read_general_string(hid_t dh, struct event *ev) -{ - htri_t v; - hid_t type; - - type = H5Dget_type(dh); - v = H5Tis_variable_str(type); - H5Tclose(type); - - if ( v < 0 ) { - return strdup("[unrecognised string type]"); - - } else if ( v > 0 ) { - /* Variable length string */ - return read_vlen_string(dh, ev); - - } else { - /* Fixed-length string */ - return read_fixed_string(dh, ev); - - } -} - - -char *hdfile_get_string_value(struct hdfile *f, const char *name, - struct event *ev) -{ - hid_t dh; - hid_t type; - hid_t class; - int buf_i; - double buf_f; - char *tmp = NULL; - char *subst_name = NULL; - - if ( (ev != NULL) && (ev->path_length != 0) ) { - subst_name = retrieve_full_path(ev, name); - } else { - subst_name = strdup(name); - } - - dh = H5Dopen2(f->fh, subst_name, H5P_DEFAULT); - if ( dh < 0 ) { - free(subst_name); - return strdup("[couldn't read string]"); - } - - type = H5Dget_type(dh); - class = H5Tget_class(type); - H5Tclose(type); - - if ( class == H5T_STRING ) { - - free(subst_name); - tmp = read_general_string(dh, ev); - H5Dclose(dh); - return tmp; - - } else { - - int r; - - H5Dclose(dh); - - switch ( class ) { - - case H5T_FLOAT : - r = hdfile_get_value(f, subst_name, ev, &buf_f, - H5T_NATIVE_DOUBLE); - free(subst_name); - if ( r == 0 ) { - tmp = malloc(256); - if ( tmp == NULL ) { - ERROR("Failed to allocate float\n"); - return NULL; - } - snprintf(tmp, 255, "%f", buf_f); - return tmp; - } else { - return NULL; - } - break; - - case H5T_INTEGER : - r = hdfile_get_value(f, subst_name, ev, &buf_i, - H5T_NATIVE_INT); - free(subst_name); - if ( r == 0 ) { - tmp = malloc(256); - if ( tmp == NULL ) { - ERROR("Failed to allocate int buf!\n"); - return NULL; - } - snprintf(tmp, 255, "%d", buf_i); - return tmp; - - } else { - return NULL; - } - break; - - default : - ERROR("Unrecognised type: %s\n", subst_name); - free(subst_name); - return NULL; - } - - } -} - - -char **hdfile_read_group(struct hdfile *f, int *n, const char *parent, - int **p_is_group, int **p_is_image) -{ - hid_t gh; - hsize_t num; - char **res; - int i; - int *is_group; - int *is_image; - H5G_info_t ginfo; - - gh = H5Gopen2(f->fh, parent, H5P_DEFAULT); - if ( gh < 0 ) { - *n = 0; - return NULL; - } - - if ( H5Gget_info(gh, &ginfo) < 0 ) { - /* Whoopsie */ - *n = 0; - return NULL; - } - num = ginfo.nlinks; - *n = num; - if ( num == 0 ) return NULL; - - res = malloc(num*sizeof(char *)); - is_image = malloc(num*sizeof(int)); - is_group = malloc(num*sizeof(int)); - *p_is_image = is_image; - *p_is_group = is_group; - - for ( i=0; i<num; i++ ) { - - char buf[256]; - hid_t dh; - H5I_type_t type; - - H5Lget_name_by_idx(gh, ".", H5_INDEX_NAME, H5_ITER_NATIVE, - i, buf, 255, H5P_DEFAULT); - res[i] = malloc(512); - if ( strlen(parent) > 1 ) { - snprintf(res[i], 511, "%s/%s", parent, buf); - } else { - snprintf(res[i], 511, "%s%s", parent, buf); - } /* ick */ - - is_image[i] = 0; - is_group[i] = 0; - dh = H5Oopen(gh, buf, H5P_DEFAULT); - if ( dh < 0 ) continue; - type = H5Iget_type(dh); - - if ( type == H5I_GROUP ) { - is_group[i] = 1; - } else if ( type == H5I_DATASET ) { - is_image[i] = looks_like_image(dh); - } - H5Oclose(dh); - - } - - H5Gclose(gh); - - return res; -} - - -int hdfile_set_first_image(struct hdfile *f, const char *group) -{ - char **names; - int *is_group; - int *is_image; - int n, i, j; - - names = hdfile_read_group(f, &n, group, &is_group, &is_image); - if ( n == 0 ) return 1; - - for ( i=0; i<n; i++ ) { - - if ( is_image[i] ) { - hdfile_set_image(f, names[i]); - for ( j=0; j<n; j++ ) free(names[j]); - free(is_image); - free(is_group); - free(names); - return 0; - } else if ( is_group[i] ) { - if ( !hdfile_set_first_image(f, names[i]) ) { - for ( j=0; j<n; j++ ) free(names[j]); - free(is_image); - free(is_group); - free(names); - return 0; - } - } - - } - - for ( j=0; j<n; j++ ) free(names[j]); - free(is_image); - free(is_group); - free(names); - - return 1; -} - - -struct parse_params { - struct hdfile *hdfile; - int path_dim; - const char *path; - struct event *curr_event; - struct event_list *ev_list; - int top_level; -}; - - -int check_path_existence(hid_t fh, const char *path) -{ - - char buffer[256]; - char buffer_full_path[2048]; - herr_t herrt; - struct H5O_info_t ob_info; - char *path_copy = strdup(path); - char *start = path_copy; - char *sep = NULL; - - buffer[0] = '\0'; - buffer_full_path[0] = '\0'; - - if ( strcmp(path_copy, "/" ) == 0 ) { - return 1; - } - - do { - - int check; - - sep = strstr(start, "/"); - if ( sep != NULL && strlen(sep) == 1 ) { - ERROR("Error: Data path ends with a / symbol\n"); - free(path_copy); - return 1; - } - - if ( sep != NULL ) { - - if ( sep == start ) { - start = sep+1; - strcat(buffer_full_path, "/"); - continue; - } - - strncpy(buffer, start, sep-start); - buffer[sep-start] = '\0'; - strcat(buffer_full_path, buffer); - - check = H5Lexists(fh, buffer_full_path, H5P_DEFAULT); - if ( check == 0 ) { - return 0; - } else { - herrt = H5Oget_info_by_name(fh, buffer_full_path, - &ob_info, - H5P_DEFAULT); - if ( herrt < 0 ) { - return -1; - } - if ( ob_info.type != H5O_TYPE_GROUP ) { - return 0; - } - - start = sep+1; - strcat(buffer_full_path, "/"); - - } - - } else { - - strcpy(buffer, start); - strcat(buffer_full_path, buffer); - - check = H5Lexists(fh, buffer_full_path, H5P_DEFAULT); - if ( check == 0 ) { - return 0; - } - - } - } while (sep); - - free(path_copy); - return 1; - -} - - -static herr_t parse_file_event_structure(hid_t loc_id, char *name, - const H5L_info_t *info, - struct parse_params *pp) - -{ - char *substituted_path; - char *ph_loc; - char *truncated_path; - htri_t check; - herr_t herrt_iterate, herrt_info; - struct H5O_info_t object_info; - - if ( !pp->top_level ) { - - int fail_push; - - fail_push = push_path_entry_to_event(pp->curr_event, name); - if ( fail_push ) { - return -1; - } - - substituted_path = event_path_placeholder_subst(name, pp->path); - - } else { - substituted_path = strdup(pp->path); - } - - if ( pp->top_level == 1 ) { - pp->top_level = 0; - } - - truncated_path = strdup(substituted_path); - ph_loc = strstr(substituted_path,"%"); - if ( ph_loc != NULL) { - truncated_path[ph_loc-substituted_path] = '\0'; - } - - herrt_iterate = 0; - herrt_info = 0; - - check = check_path_existence(pp->hdfile->fh, truncated_path); - if ( check == 0 ) { - pop_path_entry_from_event(pp->curr_event); - return 0; - } else { - - herrt_info = H5Oget_info_by_name(pp->hdfile->fh, truncated_path, - &object_info, H5P_DEFAULT); - if ( herrt_info < 0 ) { - free(truncated_path); - free(substituted_path); - return -1; - } - - if ( pp->curr_event->path_length == pp->path_dim - && object_info.type == H5O_TYPE_DATASET ) - { - - int fail_append; - - fail_append = append_event_to_event_list(pp->ev_list, - pp->curr_event); - if ( fail_append ) { - free(truncated_path); - free(substituted_path); - return -1; - } - - pop_path_entry_from_event(pp->curr_event); - return 0; - - } else { - - pp->path = substituted_path; - - if ( object_info.type == H5O_TYPE_GROUP ) { - - herrt_iterate = H5Literate_by_name(pp->hdfile->fh, - truncated_path, H5_INDEX_NAME, - H5_ITER_NATIVE, NULL, - (H5L_iterate_t)parse_file_event_structure, - (void *)pp, H5P_DEFAULT); - } - } - } - - pop_path_entry_from_event(pp->curr_event); - - free(truncated_path); - free(substituted_path); - - return herrt_iterate; -} - - -static int fill_paths(struct hdfile *hdfile, struct detector *det, int pi, - struct event_list *master_el) -{ - struct parse_params pparams; - struct event *empty_event; - struct event_list *panel_ev_list; - int ei; - int check; - - empty_event = initialize_event(); - panel_ev_list = initialize_event_list(); - if ( (empty_event == NULL) || (panel_ev_list == NULL) ) - { - ERROR("Failed to allocate memory for event list.\n"); - return 1; - } - - pparams.path = det->panels[pi].data; - pparams.hdfile = hdfile; - pparams.path_dim = det->path_dim; - pparams.curr_event = empty_event; - pparams.top_level = 1; - pparams.ev_list = panel_ev_list; - - check = parse_file_event_structure(hdfile->fh, NULL, NULL, &pparams); - if ( check < 0 ) { - free_event(empty_event); - free_event_list(panel_ev_list); - return 1; - } - - for ( ei=0; ei<panel_ev_list->num_events; ei++ ) { - - int fail_add; - - fail_add = add_non_existing_event_to_event_list(master_el, - panel_ev_list->events[ei]); - if ( fail_add ) { - free_event(empty_event); - free_event_list(panel_ev_list); - return 1; - } - - } - - free_event(empty_event); - free_event_list(panel_ev_list); - - return 0; -} - - -static int check_dims(struct hdfile *hdfile, struct panel *p, struct event *ev, - struct event_list *events, int *global_path_dim) -{ - char *full_panel_path; - hid_t dh; - hid_t sh; - int dims; - hsize_t *size; - hsize_t *max_size; - int hsdi; - int panel_path_dim = 0; - struct dim_structure *panel_dim_structure; - - /* Get the full path for this panel in this event */ - full_panel_path = retrieve_full_path(ev, p->data); - - dh = H5Dopen2(hdfile->fh, full_panel_path, H5P_DEFAULT); - if ( dh < 0 ) { - ERROR("Error opening '%s'\n", full_panel_path); - ERROR("Failed to enumerate events. " - "Check your geometry file.\n"); - return 1; - } - - sh = H5Dget_space(dh); - dims = H5Sget_simple_extent_ndims(sh); - size = malloc(dims*sizeof(hsize_t)); - max_size = malloc(dims*sizeof(hsize_t)); - if ( (size==NULL) || (max_size==NULL) ) { - ERROR("Failed to allocate memory for dimensions\n"); - return 1; - } - - dims = H5Sget_simple_extent_dims(sh, size, max_size); - - panel_dim_structure = p->dim_structure; - for ( hsdi=0; hsdi<panel_dim_structure->num_dims; hsdi++ ) { - if ( panel_dim_structure->dims[hsdi] == HYSL_PLACEHOLDER ) { - panel_path_dim = size[hsdi]; - break; - } - } - - if ( *global_path_dim == -1 ) { - - *global_path_dim = panel_path_dim; - - } else if ( panel_path_dim != *global_path_dim ) { - - ERROR("All panels must have the same number of frames\n"); - ERROR("Panel %s has %i frames in one dimension, but the first " - "panel has %i.\n", - p->name, panel_path_dim, *global_path_dim); - free(size); - free(max_size); - return 1; - } - - H5Sclose(sh); - H5Dclose(dh); - - return 0; -} - - -struct event_list *fill_event_list(struct hdfile *hdfile, struct detector *det) -{ - struct event_list *master_el; - - master_el = initialize_event_list(); - if ( master_el == NULL ) { - ERROR("Failed to allocate event list.\n"); - return NULL; - } - - /* First expand any placeholders in the HDF5 paths */ - if ( det->path_dim != 0 ) { - int pi; - for ( pi=0; pi<det->n_panels; pi++ ) { - if ( fill_paths(hdfile, det, pi, master_el) ) { - ERROR("Failed to enumerate paths.\n"); - return NULL; - } - } - } - - /* Now enumerate the placeholder dimensions */ - if ( det->dim_dim > 0 ) { - - struct event_list *master_el_with_dims; - int evi; - - /* If there were no HDF5 path placeholders, add a dummy event */ - if ( master_el->num_events == 0 ) { - struct event *empty_ev; - empty_ev = initialize_event(); - append_event_to_event_list(master_el, empty_ev); - free(empty_ev); - } - - master_el_with_dims = initialize_event_list(); - - /* For each event so far, expand the dimensions */ - for ( evi=0; evi<master_el->num_events; evi++ ) { - - int pi; - int global_path_dim = -1; - int mlwd; - - /* Check the dimensionality of each panel */ - for ( pi=0; pi<det->n_panels; pi++ ) { - if ( check_dims(hdfile, &det->panels[pi], - master_el->events[evi], - master_el_with_dims, - &global_path_dim) ) - { - ERROR("Failed to enumerate dims.\n"); - return NULL; - } - } - - /* Add this dimensionality to all events */ - for ( mlwd=0; mlwd<global_path_dim; mlwd++ ) { - - struct event *mlwd_ev; - - mlwd_ev = copy_event(master_el->events[evi]); - push_dim_entry_to_event(mlwd_ev, mlwd); - append_event_to_event_list(master_el_with_dims, - mlwd_ev); - free_event(mlwd_ev); - } - - } - - free_event_list(master_el); - return master_el_with_dims; - - } else { - - return master_el; - - } -} diff --git a/libcrystfel/src/hdf5-file.h b/libcrystfel/src/hdf5-file.h deleted file mode 100644 index 99f231e1..00000000 --- a/libcrystfel/src/hdf5-file.h +++ /dev/null @@ -1,117 +0,0 @@ -/* - * hdf5-file.h - * - * Read/write HDF5 data files - * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, - * a research centre of the Helmholtz Association. - * - * Authors: - * 2009-2017 Thomas White <taw@physics.org> - * 2014 Valerio Mariani - - * - * This file is part of CrystFEL. - * - * CrystFEL is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * CrystFEL is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. - * - */ - -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - -#ifndef HDF5_H -#define HDF5_H - -struct event_list; - -#include <stdint.h> -#include <hdf5.h> -#include "image.h" -#include "events.h" - -struct hdfile; -struct copy_hdf5_field; - -#include "image.h" -#include "events.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \file hdf5-file.h - * HDF5 abstraction layer - */ - -extern int hdf5_write(const char *filename, const void *data, - int width, int height, int type); - -extern int hdf5_write_image(const char *filename, const struct image *image, - char *element); - -extern int hdf5_read(struct hdfile *f, struct image *image, - const char* element, int satcorr); - -extern int hdf5_read2(struct hdfile *f, struct image *image, - struct event *ev, int satcorr); - -extern int check_path_existence(hid_t fh, const char *path); - -extern struct hdfile *hdfile_open(const char *filename); -int hdfile_set_image(struct hdfile *f, const char *path); - -extern int16_t *hdfile_get_image_binned(struct hdfile *hdfile, - int binning, int16_t *maxp); -extern char **hdfile_read_group(struct hdfile *f, int *n, const char *parent, - int **p_is_group, int **p_is_image); -extern int hdfile_set_first_image(struct hdfile *f, const char *group); -extern void hdfile_close(struct hdfile *f); - -extern int get_peaks(struct image *image, struct hdfile *f, const char *p); - -extern int get_peaks_2(struct image *image, struct hdfile *f, const char *p, - int half_pixel_shift); - -extern int get_peaks_cxi(struct image *image, struct hdfile *f, const char *p, - struct filename_plus_event *fpe); - -extern int get_peaks_cxi_2(struct image *image, struct hdfile *f, const char *p, - struct filename_plus_event *fpe, - int half_pixel_shift); - -extern struct copy_hdf5_field *new_copy_hdf5_field_list(void); -extern void free_copy_hdf5_field_list(struct copy_hdf5_field *f); - -extern void copy_hdf5_fields(struct hdfile *f, - const struct copy_hdf5_field *copyme, - FILE *fh, struct event *ev); -extern void add_copy_hdf5_field(struct copy_hdf5_field *copyme, - const char *name); -extern struct event_list *fill_event_list(struct hdfile* hdfile, - struct detector* det); - -extern int hdfile_get_value(struct hdfile *f, const char *name, - struct event *ev, void *val, hid_t memtype); -extern int hdfile_is_scalar(struct hdfile *f, const char *name, int verbose); -extern char *hdfile_get_string_value(struct hdfile *f, const char *name, - struct event *ev); - -#ifdef __cplusplus -} -#endif - -#endif /* HDF5_H */ diff --git a/libcrystfel/src/image-cbf.c b/libcrystfel/src/image-cbf.c new file mode 100644 index 00000000..c9de17df --- /dev/null +++ b/libcrystfel/src/image-cbf.c @@ -0,0 +1,634 @@ +/* + * image-cbf.c + * + * Image loading, CBF parts + * + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2020 Thomas White <taw@physics.org> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include <stdlib.h> +#include <assert.h> +#include <math.h> +#include <stdio.h> +#include <zlib.h> +#include <unistd.h> + +#include "image.h" +#include "utils.h" +#include "detgeom.h" + +#include "datatemplate.h" +#include "datatemplate_priv.h" + +static void add_out(float val, float *data_out, int nmemb_out, + int *outpos, int *nrej) +{ + if ( *outpos < nmemb_out ) { + data_out[(*outpos)++] = val; + } else { + (*nrej)++; + } +} + + +/* Reverses byte offset compression and converts to single precision float. + * Note that this compression scheme specifies the data format of the input + * data, therefore the X-Binary-Element-Type is completely ignored. */ +static void decode_cbf_byte_offset(float *data_out, int nmemb_out, + const int8_t *data_in, const size_t n) +{ + int inpos = 0; + int outpos = 0; + int nrej = 0; + float val = 0.0; + + while ( inpos < n ) { + + int64_t delta = data_in[inpos++]; + + if ( (delta >= -127) && (delta <= 127) ) { + val += delta; + add_out(val, data_out, nmemb_out, &outpos, &nrej); + continue; + } + + delta = *(int16_t *)(data_in+inpos); + inpos += 2; + + if ( (delta >= -32767) && (delta <= 32767) ) { + val += delta; + add_out(val, data_out, nmemb_out, &outpos, &nrej); + continue; + } + + delta = *(int32_t *)(data_in+inpos); + inpos += 4; + + if ( (delta >= -2147483647) && (delta <= 2147483647) ) { + val += delta; + add_out(val, data_out, nmemb_out, &outpos, &nrej); + continue; + } + + delta = *(int64_t *)(data_in+inpos); + inpos += 8; + val += delta; + add_out(val, data_out, nmemb_out, &outpos, &nrej); + + } + + if ( nrej > 0 ) { + STATUS("%i elements rejected\n", nrej); + } +} + + +static int binary_start(char *data) +{ + char *datac = data; + if ( (datac[0] == (char)0x0c) && (datac[1] == (char)0x1a) + && (datac[2] == (char)0x04) && (datac[3] == (char)0xd5) ) return 1; + return 0; +} + + +enum cbf_data_conversion +{ + CBF_NO_CONVERSION, + CBF_BYTE_OFFSET, + CBF_PACKED, + CBF_CANONICAL +}; + +enum cbf_data_type +{ + CBF_NO_TYPE, + CBF_ELEMENT_U8, + CBF_ELEMENT_S8, + CBF_ELEMENT_U16, + CBF_ELEMENT_S16, + CBF_ELEMENT_U32, + CBF_ELEMENT_S32, + CBF_ELEMENT_F32, + CBF_ELEMENT_F64, +}; + + +static enum cbf_data_type parse_element_type(const char *t) +{ + if ( strstr(t, "signed 8-bit integer") != NULL ) + { + return CBF_ELEMENT_S8; + } + + if ( strstr(t, "unsigned 8-bit integer") != NULL ) + { + return CBF_ELEMENT_U8; + } + + if ( strstr(t, "signed 16-bit integer") != NULL ) + { + return CBF_ELEMENT_S16; + } + + if ( strstr(t, "unsigned 16-bit integer") != NULL ) + { + return CBF_ELEMENT_U16; + } + + if ( strstr(t, "signed 32-bit integer") != NULL ) + { + return CBF_ELEMENT_S32; + } + + if ( strstr(t, "unsigned 32-bit integer") != NULL ) + { + return CBF_ELEMENT_U32; + } + + if ( strstr(t, "signed 32-bit real IEEE") != NULL ) + { + return CBF_ELEMENT_F32; + } + + if ( strstr(t, "signed 64-bit real IEEE") != NULL ) + { + return CBF_ELEMENT_F64; + } + + /* complex type is unsupported */ + + return CBF_NO_TYPE; +} + + +static size_t element_size(enum cbf_data_type t) +{ + switch ( t ) { + case CBF_ELEMENT_S8 : return 1; + case CBF_ELEMENT_U8 : return 1; + case CBF_ELEMENT_S16 : return 2; + case CBF_ELEMENT_U16 : return 2; + case CBF_ELEMENT_S32 : return 4; + case CBF_ELEMENT_U32 : return 4; + case CBF_ELEMENT_F32 : return 4; + case CBF_ELEMENT_F64 : return 8; + default : return 0; + } +} + + + +static int convert_type(float *data_out, long nmemb_exp, + enum cbf_data_type eltype, + void *data_in, size_t data_in_len) +{ + long int i; + long int o = 0; + size_t elsize = element_size(eltype); + + if ( elsize == 0 ) return 1; + + if ( nmemb_exp * elsize > data_in_len ) { + ERROR("Not enough CBF data for image size/type!\n"); + return 1; + } + + for ( i=0; i<nmemb_exp; i++ ) { + switch ( eltype ) { + + case CBF_ELEMENT_S8: + data_out[o++] = ((int8_t *)data_in)[i]; + break; + + case CBF_ELEMENT_U8: + data_out[o++] = ((uint8_t *)data_in)[i]; + break; + + case CBF_ELEMENT_S16: + data_out[o++] = ((int16_t *)data_in)[i]; + break; + + case CBF_ELEMENT_U16: + data_out[o++] = ((uint16_t *)data_in)[i]; + break; + + case CBF_ELEMENT_S32: + data_out[o++] = ((int32_t *)data_in)[i]; + break; + + case CBF_ELEMENT_U32: + data_out[o++] = ((uint32_t *)data_in)[i]; + break; + + case CBF_ELEMENT_F32: + data_out[o++] = ((float *)data_in)[i]; + break; + + case CBF_ELEMENT_F64: + data_out[o++] = ((double *)data_in)[i]; + break; + + case CBF_NO_TYPE: + break; + } + } + + return 0; +} + + +static float *read_cbf_data(const char *filename, int gz, int *w, int *h) +{ + FILE *fh; + void *buf = NULL; + char *rval; + size_t data_compressed_len = 0; + float *data_out = NULL; + enum cbf_data_conversion data_conversion = CBF_NO_CONVERSION; + enum cbf_data_type data_type = CBF_ELEMENT_U32; /* ITG (2006) 2.3.3.3 */ + int in_binary_section = 0; + + *w = 0; + *h = 0; + + if ( !gz ) { + + fh = fopen(filename, "rb"); + if ( fh == NULL ) { + ERROR("Failed to open '%s'\n", filename); + return NULL; + } + + } else { + + gzFile gzfh; + size_t len, len_read; + const size_t bufinc = 8*1024*1024; /* Allocate buffer in 8Mb chunks */ + size_t bufsz = bufinc; + + gzfh = gzopen(filename, "rb"); + if ( gzfh == NULL ) return NULL; + + /* Set larger buffer size for hopefully faster uncompression */ + gzbuffer(gzfh, 128*1024); + + buf = malloc(bufsz); + if ( buf == NULL ) return NULL; + + len = 0; + do { + + len_read = gzread(gzfh, buf+len, bufinc); + if ( len_read == -1 ) return NULL; + len += len_read; + + if ( len_read == bufinc ) { + bufsz += bufinc; + buf = realloc(buf, bufsz); + if ( buf == NULL ) return NULL; + } + + } while ( len_read == bufinc ); + + fh = fmemopen(buf, len, "rb"); + if ( fh == NULL ) return NULL; + + gzclose(gzfh); + + } + + /* This is really horrible, but there are at least three different types + * of header mingled together (CIF, MIME, DECTRIS), so a real parser + * would be very complicated and much more likely to have weird bugs. */ + do { + + char line[1024]; + long line_start; + + line_start = ftell(fh); + rval = fgets(line, 1023, fh); + if ( rval == NULL ) break; + chomp(line); + + if ( strcmp(line, "--CIF-BINARY-FORMAT-SECTION--") == 0 ) { + in_binary_section = 1; + } + + if ( strcmp(line, "--CIF-BINARY-FORMAT-SECTION----") == 0 ) { + in_binary_section = 0; + } + + if ( in_binary_section ) { + + if ( strncmp(line, "X-Binary-Size: ", 15) == 0 ) { + data_compressed_len = atoi(line+15); + } + + if ( strncmp(line, "X-Binary-Element-Byte-Order: ", 29) == 0 ) { + const char *elbo = line+29; + if ( strcmp(elbo, "LITTLE_ENDIAN") != 0 ) { + ERROR("Unsupported endianness: %s\n", elbo); + free(buf); + fclose(fh); + return NULL; + } + } + + /* Try to spot compression algorithm */ + if ( strstr(line, "conversions=\"x-CBF_BYTE_OFFSET\"") != NULL ) { + data_conversion = CBF_BYTE_OFFSET; + } else if ( strstr(line, "conversions=\"x-CBF_CANONICAL\"") != NULL ) { + data_conversion = CBF_CANONICAL; + } else if ( strstr(line, "conversions=\"x-CBF_PACKED\"") != NULL ) { + data_conversion = CBF_PACKED; + } else if ( strstr(line, "conversions=") != NULL ) { + ERROR("Unrecognised CBF content conversion: %s\n", line); + free(buf); + fclose(fh); + return NULL; + } + + /* Likewise, element type */ + if ( strncmp(line, "X-Binary-Element-Type: ", 23) == 0 ) + { + const char *eltype = (line+23); + data_type = parse_element_type(eltype); + if ( data_type == CBF_NO_TYPE ) { + ERROR("Unrecognised element type: %s\n", + eltype); + free(buf); + fclose(fh); + return NULL; + } + } + + if ( strncmp(line, "X-Binary-Size-Fastest-Dimension: ", 33) == 0 ) { + *w = atoi(line+33); + } + + if ( strncmp(line, "X-Binary-Size-Second-Dimension: ", 32) == 0 ) { + *h = atoi(line+32); + } + + } + + if ( in_binary_section && binary_start(line) ) { + + size_t len_read; + int nmemb_exp; + void *data_compressed; + int r = 0; + + if ( data_compressed_len == 0 ) { + ERROR("Found CBF data before X-Binary-Size!\n"); + free(buf); + fclose(fh); + return NULL; + } + + if ( (*w == 0) || (*h == 0) ) { + ERROR("Found CBF data before dimensions!\n"); + free(buf); + fclose(fh); + return NULL; + } + + if ( data_compressed_len > 100*1024*1024 ) { + ERROR("Stated CBF data size too big\n"); + free(buf); + fclose(fh); + return NULL; + } + + data_compressed = malloc(data_compressed_len); + if ( data_compressed == NULL ) { + ERROR("Failed to allocate memory for CBF data\n"); + free(buf); + fclose(fh); + return NULL; + } + + fseek(fh, line_start+4, SEEK_SET); + len_read = fread(data_compressed, 1, data_compressed_len, fh); + if ( len_read < data_compressed_len ) { + ERROR("Couldn't read entire CBF data\n"); + free(buf); + free(data_compressed); + fclose(fh); + return NULL; + } + + nmemb_exp = (*w) * (*h); + data_out = malloc(nmemb_exp*sizeof(float)); + if ( data_out == NULL ) { + ERROR("Failed to allocate memory for CBF data\n"); + free(buf); + free(data_compressed); + fclose(fh); + return NULL; + } + + switch ( data_conversion ) { + + case CBF_NO_CONVERSION: + r = convert_type(data_out, nmemb_exp, data_type, + data_compressed, + data_compressed_len); + break; + + case CBF_BYTE_OFFSET: + decode_cbf_byte_offset(data_out, nmemb_exp, + data_compressed, + data_compressed_len); + break; + + case CBF_PACKED: + case CBF_CANONICAL: + ERROR("Don't yet know how to decompress " + "CBF_PACKED or CBF_CANONICAL\n"); + free(buf); + free(data_compressed); + fclose(fh); + return NULL; + + } + + free(data_compressed); + + if ( r ) { + free(buf); + free(data_out); + fclose(fh); + return NULL; + } + + free(buf); + fclose(fh); + return data_out; + + } + + } while ( rval != NULL ); + + ERROR("Reached end of CBF file before finding data.\n"); + free(buf); /* might be NULL */ + return NULL; +} + + +signed int is_cbf_file(const char *filename) +{ + FILE *fh; + char line[1024]; + + fh = fopen(filename, "r"); + if ( fh == NULL ) return -1; + + if ( fgets(line, 1024, fh) == NULL ) return 0; + fclose(fh); + + if ( strstr(line, "CBF") == NULL ) { + return 0; + } + + return 1; +} + + +signed int is_cbfgz_file(const char *filename) +{ + gzFile gzfh; + char line[1024]; + + gzfh = gzopen(filename, "rb"); + if ( gzfh == NULL ) return -1; + if ( gzgets(gzfh, line, 1024) == NULL ) return 0; + gzclose(gzfh); + + if ( strstr(line, "CBF") == NULL ) { + return 0; + } + + return 1; +} + + +int image_cbf_read_mask(struct panel_template *p, + const char *filename, const char *event, + int gz, int *bad, int mask_good, int mask_bad) +{ + ERROR("Mask loading from CBF not yet supported\n"); + return 1; +} + + +static int unpack_panels(struct image *image, + const DataTemplate *dtempl, + float *data, int data_width, int data_height) +{ + int pi; + + image->dp = malloc(dtempl->n_panels * sizeof(float *)); + if ( image->dp == NULL ) { + ERROR("Failed to allocate panels.\n"); + return 1; + } + + for ( pi=0; pi<dtempl->n_panels; pi++ ) { + + struct panel_template *p; + int fs, ss; + int p_w, p_h; + + p = &dtempl->panels[pi]; + p_w = p->orig_max_fs - p->orig_min_fs + 1; + p_h = p->orig_max_ss - p->orig_min_ss + 1; + image->dp[pi] = malloc(p_w*p_h*sizeof(float)); + if ( image->dp[pi] == NULL ) { + ERROR("Failed to allocate panel\n"); + return 1; + } + + if ( (p->orig_min_fs + p_w > data_width) + || (p->orig_min_ss + p_h > data_height) ) + { + ERROR("Panel %s is outside range of data in CBF file\n", + p->name); + return 1; + } + + for ( ss=0; ss<p_h; ss++ ) { + for ( fs=0; fs<p_w; fs++ ) { + + int idx; + int cfs, css; + + cfs = fs+p->orig_min_fs; + css = ss+p->orig_min_ss; + idx = cfs + css*data_width; + + image->dp[pi][fs+p_w*ss] = data[idx]; + + } + } + + } + + return 0; +} + + +int image_cbf_read(struct image *image, + const DataTemplate *dtempl, + const char *filename, + const char *event, + int gz) +{ + float *data; + int w, h; + + if ( access(filename, R_OK) == -1 ) { + ERROR("File does not exist or cannot be read: %s\n", filename); + return 1; + } + + data = read_cbf_data(filename, gz, &w, &h); + if ( data == NULL ) { + ERROR("Failed to read CBF data\n"); + return 1; + } + + unpack_panels(image, dtempl, data, w, h); + free(data); + + //cbf_fill_in_beam_parameters(image->beam, f, image); + //cbf_fill_in_clen(image->det, f); + //fill_in_adu(image); + + return 0; +} diff --git a/libcrystfel/src/image-cbf.h b/libcrystfel/src/image-cbf.h new file mode 100644 index 00000000..ca79a1a3 --- /dev/null +++ b/libcrystfel/src/image-cbf.h @@ -0,0 +1,56 @@ +/* + * image-cbf.h + * + * Image loading, CBF parts + * + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2020 Thomas White <taw@physics.org> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +/* NB This file is NOT part of the public API, and should NOT + * be installed, but rather stays in the libcrystfel source folder. */ + +#ifndef IMAGE_CBF_H +#define IMAGE_CBF_H + +#include "datatemplate_priv.h" + +extern signed int is_cbf_file(const char *filename); + +extern signed int is_cbfgz_file(const char *filename); + +extern int load_mask_cbf(struct panel_template *p, + const char *filename, const char *event, + int gz, int *bad, int mask_good, int mask_bad); + +extern int image_cbf_read(struct image *image, + const DataTemplate *dtempl, + const char *filename, + const char *event, + int gz); + +extern int image_cbf_read_mask(struct panel_template *p, + const char *filename, const char *event, + int gz, int *bad, + int mask_good, int mask_bad); + +#endif /* IMAGE_CBF_H */ diff --git a/libcrystfel/src/image-hdf5.c b/libcrystfel/src/image-hdf5.c new file mode 100644 index 00000000..ac987970 --- /dev/null +++ b/libcrystfel/src/image-hdf5.c @@ -0,0 +1,2007 @@ +/* + * image-hdf5.c + * + * Image loading, HDF5 parts + * + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2020 Thomas White <taw@physics.org> + * 2014-2018 Valerio Mariani + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include <stdlib.h> +#include <assert.h> +#include <math.h> +#include <stdio.h> +#include <hdf5.h> +#include <unistd.h> + +#include "image.h" +#include "utils.h" +#include "detgeom.h" + +#include "datatemplate.h" +#include "datatemplate_priv.h" + + +static void close_hdf5(hid_t fh) +{ + int n_ids, i; + hid_t ids[2048]; + + n_ids = H5Fget_obj_ids(fh, H5F_OBJ_ALL, 2048, ids); + + for ( i=0; i<n_ids; i++ ) { + + hid_t id; + H5I_type_t type; + + id = ids[i]; + + type = H5Iget_type(id); + + if ( type == H5I_GROUP ) H5Gclose(id); + if ( type == H5I_DATASET ) H5Dclose(id); + if ( type == H5I_DATATYPE ) H5Tclose(id); + if ( type == H5I_DATASPACE ) H5Sclose(id); + if ( type == H5I_ATTR ) H5Aclose(id); + + } + + H5Fclose(fh); +} + + +/* Get the path parts of the event ID + * e.g. ev_orig = abc/def/ghi//5/2/7 + * -> [abc, def, ghi], with *pn_plvals=3. + * + * Not part of public API. Not "static" for testing. */ +char **read_path_parts(const char *ev_orig, int *pn_plvals) +{ + char **plvals; + char *ev; + int n_plvals = 0; + char *start; + + plvals = malloc(MAX_PATH_PARTS*sizeof(char *)); + if ( plvals == NULL ) return NULL; + + if ( ev_orig == NULL ) { + /* No ev -> no path parts */ + *pn_plvals = 0; + return plvals; + } + + ev = strdup(ev_orig); + if ( ev == NULL ) return NULL; + + start = ev; + do { + + char *sep; + + sep = strchr(start, '/'); + + if ( sep == NULL ) { + /* This would be very strange, because it + * must at least have // */ + ERROR("Couldn't read path parts ('%s')\n", + start); + free(ev); + free(plvals); + return NULL; + } + + /* Remaining string starts with '/' is end condition */ + if ( sep == start ) break; + + if ( n_plvals == MAX_PATH_PARTS ) { + ERROR("Too many path parts: %s\n", ev_orig); + free(ev); + free(plvals); + return NULL; + } + + sep[0] = '\0'; + plvals[n_plvals++] = strdup(start); + + start = sep+1; + + } while ( 1 ); + + free(ev); + *pn_plvals = n_plvals; + return plvals; +} + + +/* Get the dimension parts of the event ID + * e.g. ev_orig = abc/def/ghi//5/2/7 + * -> [5, 2, 7], with *pn_dvals=3 + * + * Not part of public API. Not "static" for testing. */ +int *read_dim_parts(const char *ev_orig, int *pn_dvals) + +{ + char *ev; + int n_dvals = 0; + int *dvals; + char *start; + int done; + + if ( ev_orig == NULL ) ev_orig = "//"; + + /* Valid event ID? (Just the part after //, please) */ + ev = strstr(ev_orig, "//"); + if ( ev == NULL ) return NULL; + + dvals = malloc(MAX_DIMS*sizeof(int)); + if ( dvals == NULL ) return NULL; + + if ( ev[2] == '\0' ) { + /* No dimension parts - early bailout */ + *pn_dvals = 0; + return dvals; /* NB Not NULL */ + } + + ev = strdup(ev+2); + if ( ev == NULL ) return NULL; + + start = ev; + done = 0; + do { + + char *sep = strchr(start, '/'); + + if ( sep != NULL ) { + sep[0] = '\0'; + } else { + done = 1; + } + + if ( n_dvals == MAX_PATH_PARTS ) { + ERROR("Too many path parts: %s\n", ev_orig); + free(ev); + free(dvals); + return NULL; + } + + if ( start[0] == '\0' ) { + ERROR("Missing dimension: %s\n", ev_orig); + free(ev); + free(dvals); + return NULL; + } + + dvals[n_dvals++] = atoi(start); + + start = sep+1; + + } while ( !done ); + + free(ev); + *pn_dvals = n_dvals; + return dvals; +} + + +static int imh_num_path_placeholders(const char *pattern) +{ + size_t l, i; + int n_pl_exp = 0; + + l = strlen(pattern); + for ( i=0; i<l; i++ ) { + if ( pattern[i] == '%' ) n_pl_exp++; + } + return n_pl_exp; +} + + +/* ev = abc/def/ghi//5/2/7 + * pattern = /data/%/somewhere/%/%/data + * output = /data/abc/somewhere/def/ghi/data + * + * Not part of public API. Not "static" for testing. + */ +char *substitute_path(const char *ev, const char *pattern) +{ + char **plvals; + int n_plvals; + int n_pl_exp; + size_t total_len; + int i; + char *subs; + const char *start; + const char *pl_pos; + + if ( pattern == NULL ) { + ERROR("Pattern cannot be NULL\n"); + return NULL; + } + + plvals = read_path_parts(ev, &n_plvals); + if ( plvals == NULL ) return NULL; + + n_pl_exp = imh_num_path_placeholders(pattern); + + if ( n_plvals != n_pl_exp ) { + ERROR("Wrong number of path placeholders: " + "'%s' (%i) into '%s' (%i)\n", + ev, n_plvals, pattern, n_pl_exp); + return NULL; + } + + if ( n_pl_exp == 0 ) { + /* No placeholders in path */ + for ( i=0; i<n_plvals; i++ ) { + free(plvals[i]); + } + free(plvals); + return strdup(pattern); + } + + total_len = strlen(pattern) - n_pl_exp; + for ( i=0; i<n_plvals; i++ ) { + total_len += strlen(plvals[i]); + } + subs = malloc(total_len+1); + if ( subs == NULL ) { + free(plvals); + return NULL; + } + + pl_pos = strchr(pattern, '%'); + if ( pl_pos == NULL ) { + ERROR("Expected a placeholder char (%): '%s'\n", + pattern); + return NULL; + } + strncpy(subs, pattern, pl_pos-pattern); + subs[pl_pos-pattern] = '\0'; + + start = pl_pos+1; + for ( i=0; i<n_plvals; i++ ) { + + /* Add the placeholder's value */ + strcat(subs, plvals[i]); + free(plvals[i]); + + /* Add the chars up to the next placeholder... */ + pl_pos = strchr(start, '%'); + if ( pl_pos == NULL ) { + /* ... or the end */ + pl_pos = start+strlen(start); + } + strncat(subs, start, pl_pos-start); + start = pl_pos+1; + } + + free(plvals); + + return subs; +} + + +static void make_placeholder_skip(signed int *dt_dims, + signed int *panel_dims) +{ + int i; + int n_dt = 0; + for ( i=0; i<MAX_DIMS; i++ ) { + if ( panel_dims[i] != DIM_PLACEHOLDER ) { + dt_dims[n_dt++] = panel_dims[i]; + } + } +} + + +static int imh_num_placeholders(const struct panel_template *p) +{ + int i; + int n_pl = 0; + for ( i=0; i<MAX_DIMS; i++ ) { + if ( p->dims[i] == DIM_PLACEHOLDER ) n_pl++; + } + return n_pl; +} + + +static int total_dimensions(const struct panel_template *p) +{ + int i; + int n_dim = 0; + for ( i=0; i<MAX_DIMS; i++ ) { + if ( p->dims[i] != DIM_UNDEFINED ) n_dim++; + } + return n_dim; +} + + +static int load_hdf5_hyperslab(struct panel_template *p, + const char *filename, + const char *event, + void **pdata, + hid_t el_type, size_t el_size, + int skip_placeholders_ok, + const char *path_spec) +{ + hid_t fh; + int total_dt_dims; + int plh_dt_dims; + int dt_dims[MAX_DIMS]; + int n_dt_dims; + herr_t r; + hsize_t *f_offset, *f_count; + hid_t dh; + herr_t check; + hid_t dataspace, memspace; + hsize_t dims[2]; + char *panel_full_path; + void *data; + int ndims; + int dim; + int *dim_vals; + int n_dim_vals; + int pl_pos; + + if ( access(filename, R_OK) == -1 ) { + ERROR("File does not exist or cannot be read: %s\n", + filename); + return 1; + } + + fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + if ( fh < 0 ) { + ERROR("Couldn't open file: %s\n", filename); + return 1; + } + + panel_full_path = substitute_path(event, path_spec); + if ( panel_full_path == NULL ) { + ERROR("Invalid path substitution: '%s' '%s'\n", + event, path_spec); + close_hdf5(fh); + return 1; + } + + dh = H5Dopen2(fh, panel_full_path, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("Cannot open data for panel %s (%s) in file %s\n", + p->name, panel_full_path, filename); + free(panel_full_path); + close_hdf5(fh); + return 1; + } + + free(panel_full_path); + + /* Set up dataspace for file + * (determine where to read the data from) */ + dataspace = H5Dget_space(dh); + ndims = H5Sget_simple_extent_ndims(dataspace); + if ( ndims < 0 ) { + ERROR("Failed to get number of dimensions for panel %s\n", + p->name); + close_hdf5(fh); + return 1; + } + + /* Does the array have the expected number of dimensions? */ + total_dt_dims = total_dimensions(p); + plh_dt_dims = imh_num_placeholders(p); + if ( ndims != total_dt_dims ) { + /* If the dimensions match after excluding + * placeholders, it's OK - probably a static mask + * in a multi-event file. */ + if ( skip_placeholders_ok + && (ndims == total_dt_dims - plh_dt_dims) ) + { + make_placeholder_skip(dt_dims, p->dims); + n_dt_dims = total_dt_dims - plh_dt_dims; + } else { + ERROR("Unexpected number of dimensions for " + "panel %s (%i, but expected %i or %i)\n", + p->name, ndims, total_dt_dims, + total_dt_dims - plh_dt_dims); + close_hdf5(fh); + return 1; + } + } else { + int i; + for ( i=0; i<MAX_DIMS; i++ ) { + dt_dims[i] = p->dims[i]; + } + n_dt_dims = total_dt_dims; + } + + f_offset = malloc(ndims*sizeof(hsize_t)); + f_count = malloc(ndims*sizeof(hsize_t)); + if ( (f_offset == NULL) || (f_count == NULL ) ) { + ERROR("Failed to allocate offset or count.\n"); + close_hdf5(fh); + return 1; + } + + /* Get those placeholder values from the event ID */ + dim_vals = read_dim_parts(event, &n_dim_vals); + + pl_pos = 0; + for ( dim=0; dim<n_dt_dims; dim++ ) { + + switch ( dt_dims[dim] ) { + + case DIM_FS: + f_offset[dim] = p->orig_min_fs; + f_count[dim] = p->orig_max_fs - p->orig_min_fs+1; + break; + + case DIM_SS: + f_offset[dim] = p->orig_min_ss; + f_count[dim] = p->orig_max_ss - p->orig_min_ss+1; + break; + + case DIM_PLACEHOLDER: + f_offset[dim] = dim_vals[pl_pos++]; + f_count[dim] = 1; + break; + + case DIM_UNDEFINED: + ERROR("Undefined dimension found!\n"); + break; + + default: + /* Fixed value */ + f_offset[dim] = dt_dims[dim]; + f_count[dim] = 1; + break; + + } + } + + free(dim_vals); + + check = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, + f_offset, NULL, f_count, NULL); + if ( check < 0 ) { + ERROR("Error selecting file dataspace for panel %s\n", + p->name); + free(f_offset); + free(f_count); + close_hdf5(fh); + return 1; + } + + dims[0] = p->orig_max_ss - p->orig_min_ss + 1; + dims[1] = p->orig_max_fs - p->orig_min_fs + 1; + memspace = H5Screate_simple(2, dims, NULL); + + data = malloc(dims[0]*dims[1]*el_size); + if ( data == NULL ) { + ERROR("Failed to allocate panel %s\n", p->name); + free(f_offset); + free(f_count); + close_hdf5(fh); + return 1; + } + + r = H5Dread(dh, el_type, memspace, dataspace, + H5P_DEFAULT, data); + if ( r < 0 ) { + ERROR("Couldn't read data for panel %s\n", + p->name); + free(f_offset); + free(f_count); + free(data); + close_hdf5(fh); + return 1; + } + + free(f_offset); + free(f_count); + close_hdf5(fh); + + *pdata = data; + return 0; +} + + +int image_hdf5_read(struct image *image, + const DataTemplate *dtempl, + const char *filename, + const char *event) +{ + int i; + + image->dp = malloc(dtempl->n_panels*sizeof(float *)); + if ( image->dp == NULL ) { + ERROR("Failed to allocate data array.\n"); + return 1; + } + + if ( event == NULL ) { + event = "//"; + } + + /* Set all pointers to NULL for easier clean-up */ + for ( i=0; i<dtempl->n_panels; i++ ) image->dp[i] = NULL; + + for ( i=0; i<dtempl->n_panels; i++ ) { + if ( load_hdf5_hyperslab(&dtempl->panels[i], filename, + event, (void *)&image->dp[i], + H5T_NATIVE_FLOAT, + sizeof(float), 0, + dtempl->panels[i].data) ) + { + ERROR("Failed to load panel data\n"); + return 1; + } + } + + image->filename = strdup(filename); + image->ev = safe_strdup(event); + + return 0; +} + + +int image_hdf5_read_mask(struct panel_template *p, + const char *filename, const char *event, + int *bad, const char *mask_location, + int mask_good, int mask_bad) +{ + int p_w, p_h; + int *mask = NULL; + long unsigned int j; + + p_w = p->orig_max_fs - p->orig_min_fs + 1; + p_h = p->orig_max_ss - p->orig_min_ss + 1; + + if ( load_hdf5_hyperslab(p, filename, event, + (void *)&mask, H5T_NATIVE_INT, + sizeof(int), 1, mask_location) ) + { + ERROR("Failed to load mask data\n"); + free(mask); + return 1; + } + + for ( j=0; j<p_w*p_h; j++ ) { + + /* Bad if it's missing any of the "good" bits */ + if ( (mask[j] & mask_good) != mask_good ) bad[j] = 1; + + /* Bad if it has any of the "bad" bits. */ + if ( mask[j] & mask_bad ) bad[j] = 1; + + } + + free(mask); + return 0; +} + + +double image_hdf5_get_value(const char *name, const char *filename, + const char *event) +{ + hid_t dh; + hid_t type; + hid_t class; + hid_t sh; + hid_t ms; + hsize_t *f_offset = NULL; + hsize_t *f_count = NULL; + hsize_t m_offset[1]; + hsize_t m_count[1]; + hsize_t msdims[1]; + hsize_t size[64]; + herr_t r; + herr_t check; + int ndims; + int i; + char *subst_name = NULL; + hid_t fh; + double val; + int *dim_vals; + int n_dim_vals; + int dim_val_pos; + + if ( access(filename, R_OK) == -1 ) { + ERROR("File does not exist or cannot be read: %s\n", filename); + return NAN; + } + + fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + if ( fh < 0 ) { + ERROR("Couldn't open file: %s\n", filename); + return NAN; + } + + subst_name = substitute_path(event, name); + if ( subst_name == NULL ) { + ERROR("Invalid event ID '%s'\n", event); + close_hdf5(fh); + return NAN; + } + + dh = H5Dopen2(fh, subst_name, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("No such numeric field '%s'\n", subst_name); + close_hdf5(fh); + return NAN; + } + + type = H5Dget_type(dh); + class = H5Tget_class(type); + + if ( (class != H5T_FLOAT) && (class != H5T_INTEGER) ) { + ERROR("Not a floating point or integer value.\n"); + close_hdf5(fh); + return NAN; + } + + /* Get the dimensionality. We have to cope with scalars expressed as + * arrays with all dimensions 1, as well as zero-d arrays. */ + sh = H5Dget_space(dh); + ndims = H5Sget_simple_extent_ndims(sh); + if ( ndims > 64 ) { + ERROR("Too many dimensions for numeric value\n"); + close_hdf5(fh); + return NAN; + } + H5Sget_simple_extent_dims(sh, size, NULL); + + /* We want to read the value as a scalar */ + m_offset[0] = 0; + m_count[0] = 1; + msdims[0] = 1; + ms = H5Screate_simple(1, msdims, NULL); + + if ( ndims == 0 ) { + /* Easy case, because value is a scalar */ + r = H5Dread(dh, H5T_NATIVE_DOUBLE, ms, sh, H5P_DEFAULT, &val); + if ( r < 0 ) { + ERROR("Couldn't read scalar value from %s.\n", + subst_name); + free(subst_name); + close_hdf5(fh); + return NAN; + } + return val; + } + + dim_vals = read_dim_parts(event, &n_dim_vals); + if ( dim_vals == NULL ) { + ERROR("Couldn't parse event '%s'\n"); + close_hdf5(fh); + return NAN; + } + + f_offset = malloc(ndims*sizeof(hsize_t)); + f_count = malloc(ndims*sizeof(hsize_t)); + if ( (f_offset == NULL) || (f_count == NULL) ) { + ERROR("Couldn't allocate dimension arrays\n"); + close_hdf5(fh); + return NAN; + } + + /* Every dimension of the dataset must either be size 1 or + * large enough to contain the next value from the event ID */ + dim_val_pos = 0; + for ( i=0; i<ndims; i++ ) { + + if ( size[i] != 1 ) { + + if ( size[i] <= dim_vals[dim_val_pos] ) { + ERROR("Array of scalar values is too " + "small (%s, dim %i, ev value %i," + " size %i)\n", + subst_name, i, + dim_vals[dim_val_pos], size[i]); + close_hdf5(fh); + return NAN; + } + + f_offset[i] = dim_vals[dim_val_pos]; + f_count[i] = 1; + dim_val_pos++; + + } else { + + f_offset[i] = 0; + f_count[i] = 1; + + } + + } + + check = H5Sselect_hyperslab(sh, H5S_SELECT_SET, + f_offset, NULL, f_count, NULL); + if ( check <0 ) { + ERROR("Error selecting dataspace for float value\n"); + free(f_offset); + free(f_count); + close_hdf5(fh); + return NAN; + } + + ms = H5Screate_simple(1,msdims,NULL); + check = H5Sselect_hyperslab(ms, H5S_SELECT_SET, + m_offset, NULL, m_count, NULL); + if ( check < 0 ) { + ERROR("Error selecting memory dataspace for float value\n"); + free(f_offset); + free(f_count); + close_hdf5(fh); + return NAN; + } + + r = H5Dread(dh, H5T_NATIVE_DOUBLE, ms, sh, H5P_DEFAULT, &val); + if ( r < 0 ) { + ERROR("Couldn't read value.\n"); + close_hdf5(fh); + return NAN; + } + + free(f_offset); + free(f_count); + free(subst_name); + close_hdf5(fh); + + return val; +} + + +static int read_peak_count(hid_t fh, char *path, int line, + int *num_peaks) +{ + + hid_t dh, sh, mh; + hsize_t size[1]; + hsize_t max_size[1]; + hsize_t offset[1], count[1]; + hsize_t m_offset[1], m_count[1], dimmh[1]; + int tw, r; + + dh = H5Dopen2(fh, path, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("Data block %s not found.\n", path); + return 1; + } + + sh = H5Dget_space(dh); + if ( sh < 0 ) { + H5Dclose(dh); + ERROR("Couldn't get dataspace for data.\n"); + return 1; + } + + if ( H5Sget_simple_extent_ndims(sh) != 1 ) { + ERROR("Data block %s has the wrong dimensionality (%i).\n", + path, H5Sget_simple_extent_ndims(sh)); + H5Sclose(sh); + H5Dclose(dh); + return 1; + } + + H5Sget_simple_extent_dims(sh, size, max_size); + + tw = size[0]; + + if ( line > tw-1 ) { + H5Sclose(sh); + H5Dclose(dh); + ERROR("Data block %s does not contain data for required event.\n", + path); + return 1; + } + + offset[0] = line; + count[0] = 1; + + r = H5Sselect_hyperslab(sh, H5S_SELECT_SET, + offset, NULL, count, NULL); + if ( r < 0 ) { + ERROR("Error selecting file dataspace " + "for data block %s\n", path); + H5Dclose(dh); + H5Sclose(sh); + return 1; + } + + m_offset[0] = 0; + m_count[0] = 1; + dimmh[0] = 1; + mh = H5Screate_simple(1, dimmh, NULL); + r = H5Sselect_hyperslab(mh, H5S_SELECT_SET, + m_offset, NULL, m_count, NULL); + if ( r < 0 ) { + ERROR("Error selecting memory dataspace " + "for data block %s\n", path); + H5Dclose(dh); + H5Sclose(sh); + H5Sclose(mh); + return 1; + } + + r = H5Dread(dh, H5T_NATIVE_INT, mh, + sh, H5P_DEFAULT, num_peaks); + if ( r < 0 ) { + ERROR("Couldn't read data for block %s, line %i\n", path, line); + H5Dclose(dh); + H5Sclose(sh); + H5Sclose(mh); + return 1; + } + + H5Dclose(dh); + H5Sclose(sh); + H5Sclose(mh); + return 0; +} + + +static float *read_peak_line(hid_t fh, char *path, int line) +{ + + hid_t dh, sh, mh; + hsize_t size[2]; + hsize_t max_size[2]; + hsize_t offset[2], count[2]; + hsize_t m_offset[2], m_count[2], dimmh[2]; + float *buf; + int tw, r; + + dh = H5Dopen2(fh, path, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("Data block (%s) not found.\n", path); + return NULL; + } + + sh = H5Dget_space(dh); + if ( sh < 0 ) { + H5Dclose(dh); + ERROR("Couldn't get dataspace for data.\n"); + return NULL; + } + + if ( H5Sget_simple_extent_ndims(sh) != 2 ) { + ERROR("Data block %s has the wrong dimensionality (%i).\n", + path, H5Sget_simple_extent_ndims(sh)); + H5Sclose(sh); + H5Dclose(dh); + return NULL; + } + + H5Sget_simple_extent_dims(sh, size, max_size); + + tw = size[0]; + if ( line> tw-1 ) { + H5Sclose(sh); + H5Dclose(dh); + ERROR("Data block %s does not contain data for required event.\n", + path); + return NULL; + } + + offset[0] = line; + offset[1] = 0; + count[0] = 1; + count[1] = size[1]; + + r = H5Sselect_hyperslab(sh, H5S_SELECT_SET, offset, NULL, count, NULL); + if ( r < 0 ) { + ERROR("Error selecting file dataspace " + "for data block %s\n", path); + H5Dclose(dh); + H5Sclose(sh); + return NULL; + } + + m_offset[0] = 0; + m_offset[1] = 0; + m_count[0] = 1; + m_count[1] = size[1]; + dimmh[0] = 1; + dimmh[1] = size[1]; + + mh = H5Screate_simple(2, dimmh, NULL); + r = H5Sselect_hyperslab(mh, H5S_SELECT_SET, + m_offset, NULL, m_count, NULL); + if ( r < 0 ) { + ERROR("Error selecting memory dataspace " + "for data block %s\n", path); + H5Dclose(dh); + H5Sclose(sh); + H5Sclose(mh); + return NULL; + } + + buf = malloc(size[1]*sizeof(float)); + if ( buf == NULL ) return NULL; + r = H5Dread(dh, H5T_NATIVE_FLOAT, mh, sh, H5P_DEFAULT, buf); + if ( r < 0 ) { + ERROR("Couldn't read data for block %s, line %i\n", path, line); + H5Dclose(dh); + H5Sclose(sh); + H5Sclose(mh); + return NULL; + } + + H5Dclose(dh); + H5Sclose(sh); + H5Sclose(mh); + return buf; +} + + +ImageFeatureList *image_hdf5_read_peaks_cxi(const DataTemplate *dtempl, + const char *filename, + const char *event, + int half_pixel_shift) +{ + ImageFeatureList *features; + hid_t fh; + char path_n[1024]; + char path_x[1024]; + char path_y[1024]; + char path_i[1024]; + int r; + int pk; + char *subst_name; + int line; + int num_peaks; + float *buf_x; + float *buf_y; + float *buf_i; + int *dim_vals; + int n_dim_vals; + + double peak_offset = half_pixel_shift ? 0.5 : 0.0; + + if ( access(filename, R_OK) == -1 ) { + ERROR("File does not exist or cannot be read: %s\n", + filename); + return NULL; + } + + subst_name = substitute_path(event, dtempl->peak_list); + if ( subst_name == NULL ) { + ERROR("Invalid peak path %s\n", subst_name); + return NULL; + } + + dim_vals = read_dim_parts(event, &n_dim_vals); + if ( dim_vals == NULL ) { + ERROR("Couldn't parse event '%s'\n"); + return NULL; + } + + if ( n_dim_vals < 1 ) { + ERROR("Not enough dimensions in event ID to use CXI " + "peak lists (%i)\n", n_dim_vals); + return NULL; + } + + line = dim_vals[0]; + free(dim_vals); + + snprintf(path_n, 1024, "%s/nPeaks", subst_name); + snprintf(path_x, 1024, "%s/peakXPosRaw", subst_name); + snprintf(path_y, 1024, "%s/peakYPosRaw", subst_name); + snprintf(path_i, 1024, "%s/peakTotalIntensity", subst_name); + + fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + if ( fh < 0 ) { + ERROR("Couldn't open file: %s\n", filename); + return NULL; + } + + r = read_peak_count(fh, path_n, line, &num_peaks); + if ( r != 0 ) { + close_hdf5(fh); + return NULL; + } + + buf_x = read_peak_line(fh, path_x, line); + if ( r != 0 ) { + close_hdf5(fh); + return NULL; + } + + buf_y = read_peak_line(fh, path_y, line); + if ( r != 0 ) { + close_hdf5(fh); + return NULL; + } + + buf_i = read_peak_line(fh, path_i, line); + if ( r != 0 ) { + close_hdf5(fh); + return NULL; + } + + features = image_feature_list_new(); + + for ( pk=0; pk<num_peaks; pk++ ) { + + float fs, ss, val; + int pn; + + fs = buf_x[pk] + peak_offset; + ss = buf_y[pk] + peak_offset; + val = buf_i[pk]; + + if ( data_template_file_to_panel_coords(dtempl, + &fs, &ss, + &pn) ) + { + ERROR("Failed to convert %i,%i to " + "panel-relative coordinates\n", fs, ss); + } else { + image_add_feature(features, fs, ss, pn, + NULL, val, NULL); + } + + } + + close_hdf5(fh); + + return features; +} + + +ImageFeatureList *image_hdf5_read_peaks_hdf5(const DataTemplate *dtempl, + const char *filename, + const char *event, + int half_pixel_shift) +{ + hid_t fh, dh, sh; + hsize_t size[2]; + hsize_t max_size[2]; + int i; + float *buf; + herr_t r; + int tw; + char *subst_name; + ImageFeatureList *features; + double peak_offset = half_pixel_shift ? 0.5 : 0.0; + + if ( dtempl->peak_list == NULL ) { + ERROR("Peak location is not given in geometry file.\n"); + return NULL; + } + + if ( access(filename, R_OK) == -1 ) { + ERROR("File does not exist or cannot be read: %s\n", + filename); + return NULL; + } + + fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + if ( fh < 0 ) { + ERROR("Couldn't open file: %s\n", filename); + return NULL; + } + + subst_name = substitute_path(event, dtempl->peak_list); + if ( subst_name == NULL ) { + ERROR("Invalid peak path: '%s' '%s'\n", + event, dtempl->peak_list); + close_hdf5(fh); + return NULL; + } + + dh = H5Dopen2(fh, subst_name, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("Peak list (%s) not found.\n", subst_name); + free(subst_name); + close_hdf5(fh); + return NULL; + } + free(subst_name); + + sh = H5Dget_space(dh); + if ( sh < 0 ) { + ERROR("Couldn't get dataspace for peak list.\n"); + close_hdf5(fh); + return NULL; + } + + if ( H5Sget_simple_extent_ndims(sh) != 2 ) { + ERROR("Peak list has the wrong dimensionality (%i).\n", + H5Sget_simple_extent_ndims(sh)); + close_hdf5(fh); + return NULL; + } + + H5Sget_simple_extent_dims(sh, size, max_size); + H5Sclose(sh); + + tw = size[1]; + if ( (tw != 3) && (tw != 4) ) { + ERROR("Peak list has the wrong dimensions.\n"); + close_hdf5(fh); + return NULL; + } + + buf = malloc(sizeof(float)*size[0]*size[1]); + if ( buf == NULL ) { + ERROR("Couldn't reserve memory for the peak list.\n"); + close_hdf5(fh); + return NULL; + } + r = H5Dread(dh, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, buf); + if ( r < 0 ) { + ERROR("Couldn't read peak list.\n"); + close_hdf5(fh); + return NULL; + } + + features = image_feature_list_new(); + if ( features == NULL ) { + ERROR("Failed to allocate peak list\n"); + close_hdf5(fh); + return NULL; + } + + for ( i=0; i<size[0]; i++ ) { + + float fs, ss, val; + int pn; + + fs = buf[tw*i+0] + peak_offset; + ss = buf[tw*i+1] + peak_offset; + val = buf[tw*i+2]; + + if ( data_template_file_to_panel_coords(dtempl, + &fs, &ss, + &pn) ) + { + ERROR("Failed to convert %i,%i to " + "panel-relative coordinates\n", fs, ss); + } else { + image_add_feature(features, fs, ss, pn, + NULL, val, NULL); + } + + } + + free(buf); + close_hdf5(fh); + + return features; +} + + +/* This could be extended, later, to include patterns other than just + * a literal string (no placeholders) and just %. However, pattern + * matching is in general not that easy. */ +static char *matches_pattern(const char *name, const char *pattern, + const char *ev_str_old) +{ + if ( strcmp(pattern, "%") == 0 ) { + char *nstr = malloc(strlen(ev_str_old)+strlen(name)+2); + if ( nstr == NULL ) { + ERROR("Couldn't allocate memory\n"); + return NULL; + } + strcpy(nstr, ev_str_old); + strcat(nstr, "/"); + strcat(nstr, name); + return nstr; + } else { + if ( strcmp(name, pattern) == 0 ) { + return strdup(ev_str_old); + } else { + return NULL; + } + } +} + + +/* Private structure, just to avoid passing char *** around */ +struct ev_list +{ + char **events; + int n_events; + int max_events; +}; + + +static int add_ev_to_list(struct ev_list *list, char *ev_str) +{ + if ( list->n_events == list->max_events ) { + char **new_events = realloc(list->events, + (list->max_events+128)*sizeof(char *)); + if ( new_events == NULL ) return 1; + list->max_events += 128; + list->events = new_events; + } + + list->events[list->n_events++] = strdup(ev_str); + + return 0; +} + + +static char *demunge_event(const char *orig) +{ + size_t len = strlen(orig); + char *slash; + + if ( len == 0 ) return strdup("//"); + + slash = malloc(len+3); + if ( slash == NULL ) return NULL; + strcpy(slash, orig+1); + strcat(slash, "//"); + return slash; +} + + +static int rec_expand_paths(hid_t gh, struct ev_list *list, + const char *ev_str, + char **pattern_bits, int n_pattern_bits) +{ + int i; + H5G_info_t group_info; + + if ( H5Gget_info(gh, &group_info) < 0 ) { + ERROR("Couldn't get group info\n"); + return 1; + } + + for ( i=0; i<group_info.nlinks; i++ ) { + + ssize_t size; + char *name; + H5O_info_t obj_info; + char *ev_str_new; + + size = H5Lget_name_by_idx(gh, ".", H5_INDEX_NAME, + H5_ITER_INC, i, NULL, 0, + H5P_DEFAULT); + if ( (size < 0) || (size > 20000) ) { + ERROR("Couldn't get link name\n"); + return 1; + } + + name = malloc(size+1); + if ( name == NULL ) { + ERROR("Couldn't allocate memory\n"); + return 1; + } + + if ( H5Lget_name_by_idx(gh, ".", H5_INDEX_NAME, + H5_ITER_INC, i, name, size+1, + H5P_DEFAULT) < 0 ) + { + ERROR("Couldn't get name\n"); + return 1; + } + + ev_str_new = matches_pattern(name, pattern_bits[0], + ev_str); + if ( ev_str_new == NULL ) { + free(name); + continue; + } + + if ( H5Oget_info_by_idx(gh, ".", H5_INDEX_NAME, + H5_ITER_INC, i, &obj_info, 0) < 0 ) + { + ERROR("Couldn't get info\n"); + free(name); + free(ev_str_new); + return 1; + } + + if ( obj_info.type == H5O_TYPE_GROUP ) { + + hid_t child_gh; + + if ( n_pattern_bits == 1 ) { + ERROR("Pattern doesn't match file" + " (too short)\n"); + free(name); + free(ev_str_new); + return 1; + } + + child_gh = H5Gopen1(gh, name); + if ( child_gh < 0 ) { + ERROR("Couldn't open '%s'\n", name); + free(name); + free(ev_str_new); + return 1; + } + + if ( rec_expand_paths(child_gh, list, + ev_str_new, + &pattern_bits[1], + n_pattern_bits - 1) ) + { + free(name); + free(ev_str_new); + return 1; + } + + free(ev_str_new); + H5Gclose(child_gh); + + } else if ( obj_info.type == H5O_TYPE_DATASET ) { + + char *addme; + + if ( n_pattern_bits != 1 ) { + ERROR("Pattern doesn't match file" + " (too long by %i)\n", + n_pattern_bits); + free(name); + free(ev_str_new); + return 1; + } + + addme = demunge_event(ev_str_new); + if ( addme != NULL ) { + add_ev_to_list(list, addme); + free(addme); + } + free(ev_str_new); + + } + + free(name); + + } + + return 0; +} + + +/* Not "static" so that ev_enumX can test it. + * Not part of public API! */ +char **expand_paths(hid_t fh, char *pattern, int *n_evs) +{ + int n_sep; + size_t len; + char **pattern_bits; + struct ev_list list; + int i; + char *start; + + if ( pattern == NULL ) return NULL; + if ( pattern[0] != '/' ) return NULL; + + /* Chop up the pattern into path bits */ + len = strlen(pattern); + n_sep = 0; + for ( i=0; i<len; i++ ) { + if ( pattern[i] == '/' ) n_sep++; + } + + pattern_bits = malloc(n_sep*sizeof(char *)); + if ( pattern_bits == NULL ) return NULL; + + start = pattern+1; + for ( i=0; i<n_sep; i++ ) { + char *sep = strchr(start, '/'); + if ( sep == NULL ) { + sep = start+strlen(start); + } + pattern_bits[i] = strndup(start, sep-start); + if ( pattern_bits[i] == NULL ) return NULL; + start = sep+1; + } + + list.n_events = 0; + list.max_events = 0; + list.events = NULL; + + rec_expand_paths(fh, &list, "", pattern_bits, n_sep); + + for ( i=0; i<n_sep; i++ ) { + free(pattern_bits[i]); + } + free(pattern_bits); + + *n_evs = list.n_events; + return list.events; +} + + +static int rec_expand_dims(struct ev_list *list, + int *placeholder_sizes, + int n_placeholder_dims, + char *path_ev) +{ + int i; + char *dim_ev; + size_t len; + + len = strlen(path_ev); + dim_ev = malloc(len+16); + if ( dim_ev == NULL ) return 1; + + if ( n_placeholder_dims == 1 ) { + for ( i=0; i<placeholder_sizes[0]; i++ ) { + snprintf(dim_ev, 16, "%s/%i", path_ev, i); + if ( add_ev_to_list(list, dim_ev) ) return 1; + } + } else { + + for ( i=0; i<placeholder_sizes[0]; i++ ) { + snprintf(dim_ev, 16, "%s/%i", path_ev, i); + if ( rec_expand_dims(list, + &placeholder_sizes[1], + n_placeholder_dims - 1, + dim_ev) ) return 1; + } + + } + + free(dim_ev); + return 0; +} + + +static char **expand_dims(int *placeholder_sizes, + int n_placeholder_dims, + char *path_ev, + int *n_evs) +{ + struct ev_list list; + + list.n_events = 0; + list.max_events = 0; + list.events = NULL; + + if ( rec_expand_dims(&list, placeholder_sizes, + n_placeholder_dims, path_ev) ) + { + *n_evs = 0; + return NULL; + } + + *n_evs = list.n_events; + return list.events; +} + + +static int n_dims_expected(struct panel_template *p) +{ + int i; + int n_dims = 0; + for ( i=0; i<MAX_DIMS; i++ ) { + if ( p->dims[i] != DIM_UNDEFINED ) n_dims++; + } + return n_dims; +} + + +char **image_hdf5_expand_frames(const DataTemplate *dtempl, + const char *filename, + int *pn_frames) +{ + char **path_evs; + int n_path_evs; + hid_t fh; + int i; + int dims_expected; + struct ev_list full_evs; + + if ( dtempl->n_panels == 0 ) return NULL; + + full_evs.events = NULL; + full_evs.n_events = 0; + full_evs.max_events = 0; + + /* If the DataTemplate already says that one frame will be + * found per file, short-circuit this whole affair */ + if ( (imh_num_placeholders(&dtempl->panels[0]) == 0) + && (imh_num_path_placeholders(dtempl->panels[0].data) == 0) ) + { + add_ev_to_list(&full_evs, "//"); + *pn_frames = full_evs.n_events; + return full_evs.events; + } + + + fh = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + if ( fh < 0 ) { + ERROR("Couldn't open file '%s'\n", filename); + return NULL; + } + + /* First, expand placeholders in the HDF5 paths. + * + * Since we require the number of placeholders to be the same + * for all panels, and the placeholders will be substituted + * with the same values for each panel (since they come from + * the same event ID), this only needs to be done for the + * first panel. */ + path_evs = expand_paths(fh, dtempl->panels[0].data, + &n_path_evs); + if ( path_evs == NULL ) { + ERROR("Failed to enumerate paths.\n"); + close_hdf5(fh); + return NULL; + } + + dims_expected = n_dims_expected(&dtempl->panels[0]); + + /* For each expanded path, enumerate the placeholder + * dimensions. Once again, since the number of placeholders + * must be the same for each panel, and the substituted values + * will be the same, this only needs to be done for one panel. + */ + for ( i=0; i<n_path_evs; i++ ) { + + hid_t dh, sh; + char *path; + hsize_t *size; + int dims; + int *placeholder_sizes; + int n_placeholder_dims; + int j; + struct panel_template *p = &dtempl->panels[0]; + + path = substitute_path(path_evs[i], p->data); + if ( path == NULL ) { + ERROR("Path substitution failed during " + "expansion of '%s' with partial event " + "ID '%s'\n", + p->data, path_evs[i]); + return NULL; + } + + dh = H5Dopen2(fh, path, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("Error opening '%s'\n", path); + ERROR("Failed to enumerate events. " + "Check your geometry file.\n"); + close_hdf5(fh); + return NULL; + } + + sh = H5Dget_space(dh); + dims = H5Sget_simple_extent_ndims(sh); + if ( dims != dims_expected ) { + ERROR("Unexpected number of dimensions" + "(%s has %i, expected %i)\n", + path, dims, dims_expected); + close_hdf5(fh); + return NULL; + } + + size = malloc(dims*sizeof(hsize_t)); + placeholder_sizes = malloc(dims*sizeof(int)); + if ( (size == NULL) || (placeholder_sizes == NULL) ) { + ERROR("Failed to allocate dimensions\n"); + close_hdf5(fh); + return NULL; + } + + if ( H5Sget_simple_extent_dims(sh, size, NULL) < 0 ) { + ERROR("Failed to get size\n"); + close_hdf5(fh); + return NULL; + } + + n_placeholder_dims = 0; + for ( j=0; j<dims; j++ ) { + if ( p->dims[j] == DIM_PLACEHOLDER ) { + placeholder_sizes[n_placeholder_dims++] = size[j]; + } + } + free(size); + + /* Path event ID ends with //, but expand_dims will + * add a slash. So, remove one slash */ + if ( n_placeholder_dims > 0 ) { + + char **evs_this_path; + int n_evs_this_path; + + path_evs[i][strlen(path_evs[i])-1] = '\0'; + evs_this_path = expand_dims(placeholder_sizes, + n_placeholder_dims, + path_evs[i], + &n_evs_this_path); + + for ( j=0; j<n_evs_this_path; j++ ) { + add_ev_to_list(&full_evs, evs_this_path[j]); + free(evs_this_path[j]); + } + + free(evs_this_path); + + } else { + + /* Easy case with no dims to expand */ + add_ev_to_list(&full_evs, path_evs[i]); + + } + + free(placeholder_sizes); + free(path); + free(path_evs[i]); + + } + + close_hdf5(fh); + free(path_evs); + *pn_frames = full_evs.n_events; + return full_evs.events; +} + + +int is_hdf5_file(const char *filename) +{ + const char *ext = NULL; + + ext = filename_extension(filename, NULL); + if ( ext == NULL ) return 0; + + return ( (strcmp(ext, ".h5") == 0) + || (strcmp(ext, ".cxi") == 0) ); +} + + +/***************************** Writing *****************************/ + +struct hdf5_write_location { + + const char *location; + int n_panels; + int *panel_idxs; + + int max_ss; + int max_fs; + +}; + + +static void add_panel_to_location(struct hdf5_write_location *loc, + struct panel_template *p, int pi) +{ + int *new_panel_idxs; + + new_panel_idxs = realloc(loc->panel_idxs, + (loc->n_panels+1)*sizeof(int)); + if ( new_panel_idxs == NULL ) { + ERROR("Error while managing write location list.\n"); + return; + } + loc->panel_idxs = new_panel_idxs; + loc->panel_idxs[loc->n_panels] = pi; + loc->n_panels += 1; + if ( p->orig_max_fs > loc->max_fs ) { + loc->max_fs = p->orig_max_fs; + } + if ( p->orig_max_ss > loc->max_ss ) { + loc->max_ss = p->orig_max_ss; + } +} + + +static void add_panel_location(struct panel_template *p, + const char *p_location, int pi, + struct hdf5_write_location **plocations, + int *pnum_locations) +{ + int li; + int num_locations = *pnum_locations; + struct hdf5_write_location *locations = *plocations; + int done = 0; + + /* Does this HDF5 path already exist in the location list? + * If so, add the new panel to it (with a unique index, we hope) */ + for ( li=0; li<num_locations; li++ ) { + if ( strcmp(p_location, locations[li].location) == 0 ) { + add_panel_to_location(&locations[li], p, pi); + done = 1; + } + } + + /* If not, add a new location to ths list */ + if ( !done ) { + + struct hdf5_write_location *new_locations; + size_t nsz; + + nsz = (num_locations+1)*sizeof(struct hdf5_write_location); + new_locations = realloc(locations, nsz); + if ( new_locations == NULL ) { + ERROR("Failed to grow location list.\n"); + return; + } + locations = new_locations; + + locations[num_locations].max_ss = p->orig_max_ss; + locations[num_locations].max_fs = p->orig_max_fs; + locations[num_locations].location = p_location; + locations[num_locations].panel_idxs = malloc(sizeof(int)); + if ( locations[num_locations].panel_idxs == NULL ) { + ERROR("Failed to allocate single idx (!)\n"); + return; + } + locations[num_locations].panel_idxs[0] = pi; + locations[num_locations].n_panels = 1; + + num_locations += 1; + + } + + *plocations = locations; + *pnum_locations = num_locations; +} + + +static struct hdf5_write_location *make_location_list(const DataTemplate *dtempl, + int *pnum_locations) +{ + int pi; + struct hdf5_write_location *locations = NULL; + int num_locations = 0; + + for ( pi=0; pi<dtempl->n_panels; pi++ ) { + + struct panel_template *p; + const char *p_location; + + p = &dtempl->panels[pi]; + + assert(p->data != NULL); + p_location = p->data; + + add_panel_location(p, p_location, pi, + &locations, &num_locations); + + } + + *pnum_locations = num_locations; + return locations; +} + + +static void write_location(hid_t fh, const DataTemplate *dtempl, + float **dp, + struct hdf5_write_location *loc) +{ + hid_t sh, dh, ph; + hid_t dh_dataspace; + hsize_t size[2]; + int pi; + + /* Note the "swap" here, according to section 3.2.5, + * "C versus Fortran Dataspaces", of the HDF5 user's guide. */ + size[0] = loc->max_ss+1; + size[1] = loc->max_fs+1; + sh = H5Screate_simple(2, size, NULL); + + ph = H5Pcreate(H5P_LINK_CREATE); + H5Pset_create_intermediate_group(ph, 1); + + dh = H5Dcreate2(fh, loc->location, H5T_NATIVE_FLOAT, sh, + ph, H5P_DEFAULT, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("Couldn't create dataset\n"); + H5Fclose(fh); + return; + } + + H5Sget_simple_extent_dims(sh, size, NULL); + + for ( pi=0; pi<loc->n_panels; pi++ ) { + + hsize_t f_offset[2], f_count[2], dims[2]; + hid_t memspace; + struct panel_template *p; + int r; + + p = &dtempl->panels[loc->panel_idxs[pi]]; + + f_offset[0] = p->orig_min_ss; + f_offset[1] = p->orig_min_fs; + f_count[0] = p->orig_max_ss - p->orig_min_ss +1; + f_count[1] = p->orig_max_fs - p->orig_min_fs +1; + + dh_dataspace = H5Dget_space(dh); + r = H5Sselect_hyperslab(dh_dataspace, H5S_SELECT_SET, + f_offset, NULL, f_count, NULL); + if ( r < 0 ) { + ERROR("Error selecting file dataspace " + "for panel %s\n", p->name); + H5Pclose(ph); + H5Dclose(dh); + H5Sclose(dh_dataspace); + H5Sclose(sh); + H5Fclose(fh); + return; + } + + dims[0] = PANEL_HEIGHT(p); + dims[1] = PANEL_WIDTH(p); + memspace = H5Screate_simple(2, dims, NULL); + + r = H5Dwrite(dh, H5T_NATIVE_FLOAT, memspace, dh_dataspace, + H5P_DEFAULT, dp[loc->panel_idxs[pi]]); + if ( r < 0 ) { + ERROR("Couldn't write data\n"); + H5Pclose(ph); + H5Dclose(dh); + H5Sclose(dh_dataspace); + H5Sclose(memspace); + H5Sclose(sh); + H5Fclose(fh); + return; + } + + H5Sclose(dh_dataspace); + H5Sclose(memspace); + } + H5Pclose(ph); + H5Sclose(sh); + H5Dclose(dh); +} + + +static void write_wavelength(hid_t fh, double wl, + const DataTemplate *dtempl) +{ + hid_t ph, sh, dh; + hsize_t size1d[1]; + int r; + + ph = H5Pcreate(H5P_LINK_CREATE); + H5Pset_create_intermediate_group(ph, 1); + + size1d[0] = 1; + sh = H5Screate_simple(1, size1d, NULL); + + dh = H5Dcreate2(fh, "/wavelength", H5T_NATIVE_DOUBLE, sh, + ph, H5S_ALL, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("Couldn't create dataset for photon energy.\n"); + return; + } + r = H5Dwrite(dh, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, + H5P_DEFAULT, &wl); + if ( r < 0 ) { + ERROR("Couldn't write photon energy.\n"); + /* carry on */ + } + + H5Pclose(ph); + H5Dclose(dh); +} + + +static void write_spectrum(hid_t fh, Spectrum *s, + const DataTemplate *dtempl) +{ + herr_t r; + double *arr; + int i; + hid_t sh, dh, ph; + double kmin, kmax, step; + const hsize_t n = 1024; + + ph = H5Pcreate(H5P_LINK_CREATE); + H5Pset_create_intermediate_group(ph, 1); + + arr = malloc(n*sizeof(double)); + if ( arr == NULL ) { + ERROR("Failed to allocate memory for spectrum.\n"); + return; + } + + /* Save the wavelength values */ + spectrum_get_range(s, &kmin, &kmax); + step = (kmax-kmin)/n; + for ( i=0; i<n; i++ ) { + arr[i] = 1.0e10/(kmin+i*step); + } + + sh = H5Screate_simple(1, &n, NULL); + + dh = H5Dcreate2(fh, "/spectrum/wavelengths_A", H5T_NATIVE_DOUBLE, + sh, ph, H5S_ALL, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("Failed to create dataset for spectrum wavelengths.\n"); + return; + } + r = H5Dwrite(dh, H5T_NATIVE_DOUBLE, H5S_ALL, + H5S_ALL, H5P_DEFAULT, arr); + if ( r < 0 ) { + ERROR("Failed to write spectrum wavelengths.\n"); + return; + } + H5Dclose(dh); + + /* Save the probability density values */ + for ( i=0; i<n; i++ ) { + arr[i] = spectrum_get_density_at_k(s, kmin+i*step); + } + + dh = H5Dcreate2(fh, "/spectrum/pdf", H5T_NATIVE_DOUBLE, sh, + H5P_DEFAULT, H5S_ALL, H5P_DEFAULT); + if ( dh < 0 ) { + ERROR("Failed to create dataset for spectrum p.d.f.\n"); + return; + } + r = H5Dwrite(dh, H5T_NATIVE_DOUBLE, H5S_ALL, + H5S_ALL, H5P_DEFAULT, arr); + if ( r < 0 ) { + ERROR("Failed to write spectrum p.d.f.\n"); + return; + } + + H5Dclose(dh); + H5Pclose(ph); + free(arr); +} + + +int image_hdf5_write(const struct image *image, + const DataTemplate *dtempl, + const char *filename) +{ + hid_t fh; + int li; + struct hdf5_write_location *locations; + int num_locations; + + if ( dtempl == NULL ) return 1; + + fh = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if ( fh < 0 ) { + ERROR("Couldn't create file: %s\n", filename); + return 1; + } + + locations = make_location_list(dtempl, &num_locations); + + for ( li=0; li<num_locations; li++ ) { + write_location(fh, dtempl, image->dp, &locations[li]); + } + + write_wavelength(fh, image->lambda, dtempl); + + if ( image->spectrum != NULL ) { + write_spectrum(fh, image->spectrum, dtempl); + } + + H5Fclose(fh); + for ( li=0; li<num_locations; li ++ ) { + free(locations[li].panel_idxs); + } + free(locations); + return 0; +} diff --git a/libcrystfel/src/image-hdf5.h b/libcrystfel/src/image-hdf5.h new file mode 100644 index 00000000..efb8a3b7 --- /dev/null +++ b/libcrystfel/src/image-hdf5.h @@ -0,0 +1,72 @@ +/* + * image-hdf5.h + * + * Image loading, HDF5 parts + * + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2020 Thomas White <taw@physics.org> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +/* NB This file is NOT part of the public API, and should NOT + * be installed, but rather stays in the libcrystfel source folder. */ + +#ifndef IMAGE_HDF5_H +#define IMAGE_HDF5_H + +#include "datatemplate_priv.h" + +extern double image_hdf5_get_value(const char *from, + const char *filename, + const char *ev); + +extern int image_hdf5_read(struct image *image, + const DataTemplate *dtempl, + const char *filename, + const char *event); + +extern int image_hdf5_read_mask(struct panel_template *p, + const char *filename, + const char *event, int *bad, + const char *mask_location, + int mask_good, int mask_bad); + +extern ImageFeatureList *image_hdf5_read_peaks_cxi(const DataTemplate *dtempl, + const char *filename, + const char *event, + int half_pixel_shift); + +extern ImageFeatureList *image_hdf5_read_peaks_hdf5(const DataTemplate *dtempl, + const char *filename, + const char *event, + int half_pixel_shift); + +extern char **image_hdf5_expand_frames(const DataTemplate *dtempl, + const char *filename, + int *n_frames); + +extern int is_hdf5_file(const char *filename); + +extern int image_hdf5_write(const struct image *image, + const DataTemplate *dtempl, + const char *filename); + +#endif /* IMAGE_HDF5_H */ diff --git a/libcrystfel/src/image-msgpack.c b/libcrystfel/src/image-msgpack.c new file mode 100644 index 00000000..420ecfb4 --- /dev/null +++ b/libcrystfel/src/image-msgpack.c @@ -0,0 +1,315 @@ +/* + * image-msgpack.c + * + * Image loading, MessagePack parts + * + * Copyright © 2017-2021 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2018-2020 Thomas White <taw@physics.org> + * 2014 Valerio Mariani + * 2017 Stijn de Graaf + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> +#include <assert.h> +#include <unistd.h> +#include <zmq.h> +#include <msgpack.h> + +#include <image.h> +#include <utils.h> +#include <msgpack.h> + +#include "datatemplate_priv.h" + + +static msgpack_object *find_msgpack_kv(msgpack_object *obj, const char *key) +{ + int i; + + if ( obj == NULL ) return NULL; + if ( obj->type != MSGPACK_OBJECT_MAP ) return NULL; + + for ( i=0; i<obj->via.map.size; i++ ) { + const char *kstr; + size_t klen; + assert(obj->via.map.ptr[i].key.type == MSGPACK_OBJECT_STR); + kstr = obj->via.map.ptr[i].key.via.str.ptr; + klen = obj->via.map.ptr[i].key.via.str.size; + if ( strncmp(kstr, key, klen) == 0 ) { + return &obj->via.map.ptr[i].val; + } + } + return NULL; +} + + +/** + * image_msgpack_read_peaks + * @dtempl: A %DataTemplate + * @obj: A %msgpack_object containing data + * @half_pixel_shift: Non-zero if 0.5 should be added to all peak coordinates + * + * Get peaks from msgpack_object. The data should be in a map, with the value + * given by "peak_list" as an array of arrays. The first of these should contain + * the list of fs positions of the peaks, the second the ss positions, and the + * third the intensities of the peaks. + * + * http://c.msgpack.org/c/ provides documentation on msgpack objects + * + * CrystFEL considers all peak locations to be distances from the corner of the + * detector panel, in pixel units, consistent with its description of detector + * geometry (see 'man crystfel_geometry'). The software which generates the + * CXI files, including Cheetah, may instead consider the peak locations to be + * pixel indices in the data array. In this case, the peak coordinates should + * have 0.5 added to them. This will be done if @half_pixel_shift is non-zero. + * + * Returns: a newly-allocated %ImageFeatureList. + * + */ +ImageFeatureList *image_msgpack_read_peaks(const DataTemplate *dtempl, + msgpack_object *obj, + int half_pixel_shift) +{ + ImageFeatureList *features; + int num_peaks; + int pk; + msgpack_object *peak_list; + msgpack_object *peak_x; + msgpack_object *peak_y; + msgpack_object *peak_i; + double peak_offset = half_pixel_shift ? 0.5 : 0.0; + + if ( obj == NULL ) { + ERROR("No MessagePack object to get peaks from.\n"); + return NULL; + } + + /* Object has structure: + * { + * "peak_list": [[peak_x], [peak_y], [peak_i]] + * "key2":val2, + * ... + * } + */ + peak_list = find_msgpack_kv(obj, "peak_list"); + peak_x = &peak_list->via.array.ptr[0]; + peak_y = &peak_list->via.array.ptr[1]; + peak_i = &peak_list->via.array.ptr[2]; + + /* Length of peak_x array gives number of peaks */ + num_peaks = peak_x->via.array.size; + + features = image_feature_list_new(); + + for ( pk=0; pk<num_peaks; pk++ ) { + + float fs, ss, val; + int pn; + + /* Retrieve data from peak_list and apply half_pixel_shift, + * if appropriate */ + fs = peak_x->via.array.ptr[pk].via.f64 + peak_offset; + ss = peak_y->via.array.ptr[pk].via.f64 + peak_offset; + val = peak_i->via.array.ptr[pk].via.f64; + + /* Convert coordinates to panel-relative */ + if ( data_template_file_to_panel_coords(dtempl, &fs, &ss, &pn) ) { + ERROR("Peak not in panel!\n"); + } else { + image_add_feature(features, fs, ss, pn, + NULL, val, NULL); + } + } + + return features; +} + + +static int unpack_slab(struct image *image, + const DataTemplate *dtempl, + double *data, + int data_width, int data_height) +{ + int pi; + + image->dp = malloc(dtempl->n_panels*sizeof(float *)); + if ( image->dp == NULL ) { + ERROR("Failed to allocate data arrays.\n"); + return 1; + } + + for ( pi=0; pi<dtempl->n_panels; pi++ ) { + + struct panel_template *p; + int fs, ss; + int p_w, p_h; + + p = &dtempl->panels[pi]; + p_w = p->orig_max_fs - p->orig_min_fs + 1; + p_h = p->orig_max_ss - p->orig_min_ss + 1; + + image->dp[pi] = malloc(p_w*p_h*sizeof(float)); + if ( image->dp[pi] == NULL ) { + ERROR("Failed to allocate panel\n"); + return 1; + } + + if ( (p->orig_min_fs + p_w > data_width) + || (p->orig_min_ss + p_h > data_height) ) + { + ERROR("Panel %s is outside range of data provided\n", + p->name); + return 1; + } + + for ( ss=0; ss<p_h; ss++) { + for ( fs=0; fs<p_w; fs++) { + + int idx; + int cfs, css; + + cfs = fs+p->orig_min_fs; + css = ss+p->orig_min_ss; + idx = cfs + css*data_width; + + image->dp[pi][fs+p_w*ss] = data[idx]; + + } + } + + } + + return 0; +} + + +static double *find_msgpack_data(msgpack_object *obj, int *width, int *height) +{ + msgpack_object *corr_data_obj; + msgpack_object *data_obj; + msgpack_object *shape_obj; + double *data; + + corr_data_obj = find_msgpack_kv(obj, "corr_data"); + if ( corr_data_obj == NULL ) { + ERROR("No corr_data MessagePack object found.\n"); + return NULL; + } + + data_obj = find_msgpack_kv(corr_data_obj, "data"); + if ( data_obj == NULL ) { + ERROR("No data MessagePack object found inside corr_data.\n"); + return NULL; + } + if ( data_obj->type != MSGPACK_OBJECT_STR ) { + ERROR("corr_data.data isn't a binary object.\n"); + return NULL; + } + data = (double *)data_obj->via.str.ptr; + + shape_obj = find_msgpack_kv(corr_data_obj, "shape"); + if ( shape_obj == NULL ) { + ERROR("No shape MessagePack object found inside corr_data.\n"); + return NULL; + } + if ( shape_obj->type != MSGPACK_OBJECT_ARRAY ) { + ERROR("corr_data.shape isn't an array object.\n"); + return NULL; + } + if ( shape_obj->via.array.size != 2 ) { + ERROR("corr_data.shape is wrong size (%i, should be 2)\n", + shape_obj->via.array.size); + return NULL; + } + if ( shape_obj->via.array.ptr[0].type != MSGPACK_OBJECT_POSITIVE_INTEGER ) { + ERROR("corr_data.shape contains wrong type of element.\n"); + return NULL; + } + *height = shape_obj->via.array.ptr[0].via.i64; + *width = shape_obj->via.array.ptr[1].via.i64; + return data; +} + + +/* Unpacks the raw panel data from a msgpack_object, applies panel geometry, + * and stores the resulting data in an image struct. Object has structure + * { + * "corr_data": + * { + * "data": binary_data, + * "shape": [data_height, data_width], + * ... + * ... + * }, + * "key2": val2, + * ... + * ... + * } + */ +struct image *image_msgpack_read(DataTemplate *dtempl, + msgpack_object *obj, + int no_image_data, + int no_mask_data) +{ + struct image *image; + int data_width, data_height; + double *data; + + if ( obj == NULL ) { + ERROR("No MessagePack object!\n"); + return NULL; + } + + if ( dtempl == NULL ) { + ERROR("NULL data template!\n"); + return NULL; + } + + image = image_new(); + if ( image == NULL ) { + ERROR("Couldn't allocate image structure.\n"); + return NULL; + } + + if ( !no_image_data ) { + data = find_msgpack_data(obj, + &data_width, &data_height); + if ( data == NULL ) { + ERROR("No image data in MessagePack object.\n"); + return NULL; + } + unpack_slab(image, dtempl, data, + data_width, data_height); + } else { + image_set_zero_data(image, dtempl); + } + + image_set_zero_mask(image, dtempl); + + return image; +} diff --git a/libcrystfel/src/image-msgpack.h b/libcrystfel/src/image-msgpack.h new file mode 100644 index 00000000..a8e5af34 --- /dev/null +++ b/libcrystfel/src/image-msgpack.h @@ -0,0 +1,64 @@ +/* + * image-msgpack.h + * + * Image loading, MessagePack parts + * + * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2020 Thomas White <taw@physics.org> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifndef IMAGE_MSGPACK_H +#define IMAGE_MSGPACK_H + +#include "datatemplate.h" + +#if defined(HAVE_MSGPACK) + +#include <msgpack.h> + +extern struct image *image_msgpack_read(DataTemplate *dtempl, + msgpack_object *obj, + int no_image_data); + +extern ImageFeatureList *image_msgpack_read_peaks(const DataTemplate *dtempl, + msgpack_object *obj, + int half_pixel_shift); + +#else /* defined(HAVE_MSGPACK) */ + +static UNUSED struct image *image_msgpack_read(DataTemplate *dtempl, + void *obj, + int no_image_data) +{ + return NULL; +} + +static UNUSED ImageFeatureList *image_msgpack_read_peaks(const DataTemplate *dtempl, + void *obj, + int half_pixel_shift) +{ + return NULL; +} + +#endif /* defined(HAVE_MSGPACK) */ + +#endif /* IMAGE_MSGPACK_H */ diff --git a/libcrystfel/src/image.c b/libcrystfel/src/image.c index 6358a9d0..db47f75d 100644 --- a/libcrystfel/src/image.c +++ b/libcrystfel/src/image.c @@ -3,12 +3,12 @@ * * Handle images and image features * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: * 2014 Kenneth Beyerlein <kenneth.beyerlein@desy.de> - * 2011-2017 Thomas White <taw@physics.org> + * 2011-2021 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -27,30 +27,27 @@ * */ +#ifdef HAVE_CONFIG_H #include <config.h> +#endif #include <stdlib.h> #include <assert.h> #include <math.h> #include <stdio.h> -#include <hdf5.h> -#include <zlib.h> +#include <sys/stat.h> +#include <fenv.h> #include "image.h" #include "utils.h" -#include "events.h" -#include "hdf5-file.h" -#include "detector.h" +#include "detgeom.h" +#include "image-hdf5.h" +#include "image-cbf.h" -/** \file image.h */ - -struct imagefile -{ - enum imagefile_type type; - char *filename; - struct hdfile *hdfile; -}; +#include "datatemplate.h" +#include "datatemplate_priv.h" +/** \file image.h */ struct _imagefeaturelist { @@ -61,7 +58,7 @@ struct _imagefeaturelist void image_add_feature(ImageFeatureList *flist, double fs, double ss, - struct panel *p, + int pn, struct image *parent, double intensity, const char *name) { if ( flist->n_features == flist->max_features ) { @@ -75,9 +72,8 @@ void image_add_feature(ImageFeatureList *flist, double fs, double ss, flist->features[flist->n_features].fs = fs; flist->features[flist->n_features].ss = ss; - flist->features[flist->n_features].p = p; + flist->features[flist->n_features].pn = pn; flist->features[flist->n_features].intensity = intensity; - flist->features[flist->n_features].parent = parent; flist->features[flist->n_features].name = name; flist->n_features++; @@ -107,10 +103,7 @@ static int comp(const void *a, const void *b) } -/** - * Strongest first. - */ -ImageFeatureList *sort_peaks(ImageFeatureList *flist) +ImageFeatureList *image_feature_list_copy(const ImageFeatureList *flist) { ImageFeatureList *n; int nf, i; @@ -128,30 +121,40 @@ ImageFeatureList *sort_peaks(ImageFeatureList *flist) nf = 0; for ( i=0; i<flist->n_features; i++ ) { - struct imagefeature *f; - f = image_get_feature(flist, i); + const struct imagefeature *f; + f = image_get_feature_const(flist, i); if ( f == NULL ) continue; n->features[nf++] = flist->features[i]; } n->n_features = nf; - qsort(n->features, nf, sizeof(struct imagefeature), comp); + return n; +} + +/** + * Strongest first. + */ +ImageFeatureList *sort_peaks(ImageFeatureList *flist) +{ + ImageFeatureList *n = image_feature_list_copy(flist); + qsort(n->features, image_feature_count(n), + sizeof(struct imagefeature), comp); return n; } void image_feature_list_free(ImageFeatureList *flist) { - if ( !flist ) return; - if ( flist->features ) free(flist->features); + if ( flist == NULL ) return; + free(flist->features); free(flist); } struct imagefeature *image_feature_closest(ImageFeatureList *flist, double fs, double ss, - struct panel *p, double *d, int *idx) + int pn, double *d, int *idx) { int i; double dmin = +HUGE_VAL; @@ -161,7 +164,7 @@ struct imagefeature *image_feature_closest(ImageFeatureList *flist, double ds; - if ( p != flist->features[i].p ) continue; + if ( pn != flist->features[i].pn ) continue; ds = distance(flist->features[i].fs, flist->features[i].ss, fs, ss); @@ -184,53 +187,21 @@ struct imagefeature *image_feature_closest(ImageFeatureList *flist, } -Reflection *image_reflection_closest(RefList *rlist, - double fs, double ss, struct panel *p, - struct detector *det, - double *d) +int image_feature_count(ImageFeatureList *flist) { - - double dmin = HUGE_VAL; - Reflection *closest = NULL; - Reflection *refl; - RefListIterator *iter; - - for ( refl = first_refl(rlist, &iter); - refl != NULL; - refl = next_refl(refl, iter) ) - { - double ds; - struct panel *p2; - double rfs, rss; - - get_detector_pos(refl, &rfs, &rss); - p2 = get_panel(refl); - - if ( p != p2 ) continue; - - ds = distance(rfs, rss, fs, ss); - - if ( ds < dmin ) { - dmin = ds; - closest = refl; - } - - } - - if ( dmin < +HUGE_VAL ) { - *d = dmin; - return closest; - } - - *d = +INFINITY; - return NULL; + if ( flist == NULL ) return 0; + return flist->n_features; } -int image_feature_count(ImageFeatureList *flist) +const struct imagefeature *image_get_feature_const(const ImageFeatureList *flist, + int idx) { - if ( flist == NULL ) return 0; - return flist->n_features; + /* Sanity check */ + if ( flist == NULL ) return NULL; + if ( idx >= flist->n_features ) return NULL; + + return &flist->features[idx]; } @@ -310,929 +281,909 @@ void free_all_crystals(struct image *image) } -/**************************** Image field lists *******************************/ - -struct imagefile_field_list +static double get_value(struct image *image, const char *from, + int *is_literal_number) { - char **fields; - int n_fields; - int max_fields; -}; - + double val; + char *rval; -struct imagefile_field_list *new_imagefile_field_list() -{ - struct imagefile_field_list *n; + if ( from == NULL ) return NAN; - n = calloc(1, sizeof(struct imagefile_field_list)); - if ( n == NULL ) return NULL; + val = strtod(from, &rval); + if ( (*rval == '\0') && (rval != from) ) { + if ( is_literal_number != NULL ) { + *is_literal_number = 1; + } + return val; + } - n->max_fields = 32; - n->fields = malloc(n->max_fields*sizeof(char *)); - if ( n->fields == NULL ) { - free(n); - return NULL; + if ( image == NULL ) { + ERROR("Attempt to retrieve a header value without an image\n"); + return NAN; } - return n; -} + if ( is_hdf5_file(image->filename) ) { + return image_hdf5_get_value(from, + image->filename, + image->ev); + } else if ( is_cbf_file(image->filename) ) { + /* FIXME: From headers */ + return NAN; -void free_imagefile_field_list(struct imagefile_field_list *n) -{ - int i; - for ( i=0; i<n->n_fields; i++ ) { - free(n->fields[i]); + } else if ( is_cbfgz_file(image->filename) ) { + /* FIXME: From headers */ + return NAN; + + } else { + ERROR("Unrecognised file type: %s\n", image->filename); + return NAN; } - free(n->fields); - free(n); } -void add_imagefile_field(struct imagefile_field_list *copyme, const char *name) +static char *get_value_and_units(struct image *image, const char *from, + double *pvalue, + int *is_literal_number) { - int i; + char *sp; + char *fromcpy; + char *unitscpy; - /* Already on the list? Don't re-add if so. */ - for ( i=0; i<copyme->n_fields; i++ ) { - if ( strcmp(copyme->fields[i], name) == 0 ) return; + if ( from == NULL ) { + *pvalue = NAN; + return NULL; } - /* Need more space? */ - if ( copyme->n_fields == copyme->max_fields ) { + fromcpy = strdup(from); + if ( fromcpy == NULL ) { + *pvalue = NAN; + return NULL; + } - char **nfields; - int nmax = copyme->max_fields + 32; + sp = strchr(fromcpy, ' '); + if ( sp == NULL ) { + unitscpy = NULL; + } else { + unitscpy = strdup(sp+1); + sp[0] = '\0'; + } - nfields = realloc(copyme->fields, nmax*sizeof(char *)); - if ( nfields == NULL ) { - ERROR("Failed to allocate space for new HDF5 field.\n"); - return; - } + *pvalue = get_value(image, fromcpy, is_literal_number); + free(fromcpy); - copyme->max_fields = nmax; - copyme->fields = nfields; + return unitscpy; +} - } - copyme->fields[copyme->n_fields] = strdup(name); - if ( copyme->fields[copyme->n_fields] == NULL ) { - ERROR("Failed to add field for copying '%s'\n", name); - return; +/* default_scale is a value to be used if both of the following + * conditions are met: + * + * 1. The value is a reference to image headers/metadata, + * rather than a literal number. + * 2. No units are specified in the number. + * + * This is totally horrible. Sorry. Blame history. + */ +static double im_get_length(struct image *image, const char *from, + double default_scale) +{ + char *units; + double value; + double scale; + int is_literal_number = 0; + + if ( from == NULL ) return NAN; + + units = get_value_and_units(image, from, + &value, &is_literal_number); + if ( units == NULL ) { + if ( is_literal_number ) { + scale = 1.0; + } else { + scale = default_scale; + } + } else { + if ( strcmp(units, "mm") == 0 ) { + scale = 1e-3; + } else if ( strcmp(units, "m") == 0 ) { + scale = 1.0; + } else { + ERROR("Invalid length unit '%s'\n", units); + free(units); + return NAN; + } } - copyme->n_fields++; + free(units); + return value * scale; } -/******************************* CBF files ************************************/ - -static int unpack_panels(struct image *image, float *data, int data_width, - int data_height) +int create_detgeom(struct image *image, const DataTemplate *dtempl) { - int pi; + struct detgeom *detgeom; + int i; - /* FIXME: Load these masks from an HDF5 file, if filenames are - * given in the geometry file */ - uint16_t *flags = NULL; - float *sat = NULL; - - image->dp = malloc(image->det->n_panels * sizeof(float *)); - image->bad = malloc(image->det->n_panels * sizeof(int *)); - image->sat = malloc(image->det->n_panels * sizeof(float *)); - if ( (image->dp == NULL) || (image->bad == NULL) - || (image->sat == NULL) ) - { - ERROR("Failed to allocate panels.\n"); + if ( dtempl == NULL ) { + ERROR("NULL data template!\n"); return 1; } - for ( pi=0; pi<image->det->n_panels; pi++ ) { - - struct panel *p; - int fs, ss; + detgeom = malloc(sizeof(struct detgeom)); + if ( detgeom == NULL ) return 1; - p = &image->det->panels[pi]; - image->dp[pi] = malloc(p->w*p->h*sizeof(float)); - image->bad[pi] = calloc(p->w*p->h, sizeof(int)); - image->sat[pi] = malloc(p->w*p->h*sizeof(float)); - if ( (image->dp[pi] == NULL) || (image->bad[pi] == NULL) - || (image->sat[pi] == NULL) ) - { - ERROR("Failed to allocate panel\n"); - return 1; - } + detgeom->panels = malloc(dtempl->n_panels*sizeof(struct detgeom_panel)); + if ( detgeom->panels == NULL ) return 1; - if ( p->mask != NULL ) { - ERROR("WARNING: Bad pixel masks do not currently work " - "with CBF files\n"); - ERROR(" (bad pixel regions specified in the geometry " - "file will be used, however)\n"); - } + detgeom->n_panels = dtempl->n_panels; - if ( p->satmap != NULL ) { - ERROR("WARNING: Saturation maps do not currently work " - "with CBF files\n"); - } + for ( i=0; i<dtempl->n_panels; i++ ) { - if ( (p->orig_min_fs + p->w > data_width) - || (p->orig_min_ss + p->h > data_height) ) - { - ERROR("Panel %s is outside range of data in CBF file\n", - p->name); - return 1; - } - - for ( ss=0; ss<p->h; ss++ ) { - for ( fs=0; fs<p->w; fs++ ) { + double shift_x, shift_y; - int idx; - int cfs, css; - int bad = 0; + detgeom->panels[i].name = safe_strdup(dtempl->panels[i].name); - cfs = fs+p->orig_min_fs; - css = ss+p->orig_min_ss; - idx = cfs + css*data_width; + detgeom->panels[i].pixel_pitch = dtempl->panels[i].pixel_pitch; - image->dp[pi][fs+p->w*ss] = data[idx]; + /* NB cnx,cny are in pixels, cnz is in m */ + detgeom->panels[i].cnx = dtempl->panels[i].cnx; + detgeom->panels[i].cny = dtempl->panels[i].cny; + detgeom->panels[i].cnz = im_get_length(image, + dtempl->panels[i].cnz_from, + 1e-3); - if ( sat != NULL ) { - image->sat[pi][fs+p->w*ss] = sat[idx]; - } else { - image->sat[pi][fs+p->w*ss] = INFINITY; - } + /* Apply offset (in m) and then convert cnz from + * m to pixels */ + detgeom->panels[i].cnz += dtempl->panels[i].cnz_offset; + detgeom->panels[i].cnz /= detgeom->panels[i].pixel_pitch; - if ( p->no_index ) bad = 1; + /* Apply overall shift (already in m) */ + shift_x = im_get_length(image, dtempl->shift_x_from, 1.0); + shift_y = im_get_length(image, dtempl->shift_y_from, 1.0); - if ( in_bad_region(image->det, p, cfs, css) ) { - bad = 1; - } + if ( !isnan(shift_x) ) { + detgeom->panels[i].cnx += shift_x; + } + if ( !isnan(shift_y) ) { + detgeom->panels[i].cny += shift_y; + } - if ( isnan(data[idx]) || isinf(data[idx]) ) bad = 1; + detgeom->panels[i].max_adu = dtempl->panels[i].max_adu; - if ( flags != NULL ) { + switch ( dtempl->panels[i].adu_scale_unit ) { - int f; + case ADU_PER_PHOTON: + detgeom->panels[i].adu_per_photon = dtempl->panels[i].adu_scale; + break; - f = flags[idx]; + case ADU_PER_EV: + detgeom->panels[i].adu_per_photon = dtempl->panels[i].adu_scale + * ph_lambda_to_eV(image->lambda); + break; - /* Bad if it's missing any of the "good" bits */ - if ( (f & image->det->mask_good) - != image->det->mask_good ) bad = 1; + default: + detgeom->panels[i].adu_per_photon = 1.0; + ERROR("Invalid ADU/ph scale unit (%i)\n", + dtempl->panels[i].adu_scale_unit); + break; - /* Bad if it has any of the "bad" bits. */ - if ( f & image->det->mask_bad ) bad = 1; + } - } - image->bad[pi][fs+p->w*ss] = bad; + detgeom->panels[i].w = dtempl->panels[i].orig_max_fs + - dtempl->panels[i].orig_min_fs + 1; + detgeom->panels[i].h = dtempl->panels[i].orig_max_ss + - dtempl->panels[i].orig_min_ss + 1; - } - } + detgeom->panels[i].fsx = dtempl->panels[i].fsx; + detgeom->panels[i].fsy = dtempl->panels[i].fsy; + detgeom->panels[i].fsz = dtempl->panels[i].fsz; + detgeom->panels[i].ssx = dtempl->panels[i].ssx; + detgeom->panels[i].ssy = dtempl->panels[i].ssy; + detgeom->panels[i].ssz = dtempl->panels[i].ssz; } + image->detgeom = detgeom; + return 0; } -static void cbf_fill_in_beam_parameters(struct beam_params *beam, - struct imagefile *f, - struct image *image) +int image_set_zero_data(struct image *image, + const DataTemplate *dtempl) { - double eV; + int pi; - if ( beam->photon_energy_from == NULL ) { + image->dp = malloc(dtempl->n_panels*sizeof(float *)); + if ( image->dp == NULL ) return 1; - /* Explicit value given */ - eV = beam->photon_energy; + for ( pi=0; pi<dtempl->n_panels; pi++ ) { - } else { + struct panel_template *p; + int p_w, p_h; + + p = &dtempl->panels[pi]; + p_w = p->orig_max_fs - p->orig_min_fs + 1; + p_h = p->orig_max_ss - p->orig_min_ss + 1; - ERROR("Can't get photon energy from CBF yet.\n"); - eV = 0.0; + image->dp[pi] = calloc(p_w*p_h, sizeof(float)); + if ( image->dp[pi] == NULL ) return 1; } - image->lambda = ph_en_to_lambda(eV_to_J(eV))*beam->photon_energy_scale; + return 0; } -static void cbf_fill_in_clen(struct detector *det, struct imagefile *f) +int image_set_zero_mask(struct image *image, + const DataTemplate *dtempl) { - int i; - - for ( i=0; i<det->n_panels; i++ ) { + int pi; - struct panel *p = &det->panels[i]; + image->bad = malloc(dtempl->n_panels*sizeof(int *)); + image->sat = malloc(dtempl->n_panels*sizeof(float *)); + if ( (image->bad == NULL) || (image->sat == NULL) ) return 1; - if ( p->clen_from != NULL ) { + for ( pi=0; pi<dtempl->n_panels; pi++ ) { - ERROR("Can't get clen from CBF yet.\n"); + struct panel_template *p; + int p_w, p_h; + long int i; - } + p = &dtempl->panels[pi]; + p_w = p->orig_max_fs - p->orig_min_fs + 1; + p_h = p->orig_max_ss - p->orig_min_ss + 1; - adjust_centering_for_rail(p); + image->bad[pi] = calloc(p_w*p_h, sizeof(int)); + image->sat[pi] = calloc(p_w*p_h, sizeof(float)); + if ( image->bad[pi] == NULL ) return 1; + if ( image->sat[pi] == NULL ) return 1; + for ( i=0; i<p_w*p_h; i++ ) { + image->sat[pi][i] = INFINITY; + } } -} - -static void add_out(float val, float *data_out, int nmemb_out, - int *outpos, int *nrej) -{ - if ( *outpos < nmemb_out ) { - data_out[(*outpos)++] = val; - } else { - (*nrej)++; - } + return 0; } -/* Reverses byte offset compression and converts to single precision float. - * Note that this compression scheme specifies the data format of the input - * data, therefore the X-Binary-Element-Type is completely ignored. */ -static void decode_cbf_byte_offset(float *data_out, int nmemb_out, - const int8_t *data_in, const size_t n) +static int file_exists(const char *filename) { - int inpos = 0; - int outpos = 0; - int nrej = 0; - float val = 0.0; - - while ( inpos < n ) { + struct stat statbuf; + int r; - int64_t delta = data_in[inpos++]; + r = stat(filename, &statbuf); + if ( r != 0 ) { + return 0; + } - if ( (delta >= -127) && (delta <= 127) ) { - val += delta; - add_out(val, data_out, nmemb_out, &outpos, &nrej); - continue; - } + return 1; +} - delta = *(int16_t *)(data_in+inpos); - inpos += 2; - if ( (delta >= -32767) && (delta <= 32767) ) { - val += delta; - add_out(val, data_out, nmemb_out, &outpos, &nrej); - continue; - } +static int image_read_image_data(struct image *image, + const DataTemplate *dtempl, + const char *filename, + const char *event) +{ + if ( !file_exists(filename) ) { + ERROR("File not found: %s\n", filename); + return image_set_zero_data(image, dtempl); + } - delta = *(int32_t *)(data_in+inpos); - inpos += 4; + if ( is_hdf5_file(filename) ) { + return image_hdf5_read(image, dtempl, filename, event); - if ( (delta >= -2147483647) && (delta <= 2147483647) ) { - val += delta; - add_out(val, data_out, nmemb_out, &outpos, &nrej); - continue; - } + } else if ( is_cbf_file(filename) ) { + return image_cbf_read(image, dtempl, filename, event, 0); - delta = *(int64_t *)(data_in+inpos); - inpos += 8; - val += delta; - add_out(val, data_out, nmemb_out, &outpos, &nrej); + } else if ( is_cbfgz_file(filename) ) { + return image_cbf_read(image, dtempl, filename, event, 1); } - if ( nrej > 0 ) { - STATUS("%i elements rejected\n", nrej); - } + ERROR("Unrecognised file type: %s\n", filename); + return 1; } -static int binary_start(char *data) +static void set_image_parameters(struct image *image, + const DataTemplate *dtempl) { - char *datac = data; - if ( (datac[0] == (char)0x0c) && (datac[1] == (char)0x1a) - && (datac[2] == (char)0x04) && (datac[3] == (char)0xd5) ) return 1; - return 0; -} + /* Wavelength might be needed to create detgeom (adu_per_eV) */ + image->lambda = convert_to_m(get_value(image, + dtempl->wavelength_from, + NULL), + dtempl->wavelength_unit); + image->bw = dtempl->bandwidth; -enum cbf_data_conversion -{ - CBF_NO_CONVERSION, - CBF_BYTE_OFFSET, - CBF_PACKED, - CBF_CANONICAL -}; + /* FIXME: Possibly load spectrum from file */ + image->spectrum = spectrum_generate_gaussian(image->lambda, + image->bw); -enum cbf_data_type -{ - CBF_NO_TYPE, - CBF_ELEMENT_U8, - CBF_ELEMENT_S8, - CBF_ELEMENT_U16, - CBF_ELEMENT_S16, - CBF_ELEMENT_U32, - CBF_ELEMENT_S32, - CBF_ELEMENT_F32, - CBF_ELEMENT_F64, -}; +} -static enum cbf_data_type parse_element_type(const char *t) +static void mark_flagged_pixels_lessthan(float *dp, int *bad, + long int n, float val) { - if ( strstr(t, "signed 8-bit integer") != NULL ) - { - return CBF_ELEMENT_S8; + long int i; + for ( i=0; i<n; i++ ) { + if ( dp[i] < val ) bad[i] = 1; } +} - if ( strstr(t, "unsigned 8-bit integer") != NULL ) - { - return CBF_ELEMENT_U8; - } - if ( strstr(t, "signed 16-bit integer") != NULL ) - { - return CBF_ELEMENT_S16; +static void mark_flagged_pixels_morethan(float *dp, int *bad, + long int n, float val) +{ + long int i; + for ( i=0; i<n; i++ ) { + if ( dp[i] > val ) bad[i] = 1; } +} - if ( strstr(t, "unsigned 16-bit integer") != NULL ) - { - return CBF_ELEMENT_U16; - } - if ( strstr(t, "signed 32-bit integer") != NULL ) - { - return CBF_ELEMENT_S32; - } - - if ( strstr(t, "unsigned 32-bit integer") != NULL ) - { - return CBF_ELEMENT_U32; - } +static void mark_flagged_pixels_equal(float *dp, int *bad, + long int n, float val) +{ + long int i; + fenv_t envp; - if ( strstr(t, "signed 32-bit real IEEE") != NULL ) - { - return CBF_ELEMENT_F32; - } + fegetenv(&envp); + fesetround(1); /* Round to nearest (for flag_value) */ - if ( strstr(t, "signed 64-bit real IEEE") != NULL ) - { - return CBF_ELEMENT_F64; + for ( i=0; i<n; i++ ) { + if ( rint(dp[i]) == val ) bad[i] = 1; } - /* complex type is unsupported */ - - return CBF_NO_TYPE; + fesetenv(&envp); } -static size_t element_size(enum cbf_data_type t) +static void mark_flagged_pixels_naninf(float *dp, int *bad, + long int n) { - switch ( t ) { - case CBF_ELEMENT_S8 : return 1; - case CBF_ELEMENT_U8 : return 1; - case CBF_ELEMENT_S16 : return 2; - case CBF_ELEMENT_U16 : return 2; - case CBF_ELEMENT_S32 : return 4; - case CBF_ELEMENT_U32 : return 4; - case CBF_ELEMENT_F32 : return 4; - case CBF_ELEMENT_F64 : return 8; - default : return 0; + long int i; + for ( i=0; i<n; i++ ) { + float val = dp[i]; + if ( isnan(val) || isinf(val) ) bad[i] = 1; } } - -static int convert_type(float *data_out, long nmemb_exp, - enum cbf_data_type eltype, - void *data_in, size_t data_in_len) +static void mark_flagged_pixels(struct panel_template *p, + float *dp, int *bad) { - long int i; - long int o = 0; - size_t elsize = element_size(eltype); - - if ( elsize == 0 ) return 1; - - if ( nmemb_exp * elsize > data_in_len ) { - ERROR("Not enough CBF data for image size/type!\n"); - return 1; - } + int p_w, p_h; + long int n; + int i; - for ( i=0; i<nmemb_exp; i++ ) { - switch ( eltype ) { + p_w = p->orig_max_fs - p->orig_min_fs + 1; + p_h = p->orig_max_ss - p->orig_min_ss + 1; + n = p_w * p_h; - case CBF_ELEMENT_S8: - data_out[o++] = ((int8_t *)data_in)[i]; - break; + mark_flagged_pixels_naninf(dp, bad, n); - case CBF_ELEMENT_U8: - data_out[o++] = ((uint8_t *)data_in)[i]; - break; + for ( i=0; i<MAX_FLAG_VALUES; i++ ) { - case CBF_ELEMENT_S16: - data_out[o++] = ((int16_t *)data_in)[i]; - break; + float fv = p->flag_values[i]; - case CBF_ELEMENT_U16: - data_out[o++] = ((uint16_t *)data_in)[i]; - break; + switch ( p->flag_types[i] ) { - case CBF_ELEMENT_S32: - data_out[o++] = ((int32_t *)data_in)[i]; + case FLAG_NOTHING: break; - case CBF_ELEMENT_U32: - data_out[o++] = ((uint32_t *)data_in)[i]; + case FLAG_LESSTHAN: + mark_flagged_pixels_lessthan(dp, bad, n, fv); break; - case CBF_ELEMENT_F32: - data_out[o++] = ((float *)data_in)[i]; + case FLAG_MORETHAN: + mark_flagged_pixels_morethan(dp, bad, n, fv); break; - case CBF_ELEMENT_F64: - data_out[o++] = ((double *)data_in)[i]; + case FLAG_EQUAL: + mark_flagged_pixels_equal(dp, bad, n, fv); break; - case CBF_NO_TYPE: - break; } } - - return 0; } -static float *read_cbf_data(struct imagefile *f, int *w, int *h) +static int region_within_panel(struct dt_badregion *region, + struct detgeom_panel *panel) { - FILE *fh; - void *buf = NULL; - char *rval; - size_t data_compressed_len = 0; - float *data_out = NULL; - enum cbf_data_conversion data_conversion = CBF_NO_CONVERSION; - enum cbf_data_type data_type = CBF_ELEMENT_U32; /* ITG (2006) 2.3.3.3 */ - int in_binary_section = 0; + assert(region->is_fsss); - *w = 0; - *h = 0; + if ( region->min_fs < 0 ) return 0; + if ( region->min_ss < 0 ) return 0; + if ( region->max_fs >= panel->w ) return 0; + if ( region->max_ss >= panel->h ) return 0; + return 1; +} - if ( f->type == IMAGEFILE_CBF ) { - fh = fopen(f->filename, "rb"); - if ( fh == NULL ) { - ERROR("Failed to open '%s'\n", f->filename); - return NULL; - } +static void draw_bad_region_fsss(struct dt_badregion *region, + int **bad, + struct detgeom *detgeom) +{ + struct detgeom_panel *panel; + int fs, ss; - } else if ( f->type == IMAGEFILE_CBFGZ ) { + panel = &detgeom->panels[region->panel_number]; - gzFile gzfh; - size_t len, len_read; - const size_t bufinc = 8*1024*1024; /* Allocate buffer in 8Mb chunks */ - size_t bufsz = bufinc; + if ( !region_within_panel(region, panel) ) { + ERROR("Bad pixel region %s is (partially) outside panel - ignoring it\n", + region->name); + return; + } - gzfh = gzopen(f->filename, "rb"); - if ( gzfh == NULL ) return NULL; + for ( ss=region->min_ss; ss<=region->max_ss; ss++ ) { + for ( fs=region->min_fs; fs<=region->max_fs; fs++ ) { + bad[region->panel_number][fs+ss*panel->w] = 1; + } + } +} - #ifdef HAVE_GZBUFFER - /* Set larger buffer size for hopefully faster uncompression */ - gzbuffer(gzfh, 128*1024); - #endif - buf = malloc(bufsz); - if ( buf == NULL ) return NULL; +static void draw_bad_region_xy(struct dt_badregion *region, + int **bad, + struct detgeom *detgeom) +{ + int i; - len = 0; - do { + for ( i=0; i<detgeom->n_panels; i++ ) { - len_read = gzread(gzfh, buf+len, bufinc); - if ( len_read == -1 ) return NULL; - len += len_read; + int fs, ss; - if ( len_read == bufinc ) { - bufsz += bufinc; - buf = realloc(buf, bufsz); - if ( buf == NULL ) return NULL; - } + struct detgeom_panel *p = &detgeom->panels[i]; + for ( ss=0; ss<p->h; ss++ ) { + for ( fs=0; fs<p->w; fs++ ) { - } while ( len_read == bufinc ); + double x, y; - fh = fmemopen(buf, len, "rb"); - if ( fh == NULL ) return NULL; + x = fs*p->fsx + ss*p->ssx + p->cnx; + y = fs*p->fsy + ss*p->ssy + p->cny; - gzclose(gzfh); + if ( (x > region->min_x ) + && (x < region->max_x) + && (y > region->min_y) + && (y < region->max_y) ) + { + bad[i][fs+ss*p->w] = 1; + } - } else { - /* Don't know how we ended up here */ - return NULL; + } + } } +} - /* This is really horrible, but there are at least three different types - * of header mingled together (CIF, MIME, DECTRIS), so a real parser - * would be very complicated and much more likely to have weird bugs. */ - do { - - char line[1024]; - long line_start; - line_start = ftell(fh); - rval = fgets(line, 1023, fh); - if ( rval == NULL ) break; - chomp(line); +static void mark_bad_regions(struct image *image, + const DataTemplate *dtempl) +{ + int i; - if ( strcmp(line, "--CIF-BINARY-FORMAT-SECTION--") == 0 ) { - in_binary_section = 1; + for ( i=0; i<dtempl->n_bad; i++ ) { + if ( dtempl->bad[i].is_fsss ) { + draw_bad_region_fsss(&dtempl->bad[i], + image->bad, + image->detgeom); + } else { + draw_bad_region_xy(&dtempl->bad[i], + image->bad, + image->detgeom); } + } +} - if ( strcmp(line, "--CIF-BINARY-FORMAT-SECTION----") == 0 ) { - in_binary_section = 0; - } - if ( in_binary_section ) { +static int load_mask(struct panel_template *p, + const char *mask_fn, + const char *ev, + int *bad, + const char *mask_location, + unsigned int mask_good, + unsigned int mask_bad) +{ + if ( is_hdf5_file(mask_fn) ) { + image_hdf5_read_mask(p, mask_fn, ev, bad, mask_location, + mask_good, mask_bad); - if ( strncmp(line, "X-Binary-Size: ", 15) == 0 ) { - data_compressed_len = atoi(line+15); - } + } else if ( is_cbf_file(mask_fn) ) { + image_cbf_read_mask(p, mask_fn, ev, 0, bad, + mask_good, mask_bad); - if ( strncmp(line, "X-Binary-Element-Byte-Order: ", 29) == 0 ) { - const char *elbo = line+29; - if ( strcmp(elbo, "LITTLE_ENDIAN") != 0 ) { - ERROR("Unsupported endianness: %s\n", elbo); - free(buf); - fclose(fh); - return NULL; - } - } + } else if ( is_cbfgz_file(mask_fn) ) { + image_cbf_read_mask(p, mask_fn, ev, 1, bad, + mask_good, mask_bad); - /* Try to spot compression algorithm */ - if ( strstr(line, "conversions=\"x-CBF_BYTE_OFFSET\"") != NULL ) { - data_conversion = CBF_BYTE_OFFSET; - } else if ( strstr(line, "conversions=\"x-CBF_CANONICAL\"") != NULL ) { - data_conversion = CBF_CANONICAL; - } else if ( strstr(line, "conversions=\"x-CBF_PACKED\"") != NULL ) { - data_conversion = CBF_PACKED; - } else if ( strstr(line, "conversions=") != NULL ) { - ERROR("Unrecognised CBF content conversion: %s\n", line); - free(buf); - fclose(fh); - return NULL; - } + } else { + ERROR("Unrecognised mask file type (%s)\n", mask_fn); + return 1; + } - /* Likewise, element type */ - if ( strncmp(line, "X-Binary-Element-Type: ", 23) == 0 ) - { - const char *eltype = (line+23); - data_type = parse_element_type(eltype); - if ( data_type == CBF_NO_TYPE ) { - ERROR("Unrecognised element type: %s\n", - eltype); - free(buf); - fclose(fh); - return NULL; - } - } + return 0; +} - if ( strncmp(line, "X-Binary-Size-Fastest-Dimension: ", 33) == 0 ) { - *w = atoi(line+33); - } - if ( strncmp(line, "X-Binary-Size-Second-Dimension: ", 32) == 0 ) { - *h = atoi(line+32); - } +static int create_badmap(struct image *image, + const DataTemplate *dtempl, + int no_mask_data) +{ + int i; - } + image->bad = malloc(dtempl->n_panels * sizeof(int *)); + if ( image->bad == NULL ) { + ERROR("Failed to allocate bad pixel mask\n"); + return 1; + } - if ( in_binary_section && binary_start(line) ) { + for ( i=0; i<dtempl->n_panels; i++ ) { - size_t len_read; - int nmemb_exp; - void *data_compressed; - int r = 0; + int p_w, p_h; + struct panel_template *p = &dtempl->panels[i]; - if ( data_compressed_len == 0 ) { - ERROR("Found CBF data before X-Binary-Size!\n"); - free(buf); - fclose(fh); - return NULL; - } + p_w = p->orig_max_fs - p->orig_min_fs + 1; + p_h = p->orig_max_ss - p->orig_min_ss + 1; - if ( (*w == 0) || (*h == 0) ) { - ERROR("Found CBF data before dimensions!\n"); - free(buf); - fclose(fh); - return NULL; - } + image->bad[i] = calloc(p_w*p_h, sizeof(int)); + if ( image->bad[i] == NULL ) { + ERROR("Failed to allocate bad pixel mask\n"); + return 1; + } - if ( data_compressed_len > 100*1024*1024 ) { - ERROR("Stated CBF data size too big\n"); - free(buf); - fclose(fh); - return NULL; - } + /* Panel marked as bad? */ + if ( p->bad ) { + /* NB this sets every element to 0x1111, + * but that's OK - value is still 'true'. */ + memset(image->bad[i], 1, p_w*p_h); + } - data_compressed = malloc(data_compressed_len); - if ( data_compressed == NULL ) { - ERROR("Failed to allocate memory for CBF data\n"); - free(buf); - fclose(fh); - return NULL; - } + /* Add bad regions (skip if panel is bad anyway) */ + if ( !p->bad ) { + mark_flagged_pixels(p, image->dp[i], + image->bad[i]); + } - fseek(fh, line_start+4, SEEK_SET); - len_read = fread(data_compressed, 1, data_compressed_len, fh); - if ( len_read < data_compressed_len ) { - ERROR("Couldn't read entire CBF data\n"); - free(buf); - free(data_compressed); - fclose(fh); - return NULL; - } + /* Load masks (skip if panel is bad anyway) */ + if ( (!no_mask_data) && (!p->bad) ) { - nmemb_exp = (*w) * (*h); - data_out = malloc(nmemb_exp*sizeof(float)); - if ( data_out == NULL ) { - ERROR("Failed to allocate memory for CBF data\n"); - free(buf); - free(data_compressed); - fclose(fh); - return NULL; - } + int j; - switch ( data_conversion ) { - - case CBF_NO_CONVERSION: - r = convert_type(data_out, nmemb_exp, data_type, - data_compressed, - data_compressed_len); - break; - - case CBF_BYTE_OFFSET: - decode_cbf_byte_offset(data_out, nmemb_exp, - data_compressed, - data_compressed_len); - break; - - case CBF_PACKED: - case CBF_CANONICAL: - ERROR("Don't yet know how to decompress " - "CBF_PACKED or CBF_CANONICAL\n"); - free(buf); - free(data_compressed); - fclose(fh); - return NULL; + for ( j=0; j<MAX_MASKS; j++ ) { - } + const char *mask_fn; - free(data_compressed); + if ( p->masks[j].data_location == NULL ) { + continue; + } - if ( r ) { - free(buf); - free(data_out); - fclose(fh); - return NULL; - } + if ( p->masks[j].filename == NULL ) { + mask_fn = image->filename; + } else { + mask_fn = p->masks[j].filename; + } - free(buf); - fclose(fh); - return data_out; + load_mask(p, mask_fn, image->ev, image->bad[i], + p->masks[j].data_location, + p->masks[j].good_bits, + p->masks[j].bad_bits); + } } + } - } while ( rval != NULL ); + mark_bad_regions(image, dtempl); - ERROR("Reached end of CBF file before finding data.\n"); - free(buf); /* might be NULL */ - return NULL; + return 0; } -static int read_cbf(struct imagefile *f, struct image *image) +static int create_satmap(struct image *image, + const DataTemplate *dtempl) { - float *data; - int w, h; - - data = read_cbf_data(f, &w, &h); - if ( data == NULL ) { - ERROR("Failed to read CBF data\n"); - return 1; - } - - unpack_panels(image, data, w, h); - free(data); - - if ( image->beam != NULL ) { - cbf_fill_in_beam_parameters(image->beam, f, image); - if ( image->lambda > 1000 ) { - ERROR("WARNING: Missing or nonsensical wavelength " - "(%e m) for %s.\n", - image->lambda, image->filename); - } - } - cbf_fill_in_clen(image->det, f); - fill_in_adu(image); - + /* FIXME: Implementation */ return 0; } -static int read_cbf_simple(struct imagefile *f, struct image *image) +/** + * Create an image structure, suitable for simulation. + * + * WARNING: This is probably not the routine you are looking for! + * If you use this routine anywhere other than a simulation program, then + * you are abusing the API and can expect breakage. In particular, your + * program will only work when the experiment is completely described by + * the DataTemplate, with no values whatsoever coming from image headers. + * + * \returns the new image structure. + * + */ +struct image *image_create_for_simulation(const DataTemplate *dtempl) { - float *data; - int w, h; + struct image *image; - data = read_cbf_data(f, &w, &h); - if ( data == NULL ) { - ERROR("Failed to read CBF data\n"); - return 1; + if ( dtempl == NULL ) { + ERROR("NULL data template!\n"); + return NULL; } - image->det = simple_geometry(image, w, h); - image->dp = malloc(sizeof(float *)); - if ( image->dp == NULL ) { - ERROR("Failed to allocate dp array\n"); - return 1; + image = image_new(); + if ( image == NULL ) { + ERROR("Couldn't allocate image structure.\n"); + return NULL; } - image->dp[0] = data; + if ( image_set_zero_data(image, dtempl) ) { + image_free(image); + return NULL; + } - if ( image->beam != NULL ) { - cbf_fill_in_beam_parameters(image->beam, f, image); - if ( image->lambda > 1000 ) { - ERROR("WARNING: Missing or nonsensical wavelength " - "(%e m) for %s.\n", - image->lambda, image->filename); - } + set_image_parameters(image, dtempl); + + if ( create_detgeom(image, dtempl) ) { + image_free(image); + return NULL; } - cbf_fill_in_clen(image->det, f); - fill_in_adu(image); - return 0; -} + if ( create_badmap(image, dtempl, 1) ) { + image_free(image); + return NULL; + } + if ( create_satmap(image, dtempl) ) { + image_free(image); + return NULL; + } -/****************************** Image files ***********************************/ + return image; +} -signed int is_cbf_file(const char *filename) +struct image *image_read(const DataTemplate *dtempl, + const char *filename, + const char *event, + int no_image_data, + int no_mask_data) { - FILE *fh; - char line[1024]; + struct image *image; + int r; - fh = fopen(filename, "r"); - if ( fh == NULL ) return -1; + if ( dtempl == NULL ) { + ERROR("NULL data template!\n"); + return NULL; + } - if ( fgets(line, 1024, fh) == NULL ) return -1; - fclose(fh); + image = image_new(); + if ( image == NULL ) { + ERROR("Couldn't allocate image structure.\n"); + return NULL; + } - if ( strstr(line, "CBF") == NULL ) { - return 0; + image->filename = strdup(filename); + if ( event != NULL ) { + image->ev = strdup(event); + } else { + image->ev = strdup("//"); /* Null event */ } - return 1; -} + /* Load the image data */ + if ( !no_image_data ) { + r = image_read_image_data(image, dtempl, + filename, event); + } else { + r = image_set_zero_data(image, dtempl); + } + if ( r ) { + image_free(image); + return NULL; + } + set_image_parameters(image, dtempl); -signed int is_cbfgz_file(const char *filename) -{ - gzFile gzfh; - char line[1024]; + if ( create_detgeom(image, dtempl) ) { + image_free(image); + return NULL; + } - gzfh = gzopen(filename, "rb"); - if ( gzfh == NULL ) return -1; - if ( gzgets(gzfh, line, 1024) == NULL ) return -1; - gzclose(gzfh); + if ( create_badmap(image, dtempl, no_mask_data) ) { + image_free(image); + return NULL; + } - if ( strstr(line, "CBF") == NULL ) { - return 0; + if ( create_satmap(image, dtempl) ) { + image_free(image); + return NULL; } - return 1; + return image; } -struct imagefile *imagefile_open(const char *filename) +void image_free(struct image *image) { - struct imagefile *f; + int i, np; + + if ( image == NULL ) return; + image_feature_list_free(image->features); + free_all_crystals(image); + spectrum_free(image->spectrum); + free(image->filename); + free(image->ev); + + if ( image->detgeom != NULL ) { + np = image->detgeom->n_panels; + detgeom_free(image->detgeom); + } else { + np = 0; + } - f = malloc(sizeof(struct imagefile)); - if ( f == NULL ) return NULL; + for ( i=0; i<np; i++ ) { + if ( image->dp != NULL ) free(image->dp[i]); + if ( image->sat != NULL ) free(image->sat[i]); + if ( image->bad != NULL ) free(image->bad[i]); + } - if ( H5Fis_hdf5(filename) > 0 ) { + free(image->dp); + free(image->sat); + free(image->bad); - /* This is an HDF5, pass through to HDF5 layer */ - f->type = IMAGEFILE_HDF5; - f->hdfile = hdfile_open(filename); + free(image); +} - if ( f->hdfile == NULL ) { - free(f); - return NULL; - } - } else if ( is_cbf_file(filename) > 0 ) { +struct image *image_new() +{ + struct image *image; + + image = malloc(sizeof(struct image)); + if ( image == NULL ) return NULL; + + image->dp = NULL; + image->bad = NULL; + image->sat = NULL; + image->hit = 0; + image->crystals = NULL; + image->n_crystals = 0; + image->indexed_by = INDEXING_NONE; + image->detgeom = NULL; + image->filename = NULL; + image->ev = NULL; + image->copied_headers = NULL; + image->id = 0; + image->serial = 0; + image->spectrum = NULL; + image->lambda = -1.0; + image->div = 0.0; + image->bw = -1.0; + image->peak_resolution = -1.0; + image->features = NULL; + + return image; +} - f->type = IMAGEFILE_CBF; - } else if ( is_cbfgz_file(filename) ) { +ImageFeatureList *image_read_peaks(const DataTemplate *dtempl, + const char *filename, + const char *event, + int half_pixel_shift) +{ + if ( is_hdf5_file(filename) ) { - f->type = IMAGEFILE_CBFGZ; + enum peak_layout layout; - } else { + if ( dtempl->peak_list_type == PEAK_LIST_AUTO ) { - ERROR("Unrecognised file type: %s\n", filename); - return NULL; + const char *ext; + ext = filename_extension(filename, NULL); - } + if ( strcmp(ext, ".cxi") == 0 ) { + layout = PEAK_LIST_CXI; + } else if ( strcmp(ext, ".h5") == 0 ) { + layout = PEAK_LIST_LIST3; + } else { + ERROR("Couldn't determine peak list layout.\n"); + ERROR("Specify peak_layout = cxi or list3n in geometry file.\n"); + return NULL; + } - f->filename = strdup(filename); - return f; -} + } else { + layout = dtempl->peak_list_type; + } + switch ( layout ) { -int imagefile_read(struct imagefile *f, struct image *image, - struct event *event) -{ - if ( f->type == IMAGEFILE_HDF5 ) { - return hdf5_read2(f->hdfile, image, event, 0); - } else if ( f->type == IMAGEFILE_CBF ) { - return read_cbf(f, image); - } else if ( f->type == IMAGEFILE_CBFGZ ) { - return read_cbf(f, image); - } else { - ERROR("Unknown file type %i\n", f->type); - return 1; - } -} + case PEAK_LIST_CXI : + return image_hdf5_read_peaks_cxi(dtempl, + filename, + event, + half_pixel_shift); + case PEAK_LIST_LIST3 : + return image_hdf5_read_peaks_hdf5(dtempl, + filename, + event, + half_pixel_shift); -/* Read a simple file, no multi-event, no prior geometry etc, and - * generate a geometry for it */ -int imagefile_read_simple(struct imagefile *f, struct image *image) -{ - if ( f->type == IMAGEFILE_HDF5 ) { - return hdf5_read(f->hdfile, image, NULL, 0); - } else if ( f->type == IMAGEFILE_CBF ) { - return read_cbf_simple(f, image); - } else if ( f->type == IMAGEFILE_CBFGZ ) { - return read_cbf_simple(f, image); - } else { - ERROR("Unknown file type %i\n", f->type); - return 1; - } -} + default : + ERROR("Invalid peak list type %i\n", layout); + return NULL; + } -enum imagefile_type imagefile_get_type(struct imagefile *f) -{ - assert(f != NULL); - return f->type; + } else { + ERROR("Peak lists can only be read from HDF5 files\n"); + return NULL; + } } -struct hdfile *imagefile_get_hdfile(struct imagefile *f) +char **image_expand_frames(const DataTemplate *dtempl, + const char *filename, int *n_frames) { - if ( f == NULL ) return NULL; - - if ( f->type != IMAGEFILE_HDF5 ) { - ERROR("Not an HDF5 file!\n"); + if ( !file_exists(filename) ) { + ERROR("File not found: %s\n", filename); return NULL; } - return f->hdfile; + if ( is_hdf5_file(filename) ) { + return image_hdf5_expand_frames(dtempl, filename, + n_frames); + } else { + char **list; + list = malloc(sizeof(char *)); + if ( list == NULL ) return NULL; + list[0] = strdup("//"); + if ( list[0] == NULL ) return NULL; + *n_frames = 1; + return list; + } } -void imagefile_copy_fields(struct imagefile *f, - const struct imagefile_field_list *copyme, - FILE *fh, struct event *ev) +void mark_resolution_range_as_bad(struct image *image, + double min, double max) { int i; - if ( copyme == NULL ) return; + if ( isinf(min) && isinf(max) ) return; /* nothing to do */ - for ( i=0; i<copyme->n_fields; i++ ) { + for ( i=0; i<image->detgeom->n_panels; i++ ) { - char *val; - char *field; - - field = copyme->fields[i]; + int fs, ss; + struct detgeom_panel *p = &image->detgeom->panels[i]; - if ( f->type == IMAGEFILE_HDF5 ) { - val = hdfile_get_string_value(f->hdfile, field, ev); - if ( field[0] == '/' ) { - fprintf(fh, "hdf5%s = %s\n", field, val); - } else { - fprintf(fh, "hdf5/%s = %s\n", field, val); + for ( ss=0; ss<p->h; ss++ ) { + for ( fs=0; fs<p->w; fs++ ) { + double q[3]; + double r; + detgeom_transform_coords(p, fs, ss, + image->lambda, + 0.0, 0.0, q); + r = modulus(q[0], q[1], q[2]); + if ( (r >= min) && (r <= max) ) { + image->bad[i][fs+p->w*ss] = 1; + } } - free(val); - - } else { - STATUS("Mock CBF variable\n"); - fprintf(fh, "cbf/%s = %s\n", field, "(FIXME)"); } - } } -void imagefile_close(struct imagefile *f) +int image_write(const struct image *image, + const DataTemplate *dtempl, + const char *filename) { - if ( f->type == IMAGEFILE_HDF5 ) { - hdfile_close(f->hdfile); + if ( is_hdf5_file(filename) ) { + return image_hdf5_write(image, dtempl, filename); } - free(f->filename); - free(f); + + ERROR("Can only write to HDF5 files.\n"); + return 1; } diff --git a/libcrystfel/src/image.h b/libcrystfel/src/image.h index d3bb56e5..c3b55ba0 100644 --- a/libcrystfel/src/image.h +++ b/libcrystfel/src/image.h @@ -3,11 +3,11 @@ * * Handle images and image features * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2009-2019 Thomas White <taw@physics.org> + * 2009-2020 Thomas White <taw@physics.org> * 2014 Valerio Mariani * * @@ -28,33 +28,23 @@ * */ -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #ifndef IMAGE_H #define IMAGE_H -struct detector; - #include <stdint.h> #include <complex.h> #include <sys/types.h> struct imagefeature; -struct sample; struct image; -struct imagefile; -struct imagefile_field_list; #include "utils.h" #include "cell.h" -#include "detector.h" #include "reflist.h" #include "crystal.h" #include "index.h" -#include "events.h" #include "spectrum.h" +#include "datatemplate.h" /** * \file image.h @@ -65,8 +55,6 @@ struct imagefile_field_list; /** Represents a peak in an image. */ struct imagefeature { - struct image *parent; /**< Pointer to image */ - /** \name Coordinates on panel (fast scan, slow scan) * Note carefully that these are the distances, measured in pixels, * from the corner of the panel. They are NOT pixel indices. @@ -77,44 +65,17 @@ struct imagefeature { double ss; /**@}*/ - struct panel *p; /**< Pointer to panel */ + int pn; /**< Panel number */ double intensity; /**< Intensity */ - /** \name Reciprocal space coordinates (m^-1) of this feature */ - /** @{ */ - double rx; - double ry; - double rz; - /** @} */ - const char *name; /**< Text name, e.g. "5,3,-1" */ }; -/** An enum representing the image file formats we can handle */ -enum imagefile_type -{ - IMAGEFILE_HDF5, /**< HDF5 file (single or multiple frames per file) */ - IMAGEFILE_CBF, /**< CBF file */ - IMAGEFILE_CBFGZ /**< gzipped CBF file, i.e. "file.cbf.gz" */ -}; - - /** An opaque type representing a list of image features */ typedef struct _imagefeaturelist ImageFeatureList; -struct beam_params -{ - double photon_energy; /**< eV per photon */ - char *photon_energy_from; /**< HDF5 dataset name */ - double photon_energy_scale; /**< Scale factor for photon energy, if it - * comes from an image header */ - double bandwidth; /**< FWHM bandwidth as a fraction of - * wavelength */ -}; - - struct image { /** The image data, by panel */ @@ -142,25 +103,16 @@ struct image int n_indexing_tries; /** The detector structure */ - struct detector *det; - - /** The nominal beam parameters (or where to get them) */ - struct beam_params *beam; + struct detgeom *detgeom; /** \name The filename and event ID for the image * @{ */ char *filename; - struct event *event; + char *ev; /** @} */ - /** A list of image file headers to copy to the stream */ - const struct imagefile_field_list *copyme; - /** A list of metadata read from the stream */ - struct stuff_from_stream *stuff_from_stream; - - /** Mean of the camera length values for all panels */ - double avg_clen; + char *copied_headers; /** ID number of the worker processing handling this image */ int id; @@ -198,7 +150,7 @@ extern ImageFeatureList *image_feature_list_new(void); extern void image_feature_list_free(ImageFeatureList *flist); extern void image_add_feature(ImageFeatureList *flist, double x, double y, - struct panel *p, + int pn, struct image *parent, double intensity, const char *name); @@ -206,42 +158,53 @@ extern void image_remove_feature(ImageFeatureList *flist, int idx); extern struct imagefeature *image_feature_closest(ImageFeatureList *flist, double fs, double ss, - struct panel *p, + int pn, double *d, int *idx); -extern Reflection *image_reflection_closest(RefList *rlist, - double fs, double ss, - struct panel *p, - struct detector *det, - double *d); - extern int image_feature_count(ImageFeatureList *flist); extern struct imagefeature *image_get_feature(ImageFeatureList *flist, int idx); +extern const struct imagefeature *image_get_feature_const(const ImageFeatureList *flist, + int idx); extern ImageFeatureList *sort_peaks(ImageFeatureList *flist); +extern ImageFeatureList *image_feature_list_copy(const ImageFeatureList *flist); extern void image_add_crystal(struct image *image, Crystal *cryst); extern int remove_flagged_crystals(struct image *image); extern void free_all_crystals(struct image *image); -/* Image files */ -extern struct imagefile *imagefile_open(const char *filename); -extern int imagefile_read(struct imagefile *f, struct image *image, - struct event *event); -extern int imagefile_read_simple(struct imagefile *f, struct image *image); -extern struct hdfile *imagefile_get_hdfile(struct imagefile *f); -extern enum imagefile_type imagefile_get_type(struct imagefile *f); -extern void imagefile_copy_fields(struct imagefile *f, - const struct imagefile_field_list *copyme, - FILE *fh, struct event *ev); -extern void imagefile_close(struct imagefile *f); -extern signed int is_cbf_file(const char *filename); - -/* Field lists */ -extern struct imagefile_field_list *new_imagefile_field_list(void); -extern void free_imagefile_field_list(struct imagefile_field_list *f); - -extern void add_imagefile_field(struct imagefile_field_list *copyme, - const char *name); +extern void mark_resolution_range_as_bad(struct image *image, + double min, double max); + +extern struct image *image_new(void); +extern struct image *image_read(const DataTemplate *dtempl, + const char *filename, + const char *event, + int no_image_data, + int no_mask_data); +extern struct image *image_create_for_simulation(const DataTemplate *dtempl); +extern void image_free(struct image *image); + +extern ImageFeatureList *image_read_peaks(const DataTemplate *dtempl, + const char *filename, + const char *event, + int half_pixel_shift); + +extern char **image_expand_frames(const DataTemplate *dtempl, + const char *filename, int *nframes); + +extern int image_set_zero_data(struct image *image, + const DataTemplate *dtempl); + +extern int image_set_zero_mask(struct image *image, + const DataTemplate *dtempl); + +extern int image_write(const struct image *image, + const DataTemplate *dtempl, + const char *filename); + +/* Use within libcrystfel only */ +extern int create_detgeom(struct image *image, + const DataTemplate *dtempl); #ifdef __cplusplus } diff --git a/libcrystfel/src/index.c b/libcrystfel/src/index.c index 7b03ba3e..58e90437 100644 --- a/libcrystfel/src/index.c +++ b/libcrystfel/src/index.c @@ -3,12 +3,12 @@ * * Perform indexing (somehow) * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Lorenzo Galli * * Authors: - * 2010-2017 Thomas White <taw@physics.org> + * 2010-2021 Thomas White <taw@physics.org> * 2010-2011 Richard Kirian <rkirian@asu.edu> * 2012 Lorenzo Galli * 2013 Cornelius Gati <cornelius.gati@cfel.de> @@ -47,19 +47,18 @@ #include "image.h" #include "utils.h" #include "peaks.h" -#include "dirax.h" -#include "asdf.h" -#include "mosflm.h" -#include "xds.h" -#include "detector.h" #include "index.h" #include "geometry.h" #include "cell-utils.h" -#include "felix.h" #include "predict-refine.h" -#include "taketwo.h" -#include "xgandalf.h" -#include "pinkindexer.h" +#include "indexers/dirax.h" +#include "indexers/asdf.h" +#include "indexers/mosflm.h" +#include "indexers/xds.h" +#include "indexers/felix.h" +#include "indexers/taketwo.h" +#include "indexers/xgandalf.h" +#include "indexers/pinkindexer.h" #include "fromfile.h" #include "uthash.h" @@ -72,6 +71,8 @@ struct _indexingprivate IndexingFlags flags; UnitCell *target_cell; double tolerance[6]; + double wavelength_estimate; + int n_threads; int n_methods; IndexingMethod *methods; @@ -113,7 +114,8 @@ static void show_indexing_flags(IndexingFlags flags) onoff(flags & INDEXING_RETRY)); } -static char *base_indexer_str(IndexingMethod indm) + +char *base_indexer_str(IndexingMethod indm) { char *str; @@ -199,7 +201,7 @@ static char *friendly_indexer_name(IndexingMethod m) static void *prepare_method(IndexingMethod *m, UnitCell *cell, - struct detector *det, struct beam_params *beam, + double wavelength_estimate, struct xgandalf_options *xgandalf_opts, struct pinkIndexer_options* pinkIndexer_opts, struct felix_options *felix_opts, @@ -250,7 +252,7 @@ static void *prepare_method(IndexingMethod *m, UnitCell *cell, case INDEXING_PINKINDEXER : priv = pinkIndexer_prepare(m, cell, pinkIndexer_opts, - det, beam); + wavelength_estimate); break; default : @@ -280,23 +282,16 @@ static void *prepare_method(IndexingMethod *m, UnitCell *cell, } -IndexingPrivate *setup_indexing(const char *method_list, UnitCell *cell, - struct detector *det, struct beam_params *beam, - float *tols, IndexingFlags flags, - struct taketwo_options *ttopts, - struct xgandalf_options *xgandalf_opts, - struct pinkIndexer_options *pinkIndexer_opts, - struct felix_options *felix_opts, - char *filename) +IndexingMethod *parse_indexing_methods(const char *method_list, + int *pn) { int i, n; char **method_strings; - IndexingPrivate *ipriv; + IndexingMethod *methods; - /* Parse indexing methods */ n = assplode(method_list, ",", &method_strings, ASSPLODE_NONE); - IndexingMethod *methods = malloc(n * sizeof(IndexingMethod)); + methods = malloc(n * sizeof(IndexingMethod)); if ( methods == NULL ) { ERROR("Failed to allocate indexing method list\n"); return NULL; @@ -323,6 +318,30 @@ IndexingPrivate *setup_indexing(const char *method_list, UnitCell *cell, } free(method_strings); + *pn = n; + return methods; +} + + +/* 'tols' is in frac (not %) and radians */ +IndexingPrivate *setup_indexing(const char *method_list, + UnitCell *cell, + float *tols, + IndexingFlags flags, + double wavelength_estimate, + int n_threads, + struct taketwo_options *ttopts, + struct xgandalf_options *xgandalf_opts, + struct pinkIndexer_options *pinkIndexer_opts, + struct felix_options *felix_opts, + char *filename) +{ + IndexingPrivate *ipriv; + IndexingMethod *methods; + int n, i; + + methods = parse_indexing_methods(method_list, &n); + /* No cell parameters -> no cell checking, no prior cell */ if ( !cell_has_parameters(cell) ) { @@ -380,7 +399,7 @@ IndexingPrivate *setup_indexing(const char *method_list, UnitCell *cell, int j; ipriv->engine_private[i] = prepare_method(&methods[i], cell, - det, beam, + wavelength_estimate, xgandalf_opts, pinkIndexer_opts, felix_opts, @@ -424,6 +443,8 @@ IndexingPrivate *setup_indexing(const char *method_list, UnitCell *cell, ipriv->methods = methods; ipriv->n_methods = n; ipriv->flags = flags; + ipriv->wavelength_estimate = wavelength_estimate; + ipriv->n_threads = n_threads; if ( cell != NULL ) { ipriv->target_cell = cell_new_from_cell(cell); @@ -518,29 +539,6 @@ void cleanup_indexing(IndexingPrivate *ipriv) } -void map_all_peaks(struct image *image) -{ - int i, n; - - n = image_feature_count(image->features); - - /* Map positions to 3D */ - for ( i=0; i<n; i++ ) { - - struct imagefeature *f; - struct rvec r; - - f = image_get_feature(image->features, i); - if ( f == NULL ) continue; - - r = get_q_for_panel(f->p, f->fs, f->ss, - NULL, 1.0/image->lambda); - f->rx = r.u; f->ry = r.v; f->rz = r.w; - - } -} - - /* Return 0 for cell OK, 1 for cell incorrect */ static int check_cell(IndexingFlags flags, Crystal *cr, UnitCell *target, double *tolerance) @@ -551,10 +549,14 @@ static int check_cell(IndexingFlags flags, Crystal *cr, UnitCell *target, /* Check at all? */ if ( !(flags & INDEXING_CHECK_CELL) ) return 0; - if ( !right_handed(target) ) STATUS("WARNING: reference cell is left handed\n"); - if ( !right_handed(crystal_get_cell(cr)) ) STATUS("WARNING: unmatched cell is left handed\n"); + if ( !right_handed(target) ) { + STATUS("WARNING: reference cell is left handed\n"); + } + if ( !right_handed(crystal_get_cell(cr)) ) { + STATUS("WARNING: unmatched cell is left handed\n"); + } out = compare_reindexed_cell_parameters(crystal_get_cell(cr), target, - tolerance, &rm); + tolerance, &rm); if ( out != NULL ) { @@ -573,6 +575,12 @@ static int check_cell(IndexingFlags flags, Crystal *cr, UnitCell *target, cell_set_lattice_type(out, cell_get_lattice_type(target)); cell_set_unique_axis(out, cell_get_unique_axis(target)); + /* Correct P to R centering, for the same reason */ + if ( (cell_get_centering(target) == 'R') + && (cell_get_centering(out) == 'P') ) { + cell_set_centering(out, 'R'); + } + return 0; } @@ -580,6 +588,16 @@ static int check_cell(IndexingFlags flags, Crystal *cr, UnitCell *target, } +#ifdef MEASURE_INDEX_TIME +static float real_time() +{ + struct timespec tp; + clock_gettime(CLOCK_MONOTONIC_RAW, &tp); + return tp.tv_sec + tp.tv_nsec*1e-9; +} +#endif + + /* Return non-zero for "success" */ static int try_indexer(struct image *image, IndexingMethod indm, IndexingPrivate *ipriv, void *mpriv, char *last_task) @@ -588,6 +606,12 @@ static int try_indexer(struct image *image, IndexingMethod indm, int n_bad = 0; int n_before = image->n_crystals; + #ifdef MEASURE_INDEX_TIME + float time_start; + float time_end; + time_start = real_time(); + #endif + switch ( indm & INDEXING_METHOD_MASK ) { case INDEXING_NONE : @@ -631,7 +655,7 @@ static int try_indexer(struct image *image, IndexingMethod indm, case INDEXING_PINKINDEXER : set_last_task(last_task, "indexing:pinkindexer"); - r = run_pinkIndexer(image, mpriv); + r = run_pinkIndexer(image, mpriv, ipriv->n_threads); break; case INDEXING_XGANDALF : @@ -647,6 +671,10 @@ static int try_indexer(struct image *image, IndexingMethod indm, set_last_task(last_task, "indexing:finalisation"); + #ifdef MEASURE_INDEX_TIME + time_end = real_time(); + #endif + /* Stop a really difficult to debug situation in its tracks */ if ( image->n_crystals - n_before != r ) { ERROR("Whoops, indexer didn't return the right number " @@ -710,6 +738,7 @@ static int try_indexer(struct image *image, IndexingMethod indm, for ( j=0; j<this_crystal; j++ ) { Crystal *that_cr = image->crystals[j]; + /* 'tols' is in frac (not %) and radians */ const double tols[] = {0.1, 0.1, 0.1, deg2rad(5.0), deg2rad(5.0), @@ -731,6 +760,16 @@ static int try_indexer(struct image *image, IndexingMethod indm, n_bad = remove_flagged_crystals(image); assert(r >= n_bad); + #ifdef MEASURE_INDEX_TIME + printf("%s took %f s, %i crystals found of which %i accepted. %s %s\n", + indexer_str(indm & INDEXING_METHOD_MASK), + time_end - time_start, + r, r - n_bad, + image->filename, + image->ev); + fflush(stdout); + #endif + return r - n_bad; } @@ -765,6 +804,7 @@ static int delete_explained_peaks(struct image *image, Crystal *cr) double ax, ay, az; double bx, by, bz; double cx, cy, cz; + double dx, dy; const double min_dist = 0.25; int i, nspots = 0, nindexed = 0; @@ -775,11 +815,13 @@ static int delete_explained_peaks(struct image *image, Crystal *cr) cell_get_cartesian(crystal_get_cell(cr), &ax, &ay, &az, &bx, &by, &bz, &cx, &cy, &cz); + crystal_get_det_shift(cr, &dx, &dy); + /* Loop over peaks, checking proximity to nearest reflection */ for ( i=0; i<image_feature_count(image->features); i++ ) { struct imagefeature *f; - struct rvec q; + double q[3]; double h, k, l, hd, kd, ld; double dsq; @@ -788,14 +830,15 @@ static int delete_explained_peaks(struct image *image, Crystal *cr) nspots++; /* Reciprocal space position of found peak */ - q = get_q_for_panel(f->p, f->fs, f->ss, - NULL, 1.0/image->lambda); + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, dx, dy, + q); /* Decimal and fractional Miller indices of nearest * reciprocal lattice point */ - hd = q.u * ax + q.v * ay + q.w * az; - kd = q.u * bx + q.v * by + q.w * bz; - ld = q.u * cx + q.v * cy + q.w * cz; + hd = q[0] * ax + q[1] * ay + q[2] * az; + kd = q[0] * bx + q[1] * by + q[2] * bz; + ld = q[0] * cx + q[1] * cy + q[2] * cz; h = lrint(hd); k = lrint(kd); l = lrint(ld); @@ -875,14 +918,23 @@ void index_pattern_3(struct image *image, IndexingPrivate *ipriv, int *ping, if ( ipriv == NULL ) return; - if ( ipriv->methods[0] != INDEXING_FILE){ - map_all_peaks(image); - orig = image->features; - } - image->crystals = NULL; image->n_crystals = 0; + if ( !isnan(ipriv->wavelength_estimate) ) { + if ( !within_tolerance(image->lambda, + ipriv->wavelength_estimate, + 10.0) ) + { + ERROR("WARNING: Wavelength for %s %s (%e) differs by " + "more than 10%% from estimated value (%e)\n", + image->filename, image->ev, + image->lambda, ipriv->wavelength_estimate); + } + } + + orig = image->features; + for ( n=0; n<ipriv->n_methods; n++ ) { int done = 0; @@ -921,13 +973,6 @@ void index_pattern_3(struct image *image, IndexingPrivate *ipriv, int *ping, } - if ( ipriv->methods[0] == INDEXING_FILE){ - map_all_peaks(image); - } - else{ - image->features = orig; - } - if ( n < ipriv->n_methods ) { image->indexed_by = ipriv->methods[n]; } else { @@ -1144,13 +1189,13 @@ char *detect_indexing_methods(UnitCell *cell) if ( methods == NULL ) return NULL; methods[0] = '\0'; + do_probe(taketwo_probe, cell, methods); + do_probe(xgandalf_probe, cell, methods); do_probe(mosflm_probe, cell, methods); - do_probe(dirax_probe, cell, methods); do_probe(asdf_probe, cell, methods); + do_probe(dirax_probe, cell, methods); do_probe(xds_probe, cell, methods); - do_probe(xgandalf_probe, cell, methods); - /* Don't automatically use TakeTwo, Felix or PinkIndexer (yet) */ - //do_probe(taketwo_probe, cell, methods); + //do_probe(felix_probe, cell, methods); //do_probe(pinkIndexer_probe, cell, methods); @@ -1161,3 +1206,15 @@ char *detect_indexing_methods(UnitCell *cell) return methods; } + + +void default_method_options(TakeTwoOptions **ttopts, + XGandalfOptions **xgandalf_opts, + PinkIndexerOptions **pinkIndexer_opts, + FelixOptions **felix_opts) +{ + taketwo_default_options(ttopts); + xgandalf_default_options(xgandalf_opts); + pinkIndexer_default_options(pinkIndexer_opts); + felix_default_options(felix_opts); +} diff --git a/libcrystfel/src/index.h b/libcrystfel/src/index.h index bd5be68e..92406604 100644 --- a/libcrystfel/src/index.h +++ b/libcrystfel/src/index.h @@ -3,13 +3,13 @@ * * Perform indexing (somehow) * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * Copyright © 2012 Lorenzo Galli * * Authors: - * 2010-2017 Thomas White <taw@physics.org> + * 2010-2021 Thomas White <taw@physics.org> * 2010 Richard Kirian * 2012 Lorenzo Galli * 2015 Kenneth Beyerlein <kenneth.beyerlein@desy.de> @@ -34,10 +34,6 @@ #ifndef INDEX_H #define INDEX_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - /** * \file index.h * The indexing subsystem @@ -144,18 +140,35 @@ extern char *indexer_str(IndexingMethod indm); extern IndexingMethod get_indm_from_string(const char *method); extern IndexingMethod get_indm_from_string_2(const char *method, int *err); -#include "detector.h" +extern IndexingMethod *parse_indexing_methods(const char *method_list, + int *pn); +extern char *base_indexer_str(IndexingMethod indm); + #include "cell.h" #include "image.h" -#include "taketwo.h" -#include "xgandalf.h" -#include "pinkindexer.h" -#include "felix.h" - -extern IndexingPrivate *setup_indexing(const char *methods, UnitCell *cell, - struct detector *det, - struct beam_params *beam, float *ltl, +#include "datatemplate.h" + +typedef struct felix_options FelixOptions; +typedef struct taketwo_options TakeTwoOptions; +typedef struct xgandalf_options XGandalfOptions; +typedef struct pinkIndexer_options PinkIndexerOptions; + +extern struct argp felix_argp; +extern struct argp pinkIndexer_argp; +extern struct argp taketwo_argp; +extern struct argp xgandalf_argp; + +extern void default_method_options(TakeTwoOptions **ttopts, + XGandalfOptions **xgandalf_opts, + PinkIndexerOptions **pinkIndexer_opts, + FelixOptions **felix_opts); + +extern IndexingPrivate *setup_indexing(const char *methods, + UnitCell *cell, + float *ltl, IndexingFlags flags, + double wavelength_estimate, + int n_threads, struct taketwo_options *ttopts, struct xgandalf_options *xgandalf_opts, struct pinkIndexer_options *pinkIndexer_opts, diff --git a/libcrystfel/src/asdf.c b/libcrystfel/src/indexers/asdf.c index 7185172d..0c57f9f0 100644 --- a/libcrystfel/src/asdf.c +++ b/libcrystfel/src/indexers/asdf.c @@ -4,12 +4,12 @@ * Alexandra's Superior Direction Finder, or * Algorithm Similar to DirAx, FFT-based * - * Copyright © 2014-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2014-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: * 2014-2015 Alexandra Tolstikova <alexandra.tolstikova@desy.de> - * 2015,2017 Thomas White <taw@physics.org> + * 2015-2017 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -42,19 +42,27 @@ #include <gsl/gsl_linalg.h> #include <gsl/gsl_multifit.h> #include <gsl/gsl_fit.h> -#include <fftw3.h> #include "image.h" -#include "dirax.h" #include "utils.h" #include "peaks.h" #include "cell-utils.h" #include "asdf.h" +#include "cell.h" /** * \file asdf.h */ +#ifdef HAVE_FFTW + +#define FFTW_NO_Complex /* Please use "double[2]", not C99 "complex", + * despite complex.h possibly already being + * included. For more information, refer to: + * http://www.fftw.org/doc/Complex-numbers.html */ + +#include <fftw3.h> + struct fftw_vars { int N; fftw_plan p; @@ -794,15 +802,18 @@ static int find_cell(struct tvector *tvectors, int N_tvectors, double IndexFit, int i_min = sl < 2 * N_tvectors ? 0 : sl - 2 * N_tvectors; int i_max = sl < N_tvectors ? sl : N_tvectors; - for ( i = i_min; i < i_max; i++) if (tvectors[i].n > acl ) { + for ( i = i_min; i < i_max; i++) { + + if (tvectors[i].n <= acl ) continue; int j_min = sl - N_tvectors - 2 * i - 1 < 0 ? i + 1 : sl - N_tvectors - i; int j_max = sl - N_tvectors - i < 0 ? sl - i : N_tvectors; - for ( j = j_min; j < j_max; j++ ) - if ( tvectors[j].n > acl ) { + for ( j = j_min; j < j_max; j++ ) { + + if ( tvectors[j].n <= acl ) continue; k = sl - i - j - 1; @@ -1083,7 +1094,7 @@ int run_asdf(struct image *image, void *ipriv) d_max = max(a, b, c) * 3 * 1e10; - double volume = cell_get_volume(dp->template) / 1e30; + double volume = cell_get_volume(dp->template) * 1e30; /* Divide volume constraints by number of lattice points per * unit cell since asdf always finds primitive cell */ @@ -1105,14 +1116,19 @@ int run_asdf(struct image *image, void *ipriv) for ( i=0; i<n; i++ ) { struct imagefeature *f; + double r[3]; f = image_get_feature(image->features, i); if ( f == NULL ) continue; + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + 0.0, 0.0, r); + reflections[N_reflections] = gsl_vector_alloc(3); - gsl_vector_set(reflections[N_reflections], 0, f->rx/1e10); - gsl_vector_set(reflections[N_reflections], 1, f->ry/1e10); - gsl_vector_set(reflections[N_reflections], 2, f->rz/1e10); + gsl_vector_set(reflections[N_reflections], 0, r[0]/1e10); + gsl_vector_set(reflections[N_reflections], 1, r[1]/1e10); + gsl_vector_set(reflections[N_reflections], 2, r[2]/1e10); N_reflections++; } @@ -1200,3 +1216,32 @@ const char *asdf_probe(UnitCell *cell) { return "asdf"; } + +#else /* HAVE_FFTW */ + +int run_asdf(struct image *image, void *ipriv) +{ + ERROR("This copy of CrystFEL was compiled without FFTW support.\n"); + return 0; +} + + +void *asdf_prepare(IndexingMethod *indm, UnitCell *cell) +{ + ERROR("This copy of CrystFEL was compiled without FFTW support.\n"); + ERROR("To use asdf indexing, recompile with FFTW.\n"); + return NULL; +} + + +const char *asdf_probe(UnitCell *cell) +{ + return NULL; +} + + +void asdf_cleanup(void *pp) +{ +} + +#endif /* HAVE_FFTW */ diff --git a/libcrystfel/src/asdf.h b/libcrystfel/src/indexers/asdf.h index d900c192..98095e3c 100644 --- a/libcrystfel/src/asdf.h +++ b/libcrystfel/src/indexers/asdf.h @@ -4,12 +4,12 @@ * Alexandra's Superior Direction Finder, or * Algorithm Similar to DirAx, FFT-based * - * Copyright © 2014-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2014-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: * 2014-2015 Alexandra Tolstikova <alexandra.tolstikova@desy.de> - * 2015,2017 Thomas White <taw@physics.org> + * 2015-2021 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -31,10 +31,6 @@ #ifndef ASDF_H #define ASDF_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include "index.h" #ifdef __cplusplus @@ -46,8 +42,6 @@ extern "C" { * The ASDF indexing algorithm. */ -#ifdef HAVE_FFTW - extern int run_asdf(struct image *image, void *ipriv); extern void *asdf_prepare(IndexingMethod *indm, UnitCell *cell); @@ -55,37 +49,6 @@ extern const char *asdf_probe(UnitCell *cell); extern void asdf_cleanup(void *pp); -#else /* HAVE_FFTW */ - -int run_asdf(struct image *image, void *ipriv) -{ - ERROR("This copy of CrystFEL was compiled without FFTW support.\n"); - return 0; -} - - -void *asdf_prepare(IndexingMethod *indm, UnitCell *cell) -{ - ERROR("This copy of CrystFEL was compiled without FFTW support.\n"); - ERROR("To use asdf indexing, recompile with FFTW.\n"); - return NULL; -} - - -const char *asdf_probe(UnitCell *cell) -{ - return NULL; -} - - -void asdf_cleanup(void *pp) -{ -} - - -#endif /* HAVE_FFTW */ - - #ifdef __cplusplus } #endif diff --git a/libcrystfel/src/dirax.c b/libcrystfel/src/indexers/dirax.c index 24be87ba..4b59f6cb 100644 --- a/libcrystfel/src/dirax.c +++ b/libcrystfel/src/indexers/dirax.c @@ -3,11 +3,11 @@ * * Invoke the DirAx auto-indexing program * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2014,2017 Thomas White <taw@physics.org> + * 2010-2021 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -490,12 +490,17 @@ static void write_drx(struct image *image) for ( i=0; i<image_feature_count(image->features); i++ ) { struct imagefeature *f; + double r[3]; f = image_get_feature(image->features, i); if ( f == NULL ) continue; + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + 0.0, 0.0, r); + fprintf(fh, "%10f %10f %10f %8f\n", - f->rx/1e10, f->ry/1e10, f->rz/1e10, 1.0); + r[0]/1e10, r[1]/1e10, r[2]/1e10, 1.0); } fclose(fh); diff --git a/libcrystfel/src/dirax.h b/libcrystfel/src/indexers/dirax.h index 33dc1189..2bb560bb 100644 --- a/libcrystfel/src/dirax.h +++ b/libcrystfel/src/indexers/dirax.h @@ -3,11 +3,11 @@ * * Invoke the DirAx auto-indexing program * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010,2012-2014,2017 Thomas White <taw@physics.org> + * 2010-2017 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -29,10 +29,6 @@ #ifndef DIRAX_H #define DIRAX_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include "index.h" #ifdef __cplusplus diff --git a/libcrystfel/src/felix.c b/libcrystfel/src/indexers/felix.c index 2e4898e6..89de5f70 100644 --- a/libcrystfel/src/felix.c +++ b/libcrystfel/src/indexers/felix.c @@ -3,11 +3,11 @@ * * Invoke Felix for multi-crystal autoindexing. * - * Copyright © 2015-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2015-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2015-2018 Thomas White <taw@physics.org> + * 2015-2021 Thomas White <taw@physics.org> * 2015 Kenneth Beyerlein <kenneth.beyerlein@desy.de> * * This file is part of CrystFEL. @@ -364,14 +364,19 @@ static void write_gve(struct image *image, struct felix_private *gp) for ( i=0; i<image_feature_count(image->features); i++ ) { struct imagefeature *f; + double r[3]; f = image_get_feature(image->features, i); if ( f == NULL ) continue; + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + 0.0, 0.0, r); + fprintf(fh, "%.6f %.6f %.6f 0 0 %.6f %.6f %.6f 0\n", - f->rz/1e10, f->rx/1e10, f->ry/1e10, - modulus(f->rx, f->ry, f->rz)/1e10, /* dstar */ - rad2deg(atan2(f->ry, f->rx)), 0.0); /* eta, omega */ + r[2]/1e10, r[0]/1e10, r[1]/1e10, + modulus(r[0], r[1], r[2])/1e10, /* dstar */ + rad2deg(atan2(r[1], r[0])), 0.0); /* eta, omega */ } fclose(fh); @@ -384,7 +389,6 @@ static char *write_ini(struct image *image, struct felix_private *gp) char *filename; char gveFilename[1024]; char logFilename[1024]; - double tt; filename = malloc(1024); if ( filename == NULL ) return NULL; @@ -400,11 +404,6 @@ static char *write_ini(struct image *image, struct felix_private *gp) return NULL; } - get_q_for_panel(image->det->furthest_out_panel, - image->det->furthest_out_fs, - image->det->furthest_out_ss, - &tt, 1.0/image->lambda); - fprintf(fh, "spacegroup %i\n", gp->spacegroup); fprintf(fh, "tthrange %f %f\n", rad2deg(gp->tthrange_min), rad2deg(gp->tthrange_max)); @@ -806,7 +805,7 @@ const char *felix_probe(UnitCell *cell) } -static void show_help() +static void felix_show_help() { printf("Parameters for the Felix indexing algorithm:\n" " --felix-domega Degree range of omega (moscaicity) to consider.\n" @@ -838,30 +837,45 @@ static void show_help() } -static error_t parse_arg(int key, char *arg, struct argp_state *state) +int felix_default_options(FelixOptions **opts_ptr) +{ + FelixOptions *opts; + + opts = malloc(sizeof(struct felix_options)); + if ( opts == NULL ) return ENOMEM; + + opts->ttmin = -1.0; + opts->ttmax = -1.0; + opts->min_visits = 0; + opts->min_completeness = -1.0; + opts->max_uniqueness = -1.0; + opts->n_voxels = 0; + opts->fraction_max_visits = -1.0; + opts->sigma = -1.0; + opts->domega = -1.0; + opts->max_internal_angle = -1.0; + + *opts_ptr = opts; + return 0; +} + + +static error_t felix_parse_arg(int key, char *arg, + struct argp_state *state) { struct felix_options **opts_ptr = state->input; float tmp; + int r; switch ( key ) { case ARGP_KEY_INIT : - *opts_ptr = malloc(sizeof(struct felix_options)); - if ( *opts_ptr == NULL ) return ENOMEM; - (*opts_ptr)->ttmin = -1.0; - (*opts_ptr)->ttmax = -1.0; - (*opts_ptr)->min_visits = 0; - (*opts_ptr)->min_completeness = -1.0; - (*opts_ptr)->max_uniqueness = -1.0; - (*opts_ptr)->n_voxels = 0; - (*opts_ptr)->fraction_max_visits = -1.0; - (*opts_ptr)->sigma = -1.0; - (*opts_ptr)->domega = -1.0; - (*opts_ptr)->max_internal_angle = -1.0; + r = felix_default_options(opts_ptr); + if ( r ) return r; break; case 1 : - show_help(); + felix_show_help(); return EINVAL; case 2 : @@ -945,7 +959,7 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) } -static struct argp_option options[] = { +static struct argp_option felix_options[] = { {"help-felix", 1, NULL, OPTION_NO_USAGE, "Show options for Felix indexing algorithm", 99}, @@ -964,4 +978,5 @@ static struct argp_option options[] = { }; -struct argp felix_argp = { options, parse_arg, NULL, NULL, NULL, NULL, NULL }; +struct argp felix_argp = { felix_options, felix_parse_arg, + NULL, NULL, NULL, NULL, NULL }; diff --git a/libcrystfel/src/felix.h b/libcrystfel/src/indexers/felix.h index 4a992548..83179d6f 100644 --- a/libcrystfel/src/felix.h +++ b/libcrystfel/src/indexers/felix.h @@ -3,12 +3,12 @@ * * Invoke Felix for multi-crystal autoindexing * - * Copyright © 2013-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2013,2017 Thomas White <taw@physics.org> - * 2013-2014 Kenneth Beyerlein <kenneth.beyerlein@desy.de> + * 2010-2017 Thomas White <taw@physics.org> + * 2013-2014 Kenneth Beyerlein <kenneth.beyerlein@desy.de> * * This file is part of CrystFEL. * @@ -30,10 +30,6 @@ #ifndef FELIX_H #define FELIX_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include <argp.h> #include "cell.h" @@ -43,8 +39,7 @@ * Felix indexer interface */ -typedef struct felix_options FelixOptions; -extern struct argp felix_argp; +extern int felix_default_options(FelixOptions **opts_ptr); extern void *felix_prepare(IndexingMethod *indm, UnitCell *cell, struct felix_options *opts); diff --git a/libcrystfel/src/mosflm.c b/libcrystfel/src/indexers/mosflm.c index ef282cbe..b0e0b121 100644 --- a/libcrystfel/src/mosflm.c +++ b/libcrystfel/src/indexers/mosflm.c @@ -3,7 +3,7 @@ * * Invoke the DPS auto-indexing algorithm through MOSFLM * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * @@ -201,10 +201,11 @@ static void mosflm_parseline(const char *line, struct image *image, } -/* This is the opposite of spacegroup_for_lattice() below. +/* This is the opposite of mosflm_spacegroup_for_lattice() below. * Note that this is not general, just a set of rules for interpreting MOSFLM's * output. */ -static LatticeType spacegroup_to_lattice(const char *sg, char *ua, char *cen) +static LatticeType mosflm_spacegroup_to_lattice(const char *sg, + char *ua, char *cen) { LatticeType latt; @@ -298,7 +299,7 @@ static int read_newmat(struct mosflm_data *mosflm, const char *filename, return 1; } //STATUS("MOSFLM says '%s'\n", symm); - latt = spacegroup_to_lattice(symm+5, &ua, &cen); + latt = mosflm_spacegroup_to_lattice(symm+5, &ua, &cen); /* MOSFLM "A" matrix is multiplied by lambda, so fix this */ c = 1.0/image->lambda; @@ -356,16 +357,21 @@ static void write_spt(struct image *image, const char *filename) struct imagefeature *f; double ttx, tty, x, y; + double r[3]; f = image_get_feature(image->features, i); if ( f == NULL ) continue; + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + 0.0, 0.0, r); + ttx = angle_between_2d(0.0, 1.0, - f->rx, 1.0/image->lambda + f->rz); + r[0], 1.0/image->lambda + r[2]); tty = angle_between_2d(0.0, 1.0, - f->ry, 1.0/image->lambda + f->rz); - if ( f->rx < 0.0 ) ttx *= -1.0; - if ( f->ry < 0.0 ) tty *= -1.0; + r[1], 1.0/image->lambda + r[2]); + if ( r[0] < 0.0 ) ttx *= -1.0; + if ( r[1] < 0.0 ) tty *= -1.0; x = -tan(ttx)*FAKE_CLEN; y = tan(tty)*FAKE_CLEN; @@ -437,7 +443,7 @@ static void mosflm_sendline(const char *line, struct mosflm_data *mosflm) /* Turn what we know about the unit cell into something which we can give to * MOSFLM to make it give us only indexing results compatible with the cell. */ -static char *spacegroup_for_lattice(UnitCell *cell) +static char *mosflm_spacegroup_for_lattice(UnitCell *cell) { LatticeType latt; char centering; @@ -530,7 +536,7 @@ static void mosflm_send_next(struct image *image, struct mosflm_data *mosflm) mosflm_sendline("CRYSTAL R\n", mosflm); } - symm = spacegroup_for_lattice(mosflm->mp->template); + symm = mosflm_spacegroup_for_lattice(mosflm->mp->template); snprintf(tmp, 255, "SYMM %s\n", symm); //STATUS("Asking MOSFLM for '%s'\n", symm); free(symm); @@ -754,8 +760,8 @@ int run_mosflm(struct image *image, void *ipriv) t.c_lflag &= ~(ECHO | ECHOE | ECHOK | ECHONL); tcsetattr(STDIN_FILENO, TCSANOW, &t); - execlp("mosflm", "mosflm", (char *)NULL); - execlp("ipmosflm", "ipmosflm", (char *)NULL); + execlp("mosflm", "mosflm", "-n", (char *)NULL); + execlp("ipmosflm", "ipmosflm", "-n", (char *)NULL); ERROR("Invocation: Failed to invoke MOSFLM: %s\n", strerror(errno)); _exit(0); diff --git a/libcrystfel/src/mosflm.h b/libcrystfel/src/indexers/mosflm.h index b6d708f5..0c64090b 100644 --- a/libcrystfel/src/mosflm.h +++ b/libcrystfel/src/indexers/mosflm.h @@ -3,13 +3,13 @@ * * Invoke the DPS auto-indexing algorithm through MOSFLM * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * * Authors: - * 2010 Richard Kirian <rkirian@asu.edu> - * 2012-2014,2017 Thomas White <taw@physics.org> + * 2010 Richard Kirian <rkirian@asu.edu> + * 2012-2017 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -31,10 +31,6 @@ #ifndef MOSFLM_H #define MOSFLM_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include "index.h" #ifdef __cplusplus diff --git a/libcrystfel/src/pinkindexer.c b/libcrystfel/src/indexers/pinkindexer.c index 22208885..b0b5a9fa 100644 --- a/libcrystfel/src/pinkindexer.c +++ b/libcrystfel/src/indexers/pinkindexer.c @@ -3,11 +3,12 @@ * * Interface to PinkIndexer * - * Copyright © 2017-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2017-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: * 2017-2019 Yaroslav Gevorkov <yaroslav.gevorkov@desy.de> + * 2021 Thomas White <thomas.white@desy.de> * * This file is part of CrystFEL. * @@ -26,34 +27,46 @@ * */ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + #include "pinkindexer.h" -#ifdef HAVE_PINKINDEXER #include <stdlib.h> +#include <sys/errno.h> +#include <argp.h> #include "utils.h" #include "cell-utils.h" #include "peaks.h" -#include "pinkIndexer/adaptions/crystfel/Lattice.h" -#include "pinkIndexer/adaptions/crystfel/ExperimentSettings.h" -#include "pinkIndexer/adaptions/crystfel/PinkIndexer.h" -#define MAX_MULTI_LATTICE_COUNT 8 +#define FAKE_CLEN (0.25) + +struct pinkIndexer_options { + unsigned int considered_peaks_count; + unsigned int angle_resolution; + unsigned int refinement_type; + float maxResolutionForIndexing_1_per_A; + float tolerance; + float reflectionRadius; /* In m^-1 */ + float customBandwidth; + float maxRefinementDisbalance; +}; + +#ifdef HAVE_PINKINDEXER + +#include <pinkIndexer/adaptions/crystfel/Lattice.h> +#include <pinkIndexer/adaptions/crystfel/ExperimentSettings.h> +#include <pinkIndexer/adaptions/crystfel/PinkIndexer.h> struct pinkIndexer_private_data { PinkIndexer *pinkIndexer; - reciprocalPeaks_1_per_A_t reciprocalPeaks_1_per_A; - float *intensities; IndexingMethod indm; UnitCell *cellTemplate; - int threadCount; - int multi; - int min_peaks; - - int no_check_indexed; float maxRefinementDisbalance; @@ -66,132 +79,182 @@ struct pinkIndexer_private_data { static void reduceReciprocalCell(UnitCell* cell, LatticeTransform_t* appliedReductionTransform); static void restoreReciprocalCell(UnitCell *cell, LatticeTransform_t* appliedReductionTransform); static void makeRightHanded(UnitCell* cell); -static void update_detector(struct detector *det, double xoffs, double yoffs); -int run_pinkIndexer(struct image *image, void *ipriv) + +static double mean_clen(struct detgeom *dg) { - struct pinkIndexer_private_data* pinkIndexer_private_data = (struct pinkIndexer_private_data*) ipriv; - reciprocalPeaks_1_per_A_t* reciprocalPeaks_1_per_A = &(pinkIndexer_private_data->reciprocalPeaks_1_per_A); - float *intensities = pinkIndexer_private_data->intensities; - - int peakCountMax = image_feature_count(image->features); - if (peakCountMax < 5) { - int goodLatticesCount = 0; - return goodLatticesCount; + int i; + double total = 0.0; + for ( i=0; i<dg->n_panels; i++ ) { + total += dg->panels[i].cnz; } - reciprocalPeaks_1_per_A->peakCount = 0; - for (int i = 0; i < peakCountMax && i < MAX_PEAK_COUNT_FOR_INDEXER; i++) { - struct imagefeature *f; - f = image_get_feature(image->features, i); - if (f == NULL) { - continue; + return total / dg->n_panels; +} + + +static void scale_detector_shift(double fake_clen, + struct detgeom *dg, + double inx, double iny, + double *pdx, double *pdy) +{ + int i; + double mean = mean_clen(dg); + for ( i=0; i<dg->n_panels; i++ ) { + if ( !within_tolerance(dg->panels[i].cnz, mean, 2.0) ) { + ERROR("WARNING: Detector is not flat enough to apply " + "detector position offset\n"); + *pdx = 0.0; + *pdy = 0.0; + return; } + } + *pdx = (mean/fake_clen)*inx; + *pdy = (mean/fake_clen)*iny; +} + - reciprocalPeaks_1_per_A->coordinates_x[reciprocalPeaks_1_per_A->peakCount] = f->rz * 1e-10; - reciprocalPeaks_1_per_A->coordinates_y[reciprocalPeaks_1_per_A->peakCount] = f->rx * 1e-10; - reciprocalPeaks_1_per_A->coordinates_z[reciprocalPeaks_1_per_A->peakCount] = f->ry * 1e-10; - intensities[reciprocalPeaks_1_per_A->peakCount] = (float) (f->intensity); - reciprocalPeaks_1_per_A->peakCount++; +int run_pinkIndexer(struct image *image, void *ipriv, int n_threads) +{ + struct pinkIndexer_private_data *pinkIndexer_private_data = ipriv; + reciprocalPeaks_1_per_A_t reciprocalPeaks_1_per_A; + float *intensities; + int npk; + int i; + + /* FIXME: Check if wavelength is too far from estimate */ + + npk = image_feature_count(image->features); + if ( npk < 5 ) return 0; + + if ( npk > MAX_PEAK_COUNT_FOR_INDEXER ) { + npk = MAX_PEAK_COUNT_FOR_INDEXER; + } + + reciprocalPeaks_1_per_A.peakCount = 0; + intensities = malloc(npk*sizeof(float)); + allocReciprocalPeaks(&reciprocalPeaks_1_per_A); + if ( intensities == NULL ) return 0; + + for ( i=0; i<npk; i++ ) { + + struct imagefeature *f; + double r[3]; + + f = image_get_feature(image->features, i); + if ( f == NULL ) continue; + + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + 0.0, 0.0, r); + reciprocalPeaks_1_per_A.coordinates_x[reciprocalPeaks_1_per_A.peakCount] = r[2] * 1e-10; + reciprocalPeaks_1_per_A.coordinates_y[reciprocalPeaks_1_per_A.peakCount] = r[0] * 1e-10; + reciprocalPeaks_1_per_A.coordinates_z[reciprocalPeaks_1_per_A.peakCount] = r[1] * 1e-10; + intensities[reciprocalPeaks_1_per_A.peakCount] = f->intensity; + reciprocalPeaks_1_per_A.peakCount++; } int indexed = 0; - Lattice_t indexedLattice[MAX_MULTI_LATTICE_COUNT]; - float center_shift[MAX_MULTI_LATTICE_COUNT][2]; + float center_shift[2]; + Lattice_t indexedLattice; + int matchedPeaksCount = PinkIndexer_indexPattern(pinkIndexer_private_data->pinkIndexer, + &indexedLattice, + center_shift, + &reciprocalPeaks_1_per_A, + intensities, + pinkIndexer_private_data->maxRefinementDisbalance, + n_threads); + free(intensities); + freeReciprocalPeaks(reciprocalPeaks_1_per_A); - do { - int peakCount = reciprocalPeaks_1_per_A->peakCount; - int matchedPeaksCount = PinkIndexer_indexPattern(pinkIndexer_private_data->pinkIndexer, - &(indexedLattice[indexed]), center_shift[indexed], reciprocalPeaks_1_per_A, intensities, - pinkIndexer_private_data->maxRefinementDisbalance, - pinkIndexer_private_data->threadCount); + if ( matchedPeaksCount == -1 ) { - if(matchedPeaksCount == -1){ - STATUS("WARNING: Indexing solution was rejected due to too large disbalance of the refinement." - "If you see this message often, check the documentation for the parameter " - "--pinkIndexer-max-refinement-disbalance\n"); + STATUS("WARNING: Indexing solution was rejected due to too " + "large imbalance of the refinement.\n" + "If you see this message often, check the documentation " + "for parameter --pinkIndexer-max-refinement-disbalance\n"); - matchedPeaksCount = 0; - } + } else { - printf("matchedPeaksCount %d from %d\n",matchedPeaksCount,peakCount); - if ((matchedPeaksCount >= 25 && matchedPeaksCount >= peakCount * 0.30) - || matchedPeaksCount >= peakCount * 0.4 - || matchedPeaksCount >= 70 - || pinkIndexer_private_data->no_check_indexed == 1) - { - UnitCell *uc; - uc = cell_new(); + UnitCell *uc; + UnitCell *new_cell_trans; - Lattice_t *l = &(indexedLattice[indexed]); + uc = cell_new(); - cell_set_reciprocal(uc, l->ay * 1e10, l->az * 1e10, l->ax * 1e10, - l->by * 1e10, l->bz * 1e10, l->bx * 1e10, - l->cy * 1e10, l->cz * 1e10, l->cx * 1e10); + cell_set_reciprocal(uc, indexedLattice.ay * 1e10, + indexedLattice.az * 1e10, + indexedLattice.ax * 1e10, + indexedLattice.by * 1e10, + indexedLattice.bz * 1e10, + indexedLattice.bx * 1e10, + indexedLattice.cy * 1e10, + indexedLattice.cz * 1e10, + indexedLattice.cx * 1e10); - restoreReciprocalCell(uc, &pinkIndexer_private_data->latticeReductionTransform); + restoreReciprocalCell(uc, &pinkIndexer_private_data->latticeReductionTransform); - UnitCell *new_cell_trans = cell_transform_intmat(uc, pinkIndexer_private_data->centeringTransformation); - cell_free(uc); - uc = new_cell_trans; + new_cell_trans = cell_transform_intmat(uc, pinkIndexer_private_data->centeringTransformation); + cell_free(uc); - cell_set_lattice_type(new_cell_trans, cell_get_lattice_type(pinkIndexer_private_data->cellTemplate)); - cell_set_centering(new_cell_trans, cell_get_centering(pinkIndexer_private_data->cellTemplate)); - cell_set_unique_axis(new_cell_trans, cell_get_unique_axis(pinkIndexer_private_data->cellTemplate)); + cell_set_lattice_type(new_cell_trans, + cell_get_lattice_type(pinkIndexer_private_data->cellTemplate)); + cell_set_centering(new_cell_trans, + cell_get_centering(pinkIndexer_private_data->cellTemplate)); + cell_set_unique_axis(new_cell_trans, + cell_get_unique_axis(pinkIndexer_private_data->cellTemplate)); - if (validate_cell(uc)) { - ERROR("pinkIndexer: problem with returned cell!\n"); - } + if ( validate_cell(new_cell_trans) ) { + ERROR("pinkIndexer: problem with returned cell!\n"); + } else { - Crystal * cr = crystal_new(); - if (cr == NULL) { + double dx, dy; + Crystal *cr = crystal_new(); + if ( cr == NULL ) { ERROR("Failed to allocate crystal.\n"); return 0; } - crystal_set_cell(cr, uc); - crystal_set_det_shift(cr, center_shift[indexed][0], center_shift[indexed][1]); - update_detector(image->det, center_shift[indexed][0], center_shift[indexed][1]); + crystal_set_cell(cr, new_cell_trans); + scale_detector_shift(FAKE_CLEN, + image->detgeom, + center_shift[0], + center_shift[1], + &dx, &dy); + crystal_set_det_shift(cr, dx, dy); image_add_crystal(image, cr); indexed++; - } else { - break; } - } while (pinkIndexer_private_data->multi - && indexed <= MAX_MULTI_LATTICE_COUNT - && reciprocalPeaks_1_per_A->peakCount >= pinkIndexer_private_data->min_peaks); + + } return indexed; } -void *pinkIndexer_prepare(IndexingMethod *indm, UnitCell *cell, + +void *pinkIndexer_prepare(IndexingMethod *indm, + UnitCell *cell, struct pinkIndexer_options *pinkIndexer_opts, - struct detector *det, struct beam_params *beam) + double wavelength_estimate) { - if ( beam->photon_energy_from != NULL && pinkIndexer_opts->customPhotonEnergy <= 0) { - ERROR("For pinkIndexer, the photon_energy must be defined as a " - "constant in the geometry file or as a parameter (see --pinkIndexer-override-photon-energy)\n"); - return NULL; - } - if ( (det->panels[0].clen_from != NULL) && pinkIndexer_opts->refinement_type == - REFINEMENT_TYPE_firstFixedThenVariableLatticeParametersCenterAdjustmentMultiSeed) { - ERROR("Using center refinement makes it necessary to have the detector distance fixed in the geometry file!"); + float beamEenergy_eV; + + if ( isnan(wavelength_estimate) ) { + ERROR("PinkIndexer requires a wavelength estimate. " + "Try again with --wavelength-estimate=xx\n"); return NULL; + } else { + beamEenergy_eV = J_to_eV(ph_lambda_to_en(wavelength_estimate)); } - if(cell == NULL){ - ERROR("Pink indexer needs a unit cell file to be specified!") + + if ( cell == NULL ) { + ERROR("Unit cell information is required for " + "PinkIndexer.\n"); return NULL; } struct pinkIndexer_private_data* pinkIndexer_private_data = malloc(sizeof(struct pinkIndexer_private_data)); - allocReciprocalPeaks(&(pinkIndexer_private_data->reciprocalPeaks_1_per_A)); - pinkIndexer_private_data->intensities = malloc(MAX_PEAK_COUNT_FOR_INDEXER * sizeof(float)); pinkIndexer_private_data->indm = *indm; pinkIndexer_private_data->cellTemplate = cell; - pinkIndexer_private_data->threadCount = pinkIndexer_opts->thread_count; - pinkIndexer_private_data->multi = pinkIndexer_opts->multi; - pinkIndexer_private_data->min_peaks = pinkIndexer_opts->min_peaks; - pinkIndexer_private_data->no_check_indexed = pinkIndexer_opts->no_check_indexed; pinkIndexer_private_data->maxRefinementDisbalance = pinkIndexer_opts->maxRefinementDisbalance; UnitCell* primitiveCell = uncenter_cell(cell, &pinkIndexer_private_data->centeringTransformation, NULL); @@ -209,21 +272,7 @@ void *pinkIndexer_prepare(IndexingMethod *indm, UnitCell *cell, .bx = bsz * 1e-10, .by = bsx * 1e-10, .bz = bsy * 1e-10, .cx = csz * 1e-10, .cy = csx * 1e-10, .cz = csy * 1e-10 }; - float detectorDistance_m; - if ( det->panels[0].clen_from != NULL ) { - detectorDistance_m = 0.25; /* fake value */ - } else { - detectorDistance_m = det->panels[0].clen + det->panels[0].coffset; - } - - float beamEenergy_eV = beam->photon_energy; - float nonMonochromaticity = beam->bandwidth*5; - if(pinkIndexer_opts->customPhotonEnergy > 0){ - beamEenergy_eV = pinkIndexer_opts->customPhotonEnergy; - } - if(pinkIndexer_opts->customBandwidth >= 0){ - nonMonochromaticity = pinkIndexer_opts->customBandwidth; - } + float nonMonochromaticity = 0.01; float reflectionRadius_1_per_A; if (pinkIndexer_opts->reflectionRadius < 0) { @@ -235,7 +284,9 @@ void *pinkIndexer_prepare(IndexingMethod *indm, UnitCell *cell, } if(beamEenergy_eV > 75000 && nonMonochromaticity < 0.02 && reflectionRadius_1_per_A < 0.0005){ - STATUS("Trying to index electron diffraction? It might be helpful to set a higher reflection radius (see documentation for --pinkIndexer-reflection-radius)") + STATUS("Trying to index electron diffraction? It might be " + " helpful to set a higher reflection radius " + "(see documentation for --pinkIndexer-reflection-radius)"); } float divergenceAngle_deg = 0.01; //fake @@ -243,9 +294,14 @@ void *pinkIndexer_prepare(IndexingMethod *indm, UnitCell *cell, float tolerance = pinkIndexer_opts->tolerance; Lattice_t sampleReciprocalLattice_1_per_A = lattice; float detectorRadius_m = 0.03; //fake, only for prediction - ExperimentSettings* experimentSettings = ExperimentSettings_new(beamEenergy_eV, detectorDistance_m, - detectorRadius_m, divergenceAngle_deg, nonMonochromaticity, sampleReciprocalLattice_1_per_A, tolerance, - reflectionRadius_1_per_A); + ExperimentSettings *experimentSettings = ExperimentSettings_new(beamEenergy_eV, + FAKE_CLEN, + detectorRadius_m, + divergenceAngle_deg, + nonMonochromaticity, + sampleReciprocalLattice_1_per_A, + tolerance, + reflectionRadius_1_per_A); consideredPeaksCount_t consideredPeaksCount = pinkIndexer_opts->considered_peaks_count; angleResolution_t angleResolution = pinkIndexer_opts->angle_resolution; @@ -341,24 +397,10 @@ static void makeRightHanded(UnitCell *cell) } } -//hack for electron crystallography while crystal_set_det_shift is not working approprietly -static void update_detector(struct detector *det, double xoffs, double yoffs) -{ - int i; - - for (i = 0; i < det->n_panels; i++) { - struct panel *p = &det->panels[i]; - p->cnx += xoffs * p->res; - p->cny += yoffs * p->res; - } -} - void pinkIndexer_cleanup(void *pp) { struct pinkIndexer_private_data* pinkIndexer_private_data = (struct pinkIndexer_private_data*) pp; - freeReciprocalPeaks(pinkIndexer_private_data->reciprocalPeaks_1_per_A); - free(pinkIndexer_private_data->intensities); intmat_free(pinkIndexer_private_data->centeringTransformation); PinkIndexer_delete(pinkIndexer_private_data->pinkIndexer); } @@ -370,15 +412,16 @@ const char *pinkIndexer_probe(UnitCell *cell) #else /* HAVE_PINKINDEXER */ -int run_pinkIndexer(struct image *image, void *ipriv) +int run_pinkIndexer(struct image *image, void *ipriv, int n_threads) { ERROR("This copy of CrystFEL was compiled without PINKINDEXER support.\n"); return 0; } -extern void *pinkIndexer_prepare(IndexingMethod *indm, UnitCell *cell, - struct pinkIndexer_options *pinkIndexer_opts, - struct detector *det, struct beam_params *beam) +extern void *pinkIndexer_prepare(IndexingMethod *indm, + UnitCell *cell, + struct pinkIndexer_options *pinkIndexer_opts, + double wavelength_estimate) { ERROR("This copy of CrystFEL was compiled without PINKINDEXER support.\n"); ERROR("To use PINKINDEXER indexing, recompile with PINKINDEXER.\n"); @@ -396,7 +439,7 @@ const char *pinkIndexer_probe(UnitCell *cell) #endif /* HAVE_PINKINDEXER */ -static void show_help() +static void pinkIndexer_show_help() { printf( "Parameters for the PinkIndexer indexing algorithm:\n" @@ -417,56 +460,49 @@ static void show_help() " Specified in 1/A. Default is 2%% of a*.\n" " --pinkIndexer-max-resolution-for-indexing=n\n" " Measured in 1/A\n" -" --pinkIndexer-multi Use pinkIndexers own multi indexing.\n" -" --pinkIndexer-thread-count=n\n" -" Thread count for internal parallelization \n" -" Default: 1\n" -" --pinkIndexer-no-check-indexed\n" -" Disable internal check for correct indexing\n" -" solutions\n" " --pinkIndexer-max-refinement-disbalance=n\n" -" Maximum disbalance after refinement:\n" -" 0 (no disbalance) to 2 (extreme disbalance), default 0.4\n" -" --pinkIndexer-override-photon-energy=ev\n" -" Mean energy in eV to use for indexing.\n" -" --pinkIndexer-override-bandwidth=n\n" -" Bandwidth in (delta energy)/(mean energy) to use for indexing.\n" -" --pinkIndexer-override-visible-energy-range=min-max\n" -" Overrides photon energy and bandwidth according to a range of \n" -" energies that have high enough intensity to produce \"visible\" \n" -" Bragg spots on the detector.\n" -" Min and max range borders are separated by a minus sign (no whitespace).\n" +" Maximum imbalance after refinement:\n" +" 0 (no imbalance) to 2 (extreme imbalance), default 0.4\n" ); } -static error_t parse_arg(int key, char *arg, struct argp_state *state) +int pinkIndexer_default_options(PinkIndexerOptions **opts_ptr) +{ + PinkIndexerOptions *opts; + + opts = malloc(sizeof(struct pinkIndexer_options)); + if ( opts == NULL ) return ENOMEM; + + opts->considered_peaks_count = 4; + opts->angle_resolution = 2; + opts->refinement_type = 1; + opts->tolerance = 0.06; + opts->maxResolutionForIndexing_1_per_A = +INFINITY; + opts->reflectionRadius = -1; + opts->maxRefinementDisbalance = 0.4; + + *opts_ptr = opts; + return 0; +} + + +static error_t pinkindexer_parse_arg(int key, char *arg, + struct argp_state *state) { - float tmp, tmp2; + float tmp; + int r; struct pinkIndexer_options **opts_ptr = state->input; switch ( key ) { case ARGP_KEY_INIT : - *opts_ptr = malloc(sizeof(struct pinkIndexer_options)); - if ( *opts_ptr == NULL ) return ENOMEM; - (*opts_ptr)->considered_peaks_count = 4; - (*opts_ptr)->angle_resolution = 2; - (*opts_ptr)->refinement_type = 1; - (*opts_ptr)->tolerance = 0.06; - (*opts_ptr)->maxResolutionForIndexing_1_per_A = +INFINITY; - (*opts_ptr)->thread_count = 1; - (*opts_ptr)->multi = 0; - (*opts_ptr)->no_check_indexed = 0; - (*opts_ptr)->min_peaks = 2; - (*opts_ptr)->reflectionRadius = -1; - (*opts_ptr)->customPhotonEnergy = -1; - (*opts_ptr)->customBandwidth = -1; - (*opts_ptr)->maxRefinementDisbalance = 0.4; + r = pinkIndexer_default_options(opts_ptr); + if ( r ) return r; break; case 1 : - show_help(); + pinkIndexer_show_help(); return EINVAL; case 2 : @@ -497,12 +533,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) break; case 5 : - if (sscanf(arg, "%d", &(*opts_ptr)->thread_count) != 1) - { - ERROR("Invalid value for --pinkIndexer-thread-count\n"); - return EINVAL; - } - break; + ERROR("Please use --max-indexer-threads instead of " + "--pinkIndexer-thread-count.\n"); + return EINVAL; case 6 : if (sscanf(arg, "%f", &(*opts_ptr)->maxResolutionForIndexing_1_per_A) != 1) @@ -522,11 +555,11 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) break; case 8 : - (*opts_ptr)->multi = 1; + ERROR("WARNING: --pinkIndexer-multi is ignored.\n"); break; case 9 : - (*opts_ptr)->no_check_indexed = 1; + ERROR("WARNING: --pinkIndexer-no-check-indexed is ignored.\n"); break; case 10 : @@ -538,32 +571,20 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) break; case 11 : - if (sscanf(arg, "%f", &(*opts_ptr)->customPhotonEnergy) != 1) - { - ERROR("Invalid value for --pinkIndexer-override-photon-energy\n"); - return EINVAL; - } - break; + ERROR("Please use --wavelength-estimate instead of " + "--pinkIndexer-override-photon-energy.\n"); + return EINVAL; case 12 : - if (sscanf(arg, "%f", &(*opts_ptr)->customBandwidth) != 1) - { - ERROR("Invalid value for --pinkIndexer-override-bandwidth\n"); - return EINVAL; - } - break; + ERROR("This CrystFEL version does not handle wide bandwidth "); + ERROR("(invalid option --pinkIndexer-override-bandwidth)\n"); + return EINVAL; + case 13 : - if (sscanf(arg, "%f-%f", &tmp, &tmp2) != 2) - { - ERROR("Invalid value for --pinkIndexer-override-visible-energy-range\n"); - return EINVAL; - } - (*opts_ptr)->customPhotonEnergy = (tmp + tmp2)/2; - (*opts_ptr)->customBandwidth = (tmp2 - tmp)/(*opts_ptr)->customPhotonEnergy; - if((*opts_ptr)->customBandwidth < 0){ - (*opts_ptr)->customBandwidth *= -1; - } - break; + ERROR("This CrystFEL version does not handle wide bandwidth "); + ERROR("(invalid option --pinkIndexer-override-visible-energy-range)\n"); + return EINVAL; + case 14 : if (sscanf(arg, "%f", &(*opts_ptr)->maxRefinementDisbalance) != 1) { @@ -576,7 +597,7 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) } -static struct argp_option options[] = { +static struct argp_option pinkindexer_options[] = { {"help-pinkindexer", 1, NULL, OPTION_NO_USAGE, "Show options for PinkIndexer indexing algorithm", 99}, @@ -617,4 +638,6 @@ static struct argp_option options[] = { }; -struct argp pinkIndexer_argp = { options, parse_arg, NULL, NULL, NULL, NULL, NULL }; +struct argp pinkIndexer_argp = { pinkindexer_options, + pinkindexer_parse_arg, + NULL, NULL, NULL, NULL, NULL }; diff --git a/libcrystfel/src/pinkindexer.h b/libcrystfel/src/indexers/pinkindexer.h index f79f4331..358a8221 100644 --- a/libcrystfel/src/pinkindexer.h +++ b/libcrystfel/src/indexers/pinkindexer.h @@ -3,11 +3,12 @@ * * Interface to PinkIndexer * - * Copyright © 2017-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2017-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: * 2017-2019 Yaroslav Gevorkov <yaroslav.gevorkov@desy.de> + * 2021 Thomas White <thomas.white@desy.de> * * This file is part of CrystFEL. * @@ -29,37 +30,19 @@ #ifndef LIBCRYSTFEL_SRC_PINKINDEXER_H_ #define LIBCRYSTFEL_SRC_PINKINDEXER_H_ -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - -typedef struct pinkIndexer_options PinkIndexerOptions; -extern struct argp pinkIndexer_argp; - -struct pinkIndexer_options { - unsigned int considered_peaks_count; - unsigned int angle_resolution; - unsigned int refinement_type; - float maxResolutionForIndexing_1_per_A; - float tolerance; - int multi; - int thread_count; - int min_peaks; - int no_check_indexed; - float reflectionRadius; /* In m^-1 */ - float customPhotonEnergy; - float customBandwidth; - float maxRefinementDisbalance; -}; - #include <stddef.h> + #include "index.h" +#include "datatemplate.h" + +extern int pinkIndexer_default_options(PinkIndexerOptions **opts_ptr); -extern int run_pinkIndexer(struct image *image, void *ipriv); +extern int run_pinkIndexer(struct image *image, void *ipriv, int n_threads); -extern void *pinkIndexer_prepare(IndexingMethod *indm, UnitCell *cell, +extern void *pinkIndexer_prepare(IndexingMethod *indm, + UnitCell *cell, struct pinkIndexer_options *pinkIndexer_opts, - struct detector *det, struct beam_params *beam); + double wavelength_estimate); extern void pinkIndexer_cleanup(void *pp); diff --git a/libcrystfel/src/taketwo.c b/libcrystfel/src/indexers/taketwo.c index a67d372c..1bccf1e0 100644 --- a/libcrystfel/src/taketwo.c +++ b/libcrystfel/src/indexers/taketwo.c @@ -4,12 +4,12 @@ * Rewrite of TakeTwo algorithm (Acta D72 (8) 956-965) for CrystFEL * * Copyright © 2016-2017 Helen Ginn - * Copyright © 2016-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2016-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: * 2016-2017 Helen Ginn <helen@strubi.ox.ac.uk> - * 2016-2017 Thomas White <taw@physics.org> + * 2016-2021 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -86,6 +86,10 @@ * * Clean up the mess (cleanup_taketwo_obs_vecs()) */ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + #include <gsl/gsl_matrix.h> #include <gsl/gsl_blas.h> #include <float.h> @@ -229,7 +233,10 @@ struct TakeTwoCell /* Maximum observed vectors before TakeTwo gives up and deals with * what is already there. */ -#define MAX_OBS_VECTORS 100000 +#define MAX_OBS_VECTORS 300 + +/* Maximum number of seeds to start from in start_seeds() */ +#define MAX_SEEDS 10 /* Tolerance for two angles to be considered the same */ #define ANGLE_TOLERANCE (deg2rad(0.6)) @@ -527,7 +534,7 @@ static double matrix_trace(gsl_matrix *a) return tr; } -static char *add_ua(const char *inp, char ua) +static char *add_unique_axis(const char *inp, char ua) { char *pg = malloc(64); if ( pg == NULL ) return NULL; @@ -540,12 +547,13 @@ static char *get_chiral_holohedry(UnitCell *cell) { LatticeType lattice = cell_get_lattice_type(cell); char *pg; - char *pgout = 0; + int add_ua = 1; switch (lattice) { case L_TRICLINIC: pg = "1"; + add_ua = 0; break; case L_MONOCLINIC: @@ -554,6 +562,7 @@ static char *get_chiral_holohedry(UnitCell *cell) case L_ORTHORHOMBIC: pg = "222"; + add_ua = 0; break; case L_TETRAGONAL: @@ -562,11 +571,13 @@ static char *get_chiral_holohedry(UnitCell *cell) case L_RHOMBOHEDRAL: pg = "3_R"; + add_ua = 0; break; case L_HEXAGONAL: if ( cell_get_centering(cell) == 'H' ) { pg = "3_H"; + add_ua = 0; } else { pg = "622"; } @@ -574,6 +585,7 @@ static char *get_chiral_holohedry(UnitCell *cell) case L_CUBIC: pg = "432"; + add_ua = 0; break; default: @@ -581,26 +593,11 @@ static char *get_chiral_holohedry(UnitCell *cell) break; } - switch (lattice) - { - case L_TRICLINIC: - case L_ORTHORHOMBIC: - case L_RHOMBOHEDRAL: - case L_CUBIC: - pgout = strdup(pg); - break; - - case L_MONOCLINIC: - case L_TETRAGONAL: - case L_HEXAGONAL: - pgout = add_ua(pg, cell_get_unique_axis(cell)); - break; - - default: - break; + if ( add_ua ) { + return add_unique_axis(pg, cell_get_unique_axis(cell)); + } else { + return strdup(pg); } - - return pgout; } @@ -1553,7 +1550,8 @@ static int find_seeds(struct TakeTwoCell *cell, struct taketwo_private *tp) cell->seeds = tmp; - for (int i = 0; i < seed_num; i++) + int i; + for ( i = 0; i < seed_num; i++) { if (seeds[i].idx1 < 0 || seeds[i].idx2 < 0) { @@ -1582,10 +1580,12 @@ static unsigned int start_seeds(gsl_matrix **rotation, struct TakeTwoCell *cell) int member_num = 0; int max_members = 0; gsl_matrix *rot = NULL; + int k; + + if ( seed_num > MAX_SEEDS ) seed_num = MAX_SEEDS; /* We have seeds! Pass each of them through the seed-starter */ /* If a seed has the highest achieved membership, make note...*/ - int k; for ( k=0; k<seed_num; k++ ) { int seed_idx1 = seeds[k].idx1; int seed_idx2 = seeds[k].idx2; @@ -2102,7 +2102,8 @@ static void partial_taketwo_cleanup(struct taketwo_private *tp) { if (tp->prevSols != NULL) { - for (int i = 0; i < tp->numPrevs; i++) + int i; + for (i = 0; i < tp->numPrevs; i++) { gsl_matrix_free(tp->prevSols[i]); } @@ -2154,18 +2155,20 @@ int taketwo_index(struct image *image, void *priv) tp->xtal_num = image->n_crystals; } - /* - STATUS("Indexing %i with %i attempts, %i crystals\n", this_serial, tp->attempts, - image->n_crystals); - */ - rlps = malloc((image_feature_count(image->features)+1)*sizeof(struct rvec)); for ( i=0; i<image_feature_count(image->features); i++ ) { + + double r[3]; struct imagefeature *pk = image_get_feature(image->features, i); if ( pk == NULL ) continue; - rlps[n_rlps].u = pk->rx; - rlps[n_rlps].v = pk->ry; - rlps[n_rlps].w = pk->rz; + + detgeom_transform_coords(&image->detgeom->panels[pk->pn], + pk->fs, pk->ss, image->lambda, + 0.0, 0.0, r); + + rlps[n_rlps].u = r[0]; + rlps[n_rlps].v = r[1]; + rlps[n_rlps].w = r[2]; n_rlps++; } rlps[n_rlps].u = 0.0; @@ -2235,6 +2238,7 @@ void *taketwo_prepare(IndexingMethod *indm, struct taketwo_options *opts, if ( tp == NULL ) return NULL; tp->cell = cell; + tp->opts = opts; tp->indm = *indm; tp->serial_num = -1; tp->xtal_num = 0; @@ -2269,7 +2273,7 @@ const char *taketwo_probe(UnitCell *cell) } -static void show_help() +static void taketwo_show_help() { printf("Parameters for the TakeTwo indexing algorithm:\n" " --taketwo-member-threshold\n" @@ -2284,24 +2288,38 @@ static void show_help() } -static error_t parse_arg(int key, char *arg, struct argp_state *state) +int taketwo_default_options(TakeTwoOptions **opts_ptr) +{ + TakeTwoOptions *opts; + + opts = malloc(sizeof(struct taketwo_options)); + if ( opts == NULL ) return ENOMEM; + opts->member_thresh = -1.0; + opts->len_tol = -1.0; + opts->angle_tol = -1.0; + opts->trace_tol = -1.0; + + *opts_ptr = opts; + return 0; +} + + +static error_t taketwo_parse_arg(int key, char *arg, + struct argp_state *state) { struct taketwo_options **opts_ptr = state->input; float tmp; + int r; switch ( key ) { case ARGP_KEY_INIT : - *opts_ptr = malloc(sizeof(struct taketwo_options)); - if ( *opts_ptr == NULL ) return ENOMEM; - (*opts_ptr)->member_thresh = -1.0; - (*opts_ptr)->len_tol = -1.0; - (*opts_ptr)->angle_tol = -1.0; - (*opts_ptr)->trace_tol = -1.0; + r = taketwo_default_options(opts_ptr); + if ( r ) return r; break; case 1 : - show_help(); + taketwo_show_help(); return EINVAL; case 2 : @@ -2348,7 +2366,7 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) } -static struct argp_option options[] = { +static struct argp_option taketwo_options[] = { {"help-taketwo", 1, NULL, OPTION_NO_USAGE, "Show options for TakeTwo indexing algorithm", 99}, @@ -2361,4 +2379,5 @@ static struct argp_option options[] = { }; -struct argp taketwo_argp = { options, parse_arg, NULL, NULL, NULL, NULL, NULL }; +struct argp taketwo_argp = { taketwo_options, taketwo_parse_arg, + NULL, NULL, NULL, NULL, NULL }; diff --git a/libcrystfel/src/taketwo.h b/libcrystfel/src/indexers/taketwo.h index 2239a77a..f3157d72 100644 --- a/libcrystfel/src/taketwo.h +++ b/libcrystfel/src/indexers/taketwo.h @@ -4,7 +4,7 @@ * Rewrite of TakeTwo algorithm (Acta D72 (8) 956-965) for CrystFEL * * Copyright © 2016-2017 Helen Ginn - * Copyright © 2016-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2016-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: @@ -38,9 +38,7 @@ /** \file taketwo.h */ -typedef struct taketwo_options TakeTwoOptions; -extern struct argp taketwo_argp; - +extern int taketwo_default_options(TakeTwoOptions **opts_ptr); extern void *taketwo_prepare(IndexingMethod *indm, struct taketwo_options *opts, UnitCell *cell); extern const char *taketwo_probe(UnitCell *cell); diff --git a/libcrystfel/src/xds.c b/libcrystfel/src/indexers/xds.c index ab6335db..3b2f2b87 100644 --- a/libcrystfel/src/xds.c +++ b/libcrystfel/src/indexers/xds.c @@ -3,13 +3,13 @@ * * Invoke xds for crystal autoindexing * - * Copyright © 2013-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2013 Cornelius Gati * * Authors: * 2010-2018 Thomas White <taw@physics.org> - * 2013 Cornelius Gati <cornelius.gati@cfel.de> + * 2013 Cornelius Gati <cornelius.gati@cfel.de> * * This file is part of CrystFEL. * @@ -56,7 +56,6 @@ #include "image.h" #include "utils.h" #include "peaks.h" -#include "detector.h" #include "cell-utils.h" /** \file xds.h */ @@ -74,7 +73,7 @@ struct xds_private }; -/* Essentially the reverse of spacegroup_for_lattice(), below */ +/* Essentially the reverse of xds_spacegroup_for_lattice(), below */ static int convert_spacegroup_number(int spg, LatticeType *lt, char *cen, char *ua) { @@ -214,17 +213,22 @@ static void write_spot(struct image *image) { struct imagefeature *f; double ttx, tty, x, y; + double r[3]; f = image_get_feature(image->features, i); if ( f == NULL ) continue; if ( f->intensity <= 0 ) continue; + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + 0.0, 0.0, r); + ttx = angle_between_2d(0.0, 1.0, - f->rx, 1.0/image->lambda + f->rz); + r[0], 1.0/image->lambda + r[2]); tty = angle_between_2d(0.0, 1.0, - f->ry, 1.0/image->lambda + f->rz); - if ( f->rx < 0.0 ) ttx *= -1.0; - if ( f->ry < 0.0 ) tty *= -1.0; + r[1], 1.0/image->lambda + r[2]); + if ( r[0] < 0.0 ) ttx *= -1.0; + if ( r[1] < 0.0 ) tty *= -1.0; x = tan(ttx)*FAKE_CLEN; y = tan(tty)*FAKE_CLEN; @@ -241,7 +245,7 @@ static void write_spot(struct image *image) /* Turn what we know about the unit cell into something which we can give to * XDS to make it give us only indexing results compatible with the cell. */ -static const char *spacegroup_for_lattice(UnitCell *cell) +static const char *xds_spacegroup_for_lattice(UnitCell *cell) { LatticeType latt; char centering; @@ -336,7 +340,7 @@ static int write_inp(struct image *image, struct xds_private *xp) if ( xp->indm & INDEXING_USE_LATTICE_TYPE ) { fprintf(fh, "SPACE_GROUP_NUMBER= %s\n", - spacegroup_for_lattice(xp->cell)); + xds_spacegroup_for_lattice(xp->cell)); } else { fprintf(fh, "SPACE_GROUP_NUMBER= 0\n"); } @@ -462,7 +466,7 @@ void *xds_prepare(IndexingMethod *indm, UnitCell *cell) return NULL; } - if ( (cell != NULL) && (spacegroup_for_lattice(cell) == NULL) ) { + if ( (cell != NULL) && (xds_spacegroup_for_lattice(cell) == NULL) ) { ERROR("Don't know how to ask XDS for your cell.\n"); return NULL; } diff --git a/libcrystfel/src/xds.h b/libcrystfel/src/indexers/xds.h index bb96feca..49a83ff5 100644 --- a/libcrystfel/src/xds.h +++ b/libcrystfel/src/indexers/xds.h @@ -3,13 +3,13 @@ * * Invoke xds for crystal autoindexing * - * Copyright © 2013-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2013 Cornelius Gati * * Authors: - * 2010-2013,2017 Thomas White <taw@physics.org> - * 2013 Cornelius Gati <cornelius.gati@cfel.de> + * 2010-2017 Thomas White <taw@physics.org> + * 2013 Cornelius Gati <cornelius.gati@cfel.de> * * This file is part of CrystFEL. * @@ -31,10 +31,6 @@ #ifndef XDS_H #define XDS_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include "cell.h" #include "index.h" diff --git a/libcrystfel/src/xgandalf.c b/libcrystfel/src/indexers/xgandalf.c index 9135c2f3..6b09f65c 100644 --- a/libcrystfel/src/xgandalf.c +++ b/libcrystfel/src/indexers/xgandalf.c @@ -3,7 +3,7 @@ * * Interface to XGANDALF indexer * - * Copyright © 2017-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2017-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: @@ -26,6 +26,10 @@ * */ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + #include "xgandalf.h" #include <stdlib.h> @@ -80,21 +84,28 @@ static void makeRightHanded(UnitCell* cell); int run_xgandalf(struct image *image, void *ipriv) { + int i; struct xgandalf_private_data *xgandalf_private_data = (struct xgandalf_private_data*) ipriv; reciprocalPeaks_1_per_A_t *reciprocalPeaks_1_per_A = &(xgandalf_private_data->reciprocalPeaks_1_per_A); int peakCountMax = image_feature_count(image->features); reciprocalPeaks_1_per_A->peakCount = 0; - for (int i = 0; i < peakCountMax && i < MAX_PEAK_COUNT_FOR_INDEXER; i++) { + for ( i = 0; i < peakCountMax && i < MAX_PEAK_COUNT_FOR_INDEXER; i++) { struct imagefeature *f; + double r[3]; + f = image_get_feature(image->features, i); if (f == NULL) { continue; } - reciprocalPeaks_1_per_A->coordinates_x[reciprocalPeaks_1_per_A->peakCount] = f->rx * 1e-10; - reciprocalPeaks_1_per_A->coordinates_y[reciprocalPeaks_1_per_A->peakCount] = f->ry * 1e-10; - reciprocalPeaks_1_per_A->coordinates_z[reciprocalPeaks_1_per_A->peakCount] = f->rz * 1e-10; + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + 0.0, 0.0, r); + + reciprocalPeaks_1_per_A->coordinates_x[reciprocalPeaks_1_per_A->peakCount] = r[0] * 1e-10; + reciprocalPeaks_1_per_A->coordinates_y[reciprocalPeaks_1_per_A->peakCount] = r[1] * 1e-10; + reciprocalPeaks_1_per_A->coordinates_z[reciprocalPeaks_1_per_A->peakCount] = r[2] * 1e-10; reciprocalPeaks_1_per_A->peakCount++; } @@ -112,7 +123,7 @@ int run_xgandalf(struct image *image, void *ipriv) } int goodLatticesCount = assembledLatticesCount; - for (int i = 0; i < assembledLatticesCount && i < 1; i++) { + for ( i = 0; i < assembledLatticesCount && i < 1; i++) { reorderLattice(&(xgandalf_private_data->sampleRealLattice_A), &assembledLattices[i]); @@ -346,7 +357,7 @@ const char *xgandalf_probe(UnitCell *cell) #endif // HAVE_XGANDALF -static void show_help() +static void xgandalf_show_help() { printf("Parameters for the TakeTwo indexing algorithm:\n" " --xgandalf-sampling-pitch\n" @@ -378,26 +389,41 @@ static void show_help() } -static error_t parse_arg(int key, char *arg, struct argp_state *state) +int xgandalf_default_options(XGandalfOptions **opts_ptr) +{ + XGandalfOptions *opts; + + opts = malloc(sizeof(struct xgandalf_options)); + if ( opts == NULL ) return ENOMEM; + + opts->sampling_pitch = 6; + opts->grad_desc_iterations = 4; + opts->tolerance = 0.02; + opts->no_deviation_from_provided_cell = 0; + opts->minLatticeVectorLength_A = 30; + opts->maxLatticeVectorLength_A = 250; + opts->maxPeaksForIndexing = 250; + + *opts_ptr = opts; + return 0; +} + + +static error_t xgandalf_parse_arg(int key, char *arg, + struct argp_state *state) { struct xgandalf_options **opts_ptr = state->input; + int r; switch ( key ) { case ARGP_KEY_INIT : - *opts_ptr = malloc(sizeof(struct xgandalf_options)); - if ( *opts_ptr == NULL ) return ENOMEM; - (*opts_ptr)->sampling_pitch = 6; - (*opts_ptr)->grad_desc_iterations = 4; - (*opts_ptr)->tolerance = 0.02; - (*opts_ptr)->no_deviation_from_provided_cell = 0; - (*opts_ptr)->minLatticeVectorLength_A = 30; - (*opts_ptr)->maxLatticeVectorLength_A = 250; - (*opts_ptr)->maxPeaksForIndexing = 250; + r = xgandalf_default_options(opts_ptr); + if ( r ) return r; break; case 1 : - show_help(); + xgandalf_show_help(); return EINVAL; case 2 : @@ -457,7 +483,7 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) } -static struct argp_option options[] = { +static struct argp_option xgandalf_options[] = { {"help-xgandalf", 1, NULL, OPTION_NO_USAGE, "Show options for XGANDALF indexing algorithm", 99}, @@ -488,4 +514,5 @@ static struct argp_option options[] = { }; -struct argp xgandalf_argp = { options, parse_arg, NULL, NULL, NULL, NULL, NULL }; +struct argp xgandalf_argp = { xgandalf_options, xgandalf_parse_arg, + NULL, NULL, NULL, NULL, NULL }; diff --git a/libcrystfel/src/xgandalf.h b/libcrystfel/src/indexers/xgandalf.h index 07fcba01..79078410 100644 --- a/libcrystfel/src/xgandalf.h +++ b/libcrystfel/src/indexers/xgandalf.h @@ -3,7 +3,7 @@ * * Interface to XGANDALF indexer * - * Copyright © 2017-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2017-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: @@ -29,10 +29,6 @@ #ifndef LIBCRYSTFEL_SRC_XGANDALF_H #define LIBCRYSTFEL_SRC_XGANDALF_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include <stddef.h> #include <argp.h> @@ -41,11 +37,10 @@ * XGANDALF indexer interface */ -typedef struct xgandalf_options XGandalfOptions; -extern struct argp xgandalf_argp; - #include "index.h" +extern int xgandalf_default_options(XGandalfOptions **opts_ptr); + extern int run_xgandalf(struct image *image, void *ipriv); extern void *xgandalf_prepare(IndexingMethod *indm, UnitCell *cell, diff --git a/libcrystfel/src/integer_matrix.c b/libcrystfel/src/integer_matrix.c index 43f41eeb..7fcd07c4 100644 --- a/libcrystfel/src/integer_matrix.c +++ b/libcrystfel/src/integer_matrix.c @@ -3,11 +3,11 @@ * * A small integer matrix library * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2012 Thomas White <taw@physics.org> + * 2012-2020 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -253,8 +253,9 @@ void intmat_zero(IntegerMatrix *m) } -static IntegerMatrix *delete_row_and_column(const IntegerMatrix *m, - unsigned int di, unsigned int dj) +static IntegerMatrix *intmat_delete_row_and_column(const IntegerMatrix *m, + unsigned int di, + unsigned int dj) { IntegerMatrix *n; unsigned int i, j; @@ -280,13 +281,13 @@ static IntegerMatrix *delete_row_and_column(const IntegerMatrix *m, } -static signed int cofactor(const IntegerMatrix *m, - unsigned int i, unsigned int j) +static signed int intmat_cofactor(const IntegerMatrix *m, + unsigned int i, unsigned int j) { IntegerMatrix *n; signed int t, C; - n = delete_row_and_column(m, i, j); + n = intmat_delete_row_and_column(m, i, j); if ( n == NULL ) { fprintf(stderr, "Failed to allocate matrix.\n"); return 0; @@ -324,7 +325,7 @@ signed int intmat_det(const IntegerMatrix *m) i = 0; /* Fixed */ for ( j=0; j<m->cols; j++ ) { - det += intmat_get(m, i, j) * cofactor(m, i, j); + det += intmat_get(m, i, j) * intmat_cofactor(m, i, j); } @@ -343,7 +344,7 @@ static IntegerMatrix *intmat_cofactors(const IntegerMatrix *m) for ( i=0; i<n->rows; i++ ) { for ( j=0; j<n->cols; j++ ) { - intmat_set(n, i, j, cofactor(m, i, j)); + intmat_set(n, i, j, intmat_cofactor(m, i, j)); } } diff --git a/libcrystfel/src/integer_matrix.h b/libcrystfel/src/integer_matrix.h index 024480c3..2ebef300 100644 --- a/libcrystfel/src/integer_matrix.h +++ b/libcrystfel/src/integer_matrix.h @@ -3,11 +3,11 @@ * * A small integer matrix library * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2012 Thomas White <taw@physics.org> + * 2012-2019 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -29,10 +29,6 @@ #ifndef INTEGER_MATRIX_H #define INTEGER_MATRIX_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - /** * \file integer_matrix.h * Matrix type containing only integers diff --git a/libcrystfel/src/integration.c b/libcrystfel/src/integration.c index 3d8258c1..4c813a11 100644 --- a/libcrystfel/src/integration.c +++ b/libcrystfel/src/integration.c @@ -3,11 +3,11 @@ * * Integration of intensities * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2016 Thomas White <taw@physics.org> + * 2010-2021 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -52,6 +52,7 @@ #include "image.h" #include "peaks.h" #include "integration.h" +#include "detgeom.h" /** \file integration.h */ @@ -110,7 +111,7 @@ struct peak_box enum boxmask_val *bm; /* Box mask */ int pn; /* Panel number */ - struct panel *p; /* The panel itself */ + struct detgeom_panel *p; /* The panel itself */ /* Fitted background parameters */ double a; @@ -273,9 +274,6 @@ static void show_peak_box(struct intcontext *ic, struct peak_box *bx, get_indices(bx->refl, &h, &k, &l); get_detector_pos(bx->refl, &fs, &ss); - /* Convert coordinates to match arrangement of panels in HDF5 file */ - fs = fs + bx->p->orig_min_fs; - ss = ss + bx->p->orig_min_ss; printw("Indices %i %i %i\nPanel %s\nPosition fs = %.1f, ss = %.1f\n\n", h, k, l, bx->p->name, fs, ss); @@ -432,33 +430,106 @@ static int alloc_boxes(struct intcontext *ic, int new_max_boxes) } -static int init_intcontext(struct intcontext *ic) +static void setup_ring_masks(struct intcontext *ic, + double ir_inn, + double ir_mid, + double ir_out) { - int i; + double lim_sq, out_lim_sq, mid_lim_sq; + int p, q; + + lim_sq = pow(ir_inn, 2.0); + mid_lim_sq = pow(ir_mid, 2.0); + out_lim_sq = pow(ir_out, 2.0); + + for ( p=0; p<ic->w; p++ ) { + for ( q=0; q<ic->w; q++ ) { + + int rsq; + + rsq = (p-ic->halfw)*(p-ic->halfw) + (q-ic->halfw)*(q-ic->halfw); + + if ( rsq > out_lim_sq ) { + /* Outside outer radius */ + ic->bm[p + ic->w*q] = BM_IG; + } else { + + if ( rsq >= mid_lim_sq ) { + /* Inside outer radius, outside middle radius */ + ic->bm[p + ic->w*q] = BM_BG; + } else if ( rsq <= lim_sq ) { + /* Inside inner radius */ + ic->bm[p + ic->w*q] = BM_PK; + } else { + /* Outside inner radius, inside middle radius */ + ic->bm[p + ic->w*q] = BM_IG; + } + } + + } + } + +} + + +void intcontext_set_diag(struct intcontext *ic, + IntDiag int_diag, + signed int idh, + signed int idk, + signed int idl) +{ + ic->int_diag = int_diag; + ic->int_diag_h = idh; + ic->int_diag_k = idk; + ic->int_diag_l = idl; +} + + +struct intcontext *intcontext_new(struct image *image, + UnitCell *cell, + IntegrationMethod meth, + int ir_inn, int ir_mid, int ir_out, + int **masks) +{ + int i; + struct intcontext *ic; + + ic = malloc(sizeof(struct intcontext)); + if ( ic == NULL ) return NULL; + + ic->halfw = ir_out; + ic->image = image; + ic->k = 1.0/image->lambda; + ic->meth = meth; + ic->n_saturated = 0; + ic->n_implausible = 0; + ic->cell = cell; + ic->masks = masks; + ic->int_diag = INTDIAG_NONE; ic->w = 2*ic->halfw + 1; ic->bm = malloc(ic->w * ic->w * sizeof(enum boxmask_val)); if ( ic->bm == NULL ) { ERROR("Failed to allocate box mask.\n"); - return 1; + return NULL; } /* How many reference profiles? */ ic->n_reference_profiles = 1; ic->reference_profiles = calloc(ic->n_reference_profiles, sizeof(double *)); - if ( ic->reference_profiles == NULL ) return 1; + if ( ic->reference_profiles == NULL ) return NULL; ic->reference_den = calloc(ic->n_reference_profiles, sizeof(double *)); - if ( ic->reference_den == NULL ) return 1; + if ( ic->reference_den == NULL ) return NULL; ic->n_profiles_in_reference = calloc(ic->n_reference_profiles, sizeof(int)); - if ( ic->n_profiles_in_reference == NULL ) return 1; + if ( ic->n_profiles_in_reference == NULL ) return NULL; for ( i=0; i<ic->n_reference_profiles; i++ ) { ic->reference_profiles[i] = malloc(ic->w*ic->w*sizeof(double)); - if ( ic->reference_profiles[i] == NULL ) return 1; + if ( ic->reference_profiles[i] == NULL ) return NULL; ic->reference_den[i] = malloc(ic->w*ic->w*sizeof(double)); - if ( ic->reference_den[i] == NULL ) return 1; + if ( ic->reference_den[i] == NULL ) return NULL; } zero_profiles(ic); @@ -466,14 +537,16 @@ static int init_intcontext(struct intcontext *ic) ic->n_boxes = 0; ic->max_boxes = 0; if ( alloc_boxes(ic, 32) ) { - return 1; + return NULL; } - return 0; + setup_ring_masks(ic, ir_inn, ir_mid, ir_out); + + return ic; } -static void free_intcontext(struct intcontext *ic) +void intcontext_free(struct intcontext *ic) { int i; @@ -494,47 +567,6 @@ static void free_intcontext(struct intcontext *ic) } -static void setup_ring_masks(struct intcontext *ic, - double ir_inn, double ir_mid, double ir_out) -{ - double lim_sq, out_lim_sq, mid_lim_sq; - int p, q; - - lim_sq = pow(ir_inn, 2.0); - mid_lim_sq = pow(ir_mid, 2.0); - out_lim_sq = pow(ir_out, 2.0); - - for ( p=0; p<ic->w; p++ ) { - for ( q=0; q<ic->w; q++ ) { - - int rsq; - - rsq = (p-ic->halfw)*(p-ic->halfw) + (q-ic->halfw)*(q-ic->halfw); - - if ( rsq > out_lim_sq ) { - /* Outside outer radius */ - ic->bm[p + ic->w*q] = BM_IG; - } else { - - if ( rsq >= mid_lim_sq ) { - /* Inside outer radius, outside middle radius */ - ic->bm[p + ic->w*q] = BM_BG; - } else if ( rsq <= lim_sq ) { - /* Inside inner radius */ - ic->bm[p + ic->w*q] = BM_PK; - } else { - /* Outside inner radius, inside middle radius */ - ic->bm[p + ic->w*q] = BM_IG; - } - - } - - } - } - -} - - static struct peak_box *add_box(struct intcontext *ic) { int idx; @@ -804,9 +836,6 @@ static int check_box(struct intcontext *ic, struct peak_box *bx, int *sat) for ( q=0; q<ic->w; q++ ) { int fs, ss; - double hd, kd, ld; - signed int h, k, l; - struct rvec dv; float lsat; fs = bx->cfs + p; @@ -862,22 +891,11 @@ static int check_box(struct intcontext *ic, struct peak_box *bx, int *sat) if ( sat != NULL ) *sat = 1; } - /* Ignore if this pixel is closer to the next reciprocal lattice - * point */ - dv = get_q_for_panel(bx->p, fs, ss, NULL, ic->k); - hd = dv.u * adx + dv.v * ady + dv.w * adz; - kd = dv.u * bdx + dv.v * bdy + dv.w * bdz; - ld = dv.u * cdx + dv.v * cdy + dv.w * cdz; - h = lrint(hd); - k = lrint(kd); - l = lrint(ld); - if ( (h != hr) || (k != kr) || (l != lr) ) { - bx->bm[p+ic->w*q] = BM_BH; - } - + /* Find brightest pixel */ if ( (bx->bm[p+ic->w*q] != BM_IG) && (bx->bm[p+ic->w*q] != BM_BH) - && (boxi(ic, bx, p, q) > bx->peak) ) { + && (boxi(ic, bx, p, q) > bx->peak) ) + { bx->peak = boxi(ic, bx, p, q); } @@ -1273,7 +1291,6 @@ static void setup_profile_boxes(struct intcontext *ic, RefList *list) double pfs, pss; struct peak_box *bx; int pn; - struct panel *p; int fid_fs, fid_ss; /* Center coordinates, rounded, * in overall data block */ int cfs, css; /* Corner coordinates */ @@ -1283,12 +1300,7 @@ static void setup_profile_boxes(struct intcontext *ic, RefList *list) set_redundancy(refl, 0); get_detector_pos(refl, &pfs, &pss); - p = get_panel(refl); - pn = panel_number(ic->image->det, p); - if ( pn == ic->image->det->n_panels ) { - ERROR("Couldn't find panel %p\n", p); - continue; - } + pn = get_panel_number(refl); /* Explicit truncation of digits after the decimal point. * This is actually the correct thing to do here, not @@ -1307,7 +1319,7 @@ static void setup_profile_boxes(struct intcontext *ic, RefList *list) bx->refl = refl; bx->cfs = cfs; bx->css = css; - bx->p = p; + bx->p = &ic->image->detgeom->panels[pn]; bx->pn = pn; /* Which reference profile? */ @@ -1345,68 +1357,58 @@ static void setup_profile_boxes(struct intcontext *ic, RefList *list) } -static void integrate_prof2d(IntegrationMethod meth, - Crystal *cr, struct image *image, IntDiag int_diag, - signed int idh, signed int idk, signed int idl, - double ir_inn, double ir_mid, double ir_out, - pthread_mutex_t *term_lock, int **masks) +void integrate_prof2d(IntegrationMethod meth, + Crystal *cr, struct image *image, IntDiag int_diag, + signed int idh, signed int idk, signed int idl, + double ir_inn, double ir_mid, double ir_out, + pthread_mutex_t *term_lock, int **masks) { RefList *list; UnitCell *cell; - struct intcontext ic; + struct intcontext *ic; int i; list = crystal_get_reflections(cr); cell = crystal_get_cell(cr); - ic.halfw = ir_out; - ic.image = image; - ic.k = 1.0/image->lambda; - ic.meth = meth; - ic.n_saturated = 0; - ic.n_implausible = 0; - ic.cell = cell; - ic.int_diag = int_diag; - ic.int_diag_h = idh; - ic.int_diag_k = idk; - ic.int_diag_l = idl; - ic.masks = masks; - if ( init_intcontext(&ic) ) { + ic = intcontext_new(image, cell, meth, + ir_inn, ir_mid, ir_out, + masks); + if ( ic == NULL ) { ERROR("Failed to initialise integration.\n"); return; } - setup_ring_masks(&ic, ir_inn, ir_mid, ir_out); - setup_profile_boxes(&ic, list); - calculate_reference_profiles(&ic); + intcontext_set_diag(ic, int_diag, idh, idk, idl); + setup_profile_boxes(ic, list); + calculate_reference_profiles(ic); - for ( i=0; i<ic.n_reference_profiles; i++ ) { - if ( ic.n_profiles_in_reference[i] == 0 ) { + for ( i=0; i<ic->n_reference_profiles; i++ ) { + if ( ic->n_profiles_in_reference[i] == 0 ) { ERROR("Reference profile %i has no contributions.\n", i); - free_intcontext(&ic); + intcontext_free(ic); return; } } - for ( i=0; i<ic.n_boxes; i++ ) { + for ( i=0; i<ic->n_boxes; i++ ) { struct peak_box *bx; - bx = &ic.boxes[i]; - integrate_prof2d_once(&ic, bx, term_lock); + bx = &ic->boxes[i]; + integrate_prof2d_once(ic, bx, term_lock); } - free_intcontext(&ic); + intcontext_free(ic); } -static int integrate_rings_once(Reflection *refl, struct image *image, - struct intcontext *ic, UnitCell *cell, - pthread_mutex_t *term_lock) +int integrate_rings_once(Reflection *refl, + struct intcontext *ic, + pthread_mutex_t *term_lock) { double pfs, pss; struct peak_box *bx; int pn; - struct panel *p; int fid_fs, fid_ss; /* Center coordinates, rounded, * in overall data block */ int cfs, css; /* Corner coordinates */ @@ -1419,12 +1421,7 @@ static int integrate_rings_once(Reflection *refl, struct image *image, set_redundancy(refl, 0); get_detector_pos(refl, &pfs, &pss); - p = get_panel(refl); - pn = panel_number(image->det, p); - if ( pn == image->det->n_panels ) { - ERROR("Couldn't find panel %p\n", p); - return 1; - } + pn = get_panel_number(refl); /* Explicit truncation of digits after the decimal point. * This is actually the correct thing to do here, not @@ -1442,7 +1439,7 @@ static int integrate_rings_once(Reflection *refl, struct image *image, bx->refl = refl; bx->cfs = cfs; bx->css = css; - bx->p = p; + bx->p = &ic->image->detgeom->panels[pn]; bx->pn = pn; if ( ic->meth & INTEGRATION_CENTER ) { @@ -1521,17 +1518,7 @@ static int integrate_rings_once(Reflection *refl, struct image *image, } -static int compare_double(const void *av, const void *bv) -{ - double a = *(double *)av; - double b = *(double *)bv; - if ( a > b ) return 1; - if ( a < b ) return -1; - return 0; -} - - -static double estimate_resolution(UnitCell *cell, ImageFeatureList *flist) +static double estimate_resolution(Crystal *cr, struct image *image) { int i; const double min_dist = 0.25; @@ -1540,6 +1527,9 @@ static double estimate_resolution(UnitCell *cell, ImageFeatureList *flist) int n_acc = 0; int max_acc = 1024; int n; + double dx, dy; + UnitCell *cell; + acc = malloc(max_acc*sizeof(double)); if ( acc == NULL ) { @@ -1547,27 +1537,36 @@ static double estimate_resolution(UnitCell *cell, ImageFeatureList *flist) return INFINITY; } - for ( i=0; i<image_feature_count(flist); i++ ) { + cell = crystal_get_cell(cr); + crystal_get_det_shift(cr, &dx, &dy); + + for ( i=0; i<image_feature_count(image->features); i++ ) { struct imagefeature *f; double h, k, l, hd, kd, ld; - - /* Assume all image "features" are genuine peaks */ - f = image_get_feature(flist, i); - if ( f == NULL ) continue; - double ax, ay, az; double bx, by, bz; double cx, cy, cz; + double r[3]; + + /* Assume all image "features" are genuine peaks */ + f = image_get_feature(image->features, i); + if ( f == NULL ) continue; cell_get_cartesian(cell, - &ax, &ay, &az, &bx, &by, &bz, &cx, &cy, &cz); + &ax, &ay, &az, + &bx, &by, &bz, + &cx, &cy, &cz); + + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + dx, dy, r); /* Decimal and fractional Miller indices of nearest * reciprocal lattice point */ - hd = f->rx * ax + f->ry * ay + f->rz * az; - kd = f->rx * bx + f->ry * by + f->rz * bz; - ld = f->rx * cx + f->ry * cy + f->rz * cz; + hd = r[0] * ax + r[1] * ay + r[2] * az; + kd = r[0] * bx + r[1] * by + r[2] * bz; + ld = r[0] * cx + r[1] * cy + r[2] * cz; h = lrint(hd); k = lrint(kd); l = lrint(ld); @@ -1619,49 +1618,39 @@ static void integrate_rings(IntegrationMethod meth, Reflection *refl; RefListIterator *iter; UnitCell *cell; - struct intcontext ic; + struct intcontext *ic; int n_rej = 0; list = crystal_get_reflections(cr); cell = crystal_get_cell(cr); - ic.halfw = ir_out; - ic.image = image; - ic.k = 1.0/image->lambda; - ic.n_saturated = 0; - ic.n_implausible = 0; - ic.cell = cell; - ic.ir_inn = ir_inn; - ic.ir_mid = ir_mid; - ic.ir_out = ir_out; - ic.int_diag = int_diag; - ic.int_diag_h = idh; - ic.int_diag_k = idk; - ic.int_diag_l = idl; - ic.meth = meth; - ic.masks = masks; - if ( init_intcontext(&ic) ) { + ic = intcontext_new(image, cell, meth, + ir_inn, ir_mid, ir_out, + masks); + if ( ic == NULL ) { ERROR("Failed to initialise integration.\n"); return; } - setup_ring_masks(&ic, ir_inn, ir_mid, ir_out); + + intcontext_set_diag(ic, int_diag, idh, idk, idl); for ( refl = first_refl(list, &iter); refl != NULL; refl = next_refl(refl, iter) ) { - n_rej += integrate_rings_once(refl, image, &ic, cell, term_lock); + n_rej += integrate_rings_once(refl, ic, + term_lock); } - free_intcontext(&ic); + intcontext_free(ic); if ( n_rej > 0 ) { ERROR("WARNING: %i reflections could not be integrated\n", n_rej); } - crystal_set_num_saturated_reflections(cr, ic.n_saturated); - crystal_set_num_implausible_reflections(cr, ic.n_implausible); + crystal_set_num_saturated_reflections(cr, ic->n_saturated); + crystal_set_num_implausible_reflections(cr, ic->n_implausible); } @@ -1673,7 +1662,7 @@ void integrate_all_5(struct image *image, IntegrationMethod meth, pthread_mutex_t *term_lock, int overpredict) { int i; - int *masks[image->det->n_panels]; + int *masks[image->detgeom->n_panels]; /* Predict all reflections */ for ( i=0; i<image->n_crystals; i++ ) { @@ -1687,8 +1676,7 @@ void integrate_all_5(struct image *image, IntegrationMethod meth, saved_R * 5); } - res = estimate_resolution(crystal_get_cell(image->crystals[i]), - image->features); + res = estimate_resolution(image->crystals[i], image); crystal_set_resolution_limit(image->crystals[i], res); list = predict_to_res(image->crystals[i], res+push_res); @@ -1700,8 +1688,9 @@ void integrate_all_5(struct image *image, IntegrationMethod meth, } - for ( i=0; i<image->det->n_panels; i++ ) { - masks[i] = make_BgMask(image, &image->det->panels[i], ir_inn); + for ( i=0; i<image->detgeom->n_panels; i++ ) { + masks[i] = make_BgMask(image, &image->detgeom->panels[i], + i, ir_inn); } for ( i=0; i<image->n_crystals; i++ ) { @@ -1737,7 +1726,7 @@ void integrate_all_5(struct image *image, IntegrationMethod meth, } - for ( i=0; i<image->det->n_panels; i++ ) { + for ( i=0; i<image->detgeom->n_panels; i++ ) { free(masks[i]); } } @@ -1852,4 +1841,3 @@ IntegrationMethod integration_method(const char *str, int *err) return meth; } - diff --git a/libcrystfel/src/integration.h b/libcrystfel/src/integration.h index 11e67dbe..f230201d 100644 --- a/libcrystfel/src/integration.h +++ b/libcrystfel/src/integration.h @@ -3,11 +3,11 @@ * * Integration of intensities * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2015 Thomas White <taw@physics.org> + * 2010-2020 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -29,10 +29,6 @@ #ifndef INTEGRATION_H #define INTEGRATION_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include "geometry.h" /** @@ -113,8 +109,22 @@ typedef enum { extern "C" { #endif +struct intcontext; + extern IntegrationMethod integration_method(const char *t, int *err); +extern struct intcontext *intcontext_new(struct image *image, + UnitCell *cell, + IntegrationMethod meth, + int ir_inn, int ir_mid, int ir_out, + int **masks); + +extern int integrate_rings_once(Reflection *refl, + struct intcontext *ic, + pthread_mutex_t *term_lock); + +extern void intcontext_free(struct intcontext *ic); + extern void integrate_all(struct image *image, IntegrationMethod meth, double ir_inn, double ir_mid, double ir_out, IntDiag int_diag, diff --git a/libcrystfel/src/libcrystfel-version.c.cmake.in b/libcrystfel/src/libcrystfel-version.c.cmake.in new file mode 100644 index 00000000..513d7871 --- /dev/null +++ b/libcrystfel/src/libcrystfel-version.c.cmake.in @@ -0,0 +1,4 @@ +const char *libcrystfel_version_string() +{ + return "${CRYSTFEL_VERSION}"; +} diff --git a/libcrystfel/src/libcrystfel-version.c.in b/libcrystfel/src/libcrystfel-version.c.in new file mode 100644 index 00000000..1cde75f7 --- /dev/null +++ b/libcrystfel/src/libcrystfel-version.c.in @@ -0,0 +1,6 @@ +#define LIBCRYSTFEL_VERSION_H + +const char *libcrystfel_version_string() +{ + return "@VCS_TAG@"; +} diff --git a/libcrystfel/src/libcrystfel-version.h b/libcrystfel/src/libcrystfel-version.h new file mode 100644 index 00000000..837b8299 --- /dev/null +++ b/libcrystfel/src/libcrystfel-version.h @@ -0,0 +1,5 @@ +#ifndef LIBCRYSTFEL_VERSION_H +#define LIBCRYSTFEL_VERSION_H +extern const char *libcrystfel_version_string(void); +extern const char *libcrystfel_licence_string(void); +#endif diff --git a/libcrystfel/src/peakfinder8.c b/libcrystfel/src/peakfinder8.c index c3f7c391..641a154b 100644 --- a/libcrystfel/src/peakfinder8.c +++ b/libcrystfel/src/peakfinder8.c @@ -3,13 +3,14 @@ * * The peakfinder8 algorithm * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2017 Valerio Mariani <valerio.mariani@desy.de> - * 2017 Anton Barty <anton.barty@desy.de> - * 2017 Oleksandr Yefanov <oleksandr.yefanov@desy.de> + * 2017-2021 Thomas White <taw@physics.org> + * 2017 Valerio Mariani <valerio.mariani@desy.de> + * 2017 Anton Barty <anton.barty@desy.de> + * 2017 Oleksandr Yefanov <oleksandr.yefanov@desy.de> * * This file is part of CrystFEL. * @@ -37,6 +38,8 @@ #include <stdlib.h> #include "peakfinder8.h" +#include "detgeom.h" +#include "image.h" /** \file peakfinder8.h */ @@ -101,10 +104,10 @@ struct peakfinder_peak_data // CrystFEL-only block 2 -static struct radius_maps *compute_radius_maps(struct detector *det) +static struct radius_maps *compute_radius_maps(struct detgeom *det) { int i, u, iss, ifs; - struct panel p; + struct detgeom_panel p; struct radius_maps *rm = NULL; rm = (struct radius_maps *)malloc(sizeof(struct radius_maps)); @@ -173,14 +176,14 @@ static struct peakfinder_mask *create_peakfinder_mask(struct image *img, struct peakfinder_mask *msk; msk = (struct peakfinder_mask *)malloc(sizeof(struct peakfinder_mask)); - msk->masks =(char **) malloc(img->det->n_panels*sizeof(char*)); - msk->n_masks = img->det->n_panels; - for ( i=0; i<img->det->n_panels; i++) { + msk->masks =(char **) malloc(img->detgeom->n_panels*sizeof(char*)); + msk->n_masks = img->detgeom->n_panels; + for ( i=0; i<img->detgeom->n_panels; i++) { - struct panel p; + struct detgeom_panel p; int iss, ifs; - p = img->det->panels[i]; + p = img->detgeom->panels[i]; msk->masks[i] = (char *)calloc(p.w*p.h,sizeof(char)); @@ -1040,14 +1043,10 @@ int peakfinder8(struct image *img, int max_n_peaks, iterations = 5; - if ( img-> det == NULL) { - return 1; - } + if ( img->detgeom == NULL) return 1; - rmaps = compute_radius_maps(img->det); - if ( rmaps == NULL ) { - return 1; - } + rmaps = compute_radius_maps(img->detgeom); + if ( rmaps == NULL ) return 1; pfmask = create_peakfinder_mask(img, rmaps, min_res, max_res); if ( pfmask == NULL ) { @@ -1055,18 +1054,18 @@ int peakfinder8(struct image *img, int max_n_peaks, return 1; } - pfdata = allocate_panel_data(img->det->n_panels); + pfdata = allocate_panel_data(img->detgeom->n_panels); if ( pfdata == NULL) { free_radius_maps(rmaps); free_peakfinder_mask(pfmask); return 1; } - for ( pi=0 ; pi<img->det->n_panels ; pi++ ) { - pfdata->panel_h[pi] = img->det->panels[pi].h; - pfdata->panel_w[pi] = img->det->panels[pi].w; + for ( pi=0 ; pi<img->detgeom->n_panels ; pi++ ) { + pfdata->panel_h[pi] = img->detgeom->panels[pi].h; + pfdata->panel_w[pi] = img->detgeom->panels[pi].w; pfdata->panel_data[pi] = img->dp[pi]; - pfdata->num_panels = img->det->n_panels; + pfdata->num_panels = img->detgeom->n_panels; } max_r = -1e9; @@ -1139,7 +1138,7 @@ int peakfinder8(struct image *img, int max_n_peaks, remaining_max_num_peaks = max_n_peaks; - for ( pi=0 ; pi<img->det->n_panels ; pi++) { + for ( pi=0 ; pi<img->detgeom->n_panels ; pi++) { int peaks_to_add; int pki; @@ -1147,10 +1146,6 @@ int peakfinder8(struct image *img, int max_n_peaks, num_found_peaks = 0; - if ( img->det->panels[pi].no_index ) { - continue; - } - ret = peakfinder8_base(rstats->roffset, rstats->rthreshold, pfdata->panel_data[pi], @@ -1192,9 +1187,9 @@ int peakfinder8(struct image *img, int max_n_peaks, for ( pki=0 ; pki<peaks_to_add ; pki++ ) { - struct panel *p; + struct detgeom_panel *p; - p = &img->det->panels[pi]; + p = &img->detgeom->panels[pi]; if ( pkdata->max_i[pki] > p->max_adu ) { if ( !use_saturated ) { @@ -1205,9 +1200,7 @@ int peakfinder8(struct image *img, int max_n_peaks, image_add_feature(img->features, pkdata->com_fs[pki]+0.5, pkdata->com_ss[pki]+0.5, - p, - img, - pkdata->tot_i[pki], + pi, img, pkdata->tot_i[pki], NULL); } } diff --git a/libcrystfel/src/peakfinder8.h b/libcrystfel/src/peakfinder8.h index b4d54fd1..28212e49 100644 --- a/libcrystfel/src/peakfinder8.h +++ b/libcrystfel/src/peakfinder8.h @@ -3,13 +3,14 @@ * * The peakfinder8 algorithm * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2017 Valerio Mariani <valerio.mariani@desy.de> - * 2017 Anton Barty <anton.barty@desy.de> - * 2017 Oleksandr Yefanov <oleksandr.yefanov@desy.de> + * 2019 Thomas White <taw@physics.org> + * 2017 Valerio Mariani <valerio.mariani@desy.de> + * 2017 Anton Barty <anton.barty@desy.de> + * 2017 Oleksandr Yefanov <oleksandr.yefanov@desy.de> * * This file is part of CrystFEL. * diff --git a/libcrystfel/src/peaks.c b/libcrystfel/src/peaks.c index 7f2526bc..d55ddd03 100644 --- a/libcrystfel/src/peaks.c +++ b/libcrystfel/src/peaks.c @@ -3,12 +3,12 @@ * * Peak search and other image analysis * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * * Authors: - * 2010-2016 Thomas White <taw@physics.org> + * 2010-2020 Thomas White <taw@physics.org> * 2012 Kenneth Beyerlein <kenneth.beyerlein@desy.de> * 2011 Andrew Martin <andrew.martin@desy.de> * 2011 Richard Kirian @@ -55,7 +55,7 @@ #include "image.h" #include "utils.h" #include "peaks.h" -#include "detector.h" +#include "detgeom.h" #include "filters.h" #include "reflist-utils.h" #include "cell-utils.h" @@ -64,94 +64,8 @@ /** \file peaks.h */ -static int cull_peaks_in_panel(struct image *image, struct panel *p) -{ - int i, n; - int nelim = 0; - - n = image_feature_count(image->features); - - for ( i=0; i<n; i++ ) { - - struct imagefeature *f; - int j, ncol; - - f = image_get_feature(image->features, i); - if ( f == NULL ) continue; - - if ( f->p != p ) continue; - - /* How many peaks are in the same column? */ - ncol = 0; - for ( j=0; j<n; j++ ) { - - struct imagefeature *g; - - if ( i==j ) continue; - - g = image_get_feature(image->features, j); - if ( g == NULL ) continue; - - if ( p->badrow == 'f' ) { - if ( fabs(f->ss - g->ss) < 2.0 ) ncol++; - } else if ( p->badrow == 's' ) { - if ( fabs(f->fs - g->fs) < 2.0 ) ncol++; - } /* else do nothing */ - - } - - /* More than three? */ - if ( ncol <= 3 ) continue; - - /* Yes? Delete them all... */ - for ( j=0; j<n; j++ ) { - struct imagefeature *g; - g = image_get_feature(image->features, j); - if ( g == NULL ) continue; - if ( p->badrow == 'f' ) { - if ( fabs(f->ss - g->ss) < 2.0 ) { - image_remove_feature(image->features, - j); - nelim++; - } - } else if ( p->badrow == 's' ) { - if ( fabs(f->fs - g->ss) < 2.0 ) { - image_remove_feature(image->features, - j); - nelim++; - } - } else { - ERROR("Invalid badrow direction.\n"); - abort(); - } - - } - - } - - return nelim; -} - - -/* Post-processing of the peak list to remove noise */ -static int cull_peaks(struct image *image) -{ - int nelim = 0; - struct panel *p; - int i; - - for ( i=0; i<image->det->n_panels; i++ ) { - p = &image->det->panels[i]; - if ( p->badrow != '-' ) { - nelim += cull_peaks_in_panel(image, p); - } - } - - return nelim; -} - - -static void add_crystal_to_mask(struct image *image, struct panel *p, +static void add_crystal_to_mask(struct image *image, + struct detgeom_panel *p, int pn, double ir_inn, int *mask, Crystal *cr) { Reflection *refl; @@ -168,7 +82,7 @@ static void add_crystal_to_mask(struct image *image, struct panel *p, get_detector_pos(refl, &pk2_fs, &pk2_ss); /* Determine if reflection is in the same panel */ - if ( get_panel(refl) != p ) continue; + if ( get_panel_number(refl) != pn ) continue; for ( dfs=-ir_inn; dfs<=ir_inn; dfs++ ) { for ( dss=-ir_inn; dss<=ir_inn; dss++ ) { @@ -197,7 +111,8 @@ static void add_crystal_to_mask(struct image *image, struct panel *p, /* cfs, css relative to panel origin */ -int *make_BgMask(struct image *image, struct panel *p, double ir_inn) +int *make_BgMask(struct image *image, struct detgeom_panel *p, + int pn, double ir_inn) { int *mask; int i; @@ -208,7 +123,7 @@ int *make_BgMask(struct image *image, struct panel *p, double ir_inn) if ( image->crystals == NULL ) return mask; for ( i=0; i<image->n_crystals; i++ ) { - add_crystal_to_mask(image, p, ir_inn, + add_crystal_to_mask(image, p, pn, ir_inn, mask, image->crystals[i]); } @@ -218,12 +133,12 @@ int *make_BgMask(struct image *image, struct panel *p, double ir_inn) /* Returns non-zero if peak has been vetoed. * i.e. don't use result if return value is not zero. */ -static int integrate_peak(struct image *image, - int p_cfs, int p_css, struct panel *p, - double *pfs, double *pss, - double *intensity, double *sigma, - double ir_inn, double ir_mid, double ir_out, - int *saturated) +int integrate_peak(struct image *image, + int p_cfs, int p_css, int pn, + double *pfs, double *pss, + double *intensity, double *sigma, + double ir_inn, double ir_mid, double ir_out, + int *saturated) { signed int dfs, dss; double lim_sq, out_lim_sq, mid_lim_sq; @@ -236,22 +151,18 @@ static int integrate_peak(struct image *image, double bg_tot_sq = 0.0; double var; double aduph; - int pn; + struct detgeom_panel *p; if ( saturated != NULL ) *saturated = 0; + p = &image->detgeom->panels[pn]; + aduph = p->adu_per_photon; lim_sq = pow(ir_inn, 2.0); mid_lim_sq = pow(ir_mid, 2.0); out_lim_sq = pow(ir_out, 2.0); - pn = panel_number(image->det, p); - if ( pn == image->det->n_panels ) { - ERROR("Couldn't find panel %p\n", p); - return 20; - } - /* Estimate the background */ for ( dss=-ir_out; dss<=+ir_out; dss++ ) { for ( dfs=-ir_out; dfs<=+ir_out; dfs++ ) { @@ -352,7 +263,7 @@ static void search_peaks_in_panel(struct image *image, float threshold, { int fs, ss, stride; float *data; - struct panel *p; + struct detgeom_panel *p; double d; int idx; double f_fs = 0.0; @@ -366,9 +277,8 @@ static void search_peaks_in_panel(struct image *image, float threshold, int nrej_snr = 0; int nrej_sat = 0; int nacc = 0; - int ncull; - p = &image->det->panels[pn]; + p = &image->detgeom->panels[pn]; data = image->dp[pn]; stride = p->w; @@ -456,7 +366,7 @@ static void search_peaks_in_panel(struct image *image, float threshold, assert(mask_ss >= 0); /* Centroid peak and get better coordinates. */ - r = integrate_peak(image, mask_fs, mask_ss, p, + r = integrate_peak(image, mask_fs, mask_ss, pn, &f_fs, &f_ss, &intensity, &sigma, ir_inn, ir_mid, ir_out, &saturated); @@ -479,7 +389,8 @@ static void search_peaks_in_panel(struct image *image, float threshold, } /* Check for a nearby feature */ - image_feature_closest(image->features, f_fs, f_ss, p, &d, &idx); + image_feature_closest(image->features, f_fs, f_ss, pn, + &d, &idx); if ( d < 2.0*ir_inn ) { nrej_pro++; continue; @@ -491,33 +402,19 @@ static void search_peaks_in_panel(struct image *image, float threshold, } /* Add using "better" coordinates */ - image_add_feature(image->features, f_fs, f_ss, p, + image_add_feature(image->features, f_fs, f_ss, pn, image, intensity, NULL); nacc++; } } - if ( image->det != NULL ) { - ncull = cull_peaks(image); - nacc -= ncull; - } else { - STATUS("Not culling peaks because I don't have a " - "detector geometry file.\n"); - ncull = 0; - } - //STATUS("%i accepted, %i box, %i proximity, %i outside panel, " // "%i failed integration, %i with SNR < %g, %i badrow culled, " // "%i saturated.\n", // nacc, nrej_dis, nrej_pro, nrej_fra, nrej_fail, - // nrej_snr, min_snr, ncull, nrej_sat); + // nrej_snr, min_snr, nrej_sat); - if ( ncull != 0 ) { - STATUS("WARNING: %i peaks were badrow culled. This feature" - " should not usually be used.\nConsider setting" - " badrow=- in the geometry file.\n", ncull); - } } @@ -532,9 +429,7 @@ void search_peaks(struct image *image, float threshold, float min_sq_gradient, } image->features = image_feature_list_new(); - for ( i=0; i<image->det->n_panels; i++ ) { - - if ( image->det->panels[i].no_index ) continue; + for ( i=0; i<image->detgeom->n_panels; i++ ) { search_peaks_in_panel(image, threshold, min_sq_gradient, min_snr, i, ir_inn, ir_mid, ir_out, @@ -606,16 +501,14 @@ int search_peaks_peakfinder9(struct image *image, float min_snr_biggest_pix, if ( allocatePeakList(&peakList, NpeaksMax) ) return 1; - for ( panel_number=0; panel_number<image->det->n_panels; panel_number++ ) { + for ( panel_number=0; panel_number<image->detgeom->n_panels; panel_number++ ) { int w, h; int peak_number; detectorRawFormat_t det_size_one_panel; - if ( image->det->panels[panel_number].no_index ) continue; - - w = image->det->panels[panel_number].w; - h = image->det->panels[panel_number].h; + w = image->detgeom->panels[panel_number].w; + h = image->detgeom->panels[panel_number].h; det_size_one_panel.asic_nx = w; det_size_one_panel.asic_ny = h; @@ -648,8 +541,7 @@ int search_peaks_peakfinder9(struct image *image, float min_snr_biggest_pix, image_add_feature(image->features, peakList.centerOfMass_rawX[peak_number], peakList.centerOfMass_rawY[peak_number], - &image->det->panels[panel_number], - image, + panel_number, image, peakList.totalIntensity[peak_number], NULL); } @@ -698,7 +590,7 @@ int indexing_peak_check(struct image *image, Crystal **crystals, int n_cryst, for ( i=0; i<image_feature_count(image->features); i++ ) { struct imagefeature *f; - struct rvec q; + double q[3]; double h,k,l,hd,kd,ld; int j; int ok = 0; @@ -708,15 +600,22 @@ int indexing_peak_check(struct image *image, Crystal **crystals, int n_cryst, if ( f == NULL ) continue; n_feat++; - /* Reciprocal space position of found peak */ - q = get_q_for_panel(f->p, f->fs, f->ss, - NULL, 1.0/image->lambda); - for ( j=0; j<n_cryst; j++ ) { double ax, ay, az; double bx, by, bz; double cx, cy, cz; + double dx, dy; + + crystal_get_det_shift(crystals[j], &dx, &dy); + + /* Reciprocal space position of found peak, + * based on a calculation including any updates to the + * detector position from the refinement of the + * current crystal. */ + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + dx, dy, q); cell_get_cartesian(crystal_get_cell(crystals[j]), &ax, &ay, &az, @@ -725,9 +624,9 @@ int indexing_peak_check(struct image *image, Crystal **crystals, int n_cryst, /* Decimal and fractional Miller indices of nearest * reciprocal lattice point */ - hd = q.u * ax + q.v * ay + q.w * az; - kd = q.u * bx + q.v * by + q.w * bz; - ld = q.u * cx + q.v * cy + q.w * cz; + hd = q[0] * ax + q[1] * ay + q[2] * az; + kd = q[0] * bx + q[1] * by + q[2] * bz; + ld = q[0] * cx + q[1] * cy + q[2] * cz; h = lrint(hd); k = lrint(kd); l = lrint(ld); @@ -797,7 +696,7 @@ void validate_peaks(struct image *image, double min_snr, continue; } - r = integrate_peak(image, f->fs, f->ss, f->p, + r = integrate_peak(image, f->fs, f->ss, f->pn, &f_fs, &f_ss, &intensity, &sigma, ir_inn, ir_mid, ir_out, &saturated); if ( r ) { @@ -818,8 +717,8 @@ void validate_peaks(struct image *image, double min_snr, } /* Add using "better" coordinates */ - image_add_feature(flist, f->fs, f->ss, f->p, image, intensity, - NULL); + image_add_feature(flist, f->fs, f->ss, f->pn, image, + intensity, NULL); } @@ -831,17 +730,8 @@ void validate_peaks(struct image *image, double min_snr, } -static int compare_double(const void *av, const void *bv) -{ - double a = *(double *)av; - double b = *(double *)bv; - if ( a > b ) return 1; - if ( a < b ) return -1; - return 0; -} - - -double estimate_peak_resolution(ImageFeatureList *peaks, double lambda) +double estimate_peak_resolution(ImageFeatureList *peaks, double lambda, + struct detgeom *det) { int i, npk, ncut; double *rns; @@ -859,12 +749,14 @@ double estimate_peak_resolution(ImageFeatureList *peaks, double lambda) for ( i=0; i<npk; i++ ) { struct imagefeature *f; - struct rvec r; + double r[3]; f = image_get_feature(peaks, i); - r = get_q_for_panel(f->p, f->fs, f->ss, NULL, 1.0/lambda); - rns[i] = modulus(r.u, r.v, r.w); + detgeom_transform_coords(&det->panels[f->pn], + f->fs, f->ss, + lambda, 0.0, 0.0, r); + rns[i] = modulus(r[0], r[1], r[2]); } @@ -877,3 +769,38 @@ double estimate_peak_resolution(ImageFeatureList *peaks, double lambda) free(rns); return max_res; } + +const char *str_peaksearch(enum peak_search_method meth) +{ + switch ( meth ) { + case PEAK_PEAKFINDER9: return "peakfinder9"; + case PEAK_PEAKFINDER8: return "peakfinder8"; + case PEAK_ZAEF: return "zaef"; + case PEAK_HDF5: return "hdf5"; + case PEAK_CXI: return "cxi"; + case PEAK_MSGPACK: return "msgpack"; + case PEAK_NONE: return "none"; + default: return "???"; + } +} + +enum peak_search_method parse_peaksearch(const char *arg) +{ + if ( strcmp(arg, "zaef") == 0 ) { + return PEAK_ZAEF; + } else if ( strcmp(arg, "peakfinder8") == 0 ) { + return PEAK_PEAKFINDER8; + } else if ( strcmp(arg, "hdf5") == 0 ) { + return PEAK_HDF5; + } else if ( strcmp(arg, "cxi") == 0 ) { + return PEAK_CXI; + } else if ( strcmp(arg, "peakfinder9") == 0 ) { + return PEAK_PEAKFINDER9; + } else if ( strcmp(arg, "msgpack") == 0 ) { + return PEAK_MSGPACK; + } else if ( strcmp(arg, "none") == 0 ) { + return PEAK_NONE; + } + + return PEAK_ERROR; +} diff --git a/libcrystfel/src/peaks.h b/libcrystfel/src/peaks.h index 6e61758c..2f100db2 100644 --- a/libcrystfel/src/peaks.h +++ b/libcrystfel/src/peaks.h @@ -3,11 +3,11 @@ * * Peak search and other image analysis * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2018 Thomas White <taw@physics.org> + * 2010-2020 Thomas White <taw@physics.org> * 2017 Valerio Mariani <valerio.mariani@desy.de> * 2017-2018 Yaroslav Gevorkov <yaroslav.gevorkov@desy.de> * @@ -31,14 +31,12 @@ #ifndef PEAKS_H #define PEAKS_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include <pthread.h> #include "reflist.h" #include "crystal.h" +#include "image.h" +#include "detgeom.h" #ifdef __cplusplus extern "C" { @@ -49,7 +47,23 @@ extern "C" { * Peak search functions */ -extern int *make_BgMask(struct image *image, struct panel *p, double ir_inn); +enum peak_search_method { + PEAK_PEAKFINDER9, + PEAK_PEAKFINDER8, + PEAK_ZAEF, + PEAK_HDF5, + PEAK_CXI, + PEAK_MSGPACK, + PEAK_NONE, + PEAK_ERROR +}; + +extern const char *str_peaksearch(enum peak_search_method meth); + +extern enum peak_search_method parse_peaksearch(const char *arg); + +extern int *make_BgMask(struct image *image, struct detgeom_panel *p, + int pn, double ir_inn); extern void search_peaks(struct image *image, float threshold, float min_gradient, float min_snr, double ir_inn, @@ -78,7 +92,9 @@ extern void validate_peaks(struct image *image, double min_snr, int ir_inn, int ir_mid, int ir_out, int use_saturated, int check_snr); -extern double estimate_peak_resolution(ImageFeatureList *peaks, double lambda); +extern double estimate_peak_resolution(ImageFeatureList *peaks, + double lambda, + struct detgeom *det); #ifdef __cplusplus } diff --git a/libcrystfel/src/predict-refine.c b/libcrystfel/src/predict-refine.c index aff5fc22..2432055f 100644 --- a/libcrystfel/src/predict-refine.c +++ b/libcrystfel/src/predict-refine.c @@ -3,11 +3,11 @@ * * Prediction refinement * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2016 Thomas White <taw@physics.org> + * 2010-2020 Thomas White <taw@physics.org> * 2016 Valerio Mariani * * This file is part of CrystFEL. @@ -73,21 +73,21 @@ struct reflpeak { Reflection *refl; struct imagefeature *peak; double Ih; /* normalised */ - struct panel *panel; /* panel the reflection appears on - * (we assume this never changes) */ + struct detgeom_panel *panel; /* panel the reflection appears on + * (we assume this never changes) */ }; static void twod_mapping(double fs, double ss, double *px, double *py, - struct panel *p) + struct detgeom_panel *p) { double xs, ys; - xs = fs*p->fsx + ss*p->ssx; - ys = fs*p->fsy + ss*p->ssy; + xs = fs*p->fsx + ss*p->ssx; /* pixels */ + ys = fs*p->fsy + ss*p->ssy; /* pixels */ - *px = (xs + p->cnx) / p->res; - *py = (ys + p->cny) / p->res; + *px = (xs + p->cnx) * p->pixel_pitch; /* metres */ + *py = (ys + p->cny) * p->pixel_pitch; /* metres */ } @@ -98,7 +98,7 @@ static double r_dev(struct reflpeak *rp) } -static double x_dev(struct reflpeak *rp, struct detector *det) +static double x_dev(struct reflpeak *rp, struct detgeom *det) { /* Peak position term */ double xpk, ypk, xh, yh; @@ -110,7 +110,7 @@ static double x_dev(struct reflpeak *rp, struct detector *det) } -static double y_dev(struct reflpeak *rp, struct detector *det) +static double y_dev(struct reflpeak *rp, struct detgeom *det) { /* Peak position term */ double xpk, ypk, xh, yh; @@ -122,50 +122,6 @@ static double y_dev(struct reflpeak *rp, struct detector *det) } -static void UNUSED write_pairs(const char *filename, struct reflpeak *rps, - int n, struct detector *det) -{ - int i; - FILE *fh; - - fh = fopen(filename, "w"); - if ( fh == NULL ) { - ERROR("Failed to open '%s'\n", filename); - return; - } - - for ( i=0; i<n; i++ ) { - - double write_fs, write_ss; - double fs, ss; - struct panel *p; - signed int h, k, l; - - fs = rps[i].peak->fs; - ss = rps[i].peak->ss; - p = rps[i].panel; - get_indices(rps[i].refl, &h, &k, &l); - - write_fs = fs + p->orig_min_fs; - write_ss = ss + p->orig_min_ss; - - fprintf(fh, "%7.2f %7.2f dev r,x,y: %9f %9f %9f %9f\n", - write_fs, write_ss, - r_dev(&rps[i])/1e9, fabs(r_dev(&rps[i])/1e9), - x_dev(&rps[i], det), - y_dev(&rps[i], det)); - - //fprintf(fh, "%4i %4i %4i 0.0 - 0.0 1 %7.2f %7.2f %s\n", - // h, k, l, write_fs, write_ss, p->name); - - } - - fclose(fh); - - STATUS("Wrote %i pairs to %s\n", n, filename); -} - - static int cmpd2(const void *av, const void *bv) { struct reflpeak *a, *b; @@ -179,14 +135,13 @@ static int cmpd2(const void *av, const void *bv) static int check_outlier_transition(struct reflpeak *rps, int n, - struct detector *det) + struct detgeom *det) { int i; if ( n < 3 ) return n; qsort(rps, n, sizeof(struct reflpeak), cmpd2); - //write_pairs("pairs-before-outlier.lst", rps, n, det); for ( i=1; i<n-1; i++ ) { @@ -223,12 +178,15 @@ static int pair_peaks(struct image *image, Crystal *cr, double ax, ay, az; double bx, by, bz; double cx, cy, cz; + double dx, dy; RefList *all_reflist; all_reflist = reflist_new(); cell_get_cartesian(crystal_get_cell(cr), &ax, &ay, &az, &bx, &by, &bz, &cx, &cy, &cz); + crystal_get_det_shift(cr, &dx, &dy); + /* First, create a RefList containing the most likely indices for each * peak, with no exclusion criteria */ for ( i=0; i<image_feature_count(image->features); i++ ) { @@ -236,16 +194,21 @@ static int pair_peaks(struct image *image, Crystal *cr, struct imagefeature *f; double h, k, l, hd, kd, ld; Reflection *refl; + double r[3]; /* Assume all image "features" are genuine peaks */ f = image_get_feature(image->features, i); if ( f == NULL ) continue; + detgeom_transform_coords(&image->detgeom->panels[f->pn], + f->fs, f->ss, image->lambda, + dx, dy, r); + /* Decimal and fractional Miller indices of nearest reciprocal * lattice point */ - hd = f->rx * ax + f->ry * ay + f->rz * az; - kd = f->rx * bx + f->ry * by + f->rz * bz; - ld = f->rx * cx + f->ry * cy + f->rz * cz; + hd = r[0] * ax + r[1] * ay + r[2] * az; + kd = r[0] * bx + r[1] * by + r[2] * bz; + ld = r[0] * cx + r[1] * cy + r[2] * cz; h = lrint(hd); k = lrint(kd); l = lrint(ld); @@ -267,11 +230,11 @@ static int pair_peaks(struct image *image, Crystal *cr, * in how far away it is from the peak location. * The predicted position and excitation errors will be * filled in by update_predictions(). */ - set_panel(refl, f->p); + set_panel_number(refl, f->pn); rps[n].refl = refl; rps[n].peak = f; - rps[n].panel = f->p; + rps[n].panel = &image->detgeom->panels[f->pn]; n++; } @@ -307,7 +270,7 @@ static int pair_peaks(struct image *image, Crystal *cr, /* Sort the pairings by excitation error and look for a transition * between good pairings and outliers */ - n_final = check_outlier_transition(rps, n_acc, image->det); + n_final = check_outlier_transition(rps, n_acc, image->detgeom); /* Add the final accepted reflections to the caller's list */ if ( reflist != NULL ) { @@ -359,16 +322,18 @@ int refine_radius(Crystal *cr, struct image *image) } -static void update_detector(struct detector *det, double xoffs, double yoffs, - double coffs) +static void update_detector(struct detgeom *det, + double xoffs, double yoffs, double coffs) { int i; for ( i=0; i<det->n_panels; i++ ) { - struct panel *p = &det->panels[i]; - p->cnx += xoffs * p->res; - p->cny += yoffs * p->res; - p->clen += coffs; + struct detgeom_panel *p = &det->panels[i]; + + /* Convert to pixels */ + p->cnx += xoffs / p->pixel_pitch; + p->cny += yoffs / p->pixel_pitch; + p->cnz += coffs / p->pixel_pitch; } } @@ -454,7 +419,7 @@ static int iterate(struct reflpeak *rps, int n, UnitCell *cell, } - v_c = x_dev(&rps[i], image->det); + v_c = x_dev(&rps[i], image->detgeom); v_c *= -gradients[k]; v_curr = gsl_vector_get(v, k); gsl_vector_set(v, k, v_curr + v_c); @@ -486,7 +451,7 @@ static int iterate(struct reflpeak *rps, int n, UnitCell *cell, } - v_c = y_dev(&rps[i], image->det); + v_c = y_dev(&rps[i], image->detgeom); v_c *= -gradients[k]; v_curr = gsl_vector_get(v, k); gsl_vector_set(v, k, v_curr + v_c); @@ -538,12 +503,13 @@ static int iterate(struct reflpeak *rps, int n, UnitCell *cell, csx += gsl_vector_get(shifts, 6); csy += gsl_vector_get(shifts, 7); csz += gsl_vector_get(shifts, 8); - update_detector(image->det, gsl_vector_get(shifts, 9), - gsl_vector_get(shifts, 10), - gsl_vector_get(shifts, 11)); + update_detector(image->detgeom, + gsl_vector_get(shifts, 9), + gsl_vector_get(shifts, 10), + 0.0); *total_x += gsl_vector_get(shifts, 9); *total_y += gsl_vector_get(shifts, 10); - *total_z += gsl_vector_get(shifts, 11); + *total_z += 0.0; cell_set_reciprocal(cell, asx, asy, asz, bsx, bsy, bsz, csx, csy, csz); @@ -555,7 +521,7 @@ static int iterate(struct reflpeak *rps, int n, UnitCell *cell, } -static double pred_residual(struct reflpeak *rps, int n, struct detector *det) +static double pred_residual(struct reflpeak *rps, int n, struct detgeom *det) { int i; double res = 0.0; @@ -621,6 +587,8 @@ int refine_prediction(struct image *image, Crystal *cr) } crystal_set_reflections(cr, reflist); + crystal_get_det_shift(cr, &total_x, &total_y); + /* Normalise the intensities to max 1 */ max_I = -INFINITY; for ( i=0; i<n; i++ ) { @@ -636,7 +604,7 @@ int refine_prediction(struct image *image, Crystal *cr) rps[i].Ih = rps[i].peak->intensity / max_I; } - //STATUS("Initial residual = %e\n", pred_residual(rps, n, image->det)); + //STATUS("Initial residual = %e\n", pred_residual(rps, n, image->detgeom)); /* Refine */ for ( i=0; i<MAX_CYCLES; i++ ) { @@ -644,12 +612,12 @@ int refine_prediction(struct image *image, Crystal *cr) if ( iterate(rps, n, crystal_get_cell(cr), image, &total_x, &total_y, &total_z) ) return 1; //STATUS("Residual after %i = %e\n", i, - // pred_residual(rps, n, image->det)); + // pred_residual(rps, n, image->detgeom)); } - //STATUS("Final residual = %e\n", pred_residual(rps, n, image->det)); + //STATUS("Final residual = %e\n", pred_residual(rps, n, image->detgeom)); snprintf(tmp, 255, "predict_refine/final_residual = %e", - pred_residual(rps, n, image->det)); + pred_residual(rps, n, image->detgeom)); crystal_add_notes(cr, tmp); crystal_set_det_shift(cr, total_x, total_y); diff --git a/libcrystfel/src/predict-refine.h b/libcrystfel/src/predict-refine.h index 6396df8a..5607e356 100644 --- a/libcrystfel/src/predict-refine.h +++ b/libcrystfel/src/predict-refine.h @@ -3,11 +3,11 @@ * * Prediction refinement * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2015 Thomas White <taw@physics.org> + * 2010-2019 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -29,11 +29,6 @@ #ifndef PREDICT_REFINE_H #define PREDICT_REFINE_H - -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include "crystal.h" struct image; diff --git a/libcrystfel/src/rational.c b/libcrystfel/src/rational.c index 05bca429..afffcf47 100644 --- a/libcrystfel/src/rational.c +++ b/libcrystfel/src/rational.c @@ -3,11 +3,11 @@ * * A small rational number library * - * Copyright © 2019-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2019-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2019 Thomas White <taw@physics.org> + * 2019-2020 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -640,8 +640,9 @@ void transform_fractional_coords_rtnl_inverse(const RationalMatrix *P, } -static RationalMatrix *delete_row_and_column(const RationalMatrix *m, - unsigned int di, unsigned int dj) +static RationalMatrix *rational_delete_row_and_column(const RationalMatrix *m, + unsigned int di, + unsigned int dj) { RationalMatrix *n; unsigned int i, j; @@ -667,13 +668,13 @@ static RationalMatrix *delete_row_and_column(const RationalMatrix *m, } -static Rational cofactor(const RationalMatrix *m, - unsigned int i, unsigned int j) +static Rational rational_cofactor(const RationalMatrix *m, + unsigned int i, unsigned int j) { RationalMatrix *n; Rational t, C; - n = delete_row_and_column(m, i, j); + n = rational_delete_row_and_column(m, i, j); if ( n == NULL ) { fprintf(stderr, "Failed to allocate matrix.\n"); return rtnl_zero(); @@ -708,7 +709,8 @@ Rational rtnl_mtx_det(const RationalMatrix *m) det = rtnl_zero(); for ( j=0; j<m->cols; j++ ) { Rational a; - a = rtnl_mul(rtnl_mtx_get(m, i, j), cofactor(m, i, j)); + a = rtnl_mul(rtnl_mtx_get(m, i, j), + rational_cofactor(m, i, j)); det = rtnl_add(det, a); } diff --git a/libcrystfel/src/rational.h b/libcrystfel/src/rational.h index f8aaa537..a083f861 100644 --- a/libcrystfel/src/rational.h +++ b/libcrystfel/src/rational.h @@ -3,7 +3,7 @@ * * A small rational number library * - * Copyright © 2019-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2019-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: @@ -29,10 +29,6 @@ #ifndef RATIONAL_H #define RATIONAL_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - /** * \file rational.h * %Rational numbers (including rational matrices) diff --git a/libcrystfel/src/reflist-utils.c b/libcrystfel/src/reflist-utils.c index 7a41f335..42c3f06d 100644 --- a/libcrystfel/src/reflist-utils.c +++ b/libcrystfel/src/reflist-utils.c @@ -3,11 +3,11 @@ * * Utilities to complement the core reflist.c * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2011-2018 Thomas White <taw@physics.org> + * 2011-2020 Thomas White <taw@physics.org> * 2014 Valerio Mariani * * This file is part of CrystFEL. @@ -41,6 +41,7 @@ #include "utils.h" #include "reflist-utils.h" #include "symmetry.h" +#include "libcrystfel-version.h" /** \file reflist-utils.h @@ -672,7 +673,7 @@ void free_contribs(RefList *list) static char *full_command_line(int argc, char *argv[]) { int i; - size_t len = 0; + size_t len = 1; char *cl; if ( argc == 0 ) return strdup(""); @@ -698,9 +699,8 @@ void reflist_add_command_and_version(RefList *list, int argc, char *argv[]) char *tmp; char vers[128]; - vers[0] = '\0'; - strcat(vers, "Generated by CrystFEL "); - strncat(vers, CRYSTFEL_VERSIONSTRING, 100); + snprintf(vers, 128, "Generated by CrystFEL %s", + libcrystfel_version_string()); reflist_add_notes(list, vers); tmp = full_command_line(argc, argv); diff --git a/libcrystfel/src/reflist-utils.h b/libcrystfel/src/reflist-utils.h index 9eb21ecb..f5177cee 100644 --- a/libcrystfel/src/reflist-utils.h +++ b/libcrystfel/src/reflist-utils.h @@ -3,11 +3,11 @@ * * Utilities to complement the core reflist.c * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2011-2018 Thomas White <taw@physics.org> + * 2011-2019 Thomas White <taw@physics.org> * 2014 Valerio Mariani * * This file is part of CrystFEL. @@ -27,10 +27,6 @@ * */ -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - /** @cond */ #ifndef REFLIST_UTILS_H #define REFLIST_UTILS_H diff --git a/libcrystfel/src/reflist.c b/libcrystfel/src/reflist.c index 9cedfc86..90c50e1f 100644 --- a/libcrystfel/src/reflist.c +++ b/libcrystfel/src/reflist.c @@ -3,11 +3,11 @@ * * Fast reflection/peak list * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2011-2018 Thomas White <taw@physics.org> + * 2011-2021 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -26,6 +26,10 @@ * */ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + #include <stdlib.h> #include <assert.h> #include <stdio.h> @@ -53,7 +57,7 @@ struct _refldata { /* Location in image */ double fs; double ss; - struct panel *panel; + int panel_number; /* Non-zero if this reflection can be used for scaling */ int scalable; @@ -111,7 +115,6 @@ struct _reflection { struct _reflist { struct _reflection *head; - struct _reflection *tail; char *notes; }; @@ -124,6 +127,7 @@ static Reflection *new_node(unsigned int serial) Reflection *new; new = calloc(1, sizeof(struct _reflection)); + if ( new == NULL ) return NULL; new->serial = serial; new->next = NULL; new->prev = NULL; @@ -308,12 +312,13 @@ void get_detector_pos(const Reflection *refl, double *fs, double *ss) /** * \param refl: Reflection * - * \returns the panel which the reflection appears on + * \returns panel number (index in detgeom/DataTemplate structure) + * which the reflection appears on * **/ -struct panel *get_panel(const Reflection *refl) +int get_panel_number(const Reflection *refl) { - return refl->data.panel; + return refl->data.panel_number; } @@ -589,14 +594,13 @@ void set_detector_pos(Reflection *refl, double fs, double ss) /** * \param refl: Reflection - * \param p: Pointer to the panel structure on which the reflection appears - * - * Note that the pointer will be stored, not the contents of the structure. + * \param pn: Panel number (index in detgeom/DataTemplate structure) of + * the panel on which the reflection appears. * **/ -void set_panel(Reflection *refl, struct panel *p) +void set_panel_number(Reflection *refl, int pn) { - refl->data.panel = p; + refl->data.panel_number = pn; } @@ -882,8 +886,11 @@ static Reflection *insert_node(Reflection *refl, Reflection *new) } -static void add_to_list(RefList *list, Reflection *new, - signed int h, signed int k, signed int l) +static void add_refl_to_list_real(RefList *list, + Reflection *new, + signed int h, + signed int k, + signed int l) { Reflection *f; @@ -929,7 +936,7 @@ Reflection *add_refl(RefList *list, signed int h, signed int k, signed int l) new = new_node(SERIAL(h, k, l)); if ( new == NULL ) return NULL; - add_to_list(list, new, h, k, l); + add_refl_to_list_real(list, new, h, k, l); return new; } @@ -948,7 +955,7 @@ void add_refl_to_list(Reflection *refl, RefList *list) get_indices(refl, &h, &k, &l); - add_to_list(list, refl, h, k, l); + add_refl_to_list_real(list, refl, h, k, l); } diff --git a/libcrystfel/src/reflist.h b/libcrystfel/src/reflist.h index 3085a2aa..864e871d 100644 --- a/libcrystfel/src/reflist.h +++ b/libcrystfel/src/reflist.h @@ -3,11 +3,11 @@ * * Fast reflection/peak list * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2011-2018 Thomas White <taw@physics.org> + * 2011-2020 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -29,10 +29,6 @@ #ifndef REFLIST_H #define REFLIST_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #define SERIAL(h, k, l) ((((h)+512)<<20) + (((k)+512)<<10) + ((l)+512)) #define GET_H(serial) ((((serial) & 0x3ff00000)>>20)-512) #define GET_K(serial) ((((serial) & 0x000ffc00)>>10)-512) @@ -109,7 +105,7 @@ extern Reflection *next_found_refl(Reflection *refl); /* Get */ extern void get_detector_pos(const Reflection *refl, double *fs, double *ss); -extern struct panel *get_panel(const Reflection *refl); +extern int get_panel_number(const Reflection *refl); extern double get_partiality(const Reflection *refl); extern double get_khalf(const Reflection *refl); extern double get_kpred(const Reflection *refl); @@ -134,7 +130,7 @@ extern struct reflection_contributions *get_contributions(const Reflection *refl /* Set */ extern void copy_data(Reflection *to, const Reflection *from); extern void set_detector_pos(Reflection *refl, double fs, double ss); -extern void set_panel(Reflection *refl, struct panel *p); +extern void set_panel_number(Reflection *refl, int pn); extern void set_kpred(Reflection *refl, double kpred); extern void set_khalf(Reflection *refl, double khalf); extern void set_exerr(Reflection *refl, double exerr); diff --git a/libcrystfel/src/spectrum.c b/libcrystfel/src/spectrum.c index e00b697f..bca293e9 100644 --- a/libcrystfel/src/spectrum.c +++ b/libcrystfel/src/spectrum.c @@ -3,7 +3,7 @@ * * A class representing a radiation spectrum * - * Copyright © 2019-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2019-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: diff --git a/libcrystfel/src/spectrum.h b/libcrystfel/src/spectrum.h index 55578974..d0fcc9c1 100644 --- a/libcrystfel/src/spectrum.h +++ b/libcrystfel/src/spectrum.h @@ -3,7 +3,7 @@ * * A class representing a radiation spectrum * - * Copyright © 2019-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2019-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: @@ -29,10 +29,6 @@ #ifndef SPECTRUM_H #define SPECTRUM_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include <gsl/gsl_rng.h> /** diff --git a/libcrystfel/src/stream.c b/libcrystfel/src/stream.c index bfe4a50a..83756c7a 100644 --- a/libcrystfel/src/stream.c +++ b/libcrystfel/src/stream.c @@ -3,12 +3,12 @@ * * Stream tools * - * Copyright © 2013-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * * Authors: - * 2010-2019 Thomas White <taw@physics.org> + * 2010-2021 Thomas White <taw@physics.org> * 2014-2016 Valerio Mariani * 2011 Richard Kirian * 2011 Andrew Aquila @@ -51,6 +51,10 @@ #include "stream.h" #include "reflist.h" #include "reflist-utils.h" +#include "datatemplate.h" +#include "detgeom.h" +#include "libcrystfel-version.h" + /** \file stream.h */ @@ -69,15 +73,21 @@ struct _stream char *audit_info; char *geometry_file; + /* The DataTemplate provided to us for writing things. + * We don't own this. */ + const DataTemplate *dtempl_write; + + /* The DataTemplate we got from the stream itself, used for + * reading things. We own this. */ + DataTemplate *dtempl_read; + long long int ln; int old_indexers; /* True if the stream reader encountered a deprecated * indexing method */ - int in_chunk; /* True if a chunk start marker has been "accidentally" - * encountered, so read_chunk() should assume a chunk is - * already in progress instead of looking for another - * marker */ + long *chunk_offsets; + int n_chunks; }; @@ -87,211 +97,136 @@ int stream_has_old_indexers(Stream *st) } -static int read_peaks(Stream *st, struct image *image) +static ImageFeatureList *read_peaks(Stream *st, + struct image *image) { char *rval = NULL; int first = 1; + ImageFeatureList *features; - image->features = image_feature_list_new(); + features = image_feature_list_new(); do { char line[1024]; float x, y, d, intensity; - int r; - struct panel *p = NULL; - float add_x, add_y; + int r, exp_n; + char panel_name[1024]; rval = fgets(line, 1023, st->fh); st->ln++; - if ( rval == NULL ) continue; + if ( rval == NULL ) { + image_feature_list_free(features); + return NULL; + } chomp(line); - if ( strcmp(line, PEAK_LIST_END_MARKER) == 0 ) return 0; - - r = sscanf(line, "%f %f %f %f", &x, &y, &d, &intensity); - if ( (r != 4) && (!first) ) { - ERROR("Failed to parse peak list line.\n"); - ERROR("The failed line was: '%s'\n", line); - return 1; + if ( strcmp(line, STREAM_PEAK_LIST_END_MARKER) == 0 ) { + return features; } - first = 0; - if ( r == 4 ) { - - if ( image->det != NULL ) { - - p = find_orig_panel(image->det, x, y); - if ( p == NULL ) { - ERROR("Panel not found\n"); - return 1; - } - - add_x = x-p->orig_min_fs; - add_y = y-p->orig_min_ss; - - image_add_feature(image->features, add_x, add_y, - p, image, intensity, NULL); - - } else { - - image_add_feature(image->features, x, y, - p, image, intensity, NULL); - } + if ( first ) { + first = 0; + continue; } - } while ( rval != NULL ); - - /* Got read error of some kind before finding PEAK_LIST_END_MARKER */ - return 1; -} - - -static int read_peaks_2_3(Stream *st, struct image *image) -{ - char *rval = NULL; - int first = 1; - - image->features = image_feature_list_new(); - - do { - - char line[1024]; - char pn[32]; - float x, y, d, intensity; - int r; - struct panel *p = NULL; - float add_x, add_y; - - rval = fgets(line, 1023, st->fh); - st->ln++; - if ( rval == NULL ) continue; - chomp(line); - - if ( strcmp(line, PEAK_LIST_END_MARKER) == 0 ) return 0; + if ( AT_LEAST_VERSION(st, 2, 3) ) { + r = sscanf(line, "%f %f %f %f %s", + &x, &y, &d, &intensity, panel_name); + exp_n = 5; + } else { + r = sscanf(line, "%f %f %f %f", + &x, &y, &d, &intensity); + exp_n = 4; + } - r = sscanf(line, "%f %f %f %f %s", &x, &y, &d, &intensity, pn); - if ( (r != 5) && (!first) ) { + if ( r != exp_n ) { ERROR("Failed to parse peak list line.\n"); ERROR("The failed line was: '%s'\n", line); - return 1; - } - - first = 0; - - if ( r == 5 ) { - - p = find_panel_by_name(image->det, pn); - if ( p == NULL ) { - ERROR("Panel not found: %s\n", pn); - return 1; - } - - add_x = x-p->orig_min_fs; - add_y = y-p->orig_min_ss; - - image_add_feature(image->features, add_x, add_y, p, - image, intensity, NULL); - + image_feature_list_free(features); + return NULL; } - } while ( rval != NULL ); - - /* Got read error of some kind before finding PEAK_LIST_END_MARKER */ - return 1; -} - - -static int write_peaks(struct image *image, FILE *ofh) -{ - int i; - - fprintf(ofh, PEAK_LIST_START_MARKER"\n"); - fprintf(ofh, " fs/px ss/px (1/d)/nm^-1 Intensity\n"); - - for ( i=0; i<image_feature_count(image->features); i++ ) { - - struct imagefeature *f; - struct rvec r; - double q; + if ( (panel_name[0] != '\0') && (st->dtempl_read != NULL) ) { - f = image_get_feature(image->features, i); - if ( f == NULL ) continue; + int pn; - r = get_q_for_panel(f->p, f->fs, f->ss, - NULL, 1.0/image->lambda); - q = modulus(r.u, r.v, r.w); + if ( data_template_panel_name_to_number(st->dtempl_read, + panel_name, + &pn) ) + { + ERROR("No such panel '%s'\n", + panel_name); + } else { - if ( image->det != NULL ) { + data_template_file_to_panel_coords(st->dtempl_read, + &x, &y, &pn); - struct panel *p; - double write_fs, write_ss; + image_add_feature(features, x, y, + pn, image, intensity, + NULL); - p = find_orig_panel(image->det, f->fs, f->ss); - if ( p == NULL ) { - ERROR("Panel not found\n"); - return 1; } - /* Convert coordinates to match arrangement of panels in - * HDF5 file */ - write_fs = f->fs + p->orig_min_fs; - write_ss = f->ss + p->orig_min_ss; - - fprintf(ofh, "%7.2f %7.2f %10.2f %10.2f\n", - write_fs, write_ss, q/1.0e9, f->intensity); - } else { - fprintf(ofh, "%7.2f %7.2f %10.2f %10.2f\n", - f->fs, f->ss, q/1.0e9, f->intensity); + /* Either it's an old format stream (in which + * case the data is probably "slabby", so no + * coordinate conversion is needed), or + * the caller isn't interested in panel + * locations */ + image_add_feature(features, x, y, 0, + image, intensity, NULL); } - } + } while ( rval != NULL ); - fprintf(ofh, PEAK_LIST_END_MARKER"\n"); - return 0; + return features; } -static int write_peaks_2_3(struct image *image, FILE *ofh) +static int write_peaks(const struct image *image, + const DataTemplate *dtempl, FILE *ofh) { int i; - fprintf(ofh, PEAK_LIST_START_MARKER"\n"); + fprintf(ofh, STREAM_PEAK_LIST_START_MARKER"\n"); fprintf(ofh, " fs/px ss/px (1/d)/nm^-1 Intensity Panel\n"); for ( i=0; i<image_feature_count(image->features); i++ ) { struct imagefeature *f; - struct rvec r; + double r[3]; double q; - double write_fs, write_ss; + float write_fs, write_ss; + struct detgeom_panel *p; f = image_get_feature(image->features, i); if ( f == NULL ) continue; - r = get_q_for_panel(f->p, f->fs, f->ss, - NULL, 1.0/image->lambda); - q = modulus(r.u, r.v, r.w); + p = &image->detgeom->panels[f->pn]; + detgeom_transform_coords(p, f->fs, f->ss, + image->lambda, 0.0, 0.0, r); + q = modulus(r[0], r[1], r[2]); - /* Convert coordinates to match arrangement of panels in HDF5 - * file */ - write_fs = f->fs + f->p->orig_min_fs; - write_ss = f->ss + f->p->orig_min_ss; + write_fs = f->fs; + write_ss = f->ss; + data_template_panel_to_file_coords(dtempl, f->pn, + &write_fs, &write_ss); fprintf(ofh, "%7.2f %7.2f %10.2f %10.2f %s\n", - write_fs, write_ss, q/1.0e9, f->intensity, f->p->name); + write_fs, write_ss, q/1.0e9, f->intensity, + data_template_panel_number_to_name(dtempl, f->pn)); } - fprintf(ofh, PEAK_LIST_END_MARKER"\n"); + fprintf(ofh, STREAM_PEAK_LIST_END_MARKER"\n"); return 0; } -static RefList *read_stream_reflections_2_3(Stream *st, struct detector *det) +static RefList *read_stream_reflections_2_3(Stream *st) { char *rval = NULL; int first = 1; @@ -317,7 +252,7 @@ static RefList *read_stream_reflections_2_3(Stream *st, struct detector *det) if ( rval == NULL ) continue; chomp(line); - if ( strcmp(line, REFLECTION_END_MARKER) == 0 ) return out; + if ( strcmp(line, STREAM_REFLECTION_END_MARKER) == 0 ) return out; r = sscanf(line, "%i %i %i %f %f %f %f %f %f %s", &h, &k, &l, &intensity, &sigma, &pk, &bg, @@ -332,24 +267,19 @@ static RefList *read_stream_reflections_2_3(Stream *st, struct detector *det) if ( r == 10 ) { - struct panel *p; - refl = add_refl(out, h, k, l); if ( refl == NULL ) { ERROR("Failed to add reflection\n"); return NULL; } set_intensity(refl, intensity); - if ( det != NULL ) { - double write_fs, write_ss; - p = find_panel_by_name(det,pn); - if ( p == NULL ) { - ERROR("Couldn't find panel '%s'\n", pn); + if ( st->dtempl_read != NULL ) { + int pn; + if ( data_template_file_to_panel_coords(st->dtempl_read, &fs, &ss, &pn) ) { + ERROR("Failed to convert\n"); } else { - write_fs = fs - p->orig_min_fs; - write_ss = ss - p->orig_min_ss; - set_detector_pos(refl, write_fs, write_ss); - set_panel(refl, p); + set_detector_pos(refl, fs, ss); + set_panel_number(refl, pn); } } set_esd_intensity(refl, sigma); @@ -361,12 +291,12 @@ static RefList *read_stream_reflections_2_3(Stream *st, struct detector *det) } while ( rval != NULL ); - /* Got read error of some kind before finding PEAK_LIST_END_MARKER */ + /* Got read error of some kind before finding STREAM_PEAK_LIST_END_MARKER */ return NULL; } -static RefList *read_stream_reflections_2_1(Stream *st, struct detector *det) +static RefList *read_stream_reflections_2_1(Stream *st) { char *rval = NULL; int first = 1; @@ -393,7 +323,7 @@ static RefList *read_stream_reflections_2_1(Stream *st, struct detector *det) if ( rval == NULL ) continue; chomp(line); - if ( strcmp(line, REFLECTION_END_MARKER) == 0 ) return out; + if ( strcmp(line, STREAM_REFLECTION_END_MARKER) == 0 ) return out; r = sscanf(line, "%i %i %i %f %s %f %i %f %f", &h, &k, &l, &intensity, phs, &sigma, &cts, @@ -416,19 +346,14 @@ static RefList *read_stream_reflections_2_1(Stream *st, struct detector *det) } set_intensity(refl, intensity); - if ( det != NULL ) { - - double write_fs, write_ss; - struct panel *p; + if ( st->dtempl_read != NULL ) { - p = find_orig_panel(det, fs, ss); - if ( p == NULL ) { - ERROR("No panel at %.2f,%.2f\n", - fs, ss); + int pn; + if ( data_template_file_to_panel_coords(st->dtempl_read, &fs, &ss, &pn) ) { + ERROR("Failed to convert\n"); } else { - write_fs = fs - p->orig_min_fs; - write_ss = ss - p->orig_min_ss; - set_detector_pos(refl, write_fs, write_ss); + set_detector_pos(refl, fs, ss); + set_panel_number(refl, pn); } } else { @@ -447,12 +372,12 @@ static RefList *read_stream_reflections_2_1(Stream *st, struct detector *det) } while ( rval != NULL ); - /* Got read error of some kind before finding PEAK_LIST_END_MARKER */ + /* Got read error of some kind before finding STREAM_PEAK_LIST_END_MARKER */ return NULL; } -static RefList *read_stream_reflections_2_2(Stream *st, struct detector *det) +static RefList *read_stream_reflections_2_2(Stream *st) { char *rval = NULL; int first = 1; @@ -473,7 +398,7 @@ static RefList *read_stream_reflections_2_2(Stream *st, struct detector *det) if ( rval == NULL ) continue; chomp(line); - if ( strcmp(line, REFLECTION_END_MARKER) == 0 ) return out; + if ( strcmp(line, STREAM_REFLECTION_END_MARKER) == 0 ) return out; r = sscanf(line, "%i %i %i %f %f %f %f %f %f", &h, &k, &l, &intensity, &sigma, &pk, &bg, &fs, &ss); @@ -492,19 +417,17 @@ static RefList *read_stream_reflections_2_2(Stream *st, struct detector *det) } set_intensity(refl, intensity); - if ( det != NULL ) { + if ( st->dtempl_read != NULL ) { - double write_fs, write_ss; - struct panel *p; + int pn; - p = find_orig_panel(det, fs, ss); - if ( p == NULL ) { - ERROR("No panel at %.2f,%.2f\n", - fs, ss); + if ( data_template_file_to_panel_coords(st->dtempl_read, &fs, &ss, &pn) ) { + ERROR("Failed to convert to " + "panel-relative coordinates: " + "%i,%i\n", fs, ss); } else { - write_fs = fs - p->orig_min_fs; - write_ss = ss - p->orig_min_ss; - set_detector_pos(refl, write_fs, write_ss); + set_detector_pos(refl, fs, ss); + set_panel_number(refl, pn); } } else { @@ -523,144 +446,13 @@ static RefList *read_stream_reflections_2_2(Stream *st, struct detector *det) } while ( rval != NULL ); - /* Got read error of some kind before finding REFLECTION_END_MARKER */ + /* Got read error of some kind before finding STREAM_REFLECTION_END_MARKER */ return NULL; } -static int write_stream_reflections_2_1(FILE *fh, RefList *list, - struct image *image) -{ - Reflection *refl; - RefListIterator *iter; - - fprintf(fh, " h k l I phase sigma(I) " - " counts fs/px ss/px\n"); - - for ( refl = first_refl(list, &iter); - refl != NULL; - refl = next_refl(refl, iter) ) - { - - signed int h, k, l; - double intensity, esd_i, ph; - int red; - double fs, ss; - char phs[16]; - int have_phase; - - get_indices(refl, &h, &k, &l); - get_detector_pos(refl, &fs, &ss); - intensity = get_intensity(refl); - esd_i = get_esd_intensity(refl); - red = get_redundancy(refl); - ph = get_phase(refl, &have_phase); - - /* Reflections with redundancy = 0 are not written */ - if ( red == 0 ) continue; - - if ( have_phase ) { - snprintf(phs, 16, "%8.2f", rad2deg(ph)); - } else { - strncpy(phs, " -", 15); - } - - if ( image->det != NULL ) { - - struct panel *p; - double write_fs, write_ss; - - p = find_orig_panel(image->det, fs, ss); - if ( p == NULL ) { - ERROR("Panel not found\n"); - return 1; - } - - /* Convert coordinates to match arrangement of panels - * in HDF5 file */ - write_fs = fs + p->orig_min_fs; - write_ss = ss + p->orig_min_ss; - - fprintf(fh, "%3i %3i %3i %10.2f %s %10.2f %7i " - "%6.1f %6.1f\n", - h, k, l, intensity, phs, esd_i, red, - write_fs, write_ss); - - } else { - - fprintf(fh, "%3i %3i %3i %10.2f %s %10.2f %7i " - "%6.1f %6.1f\n", - h, k, l, intensity, phs, esd_i, red, - fs, ss); - - } - } - return 0; -} - - -static int write_stream_reflections_2_2(FILE *fh, RefList *list, - struct image *image) -{ - Reflection *refl; - RefListIterator *iter; - - fprintf(fh, " h k l I sigma(I) " - "peak background fs/px ss/px\n"); - - for ( refl = first_refl(list, &iter); - refl != NULL; - refl = next_refl(refl, iter) ) - { - - signed int h, k, l; - double intensity, esd_i, bg, pk; - double fs, ss; - - get_indices(refl, &h, &k, &l); - get_detector_pos(refl, &fs, &ss); - intensity = get_intensity(refl); - esd_i = get_esd_intensity(refl); - pk = get_peak(refl); - bg = get_mean_bg(refl); - - /* Reflections with redundancy = 0 are not written */ - if ( get_redundancy(refl) == 0 ) continue; - - if ( image->det != NULL ) { - - struct panel *p; - double write_fs, write_ss; - - p = find_orig_panel(image->det, fs, ss); - if ( p == NULL ) { - ERROR("Panel not found\n"); - return 1; - } - - /* Convert coordinates to match arrangement of panels in HDF5 - * file */ - write_fs = fs + p->orig_min_fs; - write_ss = ss + p->orig_min_ss; - - fprintf(fh, "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f" - " %6.1f %6.1f\n", - h, k, l, intensity, esd_i, pk, bg, write_fs, - write_ss); - - } else { - - fprintf(fh, "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f" - " %6.1f %6.1f\n", - h, k, l, intensity, esd_i, pk, bg, fs, ss); - } - } - return 0; -} - - -static int write_stream_reflections_2_3(FILE *fh, RefList *list, - struct image *image) +static int write_stream_reflections(FILE *fh, RefList *list, + const DataTemplate *dtempl) { Reflection *refl; RefListIterator *iter; @@ -675,13 +467,14 @@ static int write_stream_reflections_2_3(FILE *fh, RefList *list, signed int h, k, l; double intensity, esd_i, pk, bg; - double fs, ss; - double write_fs, write_ss; - struct panel *p; + double dfs, dss; + float fs, ss; + int pn; get_indices(refl, &h, &k, &l); - get_detector_pos(refl, &fs, &ss); - p = get_panel(refl); + get_detector_pos(refl, &dfs, &dss); + fs = dfs; ss = dss; + pn = get_panel_number(refl); intensity = get_intensity(refl); esd_i = get_esd_intensity(refl); pk = get_peak(refl); @@ -690,14 +483,13 @@ static int write_stream_reflections_2_3(FILE *fh, RefList *list, /* Reflections with redundancy = 0 are not written */ if ( get_redundancy(refl) == 0 ) continue; - write_fs = fs+p->orig_min_fs; - write_ss = ss+p->orig_min_ss; + data_template_panel_to_file_coords(dtempl, pn, + &fs, &ss); - fprintf(fh, - "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f " - "%6.1f %6.1f %s\n", - h, k, l, intensity, esd_i, pk, bg, - write_fs, write_ss, p->name); + fprintf(fh, "%4i %4i %4i %10.2f %10.2f %10.2f %10.2f " + "%6.1f %6.1f %s\n", + h, k, l, intensity, esd_i, pk, bg, + fs, ss, data_template_panel_number_to_name(dtempl, pn)); } return 0; @@ -721,7 +513,8 @@ static int num_integrated_reflections(RefList *list) } -static int write_crystal(Stream *st, Crystal *cr, int include_reflections) +static int write_crystal(Stream *st, Crystal *cr, + int include_reflections) { UnitCell *cell; RefList *reflist; @@ -733,7 +526,7 @@ static int write_crystal(Stream *st, Crystal *cr, int include_reflections) double det_shift_x, det_shift_y; int ret = 0; - fprintf(st->fh, CRYSTAL_START_MARKER"\n"); + fprintf(st->fh, STREAM_CRYSTAL_START_MARKER"\n"); cell = crystal_get_cell(cr); assert(cell != NULL); @@ -792,27 +585,10 @@ static int write_crystal(Stream *st, Crystal *cr, int include_reflections) if ( reflist != NULL ) { - struct image *image; - - image = crystal_get_image(cr); - - fprintf(st->fh, REFLECTION_START_MARKER"\n"); - if ( AT_LEAST_VERSION(st, 2, 3) ) { - ret = write_stream_reflections_2_3(st->fh, - reflist, - image); - } else if ( AT_LEAST_VERSION(st, 2, 2) ) { - ret = write_stream_reflections_2_2(st->fh, - reflist, - image); - } else { - /* This function writes like a normal reflection - * list was written in stream 2.1 */ - ret = write_stream_reflections_2_1(st->fh, - reflist, - image); - } - fprintf(st->fh, REFLECTION_END_MARKER"\n"); + fprintf(st->fh, STREAM_REFLECTION_START_MARKER"\n"); + ret = write_stream_reflections(st->fh, reflist, + st->dtempl_write); + fprintf(st->fh, STREAM_REFLECTION_END_MARKER"\n"); } else { @@ -821,7 +597,7 @@ static int write_crystal(Stream *st, Crystal *cr, int include_reflections) } } - fprintf(st->fh, CRYSTAL_END_MARKER"\n"); + fprintf(st->fh, STREAM_CRYSTAL_END_MARKER"\n"); return ret; } @@ -830,29 +606,23 @@ static int write_crystal(Stream *st, Crystal *cr, int include_reflections) /** * \param st A \ref Stream * \param i An \ref image structure - * \param imfile A \ref imagefile structure - * \param include_peaks Whether to include peak search results in stream - * \param include_reflections Whether to include integration results in stream - * \param ev A \ref event strucutre + * \param srf A \ref StreamFlags enum saying what to write * * Writes a new chunk to \p st. * * \returns non-zero on error. */ -int write_chunk(Stream *st, struct image *i, struct imagefile *imfile, - int include_peaks, int include_reflections, struct event *ev) +int stream_write_chunk(Stream *st, const struct image *i, + StreamFlags srf) { int j; char *indexer; int ret = 0; - fprintf(st->fh, CHUNK_START_MARKER"\n"); + fprintf(st->fh, STREAM_CHUNK_START_MARKER"\n"); fprintf(st->fh, "Image filename: %s\n", i->filename); - if ( i->event != NULL ) { - fprintf(st->fh, "Event: %s\n", get_event_string(i->event)); - } - + fprintf(st->fh, "Event: %s\n", i->ev); fprintf(st->fh, "Image serial number: %i\n", i->serial); fprintf(st->fh, "hit = %i\n", i->hit); @@ -869,40 +639,39 @@ int write_chunk(Stream *st, struct image *i, struct imagefile *imfile, fprintf(st->fh, "beam_divergence = %.2e rad\n", i->div); fprintf(st->fh, "beam_bandwidth = %.2e (fraction)\n", i->bw); - imagefile_copy_fields(imfile, i->copyme, st->fh, ev); + /* FIXME: Better way of doing this */ + //imagefile_copy_fields(imfile, i->copyme, st->fh, ev); - if ( i->det != NULL ) { + if ( i->detgeom != NULL ) { int j; double tclen = 0.0; - for ( j=0; j<i->det->n_panels; j++ ) { - tclen += i->det->panels[j].clen; + for ( j=0; j<i->detgeom->n_panels; j++ ) { + tclen += i->detgeom->panels[j].cnz + * i->detgeom->panels[j].pixel_pitch; } fprintf(st->fh, "average_camera_length = %f m\n", - tclen / i->det->n_panels); + tclen / i->detgeom->n_panels); } fprintf(st->fh, "num_peaks = %i\n", image_feature_count(i->features)); fprintf(st->fh, "peak_resolution = %f nm^-1 or %f A\n", i->peak_resolution/1e9, 1e10/i->peak_resolution); - if ( include_peaks ) { - if ( AT_LEAST_VERSION(st, 2, 3) ) { - ret = write_peaks_2_3(i, st->fh); - } else { - ret = write_peaks(i, st->fh); - } + if ( srf & STREAM_PEAKS ) { + ret = write_peaks(i, st->dtempl_write, st->fh); } for ( j=0; j<i->n_crystals; j++ ) { - if ( crystal_get_user_flag(i->crystals[j]) == 0 ) { - ret = write_crystal(st, i->crystals[j], - include_reflections); + if ( crystal_get_user_flag(i->crystals[j]) ) { + continue; } + ret = write_crystal(st, i->crystals[j], + srf & STREAM_REFLECTIONS); } - fprintf(st->fh, CHUNK_END_MARKER"\n"); + fprintf(st->fh, STREAM_CHUNK_END_MARKER"\n"); fflush(st->fh); @@ -915,14 +684,6 @@ static int find_start_of_chunk(Stream *st) char *rval = NULL; char line[1024]; - /* Perhaps read_geometry() encountered a chunk start marker instead of a - * geometry file. In that case, we're already in a chunk, so this is - * easy. */ - if ( st->in_chunk ) { - st->in_chunk = 0; - return 0; - } - do { rval = fgets(line, 1023, st->fh); @@ -933,13 +694,14 @@ static int find_start_of_chunk(Stream *st) chomp(line); - } while ( strcmp(line, CHUNK_START_MARKER) != 0 ); + } while ( strcmp(line, STREAM_CHUNK_START_MARKER) != 0 ); return 0; } -static void read_crystal(Stream *st, struct image *image, StreamReadFlags srf) +static void read_crystal(Stream *st, struct image *image, + StreamFlags srf) { char line[1024]; char *rval = NULL; @@ -980,29 +742,25 @@ static void read_crystal(Stream *st, struct image *image, StreamReadFlags srf) if ( rval == NULL ) break; chomp(line); - if ( (srf & STREAM_READ_UNITCELL) - && (sscanf(line, "astar = %f %f %f", &u, &v, &w) == 3) ) + if ( sscanf(line, "astar = %f %f %f", &u, &v, &w) == 3 ) { as.u = u*1e9; as.v = v*1e9; as.w = w*1e9; have_as = 1; } - if ( (srf & STREAM_READ_UNITCELL) - && (sscanf(line, "bstar = %f %f %f", &u, &v, &w) == 3) ) + if ( sscanf(line, "bstar = %f %f %f", &u, &v, &w) == 3 ) { bs.u = u*1e9; bs.v = v*1e9; bs.w = w*1e9; have_bs = 1; } - if ( (srf & STREAM_READ_UNITCELL) - && (sscanf(line, "cstar = %f %f %f", &u, &v, &w) == 3) ) + if ( sscanf(line, "cstar = %f %f %f", &u, &v, &w) == 3 ) { cs.u = u*1e9; cs.v = v*1e9; cs.w = w*1e9; have_cs = 1; } - if ( (srf & STREAM_READ_UNITCELL) - && (sscanf(line, "centering = %c", &c) == 1) ) + if ( sscanf(line, "centering = %c", &c) == 1 ) { if ( !have_cen ) { centering = c; @@ -1013,8 +771,7 @@ static void read_crystal(Stream *st, struct image *image, StreamReadFlags srf) } } - if ( (srf & STREAM_READ_UNITCELL) - && (sscanf(line, "unique_axis = %c", &c) == 1) ) + if ( sscanf(line, "unique_axis = %c", &c) == 1 ) { if ( !have_ua ) { unique_axis = c; @@ -1025,8 +782,7 @@ static void read_crystal(Stream *st, struct image *image, StreamReadFlags srf) } } - if ( (srf & STREAM_READ_UNITCELL) - && (strncmp(line, "lattice_type = ", 15) == 0) ) + if ( strncmp(line, "lattice_type = ", 15) == 0 ) { if ( !have_latt ) { lattice_type = lattice_from_str(line+15); @@ -1057,8 +813,8 @@ static void read_crystal(Stream *st, struct image *image, StreamReadFlags srf) } - if ( (strcmp(line, REFLECTION_START_MARKER) == 0) - && (srf & STREAM_READ_REFLECTIONS) ) + if ( (strcmp(line, STREAM_REFLECTION_START_MARKER) == 0) + && (srf & STREAM_REFLECTIONS) ) { RefList *reflist; @@ -1066,20 +822,16 @@ static void read_crystal(Stream *st, struct image *image, StreamReadFlags srf) /* The reflection list format in the stream diverges * after 2.2 */ if ( AT_LEAST_VERSION(st, 2, 3) ) { - reflist = read_stream_reflections_2_3(st, - image->det); + reflist = read_stream_reflections_2_3(st); } else if ( AT_LEAST_VERSION(st, 2, 2) ) { - reflist = read_stream_reflections_2_2(st, - image->det); + reflist = read_stream_reflections_2_2(st); } else { - reflist = read_stream_reflections_2_1(st, - image->det); + reflist = read_stream_reflections_2_1(st); } if ( reflist == NULL ) { ERROR("Failed while reading reflections\n"); ERROR("Filename = %s\n", image->filename); - ERROR("Event = %s\n", - get_event_string(image->event)); + ERROR("Event = %s\n", image->ev); break; } @@ -1087,7 +839,7 @@ static void read_crystal(Stream *st, struct image *image, StreamReadFlags srf) } - if ( strcmp(line, CRYSTAL_END_MARKER) == 0 ) break; + if ( strcmp(line, STREAM_CRYSTAL_END_MARKER) == 0 ) break; } while ( 1 ); @@ -1139,77 +891,21 @@ static void read_crystal(Stream *st, struct image *image, StreamReadFlags srf) } -void free_stuff_from_stream(struct stuff_from_stream *sfs) -{ - int i; - if ( sfs == NULL ) return; - for ( i=0; i<sfs->n_fields; i++ ) { - free(sfs->fields[i]); - } - free(sfs->fields); - free(sfs); -} - - -static int read_and_store_field(struct image *image, const char *line) -{ - char **new_fields; - char *nf; - - if ( image->stuff_from_stream == NULL ) { - image->stuff_from_stream = - malloc(sizeof(struct stuff_from_stream)); - if ( image->stuff_from_stream == NULL) { - ERROR("Failed reading entries from stream\n"); - return 1; - } - image->stuff_from_stream->fields = NULL; - image->stuff_from_stream->n_fields = 0; - } - - new_fields = realloc(image->stuff_from_stream->fields, - (1+image->stuff_from_stream->n_fields)* - sizeof(char *)); - if ( new_fields == NULL ) { - ERROR("Failed reading entries from stream\n"); - return 1; - } - image->stuff_from_stream->fields = new_fields; - - nf = strdup(line); - if ( nf == NULL ) { - ERROR("Failed to allocate field from stream\n"); - return 1; - } - image->stuff_from_stream->fields[image->stuff_from_stream->n_fields] = nf; - image->stuff_from_stream->n_fields++; - - return 0; -} - - /** - * Read the next chunk from a stream and fill in 'image' + * Read the next chunk from a stream and return an image structure */ -int read_chunk_2(Stream *st, struct image *image, StreamReadFlags srf) +struct image *stream_read_chunk(Stream *st, StreamFlags srf) { char line[1024]; char *rval = NULL; int have_filename = 0; int have_ev = 0; + struct image *image; - if ( find_start_of_chunk(st) ) return 1; + if ( find_start_of_chunk(st) ) return NULL; - image->lambda = -1.0; - image->features = NULL; - image->crystals = NULL; - image->n_crystals = 0; - image->event = NULL; - image->stuff_from_stream = NULL; - - if ( (srf & STREAM_READ_REFLECTIONS) || (srf & STREAM_READ_UNITCELL) ) { - srf |= STREAM_READ_CRYSTALS; - } + image = image_new(); + if ( image == NULL ) return NULL; do { int ser; @@ -1229,7 +925,7 @@ int read_chunk_2(Stream *st, struct image *image, StreamReadFlags srf) } if ( strncmp(line, "Event: ", 7) == 0 ) { - image->event = get_event_from_event_string(line+7); + image->ev = strdup(line+7); } if ( strncmp(line, "indexed_by = ", 13) == 0 ) { @@ -1260,70 +956,46 @@ int read_chunk_2(Stream *st, struct image *image, StreamReadFlags srf) image->serial = ser; } - if ( strncmp(line, "camera_length_", 14) == 0 ) { - if ( image->det != NULL ) { - - int k; - char name[1024]; - struct panel *p; - - for ( k=0; k<strlen(line)-14; k++ ) { - char ch = line[k+14]; - name[k] = ch; - if ( (ch == ' ') || (ch == '=') ) { - name[k] = '\0'; - break; - } - } - - p = find_panel_by_name(image->det, name); - if ( p == NULL ) { - ERROR("No panel '%s'\n", name); - } else { - p->clen = atof(line+14+k+3); - } - - } - } - if ( strstr(line, " = ") != NULL ) { + if ( (srf & STREAM_PEAKS) + && strcmp(line, STREAM_PEAK_LIST_START_MARKER) == 0 ) { - int fail; + ImageFeatureList *peaks; + peaks = read_peaks(st, image); - fail = read_and_store_field(image, line); - if ( fail ) { - ERROR("Failed to read fields from stream.\n"); - return 1; + if ( peaks == NULL ) { + ERROR("Failed while reading peaks\n"); + image_free(image); + return NULL; } - } - if ( (srf & STREAM_READ_PEAKS) - && strcmp(line, PEAK_LIST_START_MARKER) == 0 ) { + image->features = peaks; - int fail; - if ( AT_LEAST_VERSION(st, 2, 3) ) { - fail = read_peaks_2_3(st, image); - } else { - fail = read_peaks(st, image); - } - if ( fail ) { - ERROR("Failed while reading peaks\n"); - return 1; - } } - if ( (srf & STREAM_READ_CRYSTALS) - && (strcmp(line, CRYSTAL_START_MARKER) == 0) ) { + if ( strcmp(line, STREAM_CRYSTAL_START_MARKER) == 0 ) { read_crystal(st, image, srf); } /* A chunk must have at least a filename and a wavelength, * otherwise it's incomplete */ - if ( strcmp(line, CHUNK_END_MARKER) == 0 ) { - if ( have_filename && have_ev ) return 0; + if ( strcmp(line, STREAM_CHUNK_END_MARKER) == 0 ) { + if ( have_filename && have_ev ) { + /* Success */ + if ( srf & STREAM_DATA_DETGEOM ) { + create_detgeom(image, st->dtempl_read); + image_set_zero_data(image, st->dtempl_read); + image_set_zero_mask(image, st->dtempl_read); + } + /* FIXME: Maybe arbitrary spectrum from file (?) */ + image->spectrum = spectrum_generate_gaussian(image->lambda, + image->bw); + return image; + } ERROR("Incomplete chunk found in input file.\n"); - return 1; + image_free(image); + return NULL; } } while ( 1 ); @@ -1332,38 +1004,9 @@ int read_chunk_2(Stream *st, struct image *image, StreamReadFlags srf) ERROR("Error reading stream.\n"); } - return 1; /* Either error or EOF, don't care because we will complain - * on the terminal if it was an error. */ -} - - - -/** - * \param st A \ref Stream - * \param image An \ref image structure to be filled - * - * Reads a chunk from \p st, placing the information in \p image. - * - * \returns non-zero on error. - */ -int read_chunk(Stream *st, struct image *image) -{ - return read_chunk_2(st, image, STREAM_READ_UNITCELL - | STREAM_READ_REFLECTIONS - | STREAM_READ_PEAKS); -} - - -void write_stream_header(FILE *ofh, int argc, char *argv[]) -{ - int i; - - fprintf(ofh, "Command line:"); - for ( i=0; i<argc; i++ ) { - fprintf(ofh, " %s", argv[i]); - } - fprintf(ofh, "\n"); - fflush(ofh); + image_free(image); + return NULL; /* Either error or EOF, don't care because we will complain + * on the terminal if it was an error. */ } @@ -1380,120 +1023,101 @@ char *stream_geometry_file(Stream *st) } -static void read_audit_lines(Stream *st) +static void read_geometry_file(Stream *st) { int done = 0; size_t len = 0; - int first = 1; + const size_t max_geom_len = 64*1024; + char *geom; - st->audit_info = malloc(4096); - if ( st->audit_info == NULL ) { - ERROR("Failed to allocate memory for audit information\n"); + geom = malloc(max_geom_len); + if ( geom == NULL ) { + ERROR("Failed to allocate memory for geometry file\n"); return; } - st->audit_info[0] = '\0'; + geom[0] = '\0'; - /* Read lines from stream until one of them starts with "-----", - * then rewind to the start of that line */ do { char line[1024]; char *rval; - long pos; - - pos = ftell(st->fh); rval = fgets(line, 1023, st->fh); + st->ln++; if ( rval == NULL ) { - ERROR("Failed to read stream audit info.\n"); - close_stream(st); + ERROR("Failed to read stream geometry file.\n"); + stream_close(st); + free(geom); return; } - if ( strncmp(line, "-----", 5) == 0 ) { - fseek(st->fh, pos, SEEK_SET); + if ( strcmp(line, STREAM_GEOM_END_MARKER"\n") == 0 ) { done = 1; + continue; + } + + len += strlen(line); + if ( len > max_geom_len-1 ) { + ERROR("Stream's geometry file is too long (%li > %i).\n", + (long)len, (int)max_geom_len); + free(geom); + return; } else { - chomp(line); - len += strlen(line); - if ( len > 4090 ) { - ERROR("Too much audit information.\n"); - return; - } else { - if ( !first ) { - strcat(st->audit_info, "\n"); - } - first = 0; - strcat(st->audit_info, line); - } + strcat(geom, line); } } while ( !done ); + + st->geometry_file = geom; + st->dtempl_read = data_template_new_from_string(geom); } -static void read_geometry_file(Stream *st) +static void read_headers(Stream *st) { int done = 0; size_t len = 0; - int started = 0; - const size_t max_geom_len = 64*1024; - st->geometry_file = malloc(max_geom_len); - if ( st->geometry_file == NULL ) { + st->audit_info = malloc(4096); + if ( st->audit_info == NULL ) { ERROR("Failed to allocate memory for audit information\n"); return; } - st->geometry_file[0] = '\0'; + st->audit_info[0] = '\0'; + /* Read lines from stream until one of them starts with "-----", + * then rewind to the start of that line */ do { char line[1024]; char *rval; rval = fgets(line, 1023, st->fh); + st->ln++; if ( rval == NULL ) { - ERROR("Failed to read stream geometry file.\n"); - close_stream(st); - free(st->geometry_file); - st->geometry_file = NULL; + ERROR("Failed to read stream audit info.\n"); + stream_close(st); return; } - if ( strcmp(line, GEOM_START_MARKER"\n") == 0 ) { - started = 1; - continue; - } - - if ( strcmp(line, GEOM_END_MARKER"\n") == 0 ) { - done = 1; - continue; - } - - if ( strcmp(line, CHUNK_START_MARKER"\n") == 0 ) { + if ( strcmp(line, STREAM_GEOM_START_MARKER"\n") == 0 ) { + read_geometry_file(st); done = 1; - st->in_chunk = 1; - continue; - } - - if ( !started ) continue; - - len += strlen(line); - if ( len > max_geom_len-1 ) { - ERROR("Stream's geometry file is too long (%li > %i).\n", - (long)len, (int)max_geom_len); - free(st->geometry_file); - st->geometry_file = NULL; - return; } else { - strcat(st->geometry_file, line); + len += strlen(line); + if ( len > 4090 ) { + ERROR("Too much audit information.\n"); + return; + } else { + strcat(st->audit_info, line); + } } } while ( !done ); } -Stream *open_stream_for_read(const char *filename) +Stream *stream_open_for_read(const char *filename) { Stream *st; @@ -1502,7 +1126,10 @@ Stream *open_stream_for_read(const char *filename) st->old_indexers = 0; st->audit_info = NULL; st->geometry_file = NULL; - st->in_chunk = 0; + st->n_chunks = 0; + st->chunk_offsets = NULL; + st->dtempl_read = NULL; + st->dtempl_write = NULL; if ( strcmp(filename, "-") == 0 ) { st->fh = stdin; @@ -1521,7 +1148,7 @@ Stream *open_stream_for_read(const char *filename) rval = fgets(line, 1023, st->fh); if ( rval == NULL ) { ERROR("Failed to read stream version.\n"); - close_stream(st); + stream_close(st); return NULL; } @@ -1539,14 +1166,13 @@ Stream *open_stream_for_read(const char *filename) st->minor_version = 3; } else { ERROR("Invalid stream, or stream format is too new.\n"); - close_stream(st); + stream_close(st); return NULL; } st->ln = 1; - read_audit_lines(st); - read_geometry_file(st); + read_headers(st); return st; } @@ -1556,16 +1182,16 @@ Stream *open_stream_for_read(const char *filename) * \param fd File descriptor (e.g. from open()) to use for stream data. * * Creates a new \ref Stream from \p fd, so that stream data can be written to \p fd - * using \ref write_chunk. + * using \ref stream_write_chunk. * - * In contrast to \ref open_stream_for_write, this function does not write any of + * In contrast to \ref stream_open_for_write, this function does not write any of * the usual headers. This function is mostly for use when multiple substreams * need to be multiplexed into a single master stream. The master would be - * opened using \ref open_stream_for_write, and the substreams using this function. + * opened using \ref stream_open_for_write, and the substreams using this function. * * \returns A \ref Stream, or NULL on failure. */ -Stream *open_stream_fd_for_write(int fd) +Stream *stream_open_fd_for_write(int fd, const DataTemplate *dtempl) { Stream *st; @@ -1574,7 +1200,10 @@ Stream *open_stream_fd_for_write(int fd) st->old_indexers = 0; st->audit_info = NULL; st->geometry_file = NULL; - st->in_chunk = 0; + st->n_chunks = 0; + st->chunk_offsets = NULL; + st->dtempl_read = NULL; + st->dtempl_write = NULL; st->fh = fdopen(fd, "w"); if ( st->fh == NULL ) { @@ -1582,6 +1211,7 @@ Stream *open_stream_fd_for_write(int fd) return NULL; } + st->dtempl_write = dtempl; st->major_version = LATEST_MAJOR_VERSION; st->minor_version = LATEST_MINOR_VERSION; @@ -1589,102 +1219,32 @@ Stream *open_stream_fd_for_write(int fd) } -static void write_cell_to_stream(Stream *st, UnitCell *cell) +void stream_write_target_cell(Stream *st, UnitCell *cell) { - fprintf(st->fh, CELL_START_MARKER"\n"); + if ( cell == NULL ) return; + fprintf(st->fh, STREAM_CELL_START_MARKER"\n"); write_cell(cell, st->fh); fprintf(st->fh, "; Please note: this is the target unit cell.\n"); fprintf(st->fh, "; The actual unit cells produced by indexing " "depend on many other factors.\n"); - fprintf(st->fh, CELL_END_MARKER"\n"); + fprintf(st->fh, STREAM_CELL_END_MARKER"\n"); fflush(st->fh); } /** * \param filename Filename of new stream - * \param geom_filename The geometry filename to copy - * \param cell A \ref UnitCell to write into the stream - * \param argc The number of arguments to the program - * \param argv The arguments to the program - * \param indm_str The list of indexing methods + * \param dtempl A DataTemplate * - * Creates a new stream with name \p filename, and adds the stream format - * and version header, plus a verbatim copy of the geometry file and the unit - * cell in CrystFEL format. - * - * \returns A \ref Stream, or NULL on failure. - */ -Stream *open_stream_for_write_4(const char *filename, - const char *geom_filename, UnitCell *cell, - int argc, char *argv[], const char *indm_str) - -{ - Stream *st; - - st = malloc(sizeof(struct _stream)); - if ( st == NULL ) return NULL; - st->old_indexers = 0; - st->audit_info = NULL; - st->geometry_file = NULL; - st->in_chunk = 0; - - st->fh = fopen(filename, "w"); - if ( st->fh == NULL ) { - ERROR("Failed to open stream.\n"); - free(st); - return NULL; - } - - st->major_version = LATEST_MAJOR_VERSION; - st->minor_version = LATEST_MINOR_VERSION; - - fprintf(st->fh, "CrystFEL stream format %i.%i\n", - st->major_version, st->minor_version); - fprintf(st->fh, "Generated by CrystFEL "CRYSTFEL_VERSIONSTRING"\n"); - fflush(st->fh); - - if ( (argc > 0) && (argv != NULL) ) { - write_command(st, argc, argv); - } - - if ( indm_str != NULL ) { - fprintf(st->fh, "Indexing methods selected: %s\n", indm_str); - } - if ( geom_filename != NULL ) { - write_geometry_file(st, geom_filename); - } - if ( cell != NULL ) { - write_cell_to_stream(st, cell); - } - - return st; -} - - -Stream *open_stream_for_write_3(const char *filename, - const char *geom_filename, UnitCell *cell, - int argc, char *argv[]) -{ - return open_stream_for_write_4(filename, geom_filename, cell, - argc, argv, NULL); -} - - -/** - * \param filename Filename of new stream - * \param geom_filename The geometry filename to copy - * \param argc The number of arguments to the program - * \param argv The arguments to the program + * Creates a new stream with name \p filename. If \p filename already + * exists, it will be overwritten. * - * Creates a new stream with name \p filename, and adds the stream format - * and version header, plus a verbatim copy of the geometry file + * Audit information (e.g. CrystFEL version number) will be written. * * \returns A \ref Stream, or NULL on failure. */ -Stream *open_stream_for_write_2(const char *filename, - const char *geom_filename, int argc, - char *argv[]) +Stream *stream_open_for_write(const char *filename, + const DataTemplate *dtempl) { Stream *st; @@ -1694,7 +1254,10 @@ Stream *open_stream_for_write_2(const char *filename, st->old_indexers = 0; st->audit_info = NULL; st->geometry_file = NULL; - st->in_chunk = 0; + st->n_chunks = 0; + st->chunk_offsets = NULL; + st->dtempl_write = dtempl; + st->dtempl_read = NULL; st->fh = fopen(filename, "w"); if ( st->fh == NULL ) { @@ -1708,50 +1271,15 @@ Stream *open_stream_for_write_2(const char *filename, fprintf(st->fh, "CrystFEL stream format %i.%i\n", st->major_version, st->minor_version); - fprintf(st->fh, "Generated by CrystFEL "CRYSTFEL_VERSIONSTRING"\n"); + fprintf(st->fh, "Generated by CrystFEL %s\n", + libcrystfel_version_string()); fflush(st->fh); - if ( (argc > 0) && (argv != NULL) ) { - write_command(st, argc, argv); - } - if ( geom_filename != NULL ) { - write_geometry_file(st, geom_filename); - } - return st; } -/** - * \param filename Filename of new stream - * - * Creates a new stream with name \p filename, and adds the stream format - * and version headers. - * - * You may want to follow this with a call to \ref write_command to record the - * command line. - * - * \returns A \ref Stream, or NULL on failure. - */ -Stream *open_stream_for_write(const char *filename) -{ - return open_stream_for_write_2(filename, NULL, 0, NULL); -} - - -/** - * \param st A \ref Stream - * - * This function gets the integer file descriptor for \p st, a bit like fileno(). - * - * This is useful in conjunction with \ref open_stream_fd_for_write, to get the - * underlying file descriptor to which the multiplexed stream data should be - * written. In this case, the only other operations you should ever do (or have - * done) on \p st are \ref open_stream_for_write and \ref close_stream. - * - * \returns An integer file descriptor - */ -int get_stream_fd(Stream *st) +int stream_get_fd(Stream *st) { return fileno(st->fh); } @@ -1762,46 +1290,27 @@ int get_stream_fd(Stream *st) * * Closes the stream */ -void close_stream(Stream *st) +void stream_close(Stream *st) { + if ( st == NULL ) return; free(st->audit_info); free(st->geometry_file); + data_template_free(st->dtempl_read); fclose(st->fh); free(st); } -int is_stream(const char *filename) -{ - FILE *fh; - char line[1024]; - char *rval; - - fh = fopen(filename, "r"); - if ( fh == NULL ) return 0; - - rval = fgets(line, 1023, fh); - fclose(fh); - if ( rval == NULL ) return 0; - - if ( strncmp(line, "CrystFEL stream format 2.0", 26) == 0 ) return 1; - if ( strncmp(line, "CrystFEL stream format 2.1", 26) == 0 ) return 1; - if ( strncmp(line, "CrystFEL stream format 2.2", 26) == 0 ) return 1; - - return 0; -} - - /** * \param st A \ref Stream * \param argc number of arguments * \param argv command-line arguments * - * Writes the command line to \p st. \p argc and \p argv should be exactly as were - * given to main(). This should usually be called immediately after - * ref open_stream_for_write. + * Writes the command line to \p st. \p argc and \p argv should be + * exactly as were given to main(). This should usually be called + * immediately after \ref stream_open_for_write. */ -void write_command(Stream *st, int argc, char *argv[]) +void stream_write_commandline_args(Stream *st, int argc, char *argv[]) { int i; @@ -1816,6 +1325,13 @@ void write_command(Stream *st, int argc, char *argv[]) } +void stream_write_indexing_methods(Stream *st, const char *indm_str) +{ + fprintf(st->fh, "Indexing methods selected: %s\n", indm_str); + fflush(st->fh); +} + + /** * \param st A \ref Stream * \param geom_filename geomtry file name @@ -1823,8 +1339,8 @@ void write_command(Stream *st, int argc, char *argv[]) * Writes the content of the geometry file to \p st. This should usually be * called immediately after \ref write_command. */ -void write_geometry_file(Stream *st, const char *geom_filename) { - +void stream_write_geometry_file(Stream *st, const char *geom_filename) +{ char line[2014]; FILE *geom_fh; char *rval; @@ -1838,7 +1354,7 @@ void write_geometry_file(Stream *st, const char *geom_filename) { "'%s'\n", geom_filename); return; } - fprintf(st->fh, GEOM_START_MARKER"\n"); + fprintf(st->fh, STREAM_GEOM_START_MARKER"\n"); do { rval = fgets(line, 1023, geom_fh); @@ -1853,7 +1369,7 @@ void write_geometry_file(Stream *st, const char *geom_filename) { fclose(geom_fh); - fprintf(st->fh, GEOM_END_MARKER"\n"); + fprintf(st->fh, STREAM_GEOM_END_MARKER"\n"); fflush(st->fh); } @@ -1862,16 +1378,186 @@ void write_geometry_file(Stream *st, const char *geom_filename) { * \param st A \ref Stream * * Attempts to set the file pointer for \p st to the start of the stream, so that - * later calls to \ref read_chunk will repeat the sequence of chunks from the + * later calls to \ref stream_read_chunk will repeat the sequence of chunks from the * start. * * Programs must not assume that this operation always succeeds! * * \returns Non-zero if the stream could not be rewound. */ -int rewind_stream(Stream *st) +int stream_rewind(Stream *st) { st->ln = 0; return fseek(st->fh, 0, SEEK_SET); } + +struct _streamindex +{ + char **keys; + long int *ptrs; + int n_keys; + int max_keys; +}; + + +void stream_index_free(StreamIndex *index) +{ + if ( index == NULL ) return; + free(index->keys); + free(index->ptrs); + free(index); +} + + +static char *make_key(const char *filename, + const char *ev) +{ + char *key; + + if ( ev == NULL ) ev = "//"; + + key = malloc(strlen(filename)+strlen(ev)+2); + if ( key == NULL ) return NULL; + + strcpy(key, filename); + strcat(key, " "); + strcat(key, ev); + + return key; +} + + +int stream_select_chunk(Stream *st, + StreamIndex *index, + const char *filename, + const char *ev) +{ + int i; + char *key; + + if ( index == NULL ) return 1; + + key = make_key(filename, ev); + for ( i=0; i<index->n_keys; i++ ) { + if ( strcmp(index->keys[i], key) == 0 ) { + if ( st != NULL ) { + fseek(st->fh, index->ptrs[i], SEEK_SET); + } + return 0; + } + } + return 1; +} + + +static void add_index_record(StreamIndex *index, + long int ptr, + const char *filename, + const char *ev) +{ + char *key; + + if ( index->n_keys == index->max_keys ) { + + int new_max_keys = index->max_keys + 256; + char **new_keys; + long int *new_ptrs; + + new_keys = realloc(index->keys, + new_max_keys*sizeof(char *)); + if ( new_keys == NULL ) return; + + new_ptrs = realloc(index->ptrs, + new_max_keys*sizeof(long int)); + if ( new_ptrs == NULL ) return; + + index->keys = new_keys; + index->ptrs = new_ptrs; + index->max_keys = new_max_keys; + + } + + key = make_key(filename, ev); + if ( key == NULL ) return; + + index->keys[index->n_keys] = key; + index->ptrs[index->n_keys] = ptr; + index->n_keys++; +} + + +StreamIndex *stream_make_index(const char *filename) +{ + FILE *fh; + StreamIndex *index; + long int last_start_pos = 0; + char *last_filename = NULL; + char *last_ev = NULL; + int done = 0; + + fh = fopen(filename, "r"); + if ( fh == NULL ) return NULL; + + index = malloc(sizeof(StreamIndex)); + if ( index == NULL ) { + fclose(fh); + return NULL; + } + + index->keys = NULL; + index->ptrs = NULL; + index->n_keys = 0; + index->max_keys = 0; + + STATUS("Scanning %s\n", filename); + + do { + + char *rval; + char line[1024]; + long int pos; + + pos = ftell(fh); + rval = fgets(line, 1024, fh); + if ( rval == NULL ) { + done = 1; + break; + } + chomp(line); + + if ( strcmp(line, STREAM_CHUNK_START_MARKER) == 0 ) { + last_start_pos = pos; + last_filename = NULL; + last_ev = NULL; + } + + if ( strncmp(line, "Image filename: ", 16) == 0 ) { + last_filename = strdup(line+16); + } + + if ( strncmp(line, "Event: ", 7) == 0 ) { + last_ev = strdup(line+7); + } + + if ( strcmp(line, STREAM_CHUNK_END_MARKER) == 0 ) { + if ( (last_start_pos != 0) + && (last_filename != NULL) ) + { + add_index_record(index, + last_start_pos, + last_filename, + last_ev); + } + free(last_filename); + free(last_ev); + last_start_pos = 0; + last_filename = NULL; + last_ev = NULL; + } + + } while ( !done ); + + fclose(fh); + return index; +} diff --git a/libcrystfel/src/stream.h b/libcrystfel/src/stream.h index f3e4d7e8..115d9918 100644 --- a/libcrystfel/src/stream.h +++ b/libcrystfel/src/stream.h @@ -3,11 +3,11 @@ * * Stream tools * - * Copyright © 2013-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2019 Thomas White <taw@physics.org> + * 2010-2021 Thomas White <taw@physics.org> * 2014 Valerio Mariani * 2011 Andrew Aquila * @@ -31,34 +31,28 @@ #ifndef STREAM_H #define STREAM_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - /** * \file stream.h * Stream functions (for indexing results) */ struct image; -struct hdfile; -struct event; -struct imagefile; + +#include "datatemplate.h" #include "cell.h" -#define GEOM_START_MARKER "----- Begin geometry file -----" -#define GEOM_END_MARKER "----- End geometry file -----" -#define CELL_START_MARKER "----- Begin unit cell -----" -#define CELL_END_MARKER "----- End unit cell -----" -#define CHUNK_START_MARKER "----- Begin chunk -----" -#define CHUNK_END_MARKER "----- End chunk -----" -#define PEAK_LIST_START_MARKER "Peaks from peak search" -#define PEAK_LIST_END_MARKER "End of peak list" -#define CRYSTAL_START_MARKER "--- Begin crystal" -#define CRYSTAL_END_MARKER "--- End crystal" -#define REFLECTION_START_MARKER "Reflections measured after indexing" -/* REFLECTION_END_MARKER is over in reflist-utils.h because it is also - * used to terminate a standalone list of reflections */ +#define STREAM_GEOM_START_MARKER "----- Begin geometry file -----" +#define STREAM_GEOM_END_MARKER "----- End geometry file -----" +#define STREAM_CELL_START_MARKER "----- Begin unit cell -----" +#define STREAM_CELL_END_MARKER "----- End unit cell -----" +#define STREAM_CHUNK_START_MARKER "----- Begin chunk -----" +#define STREAM_CHUNK_END_MARKER "----- End chunk -----" +#define STREAM_PEAK_LIST_START_MARKER "Peaks from peak search" +#define STREAM_PEAK_LIST_END_MARKER "End of peak list" +#define STREAM_CRYSTAL_START_MARKER "--- Begin crystal" +#define STREAM_CRYSTAL_END_MARKER "--- End crystal" +#define STREAM_REFLECTION_START_MARKER "Reflections measured after indexing" +#define STREAM_REFLECTION_END_MARKER "End of reflections" /** * An opaque structure representing a stream being read or written @@ -66,77 +60,72 @@ struct imagefile; typedef struct _stream Stream; /** - * A bitfield of things that can be read from a stream. Use this (and - * \ref read_chunk_2) to read the stream faster if you don't need the entire - * contents of the stream. + * A bitfield of things that can be read from or written to a stream. + * Use this together with stream_{read,write}_chunk to read/write the + * stream faster if you don't need all the information. * - * Using either or both of \p STREAM_READ_REFLECTIONS and \p STREAM_READ_UNITCELL - * implies \p STREAM_READ_CRYSTALS. + * General information about crystals (including unit cell parameters) + * is always read and written. **/ typedef enum { - /** Read the unit cell */ - STREAM_READ_UNITCELL = 1, - /** Read the integrated reflections */ - STREAM_READ_REFLECTIONS = 2, + STREAM_REFLECTIONS = 2, /** Read the peak search results */ - STREAM_READ_PEAKS = 4, - - /** Read the general information about crystals */ - STREAM_READ_CRYSTALS = 8, + STREAM_PEAKS = 4, -} StreamReadFlags; + /** Reconstruct the detgeom structure, + * and create (blank) data/mask arrays. + * (NB this is (currently) a slow operation) */ + STREAM_DATA_DETGEOM = 8, -struct stuff_from_stream -{ - char **fields; - int n_fields; -}; +} StreamFlags; #ifdef __cplusplus extern "C" { #endif -extern Stream *open_stream_for_read(const char *filename); -extern Stream *open_stream_for_write(const char *filename); -extern Stream *open_stream_for_write_2(const char *filename, - const char* geom_filename, int argc, - char *argv[]); -extern Stream *open_stream_for_write_3(const char *filename, - const char* geom_filename, UnitCell *cell, - int argc, char *argv[]); -extern Stream *open_stream_for_write_4(const char *filename, - const char *geom_filename, UnitCell *cell, - int argc, char *argv[], const char *indm_str); -extern Stream *open_stream_fd_for_write(int fd); -extern int get_stream_fd(Stream *st); -extern void close_stream(Stream *st); - -extern void free_stuff_from_stream(struct stuff_from_stream *sfs); - -extern int read_chunk(Stream *st, struct image *image); -extern int read_chunk_2(Stream *st, struct image *image, - StreamReadFlags srf); +/* Opening/closing streams */ +extern Stream *stream_open_for_read(const char *filename); +extern Stream *stream_open_for_write(const char *filename, + const DataTemplate *dtempl); +extern Stream *stream_open_fd_for_write(int fd, + const DataTemplate *dtempl); +extern void stream_close(Stream *st); + +/* Writing things to stream header */ +extern void stream_write_geometry_file(Stream *st, + const char *geom_filename); +extern void stream_write_target_cell(Stream *st, + UnitCell *cell); +extern void stream_write_commandline_args(Stream *st, + int argc, char *argv[]); +extern void stream_write_indexing_methods(Stream *st, + const char *indm_str); + +/* Metadata */ extern int stream_has_old_indexers(Stream *st); - -extern int write_chunk(Stream *st, struct image *image, struct imagefile *imfile, - int include_peaks, int include_reflections, - struct event *ev); - -extern int write_chunk_2(Stream *st, struct image *image, - struct imagefile *imfile, - int include_peaks, int include_reflections, - struct event *ev); - -extern void write_command(Stream *st, int argc, char *argv[]); -extern void write_geometry_file(Stream *st, const char *geom_filename); -extern int rewind_stream(Stream *st); -extern int is_stream(const char *filename); extern char *stream_audit_info(Stream *st); extern char *stream_geometry_file(Stream *st); +/* Low-level stuff used for indexamajig sandbox */ +extern int stream_get_fd(Stream *st); +extern int stream_rewind(Stream *st); + +/* Random access */ +typedef struct _streamindex StreamIndex; +extern StreamIndex *stream_make_index(const char *filename); +extern int stream_select_chunk(Stream *st, StreamIndex *index, + const char *filename, + const char *ev); +extern void stream_index_free(StreamIndex *index); + +/* Read/write chunks */ +extern struct image *stream_read_chunk(Stream *st, StreamFlags srf); +extern int stream_write_chunk(Stream *st, const struct image *image, + StreamFlags srf); + #ifdef __cplusplus } #endif diff --git a/libcrystfel/src/symmetry.c b/libcrystfel/src/symmetry.c index 32c5f6c3..bd6d2e8f 100644 --- a/libcrystfel/src/symmetry.c +++ b/libcrystfel/src/symmetry.c @@ -3,11 +3,11 @@ * * Symmetry * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2019 Thomas White <taw@physics.org> + * 2010-2020 Thomas White <taw@physics.org> * 2014 Kenneth Beyerlein <kenneth.beyerlein@desy.de> * * This file is part of CrystFEL. @@ -1684,21 +1684,24 @@ SymOpList *parse_symmetry_operations(const char *s) } -static void add_chars(char *t, const char *s, int max_len) +static void add_chars(char *t, const char *s, size_t max_len) { - char *tmp; + size_t len; - tmp = strdup(t); + len = strlen(t) + strlen(s); + if ( len > max_len ) { + ERROR("get_matrix_name: String too long!\n"); + return; + } - snprintf(t, max_len, "%s%s", tmp, s); - free(tmp); + strcat(t, s); } char *get_matrix_name(const IntegerMatrix *m, int col) { char *text; - const int max_len = 9; + const size_t max_len = 31; int i; int printed = 0; diff --git a/libcrystfel/src/symmetry.h b/libcrystfel/src/symmetry.h index f71cadb9..b8780735 100644 --- a/libcrystfel/src/symmetry.h +++ b/libcrystfel/src/symmetry.h @@ -3,7 +3,7 @@ * * Symmetry * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: @@ -30,11 +30,6 @@ #ifndef SYMMETRY_H #define SYMMETRY_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - - #include "integer_matrix.h" #include "rational.h" diff --git a/libcrystfel/src/thread-pool.c b/libcrystfel/src/thread-pool.c index 67645b1e..61849270 100644 --- a/libcrystfel/src/thread-pool.c +++ b/libcrystfel/src/thread-pool.c @@ -3,11 +3,11 @@ * * A thread pool implementation * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2012 Thomas White <taw@physics.org> + * 2010-2019 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -47,7 +47,6 @@ static int use_status_labels = 0; static pthread_key_t status_label_key; -pthread_mutex_t stderr_lock = PTHREAD_MUTEX_INITIALIZER; struct worker_args { diff --git a/libcrystfel/src/thread-pool.h b/libcrystfel/src/thread-pool.h index 16fb38bf..6bd7139c 100644 --- a/libcrystfel/src/thread-pool.h +++ b/libcrystfel/src/thread-pool.h @@ -3,11 +3,11 @@ * * A thread pool implementation * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2012 Thomas White <taw@physics.org> + * 2010-2019 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -29,18 +29,12 @@ #ifndef THREAD_POOL_H #define THREAD_POOL_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - - #include <pthread.h> #ifdef __cplusplus extern "C" { #endif -extern pthread_mutex_t stderr_lock; extern signed int get_status_label(void); /** diff --git a/libcrystfel/src/utils.c b/libcrystfel/src/utils.c index 4873b050..053e952b 100644 --- a/libcrystfel/src/utils.c +++ b/libcrystfel/src/utils.c @@ -3,11 +3,11 @@ * * Utility stuff * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2009-2014 Thomas White <taw@physics.org> + * 2009-2020 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -34,6 +34,7 @@ #include <math.h> #include <string.h> #include <stdio.h> +#include <stdarg.h> #include <unistd.h> #include <sys/types.h> #include <sys/stat.h> @@ -251,6 +252,76 @@ gsl_vector *solve_svd(gsl_vector *v, gsl_matrix *M, int *pn_filt, int verbose) } +/* ------------------------------ Message logging ---------------------------- */ + +/* Lock to keep lines serialised on the terminal */ +pthread_mutex_t stderr_lock = PTHREAD_MUTEX_INITIALIZER; + + +static void log_to_stderr(enum log_msg_type type, const char *msg, + void *vp) +{ + int error_print_val = get_status_label(); + pthread_mutex_lock(&stderr_lock); + if ( error_print_val >= 0 ) { + fprintf(stderr, "%3i: ", error_print_val); + } + fprintf(stderr, "%s", msg); + pthread_mutex_unlock(&stderr_lock); +} + + +/* Function to call with ERROR/STATUS messages */ +LogMsgFunc log_msg_func = log_to_stderr; +void *log_msg_vp = NULL; + + +void set_log_message_func(LogMsgFunc new_log_msg_func, void *vp) +{ + log_msg_func = new_log_msg_func; + log_msg_vp = vp; +} + + +void STATUS(const char *format, ...) +{ + va_list args; + char tmp[1024]; + va_start(args, format); + vsnprintf(tmp, 1024, format, args); + va_end(args); + log_msg_func(LOG_MSG_STATUS, tmp, log_msg_vp); +} + + +void ERROR(const char *format, ...) +{ + va_list args; + char tmp[1024]; + va_start(args, format); + vsnprintf(tmp, 1024, format, args); + va_end(args); + log_msg_func(LOG_MSG_ERROR, tmp, log_msg_vp); +} + + +/* ------------------------------ Useful functions ---------------------------- */ + +int convert_int(const char *str, int *pval) +{ + int val; + char *rval; + + val = strtod(str, &rval); + if ( *rval != '\0' ) { + return 1; + } else { + *pval = val; + return 0; + } +} + + size_t notrail(char *s) { ssize_t i; @@ -272,10 +343,12 @@ size_t notrail(char *s) void chomp(char *s) { size_t i; + size_t len; if ( s == NULL ) return; + len = strlen(s); - for ( i=0; i<strlen(s); i++ ) { + for ( i=0; i<len; i++ ) { if ( (s[i] == '\n') || (s[i] == '\r') ) { s[i] = '\0'; return; @@ -503,6 +576,13 @@ char *check_prefix(char *prefix) } +char *safe_strdup(const char *in) +{ + if ( in == NULL ) return NULL; + return strdup(in); +} + + char *safe_basename(const char *in) { int i; @@ -550,6 +630,29 @@ void strip_extension(char *bfn) } +const char *filename_extension(const char *fn, const char **pext2) +{ + const char *ext = NULL; + const char *ext2 = NULL; + size_t r = strlen(fn)-1; + + while ( r > 0 ) { + if ( fn[r] == '.' ) { + if ( ext != NULL ) { + ext2 = fn+r; + break; + } else { + ext = fn+r; + } + } + r--; + } + + if ( pext2 != NULL ) *pext2 = ext2; + return ext; +} + + /* Force the linker to bring in CBLAS to make GSL happy */ void utils_fudge_gslcblas() { @@ -674,3 +777,64 @@ struct rvec quat_rot(struct rvec q, struct quaternion z) return res; } + + +char *load_entire_file(const char *filename) +{ + struct stat statbuf; + int r; + char *contents; + FILE *fh; + + r = stat(filename, &statbuf); + if ( r != 0 ) { + ERROR("File '%s' not found\n", filename); + return NULL; + } + + contents = malloc(statbuf.st_size+1); + if ( contents == NULL ) { + ERROR("Failed to allocate memory for file\n"); + return NULL; + } + + fh = fopen(filename, "r"); + if ( fh == NULL ) { + ERROR("Failed to open file '%s'\n", filename); + free(contents); + return NULL; + } + + if ( fread(contents, 1, statbuf.st_size, fh) != statbuf.st_size ) { + ERROR("Failed to read file '%s'\n", filename); + free(contents); + return NULL; + } + contents[statbuf.st_size] = '\0'; + + fclose(fh); + + return contents; +} + + +int compare_double(const void *av, const void *bv) +{ + double a = *(double *)av; + double b = *(double *)bv; + if ( a > b ) return 1; + if ( a < b ) return -1; + return 0; +} + + +/* -------------------------- libcrystfel features ------------------------ */ + +int crystfel_has_peakfinder9() +{ +#ifdef HAVE_FDIP + return 1; +#else + return 0; +#endif +} diff --git a/libcrystfel/src/utils.h b/libcrystfel/src/utils.h index beb05ee0..8bcb51ff 100644 --- a/libcrystfel/src/utils.h +++ b/libcrystfel/src/utils.h @@ -3,11 +3,11 @@ * * Utility stuff * - * Copyright © 2012-2020 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2021 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2009-2014 Thomas White <taw@physics.org> + * 2009-2020 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -29,10 +29,6 @@ #ifndef UTILS_H #define UTILS_H -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - #include <math.h> #include <complex.h> #include <float.h> @@ -54,9 +50,12 @@ /* -------------------------- Fundamental constants ------------------------ */ -/* Electron charge in C */ +/* Electron charge (Coulombs) */ #define ELECTRON_CHARGE (1.6021773e-19) +/* Electron rest mass (kg) */ +#define ELECTRON_MASS (9.1093837015e-31) + /* Planck's constant (Js) */ #define PLANCK (6.62606896e-34) @@ -77,9 +76,13 @@ extern void show_matrix_eqn(gsl_matrix *M, gsl_vector *v); extern void show_matrix(gsl_matrix *M); extern gsl_vector *solve_svd(gsl_vector *v, gsl_matrix *M, int *n_filt, int verbose); + extern size_t notrail(char *s); +extern int convert_int(const char *str, int *pval); extern void chomp(char *s); +#define CLEAR_BIT(val, bit) (((val) | (bit)) ^ (bit)) + /** * Controls the behaviour of \ref assplode. **/ @@ -192,37 +195,45 @@ static inline int within_tolerance(double a, double b, double percent) /* Photon energy (eV) to k (1/m) */ #define ph_eV_to_k(a) ((a)*ELECTRON_CHARGE/PLANCK/C_VACUO) +/* Electron accelerating voltage (V) to wavelength (m) */ +static inline double el_V_to_lambda(double E) +{ + double Estar; + + /* Relativistically corrected accelerating voltage */ + Estar = E * (1.0 + E * ELECTRON_CHARGE/(2.0*ELECTRON_MASS*C_VACUO*C_VACUO)); + + return PLANCK / sqrt(2.0*ELECTRON_MASS*ELECTRON_CHARGE*Estar); +} + -/* ------------------------------ Message macros ---------------------------- */ +/* ------------------------------ Message logging ---------------------------- */ extern pthread_mutex_t stderr_lock; -#define ERROR(...) { \ - int error_print_val = get_status_label(); \ - pthread_mutex_lock(&stderr_lock); \ - if ( error_print_val >= 0 ) { \ - fprintf(stderr, "%3i: ", error_print_val); \ - } \ - fprintf(stderr, __VA_ARGS__); \ - pthread_mutex_unlock(&stderr_lock); \ - } - -#define STATUS(...) { \ - int status_print_val = get_status_label(); \ - pthread_mutex_lock(&stderr_lock); \ - if ( status_print_val >= 0 ) { \ - fprintf(stderr, "%3i: ", status_print_val); \ - } \ - fprintf(stderr, __VA_ARGS__); \ - pthread_mutex_unlock(&stderr_lock); \ - } +enum log_msg_type { + LOG_MSG_STATUS, + LOG_MSG_ERROR +}; + + +extern void STATUS(const char *format, ...); +extern void ERROR(const char *format, ...); + +typedef void (*LogMsgFunc)(enum log_msg_type type, const char *msg, void *vp); + +extern void set_log_message_func(LogMsgFunc new_log_msg_func, + void *vp); /* ------------------------------ File handling ----------------------------- */ extern char *check_prefix(char *prefix); extern char *safe_basename(const char *in); +extern char *safe_strdup(const char *in); extern void strip_extension(char *bfn); +extern char *load_entire_file(const char *filename); +extern const char *filename_extension(const char *fn, const char **ext2); /* ------------------------------ Useful stuff ------------------------------ */ @@ -258,6 +269,8 @@ extern struct quaternion random_quaternion(gsl_rng *rng); extern int quaternion_valid(struct quaternion q); extern struct rvec quat_rot(struct rvec q, struct quaternion z); +extern int compare_double(const void *av, const void *bv); + /* Keep these ones inline, to avoid function call overhead */ static inline struct quaternion invalid_quaternion(void) { @@ -293,6 +306,12 @@ static inline void mean_variance(const double x, const double w, *sumw = temp; } + +/* -------------------------- libcrystfel features ------------------------ */ + +extern int crystfel_has_peakfinder9(void); + + #ifdef __cplusplus } #endif |