From 5a9ad6f30ea2ba5599d50e847d9e9e50b9fbbe1b Mon Sep 17 00:00:00 2001 From: Thomas White Date: Tue, 3 May 2022 14:31:34 +0200 Subject: Seedee deserialization --- libcrystfel/CMakeLists.txt | 2 + libcrystfel/libcrystfel-config.h.cmake.in | 1 + libcrystfel/libcrystfel-config.h.meson.in | 1 + libcrystfel/meson.build | 11 +- libcrystfel/src/image-seedee.c | 195 ++++++++++++++++++++++++++++++ libcrystfel/src/image-seedee.h | 41 +++++++ libcrystfel/src/image.c | 6 + libcrystfel/src/image.h | 3 +- 8 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 libcrystfel/src/image-seedee.c create mode 100644 libcrystfel/src/image-seedee.h (limited to 'libcrystfel') diff --git a/libcrystfel/CMakeLists.txt b/libcrystfel/CMakeLists.txt index 3ca50b1f..ea475637 100644 --- a/libcrystfel/CMakeLists.txt +++ b/libcrystfel/CMakeLists.txt @@ -19,6 +19,7 @@ set(HAVE_FDIP ${FDIP_FOUND}) set(HAVE_MSGPACK ${MSGPACK_FOUND}) set(HAVE_LIBCCP4 ${LIBCCP4_FOUND}) set(HAVE_ZLIB ${ZLIB_FOUND}) +set(HAVE_SEEDEE 0) # Recent enough version of zlib? set(CMAKE_REQUIRED_LIBRARIES "-lz") @@ -62,6 +63,7 @@ set(LIBCRYSTFEL_SOURCES src/image-hdf5.c src/fom.c src/image-msgpack.c + src/image-seedee.c src/profile.c ${BISON_symopp_OUTPUTS} ${FLEX_symopl_OUTPUTS} diff --git a/libcrystfel/libcrystfel-config.h.cmake.in b/libcrystfel/libcrystfel-config.h.cmake.in index 830055cb..82d0611a 100644 --- a/libcrystfel/libcrystfel-config.h.cmake.in +++ b/libcrystfel/libcrystfel-config.h.cmake.in @@ -12,6 +12,7 @@ #cmakedefine HAVE_MSGPACK #cmakedefine HAVE_CLOCK_GETTIME #cmakedefine HAVE_HDF5 +#cmakedefine HAVE_SEEDEE #cmakedefine HAVE_FORKPTY_PTY_H #cmakedefine HAVE_FORKPTY_UTIL_H diff --git a/libcrystfel/libcrystfel-config.h.meson.in b/libcrystfel/libcrystfel-config.h.meson.in index 7d43147d..302fae0b 100644 --- a/libcrystfel/libcrystfel-config.h.meson.in +++ b/libcrystfel/libcrystfel-config.h.meson.in @@ -10,6 +10,7 @@ #mesondefine HAVE_MSGPACK #mesondefine HAVE_CLOCK_GETTIME #mesondefine HAVE_HDF5 +#mesondefine HAVE_SEEDEE #mesondefine HAVE_FORKPTY_PTY_H #mesondefine HAVE_FORKPTY_UTIL_H diff --git a/libcrystfel/meson.build b/libcrystfel/meson.build index b5005aa5..41cfbaa8 100644 --- a/libcrystfel/meson.build +++ b/libcrystfel/meson.build @@ -25,6 +25,14 @@ if fftwdep.found() conf_data.set10('HAVE_FFTW', true) endif +seedeedep = dependency('seedee', required: false) +cjsondep = dependency('cjson', + required: true, + fallback: ['cjson', 'libcjson_dep']) +if cjsondep.found() and seedeedep.found() + conf_data.set10('HAVE_SEEDEE', 1) +endif + xgandalfdep = dependency('xgandalf', required: false, fallback: ['xgandalf', 'xgandalf_dep']) @@ -125,6 +133,7 @@ libcrystfel_sources = ['src/image.c', 'src/image-cbf.c', 'src/image-hdf5.c', 'src/image-msgpack.c', + 'src/image-seedee.c', 'src/indexers/dirax.c', 'src/indexers/felix.c', 'src/indexers/mosflm.c', @@ -150,7 +159,7 @@ libcrystfel = library('crystfel', [libcrystfel_sources, libcrystfel_versionc], dependencies: [mdep, utildep, fftwdep, gsldep, zlibdep, hdf5dep, pthreaddep, xgandalfdep, pinkindexerdep, fdipdep, - ccp4dep, msgpackdep], + ccp4dep, msgpackdep, seedeedep, cjsondep], install: true) libcrystfeldep = declare_dependency(include_directories: libcrystfel_includes, diff --git a/libcrystfel/src/image-seedee.c b/libcrystfel/src/image-seedee.c new file mode 100644 index 00000000..1965aefa --- /dev/null +++ b/libcrystfel/src/image-seedee.c @@ -0,0 +1,195 @@ +/* + * image-seedee.c + * + * Image loading with Seedee + * + * Copyright © 2017-2022 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2018-2022 Thomas White + * 2014 Valerio Mariani + * 2017 Stijn de Graaf + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see . + * + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "datatemplate_priv.h" + + +#if defined(HAVE_SEEDEE) + +#include +#include + + +static int load_seedee_data(struct panel_template *p, + struct SeedeeNDArray *array, + float **pdata) +{ + int data_size_fs, data_size_ss; + float *data = NULL; + + data_size_ss = array->shape[0]; + data_size_fs = array->shape[1]; + + if ( (p->orig_min_fs + PANEL_WIDTH(p) > data_size_fs) + || (p->orig_min_ss + PANEL_HEIGHT(p) > data_size_ss) ) + { + ERROR("Data for panel %s (%i x %i + %i + %i) is outside data " + "array bounds (%i x %i)\n", + p->name, + PANEL_WIDTH(p), PANEL_HEIGHT(p), + p->orig_min_fs, p->orig_min_ss, + data_size_fs, data_size_ss); + return 1; + } + + if ( (array->datatype == 'u') + && (array->itemsize == 2) + && (array->byteorder == '<') ) + { + int fs, ss; + uint16_t *in_data = (uint16_t *)array->data; + + data = malloc(PANEL_WIDTH(p) * PANEL_HEIGHT(p) * sizeof(float)); + if ( data == NULL ) return 1; + + for ( ss=0; ssorig_min_fs + (ss+p->orig_min_ss)*data_size_fs; + data[fs+ss*PANEL_WIDTH(p)] = in_data[idx]; + } + } + *pdata = data; + + } else { + ERROR("Unrecognised data type %c%i%c\n", + array->datatype, array->itemsize, array->byteorder); + return 1; + } + + return 0; +} + + +/* Read the image data from 'data_block' into 'image', according to 'dtempl' */ +int image_seedee_read(struct image *image, + DataTemplate *dtempl, + void *data_block, + size_t data_block_size, + char *meta_data) +{ + struct SeedeeNDArray array; + int r; + bool zero_copy; + int i; + cJSON *json; + cJSON *data_format_str; + + json = cJSON_Parse(meta_data); + if ( json == NULL ) { + ERROR("Failed to parse JSON\n"); + return 1; + } + + data_format_str = cJSON_GetObjectItemCaseSensitive(json, "_data_format"); + if ( !cJSON_IsString(data_format_str) ) { + ERROR("_data_format isn't a string"); + cJSON_Delete(json); + return 1; + } + + profile_start("seedee-get-size"); + array.size = seedee_get_data_size(data_format_str->valuestring, + data_block, data_block_size, + &zero_copy, &array); + profile_end("seedee-get-size"); + array.data = malloc(array.size); + array.shape = malloc(array.ndims*sizeof(int)); + if ( (array.data == NULL) || (array.shape == NULL) ) { + cJSON_Delete(json); + return 1; + } + + if ( array.ndims != 2 ) { + ERROR("Seedee data has unexpected number of dimensions " + "(%i, expected 2)\n", array.ndims); + return 1; + } + + profile_start("seedee-deserialize"); + r = seedee_deserialize_ndarray(data_format_str->valuestring, + data_block, data_block_size, + 0, &array); + profile_end("seedee-deserialize"); + cJSON_Delete(json); + if ( r < 0 ) { + ERROR("Seedee deserialiation failed.\n"); + return 1; + } + + image->dp = malloc(dtempl->n_panels*sizeof(float *)); + if ( image->dp == NULL ) { + ERROR("Failed to allocate data array.\n"); + return 1; + } + + /* Set all pointers to NULL for easier clean-up */ + for ( i=0; in_panels; i++ ) image->dp[i] = NULL; + + profile_start("seedee-panel"); + for ( i=0; in_panels; i++ ) { + if ( load_seedee_data(&dtempl->panels[i], &array, &image->dp[i]) ) + { + ERROR("Failed to load data for panel '%s'\n", + dtempl->panels[i].name); + profile_end("seedee-panel"); + return 1; + } + } + profile_end("seedee-panel"); + + return 0; +} + + +#else /* defined(HAVE_SEEDEE) */ + +int image_seedee_read(struct image *image, + const DataTemplate *dtempl, + void *data, + size_t data_size, + char *meta_data) +{ + ERROR("Seedee is not supported in this installation (read).\n"); + return 1; +} + +#endif /* defined(HAVE_SEEDEE) */ diff --git a/libcrystfel/src/image-seedee.h b/libcrystfel/src/image-seedee.h new file mode 100644 index 00000000..9432e74b --- /dev/null +++ b/libcrystfel/src/image-seedee.h @@ -0,0 +1,41 @@ +/* + * image-seedee.h + * + * Image loading, SeeDee parts + * + * Copyright © 2012-2022 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2020-2022 Thomas White + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see . + * + */ + +#ifndef IMAGE_SEEDEE_H +#define IMAGE_SEEDEE_H + +#include "datatemplate.h" + + +extern int image_seedee_read(struct image *image, + const DataTemplate *dtempl, + void *data, + size_t data_size, + char *meta_data); + +#endif /* IMAGE_SEEDEE_H */ diff --git a/libcrystfel/src/image.c b/libcrystfel/src/image.c index 10c05525..f9c75b19 100644 --- a/libcrystfel/src/image.c +++ b/libcrystfel/src/image.c @@ -42,6 +42,7 @@ #include "image-hdf5.h" #include "image-cbf.h" #include "image-msgpack.h" +#include "image-seedee.h" #include "profile.h" #include "datatemplate.h" @@ -803,6 +804,11 @@ static int image_read_image_data(struct image *image, return image_msgpack_read(image, dtempl, image->data_block, image->data_block_size); + case DATA_SOURCE_TYPE_SEEDEE: + return image_seedee_read(image, dtempl, image->data_block, + image->data_block_size, + image->meta_data); + default: ERROR("Unrecognised file type %i (image_read_image_data)\n", image->data_source_type); diff --git a/libcrystfel/src/image.h b/libcrystfel/src/image.h index 6c7e2a50..654bf3b1 100644 --- a/libcrystfel/src/image.h +++ b/libcrystfel/src/image.h @@ -103,7 +103,8 @@ typedef enum DATA_SOURCE_TYPE_HDF5, DATA_SOURCE_TYPE_CBF, DATA_SOURCE_TYPE_CBFGZ, - DATA_SOURCE_TYPE_MSGPACK + DATA_SOURCE_TYPE_MSGPACK, + DATA_SOURCE_TYPE_SEEDEE } DataSourceType; -- cgit v1.2.3