Merge branch 'master' into taketwo

5 files changed, 159 insertions, 28 deletions

diff --git a/doc/examples/Eiger16M-binning2-nativefiles.geom b/doc/examples/Eiger16M-binning2-nativefiles.geom
new file mode 100644
index 00000000..0a53a74e
--- /dev/null
+++ b/doc/examples/Eiger16M-binning2-nativefiles.geom
@@ -0,0 +1,55 @@
+; Example geometry file for Eiger 16M detector, using its native file format
+; and binning 2.
+
+; Camera length (in m) and photon energy (eV)
+clen = 0.1
+photon_energy = 22000
+
+; adu_per_photon needs a relatively recent CrystFEL version.  If your version is
+; older, change it to adu_per_eV and set it to one over the photon energy in eV
+adu_per_photon = 1
+res = 13333.3   ; 75 micron pixel size
+
+; These lines describe the data layout for the Eiger native multi-event files
+dim0 = %
+dim1 = ss
+dim2 = fs
+data = /entry/data/data
+
+; Mask out strips between panels
+bad_v0/min_fs = 1030
+bad_v0/min_ss = 0
+bad_v0/max_fs = 1039
+bad_v0/max_ss = 2166
+
+bad_h0/min_fs = 0
+bad_h0/min_ss = 514
+bad_h0/max_fs = 2069
+bad_h0/max_ss = 550
+
+bad_h1/min_fs = 0
+bad_h1/min_ss = 1065
+bad_h1/max_fs = 2069
+bad_h1/max_ss = 1101
+
+bad_h2/min_fs = 0
+bad_h2/min_ss = 1616
+bad_h2/max_fs = 2069
+bad_h2/max_ss = 1652
+
+; Uncomment these lines if you have a separate bad pixel map (recommended!)
+;mask_file = eiger-badmap.h5
+;mask = /data/data
+;mask_good = 0x0
+;mask_bad = 0x1
+
+; corner_{x,y} set the position of the corner of the detector (in pixels)
+; relative to the beam
+panel0/min_fs = 0
+panel0/min_ss = 0
+panel0/max_fs = 2069
+panel0/max_ss = 2166
+panel0/corner_x = -1000.0
+panel0/corner_y = -1000.0
+panel0/fs = x
+panel0/ss = y
diff --git a/doc/man/compare_hkl.1 b/doc/man/compare_hkl.1
index 8e464d12..30fcd93e 100644
--- a/doc/man/compare_hkl.1
+++ b/doc/man/compare_hkl.1
@@ -72,9 +72,9 @@ Note that this figure of merit compares measurements within one data set, so it
 The ratio of Rano to Rsplit, as defined above.
 .IP "\fBd1sig\fR and \fBd2sig\fR"
 .PD
-The fraction of differences between intensities which are within one times (for \fBd1sig\fR) and two times (for \fBd2sig\fR) the mean of the corresponding sigma(I) values.
+The fraction of differences between intensities which are within one times (for \fBd1sig\fR) and two times (for \fBd2sig\fR) the combination of the corresponding sigma(I) values.
 .PP
-I1 and I2 are the intensities of the same reflection in both reflection lists.  The scale factor, k, is given by sum(I1*i2) / sum(I2^2), unless you use \fB-u\fR.
+I1 and I2 are the intensities of the same reflection in both reflection lists.  The two sets of reflections will be put on a common scale (linear and Debye-Waller terms) unless you use \fB-u\fR.
 .RE
 
 .PD 0
diff --git a/doc/man/crystfel.7 b/doc/man/crystfel.7
index bc545069..b7063b21 100644
--- a/doc/man/crystfel.7
+++ b/doc/man/crystfel.7
@@ -1,7 +1,7 @@
 .\"
 .\" CrystFEL main man page
 .\"
-.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY,
+.\" Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY,
 .\"                       a research centre of the Helmholtz Association.
 .\"
 .\" Part of CrystFEL - crystallography with a FEL
@@ -125,6 +125,15 @@ to this list if you experience difficulty with "-y" at any time.
 .IP Cubic
 \fB23\fR, \fBm-3\fR, \fB432\fR, \fB-43m\fR, \fBm-3m\fR.
 
+.SH FUNDING ACKNOWLEDGEMENTS
+Development of CrystFEL is primarily funded by the Helmholtz Association via programme-oriented funds.
+
+Additional funding for CrystFEL is provided by "X-Probe", a project of the European Union's 2020 Research and Innovation Program Under the Marie Skłodowska-Curie grant agreement 637295 (2015-2018).
+
+Additional funding for CrystFEL is provided by the BMBF German-Russian Cooperation "SyncFELMed", grant 05K14CHA (2014-2017).
+
+Past funding for CrystFEL has been received from BioStruct-X, a project funded by the Seventh Framework Programme (FP7) of the European Commission.
+
 .SH AUTHOR
 This page was written by Thomas White and Valerio Mariani.
 
diff --git a/doc/man/indexamajig.1 b/doc/man/indexamajig.1
index d7f59f9d..eaf812d6 100644
--- a/doc/man/indexamajig.1
+++ b/doc/man/indexamajig.1
@@ -1,7 +1,7 @@
 .\"
 .\" indexamajig man page
 .\"
-.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY,
+.\" Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY,
 .\"                       a research centre of the Helmholtz Association.
 .\"
 .\" Part of CrystFEL - crystallography with a FEL
@@ -45,8 +45,12 @@ You can control the peak detection on the command line.  Firstly, you can choose
 
 \fB--peaks=cxi\fR works similarly to this, but expects four separate HDF5 datasets beneath \fIpath\fR, \fBnPeaks\fR, \fBpeakXPosRaw\fR, \fBpeakYPosRaw\fR and \fBpeakTotalIntensity\fR.  See the specification for the CXI file format at http://www.cxidb.org/ for more details.
 
+CrystFEL considers all peak locations to be distances from the corner of the detector panel, in pixel units, consistent with its description of detector geometry (see 'man crystfel_geometry').  The software which generates the HDF5 or CXI files, including Cheetah, may instead consider the peak locations to be pixel indices in the data array.  Therefore, the peak coordinates from \fB--peaks=cxi\fR or \fB--peaks=hdf5\fR will by default have 0.5 added to them.  Use \fB--no-half-pixel-shift\fR if this isn't what you want.
+
 If you use \fB--peaks=zaef\fR, indexamajig will use a simple gradient search after Zaefferer (2000).  You can control the overall threshold and minimum squared gradient for finding a peak using \fB--threshold\fR and \fB--min-gradient\fR.  The threshold has arbitrary units matching the pixel values in the data, and the minimum gradient has the equivalent squared units.  Peaks will be rejected if the 'foot point' is further away from the 'summit' of the peak by more than the inner integration radius (see below).  They will also be rejected if the peak is closer than twice the inner integration radius from another peak.
 
+If you instead use \fB--peaks=peakfinder8\fR, indexamajig will use the "peakfinder8" peak finding algorithm describerd in Barty et al. (2014). Pixels above a radius-dependent intensity threshold are considered as candidate peaks (although the user sets an absolute minimum threshold for candidate peaks). Peaks are then only accepted if their signal to noise level over the local background is sufficiently high. Peaks can include multiple pixels and the user can reject a peak if it includes too many or too few. The distance of a peak from the center of the detector can also be used as a filtering criterion. Note that the peakfinder8 will not report more than 2048 peaks for each panel: any additional peak is ignored.
+
 You can suppress peak detection altogether for a panel in the geometry file by specifying the "no_index" value for the panel as non-zero.
 
 
@@ -229,6 +233,16 @@ these.  The defaults are probably not appropriate for your situation.
 The default is \fB--int-radius=4,5,7\fR.
 
 .PD 0
+.IP \fB--min-peaks=\fIn\fR
+.PD
+Do not try to index frames with fewer than \fIn\fR peaks.  These frames will still be described in the output stream.  To exclude them, use \fB--no-non-hits-in-stream\fR.
+
+.PD 0
+.IP \fB--no-non-hits-in-stream\fR
+.PD
+Completely exclude 'non-hit' frames in the stream.  When this option is given, frames with fewer than the number of peaks given to \fB--min-peaks\fR will not have chunks written to the stream at all.
+
+.PD 0
 .IP \fB--basename\fR
 .PD
 Remove the directory parts of the filenames taken from the input file.  If \fB--prefix\fR or \fB-x\fR is also given, the directory parts of the filename will be removed \fIbefore\fR adding the prefix.
@@ -270,7 +284,7 @@ This option is here for historical purposes only, to disable a correction which
 .PD 0
 .IP \fB--threshold=\fR\fIthres\fR
 .PD
-Set the overall threshold for peak detection using \fB--peaks=zaef\fR to \fIthres\fR, which has the same units as the detector data.  The default is \fB--threshold=800\fR.
+Set the overall threshold for peak detection using \fB--peaks=zaef\fR or \fB--peaks=peakfinder8\fR to \fIthres\fR, which has the same units as the detector data.  The default is \fB--threshold=800\fR.
 
 .PD 0
 .IP \fB--min-gradient=\fR\fIgrad\fR
@@ -280,12 +294,37 @@ Set the square of the gradient threshold for peak detection using \fB--peaks=zae
 .PD 0
 .IP \fB--min-snr=\fR\fIsnr\fR
 .PD
-Set the minimum I/sigma(I) for peak detection when using \fB--peaks=zaef\fR.  The default is \fB--min-snr=5\fR.
+Set the minimum I/sigma(I) for peak detection when using \fB--peaks=zaef\fR or \fB--peaks=peakfinder8\fR.  The default is \fB--min-snr=5\fR.
+
+.PD 0
+.IP \fB--min-pix-count=\fR\fIcnt\fR
+.PD
+Accepts peaks only if they include more than \fR\fIcnt\fR pixels, when using \fB--peaks=peakfinder8\fR.  The default is \fB--min-pix-count=2\fR.
+
+.PD 0
+.IP \fB--max-pix-count=\fR\fIcnt\fR
+.PD
+Accepts peaks only if they include less than \fR\fIcnt\fR pixels, when using \fB--peaks=peakfinder8\fR.  The default is \fB--max-pix-count=200\fR.
+
+.PD 0
+.IP \fB--local-bg-radius=\fR\fIr\fR
+.PD
+Radius (in pixels) used for the estimation of the local background when using \fB--peaks=peakfinder8\fR.  The default is \fB--local-bg-radius=3\fR.
+
+.PD 0
+.IP \fB--min-res=\fR\fIpx\fR
+.PD
+Only accept peaks if they lay at more than \fR\fIpx\fR pixels from the center of the detector when using \fB--peaks=peakfinder8\fR.  The default is \fB--min-res=0\fR.
+
+.PD 0
+.IP \fB--max-res=\fR\fIpx\fR
+.PD
+Only accept peaks if they lay at less than \fR\fIpx\fR pixels from the center of the detector when using \fB--peaks=peakfinder8\fR.  The default is \fB--max-res=1200\fR.
 
 .PD 0
 .IP \fB--copy-hdf5-field=\fR\fIpath\fR
 .PD
-Copy the information from \fIpath\fR in the HDF5 file into the output stream.  The information must be a single scalar value.  This option is sometimes useful to allow data to be separated after indexing according to some condition such the presence of an optical pump pulse.  You can give this option as many times as you need to copy multiple bits of information.
+Copy the information from \fR\fIpath\fR in the HDF5 file into the output stream.  The information must be a single scalar value.  This option is sometimes useful to allow data to be separated after indexing according to some condition such the presence of an optical pump pulse.  You can give this option as many times as you need to copy multiple bits of information.
 
 .PD 0
 .IP "\fB-j\fR \fIn\fR"
@@ -308,6 +347,11 @@ Normally, peaks which contain one or more pixels above max_adu (defined in the d
 When using \fB--peaks=hdf5\fR or \fB--peaks=cxi\fR, the peaks will be put through some of the same checks as if you were using \fB--peaks=zaef\fR.  These checks reject peaks which are too close to panel edges, are saturated (unless you use \fB--use-saturated\fR), have other nearby peaks (closer than twice the inner integration radius, see \fB--int-radius\fR), or have any part in a bad region.  Using this option skips this validation step, and uses the peaks directly.
 
 .PD 0
+.IP \fB--no-half-pixel-shift\fR
+.PD
+CrystFEL considers all peak locations to be distances from the corner of the detector panel, in pixel units, consistent with its description of detector geometry (see 'man crystfel_geometry').  The software which generates the HDF5 or CXI files, including Cheetah, may instead consider the peak locations to be pixel indices in the data array.  Therefore, the peak coordinates from \fB--peaks=cxi\fR or \fB--peaks=hdf5\fR will by default have 0.5 added to them.  This option \fBdisables\fR this half-pixel offset.
+
+.PD 0
 .IP \fB--check-hdf5-snr\fR
 .PD
 With this option with \fB--peaks=hdf5\fR, the peaks will additionally be checked to see that they satisfy the minimum SNR specified with \fB--min-snr\fR.
diff --git a/doc/reference/libcrystfel/CrystFEL-sections.txt b/doc/reference/libcrystfel/CrystFEL-sections.txt
index 4cd922d2..072f8b38 100644
--- a/doc/reference/libcrystfel/CrystFEL-sections.txt
+++ b/doc/reference/libcrystfel/CrystFEL-sections.txt
@@ -8,6 +8,8 @@ reflist_new
 reflist_free
 reflection_new
 reflection_free
+reflist_add_notes
+reflist_get_notes
 <SUBSECTION>
 add_refl
 add_refl_to_list
@@ -18,7 +20,6 @@ next_refl
 find_refl
 next_found_refl
 <SUBSECTION>
-get_excitation_error
 get_detector_pos
 get_panel
 get_partiality
@@ -77,6 +78,7 @@ copy_reflist
 find_equiv_in_list
 resolution_limits
 max_intensity
+reflist_add_command_and_version
 </SECTION>
 
 <SECTION>
@@ -151,32 +153,32 @@ solve_svd
 AssplodeFlag
 C_VACUO
 ELECTRON_CHARGE
-ERROR
-J_to_eV
 PLANCK
-STATUS
 THOMSON_LENGTH
 assplode
-biggest
-check_prefix
 chomp
-deg2rad
-eV_to_J
 gaussian_noise
-safe_basename
 progress_bar
-rad2deg
-is_odd
 poisson_noise
 notrail
+random_flat
+flat_noise
+show_matrix
+STATUS
+ERROR
+J_to_eV
+eV_to_J
+biggest
 smallest
+check_prefix
+safe_basename
+deg2rad
+rad2deg
+is_odd
 ph_en_to_lambda
 ph_lambda_to_en
 ph_eV_to_lambda
 ph_lambda_to_eV
-random_flat
-flat_noise
-show_matrix
 likely
 unlikely
 UNUSED
@@ -218,6 +220,20 @@ ImageFeatureList
 SpectrumType
 sample
 <SUBSECTION>
+imagefile
+imagefile_close
+imagefile_copy_fields
+imagefile_field_list
+imagefile_get_hdfile
+imagefile_get_type
+imagefile_open
+imagefile_read
+imagefile_read_simple
+<SUBSECTION>
+new_imagefile_field_list
+add_imagefile_field
+free_imagefile_field_list
+<SUBSECTION>
 image_add_feature
 image_feature_closest
 image_reflection_closest
@@ -268,10 +284,11 @@ INDEXING_DEFAULTS_ASDF
 INDEXING_DEFAULTS_FELIX
 INDEXING_METHOD_MASK
 INDEXING_CONTROL_FLAGS
-build_indexer_list
+setup_indexing
 cleanup_indexing
-prepare_indexing
+get_indm_from_string
 index_pattern
+index_pattern_2
 indexer_str
 dirax_prepare
 run_dirax
@@ -314,6 +331,7 @@ describe_symmetry
 symmetry_name
 set_symmetry_name
 get_matrix_name
+pointgroup_warning
 is_centrosymmetric
 is_centric
 </SECTION>
@@ -351,19 +369,17 @@ rigid_group
 rg_collection
 <SUBSECTION>
 copy_geom
-fill_in_values
+fill_in_adu
 free_detector_geometry
 get_detector_geometry
 get_detector_geometry_2
 write_detector_geometry
 write_detector_geometry_2
-find_panel
+panel_number
 find_panel_by_name
-find_panel_number
 simple_geometry
 record_image
 get_pixel_extents
-get_q
 get_q_for_panel
 get_tt
 smallest_q
@@ -378,6 +394,7 @@ rigid_group_is_in_collection
 single_panel_data_source
 find_rigid_group_collection_by_name
 detector_has_clen_references
+adjust_centering_for_rail
 </SECTION>
 
 <SECTION>
@@ -398,7 +415,6 @@ find_event
 get_event_string
 get_event_from_event_string
 event_path_placeholder_subst
-partial_event_substitution
 retrieve_full_path
 initialize_filename_plus_event
 free_filename_plus_event
@@ -439,7 +455,9 @@ add_copy_hdf5_field
 new_copy_hdf5_field_list
 free_copy_hdf5_field_list
 get_peaks
+get_peaks_2
 get_peaks_cxi
+get_peaks_cxi_2
 hdfile_is_scalar
 check_path_existence
 fill_event_list
@@ -464,6 +482,7 @@ crystal_get_resolution_limit
 crystal_get_user_flag
 crystal_get_num_implausible_reflections
 crystal_get_notes
+crystal_get_det_shift
 crystal_set_cell
 crystal_set_image
 crystal_set_mosaicity
@@ -477,6 +496,7 @@ crystal_set_user_flag
 crystal_set_num_implausible_reflections
 crystal_set_notes
 crystal_add_notes
+crystal_set_det_shift
 </SECTION>
 
 <SECTION>
@@ -498,7 +518,9 @@ y_gradient
 <FILE>peaks</FILE>
 peak_sanity_check
 search_peaks
+search_peaks_peakfinder8
 make_BgMask
+peakfinder8
 validate_peaks
 sort_peaks
 </SECTION>
@@ -555,6 +577,7 @@ close_stream
 read_chunk
 read_chunk_2
 write_chunk
+write_chunk_2
 rewind_stream
 is_stream
 write_command