diff options
66 files changed, 4022 insertions, 2573 deletions
@@ -1,3 +1,26 @@ +CrystFEL version 0.7.0, 17th May 2018 +-------------------------------------- + +- indexamajig: Auto-detection of indexing methods +- indexamajig: New options system - removes confusing "comb","bad","retry" etc. +- Most programs now use the symmetry in the reflection files, allowing "-y" + options to be omitted. +- Ginn partiality model was added. +- partialator: Switch to gradient-free minimiser. +- geoptimiser: Fixes for rotation and new features for AGIPD. +- Speed and accuracy improvements for TakeTwo indexing (Helen Ginn). +- process_hkl, partialator and ambigator now include the audit information from + the stream in their output files. +- pattern_sim: Write orientation and lattice basis vectors into HDF5 file. +- hdfsee: Add event ID and peaks/preds/rings to exported image filenames. +- indexamajig --serial-start was added (useful when using turbo-index). +- indexamajig --overpredict was added (needed for post-refinement) +- pattern_sim --flat was added (new reflection profile option) +- cell_explorer: Now shows the number of matching cells on terminal. +- Felix options were simplified. +- scripts/sum-hdf5-files and eiger-badmap were added + + CrystFEL version 0.6.3, 24th July 2017 -------------------------------------- @@ -1,7 +1,7 @@ CrystFEL - Crystallography with a FEL ------------------------------------- -Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. Authors: diff --git a/announcement-0.6.3 b/announcement-0.6.3 deleted file mode 100644 index db8d0adb..00000000 --- a/announcement-0.6.3 +++ /dev/null @@ -1,35 +0,0 @@ -Dear CrystFEL users and interested crystallographers, - -It's been a long time since the last CrystFEL release, but we have not been -resting. CrystFEL version 0.6.3 has been released. - -This version incorporates a large number of "backstage" improvements and -alterations. The most interesting changes for users are: - -- The incorporation of "peak finder 8" from Cheetah (also used in OnDA) - -- Support for images in CBF format - -- Incorporation of "TakeTwo" indexing algorithm - -- Hit finder functionality (skip frames with few peaks) - -- Better support for three-dimensional detector geometries - -See the release notes for more discussion of these improvements, and the -ChangeLog for full details of the other improvements. - -Thanks for all of your past and future feedback and contributions, and -your continued use of CrystFEL. As ever, please contact me directly if -you would prefer to be unsubscribed from this mailing list. - -CrystFEL website: -https://www.desy.de/~twhite/crystfel - -Release notes for version 0.6.3: -https://www.desy.de/~twhite/crystfel/relnotes-0.6.3 - -CrystFEL tutorial: -https://www.desy.de/~twhite/crystfel/tutorial - -Tom diff --git a/announcement-0.7.0 b/announcement-0.7.0 new file mode 100644 index 00000000..619dc570 --- /dev/null +++ b/announcement-0.7.0 @@ -0,0 +1,37 @@ +Dear CrystFEL users and interested crystallographers, + +CrystFEL version 0.7.0 has been released. + +This version adds several exciting new features while greatly simplifying the +user experience. The most interesting changes are: + +- New options system for indexamajig, removing most of the profusion of indexing + method flags "comb", "bad", "retry" and so on. Indexamajig can also now + automatically determine which indexing methods can be used if you don't tell + it which ones to use. In addition, the options for the Felix indexing method + have been simplified. + +- The spectrum-based partiality model and numerical post-refinement algorithm + from Ginn et al. [Acta D71 2015 p1400] has been incorporated. + +- The symmetry of the merged reflection list is now automatically tracked through + the subsequent data processing stages, so you no longer need to give the "-y" + option to compare_hkl, check_hkl and other programs. + +See the release notes for more discussion of these improvements, and the +ChangeLog for full details of the other improvements. + +Thanks for all of your past and future feedback and contributions, and +your continued use of CrystFEL. As ever, please contact me directly if +you would prefer to be unsubscribed from this mailing list. + +CrystFEL website: +https://www.desy.de/~twhite/crystfel + +Release notes for version 0.7.0: +https://www.desy.de/~twhite/crystfel/relnotes-0.7.0 + +CrystFEL tutorial: +https://www.desy.de/~twhite/crystfel/tutorial + +Tom diff --git a/doc/man/ambigator.1 b/doc/man/ambigator.1 index fad89641..65ef16a0 100644 --- a/doc/man/ambigator.1 +++ b/doc/man/ambigator.1 @@ -1,7 +1,7 @@ .\" .\" ambigator man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -111,7 +111,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2014-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2014-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. ambigator, and this manual, are part of CrystFEL. diff --git a/doc/man/cell_explorer.1 b/doc/man/cell_explorer.1 index 990a9108..1ce12217 100644 --- a/doc/man/cell_explorer.1 +++ b/doc/man/cell_explorer.1 @@ -1,7 +1,7 @@ .\" .\" cell_explorer man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -38,7 +38,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2014-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2014-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P cell_explorer, and this manual, are part of CrystFEL. .P diff --git a/doc/man/check_hkl.1 b/doc/man/check_hkl.1 index f02b581b..71cba848 100644 --- a/doc/man/check_hkl.1 +++ b/doc/man/check_hkl.1 @@ -1,7 +1,7 @@ .\" .\" check_hkl man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -93,7 +93,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P check_hkl, and this manual, are part of CrystFEL. .P diff --git a/doc/man/compare_hkl.1 b/doc/man/compare_hkl.1 index 30fcd93e..f85f6c90 100644 --- a/doc/man/compare_hkl.1 +++ b/doc/man/compare_hkl.1 @@ -1,7 +1,7 @@ .\" .\" compare_hkl man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -139,7 +139,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P compare_hkl, and this manual, are part of CrystFEL. .P diff --git a/doc/man/crystfel.7 b/doc/man/crystfel.7 index b7063b21..7ed67d5b 100644 --- a/doc/man/crystfel.7 +++ b/doc/man/crystfel.7 @@ -1,7 +1,7 @@ .\" .\" CrystFEL main man page .\" -.\" Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -142,7 +142,7 @@ Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel> .SH COPYRIGHT AND DISCLAIMER .PD 0 -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .PD .PP Please read the AUTHORS file in the CrystFEL source code distribution for a full list of contributions and contributors. diff --git a/doc/man/crystfel_geometry.5 b/doc/man/crystfel_geometry.5 index 47ca5138..cf64d76e 100644 --- a/doc/man/crystfel_geometry.5 +++ b/doc/man/crystfel_geometry.5 @@ -1,7 +1,7 @@ .\" .\" Geometry man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -294,7 +294,7 @@ This page was written by Thomas White and Valerio Mariani. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P CrystFEL is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. .P diff --git a/doc/man/geoptimiser.1 b/doc/man/geoptimiser.1 index e2370c7e..369a36f1 100644 --- a/doc/man/geoptimiser.1 +++ b/doc/man/geoptimiser.1 @@ -1,7 +1,7 @@ .\" .\" geoptimiser man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -139,7 +139,7 @@ This page was written by Valerio Mariani, Oleksandr Yefanov and Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2014-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2014-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P geoptimiser, and this manual, are part of CrystFEL. .P diff --git a/doc/man/get_hkl.1 b/doc/man/get_hkl.1 index a5480e20..90637a8a 100644 --- a/doc/man/get_hkl.1 +++ b/doc/man/get_hkl.1 @@ -1,7 +1,7 @@ .\" .\" get_hkl man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -96,7 +96,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P get_hkl, and this manual, are part of CrystFEL. .P diff --git a/doc/man/hdfsee.1 b/doc/man/hdfsee.1 index 8d5cccb5..3034d032 100644 --- a/doc/man/hdfsee.1 +++ b/doc/man/hdfsee.1 @@ -1,7 +1,7 @@ .\" .\" hdfsee man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -112,7 +112,7 @@ This page was written by Thomas White and Valerio Mariani. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P hdfsee, and this manual, are part of CrystFEL. .P diff --git a/doc/man/indexamajig.1 b/doc/man/indexamajig.1 index 3f49701f..c0fe8277 100644 --- a/doc/man/indexamajig.1 +++ b/doc/man/indexamajig.1 @@ -1,7 +1,7 @@ .\" .\" indexamajig man page .\" -.\" Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -89,7 +89,7 @@ You can add one or more of the following to the above indexing methods, to contr .IP \fB-latt\fR .PD -Provide the Bravais lattice type (e.g. the knowledge that the lattice tetragonal primitive), as prior information to the indexing engine. +Provide the Bravais lattice type (e.g. the knowledge that the lattice is tetragonal primitive), as prior information to the indexing engine. .IP \fB-nolatt\fR .PD @@ -101,7 +101,7 @@ Provide your unit cell parameters as prior information to the indexing engine. .IP \fB-nocell\fR .PD -The opposite of \fB-cell\fR: do not use unit cell parameters as prior information for the core indexing algorithm. +The opposite of \fB-cell\fR: do not provide unit cell parameters as prior information to the indexing engine. .PP Example: \fB--indexing=mosflm-cell-latt\fR means to use Mosflm for indexing, and provide it with unit cell parameters and Bravais lattice type information. @@ -112,7 +112,7 @@ The default indexing method is 'none', which means no indexing will be done. Th .PP You do not need to explicitly specify anything more than the indexing method itself (e.g. \fBmosflm\fR or \fBasdf\fR). The default behaviour for all indexing methods is to make the maximum possible use of prior information such as the lattice type and cell parameters. If you do not provide this information, for example if you do not give any unit cell file or if the unit cell file does not contain cell parameters (only lattice type information), the indexing methods you give will be modified accordingly. If you only specify the indexing methods themselves, in most cases \fBindexamajig\fR will do what you want and intuitively expect! However, the options are available if you need finer control. -If you don't know what to give for this option, try \fB--indexing=asdf,dirax,mosflm,xds,taketwo\fR. +If you don't know what to give for this option, leave it out completely. Indexamajig will then automatically select indexing methods based on the programs available on your computer. The indexing results from the indexing engine will be put through a number of refinement and checking stages. See the options \fB--no-check-cell, --no-cell-combinations, --no-multi, --no-retry\fR and \fB--no-refine\fR below for more details. @@ -305,7 +305,7 @@ With this option with \fB--peaks=hdf5\fR, the peaks will additionally be checked .PD 0 .IP \fB--indexing=\fR\fImethod\fR .PD -Index the patterns using \fImethod\fR. See the section titled \fBINDEXING METHODS\fR (above) for more information. The default is \fB--indexing=none\fR. +Index the patterns using \fImethod\fR. See the section titled \fBINDEXING METHODS\fR (above) for more information. The default is to automatically detect which indexing methods to use. .PD 0 .IP "\fB-p\fR \fIunitcell.cell\fR" @@ -332,9 +332,9 @@ Do not check the cell parameters against the reference unit cell (given with \fB When checking the cell parameters against the reference cell (see \fB-p\fR), do not make combinations of the axes of the candidate cell (such as \fBa'\fR=2\fBa\fR+\fBb\fR) to make it fit. Usually this reduces the success rate, but is necessary if one of the cell parameters is close to a multiple of the others. \fRThis happens for tetragonal lysozyme\fB. .PD 0 -.IP \fB--no-multi +.IP \fB--multi .PD -Disable multi-lattice indexing. This refers to the "subtract and retry" method, where after a successful indexing attempt the spots accounted for by the indexing solution are removed before trying to index again in the hope of finding a second lattice. This doesn't have anything to do with the multi-lattice indexing algorithms such as Felix. +Enable the "subtract and retry" method, where after a successful indexing attempt the spots accounted for by the indexing solution are removed before trying to index again in the hope of finding a second lattice. This doesn't have anything to do with the multi-lattice indexing algorithms such as Felix. .PD 0 .IP \fB--no-retry @@ -501,7 +501,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P indexamajig, and this manual, are part of CrystFEL. .P diff --git a/doc/man/list_events.1 b/doc/man/list_events.1 index 900288f6..5ce086d1 100644 --- a/doc/man/list_events.1 +++ b/doc/man/list_events.1 @@ -1,8 +1,8 @@ .\" .\" list_events man page .\" -.\" Copyright © 2015 Deutsches Elektronen-Synchrotron DESY, -.\" a research centre of the Helmholtz Association. +.\" Copyright © 2015-2018 Deutsches Elektronen-Synchrotron DESY, +.\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL .\" @@ -43,7 +43,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2015-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P list_events, and this manual, are part of CrystFEL. .P diff --git a/doc/man/partial_sim.1 b/doc/man/partial_sim.1 index 88148776..72655758 100644 --- a/doc/man/partial_sim.1 +++ b/doc/man/partial_sim.1 @@ -1,7 +1,7 @@ .\" .\" partial_sim man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -155,7 +155,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P partial_sim, and this manaul, are part of CrystFEL. .P diff --git a/doc/man/partialator.1 b/doc/man/partialator.1 index decaa07c..aa25d2a7 100644 --- a/doc/man/partialator.1 +++ b/doc/man/partialator.1 @@ -1,7 +1,7 @@ .\" .\" partialator man page .\" -.\" Copyright © 2012-2016 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -132,29 +132,45 @@ Reject crystals if the absolute values of their relative Debye-Waller ("B") fact .PD Write out the per-crystal parameters and reflection lists after every cycle of refinement, instead of only at the end. The intermediate reflection lists and parameter filenames will be prefixed with \fBiter\fIN\fB_\fR (note the trailing underscore), where \fIN\fR is the iteration number. If you use \fB--custom-split\fR, intermediate results will also be output for each custom dataset. +.PD 0 +.IP \fB--no-logs\fR +.PD +Do not write the extensive log files needed for plotting contour maps and spectrum graphs. This makes the process a lot faster, but you probably do want these logs to check that post-refinement is working reasonably. + +.PD 0 +.IP "\fB-w\fR \fIpg\fR" +.PD +Set the apparent symmetry of the crystals. The ambiguity operator will be determined by comparing this to the actual symmetry. +.IP +If you prefer (or the scenario demands it), you can specify the ambiguity operator directly using \fB--operator\fR. + +.PD 0 +.IP \fB--operator=\fR\fIop\fR +.PD +Specify the indexing ambiguity operator. Example: \fB--operator=k,h,-l\fR. +.IP +If you prefer, you can specify the ambiguity operator by specifying the apparent symmetry using \fB-w\fR. + +.PD 0 +.IP \fB--force-bandwidth=\fIbw\fR +.IP \fB--force-radius=\fIR\fR +.PD +Set the X-ray bandwidth or initial profile radius for all crystals before proceeding, overriding the values from the stream. Bandwidth is given as a fraction, i.e. \fB--force-bandwidth=0.0013\fR means 0.13 percent (approximate FWHM). Radius is given in nm^-1. + .SH PARTIALITY MODELS The available partiality models are: -.IP \fBscsphere\fR +.IP \fBxsphere\fR .PD The volume of intersection between a sphere centered on each reciprocal lattice point and the part of reciprocal space excited by the Ewald sphere taking into account the finite bandwidth and convergence angle. A "source coverage factor" is included to take into account the spectral brightness of the effective -source for the reflection. +source for the reflection. The X-ray spectrum is modelled as a super-Gaussian +with exponent 1.5, and the overlap integral is evaluated numerically. -This model is similar to that described in Acta Cryst. D69 (2013) p1231-1240, -and in Phil. Trans. Roy. Soc. B 369 (2014) 20130330, except that the "Lorentz -factor" described there is no longer treated as a separate factor. - - -.IP \fBscgaussian\fR -.PD -As \fBscsphere\fR, except that the shape of the scattering density centered on -each reciprocal lattice point is taken to be a 3D Gaussian distribution instead -of a sphere. The standard deviation of the distribution will be the profile -radius (determined by indexamajig) divided by 2.6. +This model is the same as that described in Acta Cryst. D71 (2015) p1400. .IP \fBunity\fR .PD @@ -174,15 +190,15 @@ partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=uni .IP "Merging with partialities, but without post-refinement and without scaling:" .PD -partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=scsphere --iterations=0\fR +partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=xsphere --iterations=0\fR .IP "Merging with partialities, with scaling but without post-refinement:" .PD -partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=scsphere --iterations=1 --no-pr\fR +partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=xsphere --iterations=1 --no-pr\fR .IP "Merging with partialities, post-refinement and scaling:" .PD -partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=scsphere --iterations=1\fR +partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=xsphere --iterations=1\fR .IP (Use a higher number of iterations to increase the accuracy of scaling and post-refinement, but at a cost of more CPU time and possibly more rejected crystals) @@ -190,11 +206,9 @@ partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=scs .PD This would be a strange thing to want to do, however: .IP -partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=scsphere --iterations=1 --no-scale\fR +partialator -i \fImy.stream \fR-o \fImy.hkl\fR -y \fImypointgroup \fB--model=xsphere --iterations=1 --no-scale\fR .IP (Use a higher number of iterations to increase the accuracy of post-refinement, but at a cost of more CPU time and possibly more rejected crystals) -.PP -\fBscguassian\fR could be substituted for \fBscsphere\fR in the above examples to use the Gaussian partiality model instead of the spherical one. .SH CUSTOM DATASET SPLITTING When performing a time-resolved experiment (for example), it is preferable to ensure that the data for all time points has been processed identically. Rather than processing each time point independently with separate runs of partialator, it is better to process them all together and do the splitting into time points just before the final output. Consider, for example, the case of simple scaling (without a B factor): when merging independently, the resulting datasets would probably end up with different overall scaling factors. When comparing the results, you would need to take this difference into account. In practice, most programs can do that job easily, but what about if a B factor is included? And what if partialities are included - how unique is the solution? @@ -216,7 +230,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P partialator, and this manual, are part of CrystFEL. .P diff --git a/doc/man/pattern_sim.1 b/doc/man/pattern_sim.1 index 7c761c71..c93c5e4c 100644 --- a/doc/man/pattern_sim.1 +++ b/doc/man/pattern_sim.1 @@ -1,7 +1,7 @@ .\" .\" pattern_sim man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -246,7 +246,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P pattern_sim, and this manual, are part of CrystFEL. .P diff --git a/doc/man/process_hkl.1 b/doc/man/process_hkl.1 index 3876005a..658f42bf 100644 --- a/doc/man/process_hkl.1 +++ b/doc/man/process_hkl.1 @@ -1,7 +1,7 @@ .\" .\" process_hkl man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -141,7 +141,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P qprocess_hkl, and this manual, are part of CrystFEL. .P diff --git a/doc/man/render_hkl.1 b/doc/man/render_hkl.1 index eadbba34..a7e67442 100644 --- a/doc/man/render_hkl.1 +++ b/doc/man/render_hkl.1 @@ -1,7 +1,7 @@ .\" .\" render_hkl man page .\" -.\" Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, +.\" Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, .\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL @@ -125,7 +125,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P render_hkl, and this manual, are part of CrystFEL. .P diff --git a/doc/man/whirligig.1 b/doc/man/whirligig.1 index e213a007..938ac5d0 100644 --- a/doc/man/whirligig.1 +++ b/doc/man/whirligig.1 @@ -1,8 +1,8 @@ .\" .\" whirligig man page .\" -.\" Copyright © 2015 Deutsches Elektronen-Synchrotron DESY, -.\" a research centre of the Helmholtz Association. +.\" Copyright © 2015-2018 Deutsches Elektronen-Synchrotron DESY, +.\" a research centre of the Helmholtz Association. .\" .\" Part of CrystFEL - crystallography with a FEL .\" @@ -38,7 +38,7 @@ This page was written by Thomas White. Report bugs to <taw@physics.org>, or visit <http://www.desy.de/~twhite/crystfel>. .SH COPYRIGHT AND DISCLAIMER -Copyright © 2015 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. +Copyright © 2015-2018 Deutsches Elektronen-Synchrotron DESY, a research centre of the Helmholtz Association. .P whirligig, and this manual, are part of CrystFEL. .P diff --git a/doc/reference/libcrystfel/CrystFEL-docs.sgml b/doc/reference/libcrystfel/CrystFEL-docs.sgml index c6131ae7..15d9f43f 100644 --- a/doc/reference/libcrystfel/CrystFEL-docs.sgml +++ b/doc/reference/libcrystfel/CrystFEL-docs.sgml @@ -8,7 +8,7 @@ <bookinfo> <title>CrystFEL Reference Manual</title> <releaseinfo> - For libcrystfel from CrystFEL 0.6.3. + For libcrystfel from CrystFEL 0.7.0. </releaseinfo> <abstract> This is the internal documentation for CrystFEL. Unless you are looking at diff --git a/doc/reference/libcrystfel/CrystFEL-sections.txt b/doc/reference/libcrystfel/CrystFEL-sections.txt index 206ba4df..4a436e34 100644 --- a/doc/reference/libcrystfel/CrystFEL-sections.txt +++ b/doc/reference/libcrystfel/CrystFEL-sections.txt @@ -15,7 +15,9 @@ add_refl add_refl_to_list <SUBSECTION> first_refl +first_refl_const next_refl +next_refl_const <SUBSECTION> find_refl next_found_refl @@ -24,7 +26,6 @@ get_detector_pos get_panel get_partiality get_lorentz -get_partial get_indices get_symmetric_indices get_intensity @@ -36,12 +37,14 @@ get_peak get_temp1 get_temp2 get_flag +get_exerr +get_khalf +get_kpred <SUBSECTION> set_detector_pos set_panel set_partiality set_lorentz -set_partial set_intensity set_redundancy set_esd_intensity @@ -52,6 +55,9 @@ set_symmetric_indices set_temp1 set_temp2 set_flag +set_exerr +set_khalf +set_kpred <SUBSECTION> copy_data num_reflections @@ -130,6 +136,7 @@ rotate_cell cell_print resolution match_cell +compare_cells match_cell_ab cell_is_sensible validate_cell @@ -143,6 +150,7 @@ load_cell_from_file write_cell lattice_from_str cell_get_volume +transform_cell_gsl </SECTION> <SECTION> @@ -228,6 +236,7 @@ imagefile_get_type imagefile_open imagefile_read imagefile_read_simple +is_cbf_file <SUBSECTION> new_imagefile_field_list add_imagefile_field @@ -276,34 +285,48 @@ stat_scale_intensity <FILE>indexing</FILE> IndexingMethod IndexingPrivate +IndexingFlags INDEXING_DEFAULTS_DIRAX INDEXING_DEFAULTS_MOSFLM INDEXING_DEFAULTS_XDS INDEXING_DEFAULTS_ASDF INDEXING_DEFAULTS_FELIX +INDEXING_DEFAULTS_TAKETWO INDEXING_METHOD_MASK -INDEXING_CONTROL_FLAGS +detect_indexing_methods setup_indexing cleanup_indexing get_indm_from_string +get_indm_from_string_2 index_pattern index_pattern_2 indexer_str +dirax_probe dirax_prepare run_dirax dirax_cleanup +mosflm_probe mosflm_prepare run_mosflm mosflm_cleanup +xds_probe xds_prepare run_xds xds_cleanup +asdf_probe asdf_prepare run_asdf asdf_cleanup +felix_options +felix_probe felix_prepare felix_index felix_cleanup +taketwo_options +taketwo_probe +taketwo_prepare +taketwo_index +taketwo_cleanup </SECTION> <SECTION> @@ -470,6 +493,7 @@ crystal_copy crystal_free <SUBSECTION> crystal_get_cell +crystal_get_cell_const crystal_get_image crystal_get_mosaicity crystal_get_num_saturated_reflections @@ -502,9 +526,9 @@ crystal_set_det_shift <FILE>geometry</FILE> PartialityModel gparam -find_intersections -find_intersections_to_res -update_partialities +predict_to_res +update_predictions +calculate_partialities polarisation_correction sphere_fraction gaussian_fraction @@ -547,6 +571,7 @@ integrate_all integrate_all_2 integrate_all_3 integrate_all_4 +integrate_all_5 integration_method </SECTION> @@ -571,6 +596,7 @@ open_stream_for_read open_stream_for_write open_stream_for_write_2 open_stream_for_write_3 +open_stream_for_write_4 get_stream_fd close_stream read_chunk @@ -582,6 +608,8 @@ is_stream write_command write_geometry_file stuff_from_stream +stream_audit_info +stream_has_old_indexers </SECTION> <SECTION> diff --git a/libcrystfel/src/cell.c b/libcrystfel/src/cell.c index 2a4e45be..cc18b49d 100644 --- a/libcrystfel/src/cell.c +++ b/libcrystfel/src/cell.c @@ -382,7 +382,7 @@ static int cell_invert(double ax, double ay, double az, return 1; } if ( gsl_linalg_LU_invert(m, perm, inv) ) { - ERROR("Couldn't invert matrix\n"); + ERROR("Couldn't invert cell matrix:\n"); gsl_matrix_free(m); gsl_permutation_free(perm); return 1; @@ -655,7 +655,7 @@ UnitCellTransformation *tfn_inverse(UnitCellTransformation *t) return NULL; } if ( gsl_linalg_LU_invert(m, perm, inv) ) { - ERROR("Couldn't invert matrix\n"); + ERROR("Couldn't invert transformation matrix\n"); gsl_permutation_free(perm); return NULL; } diff --git a/libcrystfel/src/events.c b/libcrystfel/src/events.c index 8e4eb861..d0e521d3 100644 --- a/libcrystfel/src/events.c +++ b/libcrystfel/src/events.c @@ -150,6 +150,8 @@ struct event *copy_event(struct event *ev) struct event *new_ev; int pi, di; + if ( ev == NULL ) return NULL; + if ( ev->dim_length == 0 && ev->path_length == 0) { new_ev = initialize_event(); @@ -252,6 +254,8 @@ void free_event(struct event *ev) { int pi; + if ( ev == NULL ) return; + if ( ev->path_length != 0 ) { for ( pi=0; pi<ev->path_length; pi++ ) { free(ev->path_entries[pi]); diff --git a/libcrystfel/src/geometry.c b/libcrystfel/src/geometry.c index 576655de..9ae6d96f 100644 --- a/libcrystfel/src/geometry.c +++ b/libcrystfel/src/geometry.c @@ -238,76 +238,38 @@ static double random_partiality(signed int h, signed int k, signed int l, } -static double partiality(PartialityModel pmodel, - signed int h, signed int k, signed int l, - int serial, - double rlow, double rhigh, double pr) -{ - double D = rlow - rhigh; - - /* Convert to partiality */ - switch ( pmodel ) { - - default: - case PMODEL_UNITY: - return 1.0; - - case PMODEL_SCSPHERE: - return 4.0*sphere_fraction(rlow, rhigh, pr)*pr / (3.0*D); - - case PMODEL_SCGAUSSIAN: - return 4.0*gaussian_fraction(rlow, rhigh, pr)*pr / (3.0*D); - - case PMODEL_RANDOM: - return random_partiality(h, k, l, serial); - - } -} - - static Reflection *check_reflection(struct image *image, Crystal *cryst, signed int h, signed int k, signed int l, double xl, double yl, double zl, Reflection *updateme) { - double tl; - double rlow, rhigh; /* "Excitation error" */ - double klow, khigh; /* Wavenumber */ Reflection *refl; - double cet, cez; /* Centre of Ewald sphere */ - double pr; - double del; + double R, top; + double kmin, kmax, k0, knom, k1, khalf; + double dcs, exerr; /* Don't predict 000 */ if ( (updateme == NULL) && (abs(h)+abs(k)+abs(l) == 0) ) return NULL; - pr = crystal_get_profile_radius(cryst); - del = image->div + crystal_get_mosaicity(cryst); - - /* "low" gives the largest Ewald sphere (wavelength short => k large) - * "high" gives the smallest Ewald sphere (wavelength long => k small) - */ - klow = 1.0/(image->lambda - image->lambda*image->bw/2.0); - khigh = 1.0/(image->lambda + image->lambda*image->bw/2.0); - - /* If the point is looking "backscattery", reject it straight away */ - if ( (updateme == NULL) && (zl < -khigh/2.0) ) return NULL; - - tl = sqrt(xl*xl + yl*yl); - - cet = -sin(del/2.0) * khigh; - cez = -cos(del/2.0) * khigh; - rhigh = khigh - distance(cet, cez, tl, zl); /* Loss of precision */ - - cet = sin(del/2.0) * klow; - cez = -cos(del/2.0) * klow; - rlow = klow - distance(cet, cez, tl, zl); /* Loss of precision */ - - /* Condition for reflection to be excited at all */ - if ( (updateme == NULL) - && (signbit(rlow) == signbit(rhigh)) - && (fabs(rlow) > pr) - && (fabs(rhigh) > pr) ) return NULL; + /* Calculate the limiting wavelengths, lambda0 and lambda1 + * = 1/k0 and 1/k1 respectively */ + R = fabs(crystal_get_profile_radius(cryst)); + top = R*R - xl*xl - yl*yl - zl*zl; + k0 = top/(2.0*(zl+R)); + khalf = (- xl*xl - yl*yl - zl*zl) / (2.0*zl); + k1 = top/(2.0*(zl-R)); + + /* The reflection is excited if any of the reflection is within 2sigma + * of the nominal * wavelength of the X-ray beam + * (NB image->bw is full width) */ + kmin = 1.0/(image->lambda + image->lambda*image->bw); + knom = 1.0/image->lambda; + kmax = 1.0/(image->lambda - image->lambda*image->bw); + if ( (updateme == NULL) && ((k1>kmax) || (k0<kmin)) ) return NULL; + + /* Calculate excitation error */ + dcs = distance3d(0.0, 0.0, -knom, xl, yl, zl); + exerr = 1.0/image->lambda - dcs; /* Loss of precision */ if ( updateme == NULL ) { refl = reflection_new(h, k, l); @@ -321,7 +283,7 @@ static Reflection *check_reflection(struct image *image, Crystal *cryst, if ( (image->det != NULL) && (updateme != NULL) ) { double fs, ss; - locate_peak_on_panel(xl, yl, zl, 1.0/image->lambda, + locate_peak_on_panel(xl, yl, zl, knom, get_panel(updateme), &fs, &ss); set_detector_pos(refl, fs, ss); @@ -333,7 +295,7 @@ static Reflection *check_reflection(struct image *image, Crystal *cryst, double fs, ss; /* Position on detector */ signed int p; /* Panel number */ - p = locate_peak(xl, yl, zl, 1.0/image->lambda, + p = locate_peak(xl, yl, zl, knom, image->det, &fs, &ss); if ( p == -1 ) { reflection_free(refl); @@ -344,20 +306,9 @@ static Reflection *check_reflection(struct image *image, Crystal *cryst, } - if ( unlikely(rlow < rhigh) ) { - ERROR("Reflection with rlow < rhigh!\n"); - ERROR("%3i %3i %3i rlow = %e, rhigh = %e\n", - h, k, l, rlow, rhigh); - ERROR("div + m = %e, R = %e, bw = %e\n", del, pr, image->bw); - /* If we are updating, this is (kind of) OK */ - if ( updateme == NULL ) { - reflection_free(refl); - return NULL; - } - } - - set_partial(refl, rlow, rhigh, 1.0); /* Actual partiality set by - * calculate_partialities() */ + set_kpred(refl, knom); + set_khalf(refl, khalf); + set_exerr(refl, exerr); set_lorentz(refl, 1.0); set_symmetric_indices(refl, h, k, l); set_redundancy(refl, 1); @@ -368,18 +319,12 @@ static Reflection *check_reflection(struct image *image, Crystal *cryst, double r_gradient(UnitCell *cell, int k, Reflection *refl, struct image *image) { - double azi; double asx, asy, asz; double bsx, bsy, bsz; double csx, csy, csz; double xl, yl, zl; signed int hs, ks, ls; - double rlow, rhigh, p; - double philow, phihigh, phi; - double khigh, klow; - double tl, cet, cez; - - get_partial(refl, &rlow, &rhigh, &p); + double tl, phi, azi; get_symmetric_indices(refl, &hs, &ks, &ls); @@ -390,26 +335,9 @@ double r_gradient(UnitCell *cell, int k, Reflection *refl, struct image *image) yl = hs*asy + ks*bsy + ls*csy; zl = hs*asz + ks*bsz + ls*csz; - /* "low" gives the largest Ewald sphere (wavelength short => k large) - * "high" gives the smallest Ewald sphere (wavelength long => k small) - */ - klow = 1.0/(image->lambda - image->lambda*image->bw/2.0); - khigh = 1.0/(image->lambda + image->lambda*image->bw/2.0); - tl = sqrt(xl*xl + yl*yl); - - cet = -sin(image->div/2.0) * klow; - cez = -cos(image->div/2.0) * klow; - philow = angle_between_2d(tl-cet, zl-cez, 0.0, 1.0); - - cet = -sin(image->div/2.0) * khigh; - cez = -cos(image->div/2.0) * khigh; - phihigh = angle_between_2d(tl-cet, zl-cez, 0.0, 1.0); - - /* Approximation: philow and phihigh are very similar */ - phi = (philow + phihigh) / 2.0; - - azi = atan2(yl, xl); + phi = angle_between_2d(tl, zl+1.0/image->lambda, 0.0, 1.0); /* 2theta */ + azi = atan2(yl, xl); /* azimuth */ switch ( k ) { @@ -531,11 +459,17 @@ RefList *predict_to_res(Crystal *cryst, double max_res) } -static void set_unity_partialities(RefList *list) +static void set_unity_partialities(Crystal *cryst) { + RefList *list; Reflection *refl; RefListIterator *iter; + list = crystal_get_reflections(cryst); + if ( list == NULL ) { + ERROR("No reflections for partiality calculation!\n"); + return; + } for ( refl = first_refl(list, &iter); refl != NULL; refl = next_refl(refl, iter) ) @@ -546,25 +480,11 @@ static void set_unity_partialities(RefList *list) } -/** - * calculate_partialities: - * @cryst: A %Crystal - * @pmodel: A %PartialityModel - * - * Calculates the partialities for the reflections in @cryst, given the current - * crystal and image parameters. The crystal's image and reflection lists - * must be set. The specified %PartialityModel will be used. - * - * You must not have changed the crystal or image parameters since you last - * called predict_to_res() or update_predictions(), because this function - * relies on the limiting wavelength values calculated by those functions. - */ -void calculate_partialities(Crystal *cryst, PartialityModel pmodel) +static void set_random_partialities(Crystal *cryst) { RefList *list; Reflection *refl; RefListIterator *iter; - double pr; struct image *image; list = crystal_get_reflections(cryst); @@ -573,8 +493,166 @@ void calculate_partialities(Crystal *cryst, PartialityModel pmodel) return; } - if ( pmodel == PMODEL_UNITY ) { - set_unity_partialities(list); + image = crystal_get_image(cryst); + if ( image == NULL ) { + ERROR("No image structure for partiality calculation!\n"); + return; + } + + for ( refl = first_refl(list, &iter); + refl != NULL; + refl = next_refl(refl, iter) ) + { + signed int h, k, l; + get_symmetric_indices(refl, &h, &k, &l); + set_partiality(refl, random_partiality(h, k, l, image->serial)); + set_lorentz(refl, 1.0); + } +} + + +static double do_integral(double q2, double zl, double R, + double lambda, double sig, char *verbose) +{ + int i; + double kmin, kmax, kstart, kfinis; + double inc; + double total = 0.0; + double k0, k1; + const int SAMPLES = 50; /* Number of samples for integration */ + const double N = 1.5; /* Pointiness of spectrum */ + FILE *fh = NULL; + + assert(R*R < q2); + assert(R > 0.0); + + /* Range over which P is different from zero */ + k0 = (R*R - q2)/(2.0*(zl+R)); + k1 = (R*R - q2)/(2.0*(zl-R)); + + /* Check for reflections partially "behind the beam" */ + if ( k0 < 0.0 ) k0 = +INFINITY; + if ( k1 < 0.0 ) k1 = +INFINITY; + + /* Range over which E is significantly different from zero */ + kmin = 1.0 / (lambda + 3.0*sig); + kmax = 1.0 / (lambda - 3.0*sig); + + /* Calculate range over which E*P is different from zero */ + if ( k0 < k1 ) { + STATUS("%e %e\n", k0, k1); + STATUS("%e %e %e\n", q2, zl, R); + STATUS("%e %e\n", kmin, kmax); + } + assert((isinf(k0) && isinf(k1)) || (k0 > k1)); + assert(kmax > kmin); + if ( kmin < k1 ) { + if ( kmax < k1 ) { + /* Case 1 */ + kstart = k1; kfinis = k0; + return 0.0; + } else if ( kmax < k0 ) { + /* Case 2 (kmax > k1)*/ + kstart = k1; kfinis = kmax; + } else { + /* Case 3 (kmax > k1 and kmax > k0) */ + kstart = k1; kfinis = k0; + } + } else if ( kmin < k0 ) { /* kmin > k1 */ + if ( kmax < k0 ) { + /* Case 4 */ + kstart = kmin; kfinis = kmax; + } else { + /* Case 5 (kmax > k0) */ + kstart = kmin; kfinis = k0; + } + } else { + /* Case 6 (kmin > k1 and (kmax>)kmin > k0) */ + kstart = k1; kfinis = k0; + return 0.0; + } + + if ( kstart < 0.0 ) kstart = kmin; + if ( kfinis < 0.0 ) kfinis = kmax; + + inc = (kfinis - kstart) / SAMPLES; + + if ( verbose ) { + char fn[64]; + snprintf(fn, 63, "partial%s.graph", verbose); + fh = fopen(fn, "w"); + fprintf(fh, " n p wavelength E P\n"); + STATUS("Nominal k = %e m^-1\n", 1.0/lambda); + STATUS(" (wavelength %e m)\n", lambda); + STATUS("Bandwidth %e m\n", sig); + STATUS("k1/2 = %e m^-1\n", -q2/(2.0*zl)); + STATUS(" (wavelength %e m)\n", 1.0/(-q2/(2.0*zl))); + STATUS("Reflection k goes from %e to %e m^-1\n", k1, k0); + STATUS(" (wavelengths from %e to %e m\n", 1.0/k1, 1.0/k0); + STATUS("Beam goes from %e to %e m^-1\n", kmin, kmax); + STATUS(" (wavelengths from %e to %e m\n", 1.0/kmin, 1.0/kmax); + STATUS("Integration goes from %e to %e m^-1\n", kstart, kfinis); + STATUS(" (wavelengths from %e to %e m\n", 1.0/kstart, 1.0/kfinis); + } + + for ( i=0; i<SAMPLES; i++ ) { + + double p, kp, lrel; + double E, P; + + kp = kstart + i*inc; + double pref = sqrt(q2 + kp*kp + 2.0*zl*kp)/(2.0*R); + p = pref + 0.5 - kp/(2.0*R); + + /* Spectral energy term */ + lrel = fabs(1.0/kp - lambda); + E = exp(-0.5 * pow(lrel / sig, N)); + E /= sqrt(2.0 * M_PI * sig); + + /* RLP profile term */ + P = 4.0*p * (1.0 - p); + + total += E*P*inc; + + if ( fh != NULL ) { + fprintf(fh, "%3i %f %e %e %e\n", i, p, 1.0/kp, E, P); + } + } + + if ( fh != NULL ) fclose(fh); + + if ( isnan(total) ) { + ERROR("NaN total!\n"); + STATUS("Nominal k = %e m^-1\n", 1.0/lambda); + STATUS(" (wavelength %e m)\n", lambda); + STATUS("Bandwidth %e m\n", sig); + STATUS("k1/2 = %e m^-1\n", -q2/(2.0*zl)); + STATUS(" (wavelength %e m)\n", 1.0/(-q2/(2.0*zl))); + STATUS("Reflection k goes from %e to %e m^-1\n", k1, k0); + STATUS(" (wavelengths from %e to %e m\n", 1.0/k1, 1.0/k0); + STATUS("Beam goes from %e to %e m^-1\n", kmin, kmax); + STATUS(" (wavelengths from %e to %e m\n", 1.0/kmin, 1.0/kmax); + STATUS("Integration goes from %e to %e m^-1\n", kstart, kfinis); + STATUS(" (wavelengths from %e to %e m\n", 1.0/kstart, 1.0/kfinis); + } + + return total; +} + + +static void ginn_spectrum_partialities(Crystal *cryst) +{ + RefList *list; + Reflection *refl; + RefListIterator *iter; + double r0, m, lambda, sig; + struct image *image; + UnitCell *cell; + double asx, asy, asz, bsx, bsy, bsz, csx, csy, csz; + + list = crystal_get_reflections(cryst); + if ( list == NULL ) { + ERROR("No reflections for partiality calculation!\n"); return; } @@ -584,31 +662,88 @@ void calculate_partialities(Crystal *cryst, PartialityModel pmodel) return; } - pr = crystal_get_profile_radius(cryst); + cell = crystal_get_cell(cryst); + if ( cell == NULL ) { + ERROR("No unit cell for partiality calculation!\n"); + return; + } + cell_get_reciprocal(cell, &asx, &asy, &asz, + &bsx, &bsy, &bsz, + &csx, &csy, &csz); + + r0 = fabs(crystal_get_profile_radius(cryst)); + m = crystal_get_mosaicity(cryst); + lambda = image->lambda; + sig = image->bw * lambda / 2.0; for ( refl = first_refl(crystal_get_reflections(cryst), &iter); refl != NULL; refl = next_refl(refl, iter) ) { - double rlow, rhigh, part; + double R; signed int h, k, l; + double xl, yl, zl; + double q2; + double total, norm; get_symmetric_indices(refl, &h, &k, &l); - get_partial(refl, &rlow, &rhigh, &part); + xl = h*asx + k*bsx + l*csx; + yl = h*asy + k*bsy + l*csy; + zl = h*asz + k*bsz + l*csz; + + /* Radius of rlp profile */ + q2 = xl*xl + yl*yl + zl*zl; + + R = r0 + m * sqrt(q2); + + //char tmp[256]; + //snprintf(tmp, 255, "-%i,%i,%i", h, k, l); + char *tmp = NULL; - part = partiality(pmodel, h, k, l, image->serial, - rlow, rhigh, pr); + total = do_integral(q2, zl, R, lambda, sig, tmp); + norm = do_integral(q2, -0.5*q2*lambda, R, lambda, sig, NULL); - if ( isnan(part) && ((h!=0) || (k!=0) || (l!=0)) ) { - ERROR("Assigning NAN partiality!\n"); - ERROR("%3i %3i %3i rlow = %e, rhigh = %e\n", - h, k, l, rlow, rhigh); - ERROR("R = %e, bw = %e\n", pr, image->bw); - ERROR("D = %e\n", rlow - rhigh); - abort(); - } + set_partiality(refl, total/norm); + set_lorentz(refl, 1.0); - set_partiality(refl, part); + assert(total <= 2.0*norm); + + } +} + + +/** + * calculate_partialities: + * @cryst: A %Crystal + * @pmodel: A %PartialityModel + * + * Calculates the partialities for the reflections in @cryst, given the current + * crystal and image parameters. The crystal's image and reflection lists + * must be set. The specified %PartialityModel will be used. + * + * You must not have changed the crystal or image parameters since you last + * called predict_to_res() or update_predictions(), because this function + * relies on the limiting wavelength values calculated by those functions. + */ +void calculate_partialities(Crystal *cryst, PartialityModel pmodel) +{ + switch ( pmodel ) { + + case PMODEL_UNITY : + set_unity_partialities(cryst); + break; + + case PMODEL_XSPHERE : + ginn_spectrum_partialities(cryst); + break; + + case PMODEL_RANDOM : + set_random_partialities(cryst); + break; + + default : + ERROR("Unknown partiality model %i\n", pmodel); + break; } } @@ -694,29 +829,30 @@ void polarisation_correction(RefList *list, UnitCell *cell, struct image *image) /* Returns dx_h/dP, where P = any parameter */ -double x_gradient(int param, Reflection *refl, UnitCell *cell, - struct panel *p, double lambda) +double x_gradient(int param, Reflection *refl, UnitCell *cell, struct panel *p) { signed int h, k, l; - double x, z, wn; - double ax, ay, az, bx, by, bz, cx, cy, cz; + double xl, zl, kpred; + double asx, asy, asz, bsx, bsy, bsz, csx, csy, csz; get_indices(refl, &h, &k, &l); - wn = 1.0 / lambda; - cell_get_reciprocal(cell, &ax, &ay, &az, &bx, &by, &bz, &cx, &cy, &cz); - x = h*ax + k*bx + l*cx; - z = h*az + k*bz + l*cz; + kpred = get_kpred(refl); + cell_get_reciprocal(cell, &asx, &asy, &asz, + &bsx, &bsy, &bsz, + &csx, &csy, &csz); + xl = h*asx + k*bsx + l*csx; + zl = h*asz + k*bsz + l*csz; switch ( param ) { case GPARAM_ASX : - return h * p->clen / (wn+z); + return h * p->clen / (kpred + zl); case GPARAM_BSX : - return k * p->clen / (wn+z); + return k * p->clen / (kpred + zl); case GPARAM_CSX : - return l * p->clen / (wn+z); + return l * p->clen / (kpred + zl); case GPARAM_ASY : return 0.0; @@ -728,13 +864,13 @@ double x_gradient(int param, Reflection *refl, UnitCell *cell, return 0.0; case GPARAM_ASZ : - return -h * x * p->clen / (wn*wn + 2*wn*z + z*z); + return -h * xl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_BSZ : - return -k * x * p->clen / (wn*wn + 2*wn*z + z*z); + return -k * xl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_CSZ : - return -l * x * p->clen / (wn*wn + 2*wn*z + z*z); + return -l * xl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_DETX : return -1; @@ -743,7 +879,7 @@ double x_gradient(int param, Reflection *refl, UnitCell *cell, return 0; case GPARAM_CLEN : - return x / (wn+z); + return xl / (kpred+zl); } @@ -753,18 +889,19 @@ double x_gradient(int param, Reflection *refl, UnitCell *cell, /* Returns dy_h/dP, where P = any parameter */ -double y_gradient(int param, Reflection *refl, UnitCell *cell, - struct panel *p, double lambda) +double y_gradient(int param, Reflection *refl, UnitCell *cell, struct panel *p) { signed int h, k, l; - double y, z, wn; - double ax, ay, az, bx, by, bz, cx, cy, cz; + double yl, zl, kpred; + double asx, asy, asz, bsx, bsy, bsz, csx, csy, csz; get_indices(refl, &h, &k, &l); - wn = 1.0 / lambda; - cell_get_reciprocal(cell, &ax, &ay, &az, &bx, &by, &bz, &cx, &cy, &cz); - y = h*ay + k*by + l*cy; - z = h*az + k*bz + l*cz; + kpred = get_kpred(refl); + cell_get_reciprocal(cell, &asx, &asy, &asz, + &bsx, &bsy, &bsz, + &csx, &csy, &csz); + yl = h*asy + k*bsy + l*csy; + zl = h*asz + k*bsz + l*csz; switch ( param ) { @@ -778,22 +915,22 @@ double y_gradient(int param, Reflection *refl, UnitCell *cell, return 0.0; case GPARAM_ASY : - return h * p->clen / (wn+z); + return h * p->clen / (kpred + zl); case GPARAM_BSY : - return k * p->clen / (wn+z); + return k * p->clen / (kpred + zl); case GPARAM_CSY : - return l * p->clen / (wn+z); + return l * p->clen / (kpred + zl); case GPARAM_ASZ : - return -h * y * p->clen / (wn*wn + 2*wn*z + z*z); + return -h * yl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_BSZ : - return -k * y * p->clen / (wn*wn + 2*wn*z + z*z); + return -k * yl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_CSZ : - return -l * y * p->clen / (wn*wn + 2*wn*z + z*z); + return -l * yl * p->clen / (kpred*kpred + 2.0*kpred*zl + zl*zl); case GPARAM_DETX : return 0; @@ -802,7 +939,7 @@ double y_gradient(int param, Reflection *refl, UnitCell *cell, return -1; case GPARAM_CLEN : - return y / (wn+z); + return yl / (kpred+zl); } diff --git a/libcrystfel/src/geometry.h b/libcrystfel/src/geometry.h index 4fbd0bca..529f112e 100644 --- a/libcrystfel/src/geometry.h +++ b/libcrystfel/src/geometry.h @@ -3,7 +3,7 @@ * * Geometry of diffraction * - * Copyright © 2013-2016 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2017 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * @@ -47,8 +47,7 @@ extern "C" { /** * PartialityModel: * @PMODEL_UNITY : Set all all partialities and Lorentz factors to 1. - * @PMODEL_SCSPHERE : Sphere model with source coverage factor included - * @PMODEL_SCGAUSSIAN : Gaussian model with source coverage factor included + * @PMODEL_XSPHERE : Flat sphere model with super-Gaussian spectrum * @PMODEL_RANDOM : Randomly assigned partialities * * A %PartialityModel describes a geometrical model which can be used to @@ -57,8 +56,7 @@ extern "C" { typedef enum { PMODEL_UNITY, - PMODEL_SCSPHERE, - PMODEL_SCGAUSSIAN, + PMODEL_XSPHERE, PMODEL_RANDOM, } PartialityModel; @@ -80,8 +78,13 @@ enum gparam { GPARAM_DETX, GPARAM_DETY, GPARAM_CLEN, - GPARAM_OSF, - GPARAM_BFAC + GPARAM_OSF, /* Linear scale factor */ + GPARAM_BFAC, /* D-W scale factor */ + GPARAM_ANG1, /* Out of plane rotation angles of crystal */ + GPARAM_ANG2, + GPARAM_WAVELENGTH, + + GPARAM_EOL /* End of list */ }; @@ -99,9 +102,9 @@ extern double sphere_fraction(double rlow, double rhigh, double pr); extern double gaussian_fraction(double rlow, double rhigh, double pr); extern double x_gradient(int param, Reflection *refl, UnitCell *cell, - struct panel *p, double lambda); + struct panel *p); extern double y_gradient(int param, Reflection *refl, UnitCell *cell, - struct panel *p, double lambda); + struct panel *p); #ifdef __cplusplus } diff --git a/libcrystfel/src/image.h b/libcrystfel/src/image.h index 99c3d57e..9769c66a 100644 --- a/libcrystfel/src/image.h +++ b/libcrystfel/src/image.h @@ -216,7 +216,7 @@ struct image { /* Per-shot radiation values */ double lambda; /* Wavelength in m */ double div; /* Divergence in radians */ - double bw; /* Bandwidth as a fraction */ + double bw; /* FWHM bandwidth as a fraction */ /* Detected peaks */ long long num_peaks; diff --git a/libcrystfel/src/index.c b/libcrystfel/src/index.c index 20595604..b20c689a 100644 --- a/libcrystfel/src/index.c +++ b/libcrystfel/src/index.c @@ -316,9 +316,9 @@ IndexingPrivate *setup_indexing(const char *method_list, UnitCell *cell, ERROR("To disable prediction refinement ('norefine'), use --no-refine.\n"); ERROR("To check cell axes only ('axes'), use --no-cell-combinations.\n"); ERROR("To disable all unit cell checks ('raw'), use --no-check-cell.\n"); + ERROR("To disable peak alignment check ('bad'), use --no-check-peaks.\n"); ERROR("To disable indexing retry ('noretry'), use --no-retry.\n"); - ERROR("Multi-lattice indexing ('multi') is now the default: " - "use --no-multi to disable it.\n"); + ERROR("To enable multi-lattice indexing by 'delete and retry', use --multi\n"); ERROR("------------------\n"); free(methods); return NULL; @@ -1044,8 +1044,8 @@ char *detect_indexing_methods(UnitCell *cell) do_probe(dirax_probe, cell, methods); do_probe(asdf_probe, cell, methods); do_probe(xds_probe, cell, methods); - do_probe(taketwo_probe, cell, methods); - /* Don't automatically use Felix (yet) */ + /* Don't automatically use TakeTwo or Felix (yet) */ + //do_probe(taketwo_probe, cell, methods); //do_probe(felix_probe, cell, methods); if ( strlen(methods) == 0 ) { diff --git a/libcrystfel/src/integration.c b/libcrystfel/src/integration.c index d7a34954..880ae87e 100644 --- a/libcrystfel/src/integration.c +++ b/libcrystfel/src/integration.c @@ -1760,7 +1760,7 @@ void integrate_all_2(struct image *image, IntegrationMethod meth, IntDiag int_diag, signed int idh, signed int idk, signed int idl) { - integrate_all_3(image, meth, PMODEL_SCSPHERE, 0.0, + integrate_all_3(image, meth, PMODEL_XSPHERE, 0.0, ir_inn, ir_mid, ir_out, int_diag, idh, idk, idl); } diff --git a/libcrystfel/src/mosflm.c b/libcrystfel/src/mosflm.c index e59cf4e2..d1b7ffed 100644 --- a/libcrystfel/src/mosflm.c +++ b/libcrystfel/src/mosflm.c @@ -500,12 +500,18 @@ static void mosflm_send_next(struct image *image, struct mosflm_data *mosflm) switch ( mosflm->step ) { case 1 : + /* Backwards-compatible workaround for different Mosflm behaviour + * in version 7.2.2 */ + mosflm_sendline("DETECTOR ADSC\n", mosflm); + break; + + case 2 : mosflm_sendline("DETECTOR ROTATION HORIZONTAL" " ANTICLOCKWISE ORIGIN LL FAST HORIZONTAL" " RECTANGULAR\n", mosflm); break; - case 2 : + case 3 : if ( (mosflm->mp->indm & INDEXING_USE_LATTICE_TYPE) && (mosflm->mp->template != NULL) ) { @@ -527,32 +533,32 @@ static void mosflm_send_next(struct image *image, struct mosflm_data *mosflm) } break; - case 3 : + case 4 : snprintf(tmp, 255, "DISTANCE %f\n", FAKE_CLEN*1e3); mosflm_sendline(tmp, mosflm); break; - case 4 : + case 5 : mosflm_sendline("BEAM 0.0 0.0\n", mosflm); break; - case 5 : + case 6 : wavelength = image->lambda*1e10; snprintf(tmp, 255, "WAVELENGTH %10.5f\n", wavelength); mosflm_sendline(tmp, mosflm); break; - case 6 : + case 7 : snprintf(tmp, 255, "NEWMAT %s\n", mosflm->newmatfile); mosflm_sendline(tmp, mosflm); break; - case 7 : + case 8 : snprintf(tmp, 255, "IMAGE %s phi 0 0\n", mosflm->imagefile); mosflm_sendline(tmp, mosflm); break; - case 8 : + case 9 : if ( mosflm->mp->indm & INDEXING_USE_CELL_PARAMETERS ) { cell_get_parameters(mosflm->mp->template, &a, &b, &c, &alpha, &beta, &gamma); @@ -572,7 +578,7 @@ static void mosflm_send_next(struct image *image, struct mosflm_data *mosflm) mosflm_sendline(tmp, mosflm); break; - case 9 : + case 10 : mosflm_sendline("GO\n", mosflm); mosflm->finished_ok = 1; break; diff --git a/libcrystfel/src/peakfinder8.c b/libcrystfel/src/peakfinder8.c index 417465df..f96750de 100644 --- a/libcrystfel/src/peakfinder8.c +++ b/libcrystfel/src/peakfinder8.c @@ -32,6 +32,7 @@ #include <config.h> #endif +#include <float.h> #include <math.h> #include <stdlib.h> @@ -363,13 +364,15 @@ static void fill_radial_bins(float *data, int curr_r; float value; - for ( iss = 0; iss<h ; iss++ ) { - for ( ifs = 0; ifs<w ; ifs++ ) { + for ( iss=0; iss<h; iss++ ) { + for ( ifs=0; ifs<w; ifs++ ) { pidx = iss * w + ifs; if ( mask[pidx] != 0 ) { curr_r = (int)rint(r_map[pidx]); value = data[pidx]; - if ( value < rthreshold[curr_r ] && value>lthreshold[curr_r]) { + if ( value < rthreshold[curr_r] + && value > lthreshold[curr_r] ) + { roffset[curr_r] += value; rsigma[curr_r] += (value * value); rcount[curr_r] += 1; @@ -397,8 +400,8 @@ static void compute_radial_stats(float *rthreshold, if ( rcount[ri] == 0 ) { roffset[ri] = 0; rsigma[ri] = 0; - rthreshold[ri] = 1e9; - lthreshold[ri] = -1e9; + rthreshold[ri] = FLT_MAX; + lthreshold[ri] = FLT_MIN; } else { this_offset = roffset[ri] / rcount[ri]; this_sigma = rsigma[ri] / rcount[ri] - (this_offset * this_offset); @@ -1064,6 +1067,7 @@ int peakfinder8(struct image *img, int max_n_peaks, for ( i=0 ; i<rstats->n_rad_bins ; i++) { rstats->rthreshold[i] = 1e9; + rstats->lthreshold[i] = -1e9; } for ( it_counter=0 ; it_counter<iterations ; it_counter++ ) { diff --git a/libcrystfel/src/predict-refine.c b/libcrystfel/src/predict-refine.c index 3246dbec..14a60bc7 100644 --- a/libcrystfel/src/predict-refine.c +++ b/libcrystfel/src/predict-refine.c @@ -91,9 +91,7 @@ static void twod_mapping(double fs, double ss, double *px, double *py, static double r_dev(struct reflpeak *rp) { /* Excitation error term */ - double rlow, rhigh, p; - get_partial(rp->refl, &rlow, &rhigh, &p); - return (rlow+rhigh)/2.0; + return get_exerr(rp->refl); } @@ -425,7 +423,7 @@ static int iterate(struct reflpeak *rps, int n, UnitCell *cell, /* Positional x terms */ for ( k=0; k<num_params; k++ ) { gradients[k] = x_gradient(rv[k], rps[i].refl, cell, - rps[i].panel, image->lambda); + rps[i].panel); } for ( k=0; k<num_params; k++ ) { @@ -457,7 +455,7 @@ static int iterate(struct reflpeak *rps, int n, UnitCell *cell, /* Positional y terms */ for ( k=0; k<num_params; k++ ) { gradients[k] = y_gradient(rv[k], rps[i].refl, cell, - rps[i].panel, image->lambda); + rps[i].panel); } for ( k=0; k<num_params; k++ ) { diff --git a/libcrystfel/src/reflist.c b/libcrystfel/src/reflist.c index 707c802e..a35aa575 100644 --- a/libcrystfel/src/reflist.c +++ b/libcrystfel/src/reflist.c @@ -67,10 +67,11 @@ struct _refldata { signed int ls; /* Partiality and related geometrical stuff */ - double rlow; /* Low excitation error */ - double rhigh; /* High excitation error */ - double p; /* Partiality */ - double L; /* Lorentz factor */ + double khalf; /* Wavenumber of middle of reflection */ + double kpred; /* Wavenumber for prediction */ + double exerr; /* Excitation error */ + double p; /* Partiality */ + double L; /* Lorentz factor */ /* Location in image */ double fs; @@ -422,22 +423,43 @@ double get_intensity(const Reflection *refl) /** - * get_partial: + * get_khalf * @refl: A %Reflection - * @rlow: Location at which to store the "low" excitation error - * @rhigh: Location at which to store the "high" excitation error - * @p: Location at which to store the partiality * - * This function is used during post refinement (in conjunction with - * set_partial()) to get access to the details of the partiality calculation. + * Returns: the wavenumber at the centre of the reflection * **/ -void get_partial(const Reflection *refl, double *rlow, double *rhigh, - double *p) +double get_khalf(const Reflection *refl) { - *rlow = refl->data.rlow; - *rhigh = refl->data.rhigh; - *p = get_partiality(refl); + return refl->data.khalf; +} + + + + +/** + * get_kpred: + * @refl: A %Reflection + * + * Returns: the wavenumber which should be used for prediction of this reflection + * + **/ +double get_kpred(const Reflection *refl) +{ + return refl->data.kpred; +} + + +/** + * get_exerr: + * @refl: A %Reflection + * + * Returns: the excitation error (in m^-1) for this reflection + * + **/ +double get_exerr(const Reflection *refl) +{ + return refl->data.exerr; } @@ -612,23 +634,43 @@ void set_panel(Reflection *refl, struct panel *p) refl->data.panel = p; } +/** + * set_khalf: + * @refl: A %Reflection + * @khalf: The wavenumber at which the reflection should be predicted + * + * Sets the wavenumber at the centre of the reflection. + **/ +void set_khalf(Reflection *refl, double khalf) +{ + refl->data.khalf = khalf; +} + + /** - * set_partial: + * set_kpred: * @refl: A %Reflection - * @rlow: The "low" excitation error - * @rhigh: The "high" excitation error - * @p: The partiality + * @kpred: The wavenumber at which the reflection should be predicted * - * This function is used during post refinement (in conjunction with - * get_partial()) to get access to the details of the partiality calculation. + * Sets the wavenumber at which the reflection should be predicted. + * Used by predict_to_res() and update_predictions() + **/ +void set_kpred(Reflection *refl, double kpred) +{ + refl->data.kpred = kpred; +} + + +/** + * set_exerr: + * @refl: A %Reflection + * @exerr: The excitation error for the reflection * **/ -void set_partial(Reflection *refl, double rlow, double rhigh, double p) +void set_exerr(Reflection *refl, double exerr) { - refl->data.rlow = rlow; - refl->data.rhigh = rhigh; - refl->data.p = p; + refl->data.exerr = exerr; } diff --git a/libcrystfel/src/reflist.h b/libcrystfel/src/reflist.h index f2126ac9..9f494474 100644 --- a/libcrystfel/src/reflist.h +++ b/libcrystfel/src/reflist.h @@ -3,11 +3,11 @@ * * Fast reflection/peak list * - * Copyright © 2012-2016 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2011-2016 Thomas White <taw@physics.org> + * 2011-2017 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -87,6 +87,9 @@ extern Reflection *next_found_refl(Reflection *refl); extern void get_detector_pos(const Reflection *refl, double *fs, double *ss); extern struct panel *get_panel(const Reflection *refl); extern double get_partiality(const Reflection *refl); +extern double get_khalf(const Reflection *refl); +extern double get_kpred(const Reflection *refl); +extern double get_exerr(const Reflection *refl); extern double get_lorentz(const Reflection *refl); extern void get_indices(const Reflection *refl, signed int *h, signed int *k, signed int *l); @@ -94,8 +97,6 @@ extern void get_symmetric_indices(const Reflection *refl, signed int *hs, signed int *ks, signed int *ls); extern double get_intensity(const Reflection *refl); -extern void get_partial(const Reflection *refl, double *rlow, double *rhigh, - double *p); extern int get_redundancy(const Reflection *refl); extern double get_temp1(const Reflection *refl); extern double get_temp2(const Reflection *refl); @@ -109,7 +110,9 @@ extern int get_flag(const Reflection *refl); extern void copy_data(Reflection *to, const Reflection *from); extern void set_detector_pos(Reflection *refl, double fs, double ss); extern void set_panel(Reflection *refl, struct panel *p); -extern void set_partial(Reflection *refl, double rlow, double rhigh, double p); +extern void set_kpred(Reflection *refl, double kpred); +extern void set_khalf(Reflection *refl, double khalf); +extern void set_exerr(Reflection *refl, double exerr); extern void set_partiality(Reflection *refl, double p); extern void set_lorentz(Reflection *refl, double L); extern void set_intensity(Reflection *refl, double intensity); diff --git a/libcrystfel/src/stream.c b/libcrystfel/src/stream.c index 0f4dcd6b..9d97ff98 100644 --- a/libcrystfel/src/stream.c +++ b/libcrystfel/src/stream.c @@ -64,6 +64,7 @@ struct _stream int major_version; int minor_version; + char *audit_info; long long int ln; @@ -1334,6 +1335,64 @@ void write_stream_header(FILE *ofh, int argc, char *argv[]) } +char *stream_audit_info(Stream *st) +{ + if ( st->audit_info == NULL ) return NULL; + return strdup(st->audit_info); +} + + +static void read_audit_lines(Stream *st) +{ + int done = 0; + size_t len = 0; + int first = 1; + + st->audit_info = malloc(4096); + if ( st->audit_info == NULL ) { + ERROR("Failed to allocate memory for audit information\n"); + return; + } + + /* Read lines from stream until one of them starts with "-----", + * then rewind to the start of that line */ + do { + + char line[1024]; + char *rval; + long pos; + + pos = ftell(st->fh); + + rval = fgets(line, 1023, st->fh); + if ( rval == NULL ) { + ERROR("Failed to read stream audit info.\n"); + close_stream(st); + return; + } + + if ( strncmp(line, "-----", 5) == 0 ) { + fseek(st->fh, pos, SEEK_SET); + done = 1; + } else { + chomp(line); + len += strlen(line); + if ( len > 4090 ) { + ERROR("Too much audit information.\n"); + return; + } else { + if ( !first ) { + strcat(st->audit_info, "\n"); + } + first = 0; + strcat(st->audit_info, line); + } + } + + } while ( !done ); +} + + Stream *open_stream_for_read(const char *filename) { Stream *st; @@ -1341,6 +1400,7 @@ Stream *open_stream_for_read(const char *filename) st = malloc(sizeof(struct _stream)); if ( st == NULL ) return NULL; st->old_indexers = 0; + st->audit_info = NULL; if ( strcmp(filename, "-") == 0 ) { st->fh = stdin; @@ -1383,6 +1443,8 @@ Stream *open_stream_for_read(const char *filename) st->ln = 1; + read_audit_lines(st); + return st; } @@ -1408,6 +1470,7 @@ Stream *open_stream_fd_for_write(int fd) st = malloc(sizeof(struct _stream)); if ( st == NULL ) return NULL; st->old_indexers = 0; + st->audit_info = NULL; st->fh = fdopen(fd, "w"); if ( st->fh == NULL ) { @@ -1435,12 +1498,13 @@ static void write_cell_to_stream(Stream *st, UnitCell *cell) /** - * open_stream_for_write_3 + * open_stream_for_write_4 * @filename: Filename of new stream * @geom_filename: The geometry filename to copy * @cell: A %UnitCell to write into the stream * @argc: The number of arguments to the program * @argv: The arguments to the program + * @indm_str: The list of indexing methods * * Creates a new stream with name @filename, and adds the stream format * and version header, plus a verbatim copy of the geometry file and the unit @@ -1448,9 +1512,9 @@ static void write_cell_to_stream(Stream *st, UnitCell *cell) * * Returns: a %Stream, or NULL on failure. */ -Stream *open_stream_for_write_3(const char *filename, +Stream *open_stream_for_write_4(const char *filename, const char *geom_filename, UnitCell *cell, - int argc, char *argv[]) + int argc, char *argv[], const char *indm_str) { Stream *st; @@ -1458,6 +1522,7 @@ Stream *open_stream_for_write_3(const char *filename, st = malloc(sizeof(struct _stream)); if ( st == NULL ) return NULL; st->old_indexers = 0; + st->audit_info = NULL; st->fh = fopen(filename, "w"); if ( st->fh == NULL ) { @@ -1477,6 +1542,10 @@ Stream *open_stream_for_write_3(const char *filename, if ( (argc > 0) && (argv != NULL) ) { write_command(st, argc, argv); } + + if ( indm_str != NULL ) { + fprintf(st->fh, "Indexing methods selected: %s\n", indm_str); + } if ( geom_filename != NULL ) { write_geometry_file(st, geom_filename); } @@ -1488,6 +1557,15 @@ Stream *open_stream_for_write_3(const char *filename, } +Stream *open_stream_for_write_3(const char *filename, + const char *geom_filename, UnitCell *cell, + int argc, char *argv[]) +{ + return open_stream_for_write_4(filename, geom_filename, cell, + argc, argv, NULL); +} + + /** * open_stream_for_write_2 * @filename: Filename of new stream @@ -1510,6 +1588,7 @@ Stream *open_stream_for_write_2(const char *filename, st = malloc(sizeof(struct _stream)); if ( st == NULL ) return NULL; st->old_indexers = 0; + st->audit_info = NULL; st->fh = fopen(filename, "w"); if ( st->fh == NULL ) { @@ -1576,6 +1655,7 @@ int get_stream_fd(Stream *st) void close_stream(Stream *st) { + free(st->audit_info); fclose(st->fh); free(st); } diff --git a/libcrystfel/src/stream.h b/libcrystfel/src/stream.h index 2ed7b050..a8e3e2ee 100644 --- a/libcrystfel/src/stream.h +++ b/libcrystfel/src/stream.h @@ -3,11 +3,11 @@ * * Stream tools * - * Copyright © 2013-2017 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2018 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2017 Thomas White <taw@physics.org> + * 2010-2018 Thomas White <taw@physics.org> * 2014 Valerio Mariani * 2011 Andrew Aquila * @@ -99,6 +99,9 @@ extern Stream *open_stream_for_write_2(const char *filename, extern Stream *open_stream_for_write_3(const char *filename, const char* geom_filename, UnitCell *cell, int argc, char *argv[]); +extern Stream *open_stream_for_write_4(const char *filename, + const char *geom_filename, UnitCell *cell, + int argc, char *argv[], const char *indm_str); extern Stream *open_stream_fd_for_write(int fd); extern int get_stream_fd(Stream *st); extern void close_stream(Stream *st); @@ -121,6 +124,7 @@ extern void write_command(Stream *st, int argc, char *argv[]); extern void write_geometry_file(Stream *st, const char *geom_filename); extern int rewind_stream(Stream *st); extern int is_stream(const char *filename); +extern char *stream_audit_info(Stream *st); #ifdef __cplusplus } diff --git a/libcrystfel/src/taketwo.c b/libcrystfel/src/taketwo.c index 0857f813..234b00c3 100644 --- a/libcrystfel/src/taketwo.c +++ b/libcrystfel/src/taketwo.c @@ -28,6 +28,69 @@ * */ +/** + * \class TakeTwo + * Code outline. + * --- Get ready for calculation --- + * Pre-calculate symmetry operations (generate_rotation_symops()) + * Pre-calculate theoretical vectors from unit cell dimensions + * (gen_theoretical_vecs()) + * Generate observed vectors from data (gen_observed_vecs()) + * Match observed vectors to theoretical vectors (match_obs_to_cell_vecs()) + * + * --- Business bit --- + * ... n.b. rearranging to find all seeds in advance. + * + * Find starting seeds (find_seeds()): + * - Loop through pairs of observed vectors + * - If they share a spot, find matching pairs of theoretical vectors + * - Remove all duplicate matches due to symmetry operations + * - For the remainder, loop through the matches and extend the seeds + * (start_seed()). + * - If it returns a membership greater than the highest member threshold, + * return the matrix to CrystFEL. + * + * Extending a seed (start_seed()): + * - Generate a rotation matrix which matches the chosen start seed. + * - Loop through all observed vectors starting from 0. + * - Find another vector to add to the network, if available + * (find_next_index()). + * - If the index is not available, then give up if there were too many dead + * ends. Otherwise, remove the last member and pretend like that didn't + * happen, so it won't happen again. + * - Add the vector to increment the membership list. + * - If the membership number exceeds the maximum, tidy up the solution and + * return a success. + * - If the membership does not, then resume the loop and search for the + * next vector. + * + * Finding the next member (find_next_index()): + * - Go through the observed vectors, starting from the last index + 1 to + * explore only the "new" vectors. + * - If the vector does not share a spot with the current array of vectors, + * then skip it. + * - We must loop through all the current vectors in the network, and see if + * they match the newcomer for a given matching theoretical vector. + * - We only accept a match if the rotation matrix matches the seed matrix + * for a single matching theoretical vector. + * - If it does match, we can return a success. + * + * Tidying the solution (finish_solution()): + * - This chooses the most common rotation matrix of the bunch to choose to + * send to CrystFEL. But this should probably take the average instead, + * which is very possible. + * + * Clean up the mess (cleanup_taketwo_obs_vecs()) + */ + +/** + * Helen's to-do list + * - + * + * + * - Improve the final solution + */ + #include <gsl/gsl_matrix.h> #include <gsl/gsl_blas.h> #include <float.h> @@ -57,40 +120,90 @@ struct SpotVec { struct rvec obsvec; - struct rvec *matches; + struct TheoryVec *matches; int match_num; - struct rvec *asym_matches; - int asym_match_num; + int assignment; + int in_network; double distance; struct rvec *her_rlp; struct rvec *his_rlp; }; +/** + * theoryvec + */ + +struct TheoryVec +{ + struct rvec vec; + int asym; +}; + + +/** + * seed + */ + +struct Seed +{ + int obs1; + int obs2; + int idx1; + int idx2; + double score; +}; struct taketwo_private { IndexingMethod indm; UnitCell *cell; + int serial_num; /**< Serial of last image, -1 if unassigned */ + unsigned int xtal_num; /**< last number of crystals recorded */ + + struct TheoryVec *theory_vecs; /**< Theoretical vectors for given unit cell */ + unsigned int vec_count; /**< Number of theoretical vectors */ + + gsl_matrix **prevSols; /**< Previous solutions to be ignored */ + unsigned int numPrevs; /**< Previous solution count */ + double *prevScores; /**< previous solution scores */ + unsigned int *membership; /**< previous solution was success or failure */ + }; -// These rotation symmetry operators +/** + * Internal structure which gets passed the various functions looking after + * the indexing bits and bobs. */ struct TakeTwoCell { - UnitCell *cell; + UnitCell *cell; /**< Contains unit cell dimensions */ gsl_matrix **rotSymOps; unsigned int numOps; - struct SpotVec **obs_vecs; // Pointer to an array + + struct SpotVec *obs_vecs; + struct Seed *seeds; + int seed_count; int obs_vec_count; /* Options */ int member_thresh; - double len_tol; /* In reciprocal metres */ - double angle_tol; /* In radians */ - double trace_tol; /* Contains sqrt(4*(1-cos(angle))) */ + double len_tol; /**< In reciprocal metres */ + double angle_tol; /**< In radians */ + double trace_tol; /**< Contains sqrt(4*(1-cos(angle))) */ + + /** A given solution to refine */ + gsl_matrix *solution; + double x_ang; /**< Rotations in radians to apply to x axis of solution */ + double y_ang; /**< Rotations in radians to apply to y axis of solution */ + double z_ang; /**< Rotations in radians to apply to z axis of solution */ + + /**< Temporary memory always allocated for calculations */ gsl_matrix *twiz1Tmp; + /**< Temporary memory always allocated for calculations */ gsl_matrix *twiz2Tmp; + /**< Temporary memory always allocated for calculations */ gsl_vector *vec1Tmp; + /**< Temporary memory always allocated for calculations */ gsl_vector *vec2Tmp; }; @@ -104,6 +217,9 @@ struct TakeTwoCell /* Threshold for network members to consider a potential solution */ #define NETWORK_MEMBER_THRESHOLD (20) +/* Minimum for network members to consider a potential solution */ +#define MINIMUM_MEMBER_THRESHOLD (5) + /* Maximum dead ends for a single branch extension during indexing */ #define MAX_DEAD_ENDS (10) @@ -117,6 +233,12 @@ struct TakeTwoCell /* Tolerance for rot_mats_are_similar */ #define TRACE_TOLERANCE (deg2rad(3.0)) +/* Initial step size for refinement of solutions */ +#define ANGLE_STEP_SIZE (deg2rad(0.5)) + +/* Final required step size for refinement of solutions */ +#define ANGLE_CONVERGE_SIZE (deg2rad(0.01)) + /* TODO: Multiple lattices */ @@ -145,6 +267,15 @@ static struct rvec new_rvec(double new_u, double new_v, double new_w) return new_rvector; } +static struct rvec rvec_add_rvec(struct rvec first, struct rvec second) +{ + struct rvec diff = new_rvec(second.u + first.u, + second.v + first.v, + second.w + first.w); + + return diff; +} + static struct rvec diff_vec(struct rvec from, struct rvec to) { @@ -240,6 +371,41 @@ static void rotation_around_axis(struct rvec c, double th, gsl_matrix_set(res, 2, 2, cos(th) + c.w*c.w*omc); } +/** Rotate GSL matrix by three angles along x, y and z axes */ +static void rotate_gsl_by_angles(gsl_matrix *sol, double x, double y, + double z, gsl_matrix *result) +{ + gsl_matrix *x_rot = gsl_matrix_alloc(3, 3); + gsl_matrix *y_rot = gsl_matrix_alloc(3, 3); + gsl_matrix *z_rot = gsl_matrix_alloc(3, 3); + gsl_matrix *xy_rot = gsl_matrix_alloc(3, 3); + gsl_matrix *xyz_rot = gsl_matrix_alloc(3, 3); + + struct rvec x_axis = new_rvec(1, 0, 0); + struct rvec y_axis = new_rvec(1, 0, 0); + struct rvec z_axis = new_rvec(1, 0, 0); + + rotation_around_axis(x_axis, x, x_rot); + rotation_around_axis(y_axis, y, y_rot); + rotation_around_axis(z_axis, z, z_rot); + + /* Collapse the rotations in x and y onto z */ + gsl_blas_dgemm(CblasNoTrans, CblasNoTrans, 1.0, x_rot, + y_rot, 0.0, xy_rot); + gsl_blas_dgemm(CblasNoTrans, CblasNoTrans, 1.0, xy_rot, + z_rot, 0.0, xyz_rot); + + /* Apply the whole rotation offset to the solution */ + gsl_blas_dgemm(CblasNoTrans, CblasNoTrans, 1.0, xyz_rot, + sol, 0.0, result); + + gsl_matrix_free(x_rot); + gsl_matrix_free(y_rot); + gsl_matrix_free(z_rot); + gsl_matrix_free(xy_rot); + gsl_matrix_free(xyz_rot); +} + /* Rotate vector (vec1) around axis (axis) by angle theta. Find value of * theta for which the angle between (vec1) and (vec2) is minimised. */ @@ -297,6 +463,34 @@ static void closest_rot_mat(struct rvec vec1, struct rvec vec2, rotation_around_axis(axis, bestAngle, twizzle); } +/* +static double matrix_angle(gsl_matrix *m) +{ + double a = gsl_matrix_get(m, 0, 0); + double b = gsl_matrix_get(m, 1, 1); + double c = gsl_matrix_get(m, 2, 2); + + double cos_t = (a + b + c - 1) / 2; + double theta = acos(cos_t); + + return theta; +} + +static struct rvec matrix_axis(gsl_matrix *a) +{ + double ang = matrix_angle(a); + double cos_t = cos(ang); + double p = gsl_matrix_get(a, 0, 0); + double q = gsl_matrix_get(a, 1, 1); + double r = gsl_matrix_get(a, 2, 2); + double x = sqrt((p - cos_t) / (1 - cos_t)); + double y = sqrt((q - cos_t) / (1 - cos_t)); + double z = sqrt((r - cos_t) / (1 - cos_t)); + + struct rvec v = new_rvec(x, y, z); + return v; +} +*/ static double matrix_trace(gsl_matrix *a) { @@ -487,9 +681,6 @@ static gsl_matrix *generate_rot_mat(struct rvec obs1, struct rvec obs2, gsl_matrix *fullMat; rvec_to_gsl(cell->vec1Tmp, cell2); -// gsl_vector *cell2v = rvec_to_gsl(cell2); -// gsl_vector *cell2vr = gsl_vector_calloc(3); - normalise_rvec(&obs1); normalise_rvec(&obs2); normalise_rvec(&cell1); @@ -553,96 +744,118 @@ static int obs_shares_spot_w_array(struct SpotVec *obs_vecs, int test_idx, } -static int obs_vecs_match_angles(struct SpotVec *her_obs, - struct SpotVec *his_obs, - int **her_match_idxs, int **his_match_idxs, - int *match_count, struct TakeTwoCell *cell) +static int obs_vecs_match_angles(int her, int his, + struct Seed **seeds, int *match_count, + struct TakeTwoCell *cell) { - int i, j; - *match_count = 0; + struct SpotVec *obs_vecs = cell->obs_vecs; + struct SpotVec *her_obs = &obs_vecs[her]; + struct SpotVec *his_obs = &obs_vecs[his]; + + *match_count = 0; + + double min_angle = deg2rad(2.5); + double max_angle = deg2rad(187.5); - double min_angle = deg2rad(2.5); - double max_angle = deg2rad(187.5); + /* calculate angle between observed vectors */ + double obs_angle = rvec_angle(her_obs->obsvec, his_obs->obsvec); - /* calculate angle between observed vectors */ - double obs_angle = rvec_angle(her_obs->obsvec, his_obs->obsvec); + /* calculate angle between all potential theoretical vectors */ + + int i, j; + for ( i=0; i<her_obs->match_num; i++ ) { + for ( j=0; j<his_obs->match_num; j++ ) { + double score = 0; - /* calculate angle between all potential theoretical vectors */ + struct rvec *her_match = &her_obs->matches[i].vec; + struct rvec *his_match = &his_obs->matches[j].vec; - for ( i=0; i<her_obs->match_num; i++ ) { - for ( j=0; j<his_obs->match_num; j++ ) { + double her_dist = rvec_length(*her_match); + double his_dist = rvec_length(*his_match); - struct rvec *her_match = &her_obs->matches[i]; - struct rvec *his_match = &his_obs->matches[j]; + double theory_angle = rvec_angle(*her_match, + *his_match); - double theory_angle = rvec_angle(*her_match, - *his_match); + /* is this angle a match? */ - /* is this angle a match? */ + double angle_diff = fabs(theory_angle - obs_angle); - double angle_diff = fabs(theory_angle - obs_angle); + /* within basic tolerance? */ + if ( angle_diff != angle_diff || + angle_diff > cell->angle_tol ) { + continue; + } + + double add = angle_diff; + if (add == add) { + score += add * her_dist * his_dist; + } - if ( angle_diff < cell->angle_tol ) { + /* If the angles are too close to 0 + or 180, one axis ill-determined */ + if (theory_angle < min_angle || + theory_angle > max_angle) { + continue; + } - // in the case of a brief check only - if (!her_match_idxs || !his_match_idxs) { - return 1; - } + /* check that third vector adequately completes + * triangle */ - /* If the angles are too close to 0 - or 180, one axis ill-determined */ - if (theory_angle < min_angle || - theory_angle > max_angle) { - continue; - } + struct rvec theory_diff = diff_vec(*his_match, *her_match); + struct rvec obs_diff = diff_vec(his_obs->obsvec, + her_obs->obsvec); - // check the third vector + theory_angle = rvec_angle(*her_match, + theory_diff); + obs_angle = rvec_angle(her_obs->obsvec, obs_diff); + angle_diff = fabs(obs_angle - theory_angle); - struct rvec theory_diff = diff_vec(*his_match, *her_match); - struct rvec obs_diff = diff_vec(his_obs->obsvec, - her_obs->obsvec); + double diff_dist = rvec_length(obs_diff); - theory_angle = rvec_angle(*her_match, - theory_diff); - obs_angle = rvec_angle(her_obs->obsvec, obs_diff); + if (angle_diff > ANGLE_TOLERANCE) { + continue; + } - if (fabs(obs_angle - theory_angle) > ANGLE_TOLERANCE) { - continue; - } + add = angle_diff; + if (add == add) { + score += add * her_dist * diff_dist; + } - theory_angle = rvec_angle(*his_match, - theory_diff); - obs_angle = rvec_angle(his_obs->obsvec, obs_diff); + theory_angle = rvec_angle(*his_match, + theory_diff); + obs_angle = rvec_angle(his_obs->obsvec, obs_diff); - if (fabs(obs_angle - theory_angle) > ANGLE_TOLERANCE) { - continue; - } + if (fabs(obs_angle - theory_angle) > ANGLE_TOLERANCE) { + continue; + } - size_t new_size = (*match_count + 1) * - sizeof(int); - if (her_match_idxs && his_match_idxs) - { - /* Reallocate the array to fit in another match */ - int *temp_hers; - int *temp_his; - temp_hers = realloc(*her_match_idxs, new_size); - temp_his = realloc(*his_match_idxs, new_size); + add = angle_diff; + if (add == add) { + score += add * his_dist * diff_dist; + } - if ( temp_hers == NULL || temp_his == NULL ) { - apologise(); - } + /* we add a new seed to the array */ + size_t new_size = (*match_count + 1); + new_size *= sizeof(struct Seed); - (*her_match_idxs) = temp_hers; - (*his_match_idxs) = temp_his; + /* Reallocate the array to fit in another match */ + struct Seed *tmp_seeds = realloc(*seeds, new_size); - (*her_match_idxs)[*match_count] = i; - (*his_match_idxs)[*match_count] = j; - } + if ( tmp_seeds == NULL ) { + apologise(); + } + + (*seeds) = tmp_seeds; + + (*seeds)[*match_count].obs1 = her; + (*seeds)[*match_count].obs2 = his; + (*seeds)[*match_count].idx1 = i; + (*seeds)[*match_count].idx2 = j; + (*seeds)[*match_count].score = score * 1000; (*match_count)++; } } - } return (*match_count > 0); } @@ -672,8 +885,8 @@ static signed int finish_solution(gsl_matrix *rot, struct SpotVec *obs_vecs, struct rvec i_obsvec = i_vec.obsvec; struct rvec j_obsvec = j_vec.obsvec; - struct rvec i_cellvec = i_vec.matches[match_members[i]]; - struct rvec j_cellvec = j_vec.matches[match_members[j]]; + struct rvec i_cellvec = i_vec.matches[match_members[i]].vec; + struct rvec j_cellvec = j_vec.matches[match_members[j]].vec; rotations[count] = generate_rot_mat(i_obsvec, j_obsvec, i_cellvec, j_cellvec, @@ -716,10 +929,26 @@ static signed int finish_solution(gsl_matrix *rot, struct SpotVec *obs_vecs, return 1; } -static int weed_duplicate_matches(struct SpotVec *her_obs, - struct SpotVec *his_obs, - int **her_match_idxs, int **his_match_idxs, - int *match_count, struct TakeTwoCell *cell) +gsl_matrix *rot_mat_from_indices(int her, int his, + int her_match, int his_match, + struct TakeTwoCell *cell) +{ + struct SpotVec *obs_vecs = cell->obs_vecs; + struct SpotVec *her_obs = &obs_vecs[her]; + struct SpotVec *his_obs = &obs_vecs[his]; + struct rvec i_obsvec = her_obs->obsvec; + struct rvec j_obsvec = his_obs->obsvec; + struct rvec i_cellvec = her_obs->matches[her_match].vec; + struct rvec j_cellvec = his_obs->matches[his_match].vec; + + gsl_matrix *mat = generate_rot_mat(i_obsvec, j_obsvec, + i_cellvec, j_cellvec, cell); + + return mat; +} + +static int weed_duplicate_matches(struct Seed **seeds, + int *match_count, struct TakeTwoCell *cell) { int num_occupied = 0; gsl_matrix **old_mats = calloc(*match_count, sizeof(gsl_matrix *)); @@ -731,19 +960,18 @@ static int weed_duplicate_matches(struct SpotVec *her_obs, } signed int i, j; + int duplicates = 0; - for (i = *match_count - 1; i >= 0; i--) { - int her_match = (*her_match_idxs)[i]; - int his_match = (*his_match_idxs)[i]; + /* Now we weed out the self-duplicates from the remaining batch */ - struct rvec i_obsvec = her_obs->obsvec; - struct rvec j_obsvec = his_obs->obsvec; - struct rvec i_cellvec = her_obs->matches[her_match]; - struct rvec j_cellvec = his_obs->matches[his_match]; + for (i = *match_count - 1; i >= 0; i--) { + int her_match = (*seeds)[i].idx1; + int his_match = (*seeds)[i].idx2; - gsl_matrix *mat = generate_rot_mat(i_obsvec, j_obsvec, - i_cellvec, j_cellvec, cell); + gsl_matrix *mat; + mat = rot_mat_from_indices((*seeds)[i].obs1, (*seeds)[i].obs2, + her_match, his_match, cell); int found = 0; @@ -752,8 +980,8 @@ static int weed_duplicate_matches(struct SpotVec *her_obs, symm_rot_mats_are_similar(old_mats[j], mat, cell)) { // we have found a duplicate, so flag as bad. - (*her_match_idxs)[i] = -1; - (*his_match_idxs)[i] = -1; + (*seeds)[i].idx1 = -1; + (*seeds)[i].idx2 = -1; found = 1; duplicates++; @@ -785,7 +1013,7 @@ static signed int find_next_index(gsl_matrix *rot, int *obs_members, int *match_members, int start, int member_num, int *match_found, struct TakeTwoCell *cell) { - struct SpotVec *obs_vecs = *(cell->obs_vecs); + struct SpotVec *obs_vecs = cell->obs_vecs; int obs_vec_count = cell->obs_vec_count; gsl_matrix *sub = gsl_matrix_calloc(3, 3); gsl_matrix *mul = gsl_matrix_calloc(3, 3); @@ -794,42 +1022,42 @@ static signed int find_next_index(gsl_matrix *rot, int *obs_members, for ( i=start; i<obs_vec_count; i++ ) { + /* If we've considered this vector before, ignore it */ + if (obs_vecs[i].in_network == 1) + { + continue; + } + /* first we check for a shared spot - harshest condition */ int shared = obs_shares_spot_w_array(obs_vecs, i, obs_members, member_num); if ( !shared ) continue; - int skip = 0; - for ( j=0; j<member_num && skip == 0; j++ ) { - if (i == obs_members[j]) { - skip = 1; - } - } - - if (skip) { - continue; - } - int all_ok = 1; int matched = -1; + /* Check all existing members are happy to let in the newcomer */ for ( j=0; j<member_num && all_ok; j++ ) { struct SpotVec *me = &obs_vecs[i]; struct SpotVec *you = &obs_vecs[obs_members[j]]; - struct rvec you_cell = you->matches[match_members[j]]; + struct rvec you_cell; + you_cell = you->matches[match_members[j]].vec; struct rvec me_obs = me->obsvec; struct rvec you_obs = you->obsvec; int one_is_okay = 0; + /* Loop through all possible theoretical vector + * matches for the newcomer.. */ + for ( k=0; k<me->match_num; k++ ) { gsl_matrix *test_rot; - struct rvec me_cell = me->matches[k]; + struct rvec me_cell = me->matches[k].vec; test_rot = generate_rot_mat(me_obs, you_obs, me_cell, you_cell, @@ -844,6 +1072,11 @@ static signed int find_next_index(gsl_matrix *rot, int *obs_members, if (ok) { one_is_okay = 1; + /* We are only happy if the vector + * matches for only one kind of + * theoretical vector. We don't want to + * accept mixtures of theoretical vector + * matches. */ if (matched >= 0 && k == matched) { *match_found = k; } else if (matched < 0) { @@ -863,12 +1096,6 @@ static signed int find_next_index(gsl_matrix *rot, int *obs_members, if (all_ok) { - - for ( j=0; j<member_num; j++ ) { - // STATUS("%i ", obs_members[j]); - } - //STATUS("%i\n", i); - return i; } } @@ -878,16 +1105,212 @@ static signed int find_next_index(gsl_matrix *rot, int *obs_members, return -1; } +/** + * Reward target function for refining solution to all vectors in a + * given image. Sum of exponentials of the negative distances, which + * means that the reward decays as the distance from the nearest + * theoretical vector reduces. */ +static double obs_to_sol_score(struct TakeTwoCell *ttCell) +{ + double total = 0; + int count = 0; + int i; + gsl_matrix *solution = ttCell->solution; + gsl_matrix *full_rot = gsl_matrix_alloc(3, 3); + rotate_gsl_by_angles(solution, ttCell->x_ang, ttCell->y_ang, + ttCell->z_ang, full_rot); + + for (i = 0; i < ttCell->obs_vec_count; i++) + { + struct rvec *obs = &ttCell->obs_vecs[i].obsvec; + rvec_to_gsl(ttCell->vec1Tmp, *obs); + + /* Rotate all the observed vectors by the modified soln */ + /* ttCell->vec2Tmp = 1.0 * full_rot * ttCell->vec1Tmp */ + gsl_blas_dgemv(CblasTrans, 1.0, full_rot, ttCell->vec1Tmp, + 0.0, ttCell->vec2Tmp); + struct rvec rotated = gsl_to_rvec(ttCell->vec2Tmp); + + int j = ttCell->obs_vecs[i].assignment; + + if (j < 0) continue; + + struct rvec *match = &ttCell->obs_vecs[i].matches[j].vec; + struct rvec diff = diff_vec(rotated, *match); + + double length = rvec_length(diff); + + double addition = exp(-(1 / RECIP_TOLERANCE) * length); + total += addition; + count++; + } + + total /= (double)-count; + + gsl_matrix_free(full_rot); + + return total; +} + +/** + * Matches every observed vector in the image to its closest theoretical + * neighbour after applying the rotation matrix, in preparation for + * refining the rotation matrix to match these. */ +static void match_all_obs_to_sol(struct TakeTwoCell *ttCell) +{ + int i, j; + double total = 0; + int count = 0; + gsl_matrix *solution = ttCell->solution; + + for (i = 0; i < ttCell->obs_vec_count; i++) + { + struct rvec *obs = &ttCell->obs_vecs[i].obsvec; + rvec_to_gsl(ttCell->vec1Tmp, *obs); + + /* ttCell->vec2Tmp = 1.0 * solution * ttCell->vec1Tmp */ + gsl_blas_dgemv(CblasTrans, 1.0, solution, ttCell->vec1Tmp, + 0.0, ttCell->vec2Tmp); + struct rvec rotated = gsl_to_rvec(ttCell->vec2Tmp); + + double smallest = FLT_MAX; + int assigned = -1; + + for (j = 0; j < ttCell->obs_vecs[i].match_num; j++) + { + struct rvec *match = &ttCell->obs_vecs[i].matches[j].vec; + struct rvec diff = diff_vec(rotated, *match); + + double length = rvec_length(diff); + if (length < smallest) + { + smallest = length; + assigned = j; + } + } + + ttCell->obs_vecs[i].assignment = assigned; + + if (smallest != FLT_MAX) + { + double addition = exp(-(1 / RECIP_TOLERANCE) * smallest); + total += addition; + count++; -static int grow_network(gsl_matrix *rot, int obs_idx1, int obs_idx2, - int match_idx1, int match_idx2, int *max_members, - struct TakeTwoCell *cell) + } + } + + total /= (double)count; +} + +/** + * Refines a matrix against all of the observed vectors against their + * closest theoretical neighbour, by perturbing the matrix along the principle + * axes until it maximises a reward function consisting of the sum of + * the distances of individual observed vectors to their closest + * theoretical neighbour. Reward function means that noise and alternative + * lattices do not dominate the equation! + **/ +static void refine_solution(struct TakeTwoCell *ttCell) { - struct SpotVec *obs_vecs = *(cell->obs_vecs); + match_all_obs_to_sol(ttCell); + + int i, j, k; + const int total = 3 * 3 * 3; + const int middle = (total - 1) / 2; + + struct rvec steps[total]; + double start = obs_to_sol_score(ttCell); + const int max_tries = 100; + + int count = 0; + double size = ANGLE_STEP_SIZE; + + /* First we create our combinations of steps */ + for (i = -1; i <= 1; i++) { + for (j = -1; j <= 1; j++) { + for (k = -1; k <= 1; k++) { + struct rvec vec = new_rvec(i, j, k); + steps[count] = vec; + count++; + } + } + } + + struct rvec current = new_rvec(ttCell->x_ang, ttCell->y_ang, + ttCell->z_ang); + + double best = start; + count = 0; + + while (size > ANGLE_CONVERGE_SIZE && count < max_tries) + { + struct rvec sized[total]; + + int best_num = middle; + for (i = 0; i < total; i++) + { + struct rvec sized_step = steps[i]; + sized_step.u *= size; + sized_step.v *= size; + sized_step.w *= size; + + struct rvec new_angles = rvec_add_rvec(current, + sized_step); + + sized[i] = new_angles; + + ttCell->x_ang = sized[i].u; + ttCell->y_ang = sized[i].v; + ttCell->z_ang = sized[i].w; + + double score = obs_to_sol_score(ttCell); + + if (score < best) + { + best = score; + best_num = i; + } + } + + if (best_num == middle) + { + size /= 2; + } + + current = sized[best_num]; + count++; + } + + ttCell->x_ang = 0; + ttCell->y_ang = 0; + ttCell->z_ang = 0; + + gsl_matrix *tmp = gsl_matrix_alloc(3, 3); + rotate_gsl_by_angles(ttCell->solution, current.u, + current.v, current.w, tmp); + gsl_matrix_free(ttCell->solution); + ttCell->solution = tmp; +} + + +static unsigned int grow_network(gsl_matrix *rot, int obs_idx1, int obs_idx2, + int match_idx1, int match_idx2, + struct TakeTwoCell *cell) +{ + + struct SpotVec *obs_vecs = cell->obs_vecs; int obs_vec_count = cell->obs_vec_count; int *obs_members; int *match_members; + /* Clear the in_network status of all vectors to start */ + int i; + for (i = 0; i < obs_vec_count; i++) + { + obs_vecs[i].in_network = 0; + } + /* indices of members of the self-consistent network of vectors */ obs_members = malloc((cell->member_thresh+3)*sizeof(int)); match_members = malloc((cell->member_thresh+3)*sizeof(int)); @@ -902,7 +1325,6 @@ static int grow_network(gsl_matrix *rot, int obs_idx1, int obs_idx2, match_members[0] = match_idx1; match_members[1] = match_idx2; int member_num = 2; - *max_members = 2; /* counter for dead ends which must not exceed MAX_DEAD_ENDS * before it is reset in an additional branch */ @@ -923,7 +1345,7 @@ static int grow_network(gsl_matrix *rot, int obs_idx1, int obs_idx2, signed int next_index = find_next_index(rot, obs_members, match_members, - start, member_num, + 0, member_num, &match_found, cell); if ( member_num < 2 ) { @@ -942,25 +1364,19 @@ static int grow_network(gsl_matrix *rot, int obs_idx1, int obs_idx2, /* We have not had too many dead ends. Try removing the last member and continue. */ - start = obs_members[member_num - 1] + 1; member_num--; dead_ends++; continue; } - /* we have elongated membership - so reset dead_ends counter */ - // dead_ends = 0; - + /* Elongation of the network was successful */ + obs_vecs[next_index].in_network = 1; obs_members[member_num] = next_index; match_members[member_num] = match_found; member_num++; - if (member_num > *max_members) { - *max_members = member_num; - } - /* If member_num is high enough, we want to return a yes */ if ( member_num > cell->member_thresh ) break; @@ -972,126 +1388,210 @@ static int grow_network(gsl_matrix *rot, int obs_idx1, int obs_idx2, free(obs_members); free(match_members); - return ( member_num > cell->member_thresh ); + return ( member_num ); } -static int start_seed(int i, int j, int i_match, int j_match, - gsl_matrix **rotation, int *max_members, - struct TakeTwoCell *cell) +static unsigned int start_seed(int i, int j, int i_match, int j_match, + gsl_matrix **rotation, struct TakeTwoCell *cell) { - struct SpotVec *obs_vecs = *(cell->obs_vecs); + struct SpotVec *obs_vecs = cell->obs_vecs; gsl_matrix *rot_mat; rot_mat = generate_rot_mat(obs_vecs[i].obsvec, obs_vecs[j].obsvec, - obs_vecs[i].matches[i_match], - obs_vecs[j].matches[j_match], + obs_vecs[i].matches[i_match].vec, + obs_vecs[j].matches[j_match].vec, cell); /* Try to expand this rotation matrix to a larger network */ - int success = grow_network(rot_mat, i, j, i_match, j_match, max_members, - cell); + int member_num = grow_network(rot_mat, i, j, i_match, j_match, + cell); /* return this matrix and if it was immediately successful */ *rotation = rot_mat; - return success; + return member_num; } -static int find_seed(gsl_matrix **rotation, struct TakeTwoCell *cell) +static int sort_seed_by_score(const void *av, const void *bv) { - struct SpotVec *obs_vecs = *(cell->obs_vecs); - int obs_vec_count = cell->obs_vec_count; + struct Seed *a = (struct Seed *)av; + struct Seed *b = (struct Seed *)bv; + return a->score > b->score; +} + +static void remove_old_solutions(struct TakeTwoCell *cell, + struct taketwo_private *tp) +{ + int duplicates = 0; + struct Seed *seeds = cell->seeds; + unsigned int total = cell->seed_count; + + /* First we remove duplicates with previous solutions */ + + int i, j; + for (i = total - 1; i >= 0; i--) { + int her_match = seeds[i].idx1; + int his_match = seeds[i].idx2; + + gsl_matrix *mat; + mat = rot_mat_from_indices(seeds[i].obs1, seeds[i].obs2, + her_match, his_match, cell); + + if (mat == NULL) + { + continue; + } + + for (j = 0; j < tp->numPrevs; j++) + { + int sim = symm_rot_mats_are_similar(tp->prevSols[j], + mat, cell); + + /* Found a duplicate with a previous solution */ + if (sim) + { + seeds[i].idx1 = -1; + seeds[i].idx2 = -1; + duplicates++; + break; + } + } - /* META: Maximum achieved maximum network membership */ - int max_max_members = 0; - gsl_matrix *best_rotation = NULL; + gsl_matrix_free(mat); + } + +// STATUS("Removing %i duplicates due to prev solutions.\n", duplicates); +} -// unsigned long start_time = time(NULL); +static int find_seeds(struct TakeTwoCell *cell, struct taketwo_private *tp) +{ + struct SpotVec *obs_vecs = cell->obs_vecs; + int obs_vec_count = cell->obs_vec_count; /* loop round pairs of vectors to try and find a suitable * seed to start building a self-consistent network of vectors */ int i, j; - for ( i=0; i<obs_vec_count; i++ ) { + for ( i=1; i<obs_vec_count; i++ ) { + for ( j=0; j<i; j++ ) { + /** Only check distances which are accumulatively less + * than the limit if we can easily generate seeds */ + if (obs_vecs[j].distance + obs_vecs[i].distance > + MAX_RECIP_DISTANCE && cell->seed_count > 100) { + continue; + } + /** Check to see if there is a shared spot - opportunity * for optimisation by generating a look-up table * by spot instead of by vector. */ - int shared = obs_vecs_share_spot(&obs_vecs[i], &obs_vecs[j]); - + int shared = obs_vecs_share_spot(&obs_vecs[i], + &obs_vecs[j]); if ( !shared ) continue; /* cell vector index matches stored in i, j and total * number stored in int matches. */ - int *i_idx = 0; - int *j_idx = 0; - int matches; - - /* Check to see if any angles match from the cell vectors */ - obs_vecs_match_angles(&obs_vecs[i], &obs_vecs[j], - &i_idx, &j_idx, &matches, cell); - if ( matches == 0 ) { - free(i_idx); - free(j_idx); + int seed_num = 0; + struct Seed *seeds = NULL; + + /* Check to see if any angles match from the cell + * vectors */ + obs_vecs_match_angles(i, j, &seeds, &seed_num, cell); + + if (seed_num == 0) + { + /* Nothing to clean up here */ continue; } /* Weed out the duplicate seeds (from symmetric - * reflection pairs) - */ + * reflection pairs) */ + weed_duplicate_matches(&seeds, &seed_num, cell); + + /* Add all the new seeds to the full list */ + + size_t new_size = cell->seed_count + seed_num; + new_size *= sizeof(struct Seed); + + struct Seed *tmp = realloc(cell->seeds, new_size); - weed_duplicate_matches(&obs_vecs[i], &obs_vecs[j], - &i_idx, &j_idx, &matches, cell); + if (tmp == NULL) { + apologise(); + } + + cell->seeds = tmp; - /* We have seeds! Pass each of them through the seed-starter */ - /* If a seed has the highest achieved membership, make note...*/ - int k; - for ( k=0; k<matches; k++ ) { - if (i_idx[k] < 0 || j_idx[k] < 0) { + for (int i = 0; i < seed_num; i++) + { + if (seeds[i].idx1 < 0 || seeds[i].idx2 < 0) + { continue; } - int max_members = 0; - - int success = start_seed(i, j, - i_idx[k], j_idx[k], - rotation, &max_members, - cell); - - if (success) { - free(i_idx); free(j_idx); - gsl_matrix_free(best_rotation); - return success; - } else { - if (max_members > max_max_members) { - max_max_members = max_members; - gsl_matrix_free(best_rotation); - best_rotation = *rotation; - *rotation = NULL; - } else { - gsl_matrix_free(*rotation); - *rotation = NULL; - } - } + cell->seeds[cell->seed_count] = seeds[i]; + cell->seed_count++; } + } + } + + qsort(cell->seeds, cell->seed_count, sizeof(struct Seed), + sort_seed_by_score); + + + return 1; +} + +static unsigned int start_seeds(gsl_matrix **rotation, struct TakeTwoCell *cell) +{ + struct Seed *seeds = cell->seeds; + int seed_num = cell->seed_count; + int member_num = 0; + int max_members = 0; + gsl_matrix *rot = NULL; + + /* We have seeds! Pass each of them through the seed-starter */ + /* If a seed has the highest achieved membership, make note...*/ + int k; + for ( k=0; k<seed_num; k++ ) { + int seed_idx1 = seeds[k].idx1; + int seed_idx2 = seeds[k].idx2; + + if (seed_idx1 < 0 || seed_idx2 < 0) { + continue; + } + + int seed_obs1 = seeds[k].obs1; + int seed_obs2 = seeds[k].obs2; - free(i_idx); - free(j_idx); + member_num = start_seed(seed_obs1, seed_obs2, seed_idx1, + seed_idx2, &rot, cell); + + if (member_num > max_members) + { + *rotation = rot; + max_members = member_num; + } + + if (member_num >= NETWORK_MEMBER_THRESHOLD) { + free(seeds); + return max_members; } - } /* yes this } is meant to be here */ + } + + free(seeds); - *rotation = best_rotation; - return (best_rotation != NULL); + return max_members; } + static void set_gsl_matrix(gsl_matrix *mat, double asx, double asy, double asz, double bsx, double bsy, double bsz, double csx, double csy, double csz) @@ -1184,25 +1684,24 @@ static int generate_rotation_sym_ops(struct TakeTwoCell *ttCell) return 1; } - struct sortme { - struct rvec v; + struct TheoryVec v; double dist; }; -static int sort_func(const void *av, const void *bv) +static int sort_theory_distances(const void *av, const void *bv) { struct sortme *a = (struct sortme *)av; struct sortme *b = (struct sortme *)bv; return a->dist > b->dist; } - -static int match_obs_to_cell_vecs(struct rvec *cell_vecs, int cell_vec_count, - struct SpotVec *obs_vecs, int obs_vec_count, - int is_asymmetric, struct TakeTwoCell *cell) +static int match_obs_to_cell_vecs(struct TheoryVec *cell_vecs, int cell_vec_count, + struct TakeTwoCell *cell) { + struct SpotVec *obs_vecs = cell->obs_vecs; + int obs_vec_count = cell->obs_vec_count; int i, j; for ( i=0; i<obs_vec_count; i++ ) { @@ -1212,7 +1711,7 @@ static int match_obs_to_cell_vecs(struct rvec *cell_vecs, int cell_vec_count, for ( j=0; j<cell_vec_count; j++ ) { /* get distance for unit cell vector */ - double cell_length = rvec_length(cell_vecs[j]); + double cell_length = rvec_length(cell_vecs[j].vec); double obs_length = obs_vecs[i].distance; /* check if this matches the observed length */ @@ -1235,20 +1734,15 @@ static int match_obs_to_cell_vecs(struct rvec *cell_vecs, int cell_vec_count, /* Pointers to relevant things */ - struct rvec **match_array; + struct TheoryVec **match_array; int *match_count; - if (!is_asymmetric) { - match_array = &(obs_vecs[i].matches); - match_count = &(obs_vecs[i].match_num); - } else { - match_array = &(obs_vecs[i].asym_matches); - match_count = &(obs_vecs[i].asym_match_num); - } + match_array = &(obs_vecs[i].matches); + match_count = &(obs_vecs[i].match_num); /* Sort in order to get most agreeable matches first */ - qsort(for_sort, count, sizeof(struct sortme), sort_func); - *match_array = malloc(count*sizeof(struct rvec)); + qsort(for_sort, count, sizeof(struct sortme), sort_theory_distances); + *match_array = malloc(count*sizeof(struct TheoryVec)); *match_count = count; for ( j=0; j<count; j++ ) { (*match_array)[j] = for_sort[j].v; @@ -1268,7 +1762,7 @@ static int compare_spot_vecs(const void *av, const void *bv) } static int gen_observed_vecs(struct rvec *rlps, int rlp_count, - struct SpotVec **obs_vecs, int *obs_vec_count) + struct TakeTwoCell *cell) { int i, j; int count = 0; @@ -1279,7 +1773,6 @@ static int gen_observed_vecs(struct rvec *rlps, int rlp_count, for ( i=0; i<rlp_count-1 && count < MAX_OBS_VECTORS; i++ ) { for ( j=i+1; j<rlp_count; j++ ) { - /* calculate difference vector between rlps */ struct rvec diff = diff_vec(rlps[i], rlps[j]); @@ -1291,13 +1784,13 @@ static int gen_observed_vecs(struct rvec *rlps, int rlp_count, count++; struct SpotVec *temp_obs_vecs; - temp_obs_vecs = realloc(*obs_vecs, + temp_obs_vecs = realloc(cell->obs_vecs, count*sizeof(struct SpotVec)); if ( temp_obs_vecs == NULL ) { return 0; } else { - *obs_vecs = temp_obs_vecs; + cell->obs_vecs = temp_obs_vecs; /* initialise all SpotVec struct members */ @@ -1305,29 +1798,27 @@ static int gen_observed_vecs(struct rvec *rlps, int rlp_count, spot_vec.obsvec = diff; spot_vec.distance = sqrt(sqlength); spot_vec.matches = NULL; + spot_vec.assignment = -1; spot_vec.match_num = 0; spot_vec.her_rlp = &rlps[i]; spot_vec.his_rlp = &rlps[j]; - (*obs_vecs)[count - 1] = spot_vec; + cell->obs_vecs[count - 1] = spot_vec; } } } - /* Sort such that the shortest and least error-prone distances - are searched first. - */ - qsort(*obs_vecs, count, sizeof(struct SpotVec), compare_spot_vecs); + /* Sort such that the shortest distances are searched first. */ + qsort(cell->obs_vecs, count, sizeof(struct SpotVec), compare_spot_vecs); - *obs_vec_count = count; + cell->obs_vec_count = count; return 1; } -static int gen_theoretical_vecs(UnitCell *cell, struct rvec **cell_vecs, - struct rvec **asym_vecs, int *vec_count, - int *asym_vec_count) +static int gen_theoretical_vecs(UnitCell *cell, struct TheoryVec **cell_vecs, + unsigned int *vec_count) { double a, b, c, alpha, beta, gamma; int h_max, k_max, l_max; @@ -1351,7 +1842,6 @@ static int gen_theoretical_vecs(UnitCell *cell, struct rvec **cell_vecs, int h, k, l; int _h, _k, _l; int count = 0; - int asym_count = 0; for ( h=-h_max; h<=+h_max; h++ ) { for ( k=-k_max; k<=+k_max; k++ ) { @@ -1367,50 +1857,37 @@ static int gen_theoretical_vecs(UnitCell *cell, struct rvec **cell_vecs, if (h == _h && k == _k && l == _l) { asymmetric = 1; - asym_count++; } cell_vec.u = h*asx + k*bsx + l*csx; cell_vec.v = h*asy + k*bsy + l*csy; cell_vec.w = h*asz + k*bsz + l*csz; + struct TheoryVec theory; + theory.vec = cell_vec; + theory.asym = asymmetric; + /* add this to our array - which may require expanding */ count++; - struct rvec *temp_cell_vecs; - temp_cell_vecs = realloc(*cell_vecs, count*sizeof(struct rvec)); - struct rvec *temp_asym_vecs = NULL; - - if (asymmetric) { - temp_asym_vecs = realloc(*asym_vecs, - count*sizeof(struct rvec)); - } + struct TheoryVec *temp_cell_vecs; + temp_cell_vecs = realloc(*cell_vecs, + count*sizeof(struct TheoryVec)); if ( temp_cell_vecs == NULL ) { return 0; - } else if (asymmetric && temp_asym_vecs == NULL) { - return 0; } else { *cell_vecs = temp_cell_vecs; - (*cell_vecs)[count - 1] = cell_vec; - - if (!asymmetric) { - continue; - } - - *asym_vecs = temp_asym_vecs; - (*asym_vecs)[asym_count - 1] = cell_vec; + (*cell_vecs)[count - 1] = theory; } } } } *vec_count = count; - *asym_vec_count = asym_count; free_symoplist(rawList); - return 1; } @@ -1425,7 +1902,6 @@ static void cleanup_taketwo_obs_vecs(struct SpotVec *obs_vecs, int i; for ( i=0; i<obs_vec_count; i++ ) { free(obs_vecs[i].matches); - free(obs_vecs[i].asym_matches); } free(obs_vecs); @@ -1433,11 +1909,16 @@ static void cleanup_taketwo_obs_vecs(struct SpotVec *obs_vecs, static void cleanup_taketwo_cell(struct TakeTwoCell *ttCell) { + /* n.b. solutions in ttCell are taken care of in the + * partial taketwo cleanup. */ int i; for ( i=0; i<ttCell->numOps; i++ ) { gsl_matrix_free(ttCell->rotSymOps[i]); } + cleanup_taketwo_obs_vecs(ttCell->obs_vecs, + ttCell->obs_vec_count); + free(ttCell->vec1Tmp); free(ttCell->vec2Tmp); free(ttCell->twiz1Tmp); @@ -1459,39 +1940,38 @@ static void cleanup_taketwo_cell(struct TakeTwoCell *ttCell) * successful. **/ static UnitCell *run_taketwo(UnitCell *cell, const struct taketwo_options *opts, - struct rvec *rlps, int rlp_count) + struct rvec *rlps, int rlp_count, + struct taketwo_private *tp) { - int cell_vec_count = 0; - int asym_vec_count = 0; - struct rvec *cell_vecs = NULL; - struct rvec *asym_vecs = NULL; UnitCell *result; - int obs_vec_count = 0; - struct SpotVec *obs_vecs = NULL; int success = 0; gsl_matrix *solution = NULL; + /* Initialise TakeTwoCell */ struct TakeTwoCell ttCell; ttCell.cell = cell; + ttCell.seeds = NULL; + ttCell.seed_count = 0; ttCell.rotSymOps = NULL; + ttCell.obs_vecs = NULL; ttCell.twiz1Tmp = gsl_matrix_calloc(3, 3); ttCell.twiz2Tmp = gsl_matrix_calloc(3, 3); ttCell.vec1Tmp = gsl_vector_calloc(3); ttCell.vec2Tmp = gsl_vector_calloc(3); ttCell.numOps = 0; + ttCell.obs_vec_count = 0; + ttCell.solution = NULL; + ttCell.x_ang = 0; + ttCell.y_ang = 0; + ttCell.z_ang = 0; success = generate_rotation_sym_ops(&ttCell); - success = gen_theoretical_vecs(cell, &cell_vecs, &asym_vecs, - &cell_vec_count, &asym_vec_count); if ( !success ) return NULL; - success = gen_observed_vecs(rlps, rlp_count, &obs_vecs, &obs_vec_count); + success = gen_observed_vecs(rlps, rlp_count, &ttCell); if ( !success ) return NULL; - ttCell.obs_vecs = &obs_vecs; - ttCell.obs_vec_count = obs_vec_count; - if ( opts->member_thresh < 0 ) { ttCell.member_thresh = NETWORK_MEMBER_THRESHOLD; } else { @@ -1516,32 +1996,79 @@ static UnitCell *run_taketwo(UnitCell *cell, const struct taketwo_options *opts, ttCell.trace_tol = sqrt(4.0*(1.0-cos(opts->trace_tol))); } - success = match_obs_to_cell_vecs(asym_vecs, asym_vec_count, - obs_vecs, obs_vec_count, 1, &ttCell); - - success = match_obs_to_cell_vecs(cell_vecs, cell_vec_count, - obs_vecs, obs_vec_count, 0, &ttCell); - - free(cell_vecs); - free(asym_vecs); + success = match_obs_to_cell_vecs(tp->theory_vecs, tp->vec_count, + &ttCell); if ( !success ) return NULL; - find_seed(&solution, &ttCell); + /* Find all the seeds, then take each one and extend them, returning + * a solution if it exceeds the NETWORK_MEMBER_THRESHOLD. */ + find_seeds(&ttCell, tp); + remove_old_solutions(&ttCell, tp); + unsigned int members = start_seeds(&solution, &ttCell); if ( solution == NULL ) { - cleanup_taketwo_obs_vecs(obs_vecs, obs_vec_count); return NULL; } + /* If we have a solution, refine against vectors in the entire image */ + ttCell.solution = solution; + refine_solution(&ttCell); + solution = ttCell.solution; + double score = obs_to_sol_score(&ttCell); + + /* Add the current solution to the previous solutions list */ + int new_size = (tp->numPrevs + 1) * sizeof(gsl_matrix *); + gsl_matrix **tmp = realloc(tp->prevSols, new_size); + double *tmpScores = realloc(tp->prevScores, + (tp->numPrevs + 1) * sizeof(double)); + unsigned int *tmpSuccesses; + tmpSuccesses = realloc(tp->membership, + (tp->numPrevs + 1) * sizeof(unsigned int)); + + if (!tmp) { + apologise(); + } + + tp->prevSols = tmp; + tp->prevScores = tmpScores; + tp->membership = tmpSuccesses; + + tp->prevSols[tp->numPrevs] = solution; + tp->prevScores[tp->numPrevs] = score; + tp->membership[tp->numPrevs] = members; + tp->numPrevs++; + + /* Prepare the solution for CrystFEL friendliness */ result = transform_cell_gsl(cell, solution); - gsl_matrix_free(solution); - cleanup_taketwo_obs_vecs(obs_vecs, obs_vec_count); cleanup_taketwo_cell(&ttCell); return result; } +/* Cleans up the per-image information for taketwo */ + +static void partial_taketwo_cleanup(struct taketwo_private *tp) +{ + if (tp->prevSols != NULL) + { + for (int i = 0; i < tp->numPrevs; i++) + { + gsl_matrix_free(tp->prevSols[i]); + } + + free(tp->prevSols); + } + + free(tp->prevScores); + free(tp->membership); + tp->prevScores = NULL; + tp->membership = NULL; + tp->xtal_num = 0; + tp->numPrevs = 0; + tp->prevSols = NULL; + +} /* CrystFEL interface hooks */ @@ -1555,6 +2082,34 @@ int taketwo_index(struct image *image, const struct taketwo_options *opts, int i; struct taketwo_private *tp = (struct taketwo_private *)priv; + /* Check serial number against previous for solution tracking */ + int this_serial = image->serial; + + if (tp->serial_num == this_serial) + { + tp->xtal_num = image->n_crystals; + } + else + { + /* + for (i = 0; i < tp->numPrevs; i++) + { + STATUS("score, %i, %.5f, %i\n", + this_serial, tp->prevScores[i], + tp->membership[i]); + } + */ + + partial_taketwo_cleanup(tp); + tp->serial_num = this_serial; + tp->xtal_num = image->n_crystals; + } + + /* + STATUS("Indexing %i with %i attempts, %i crystals\n", this_serial, tp->attempts, + image->n_crystals); + */ + rlps = malloc((image_feature_count(image->features)+1)*sizeof(struct rvec)); for ( i=0; i<image_feature_count(image->features); i++ ) { struct imagefeature *pk = image_get_feature(image->features, i); @@ -1568,7 +2123,7 @@ int taketwo_index(struct image *image, const struct taketwo_options *opts, rlps[n_rlps].v = 0.0; rlps[n_rlps++].w = 0.0; - cell = run_taketwo(tp->cell, opts, rlps, n_rlps); + cell = run_taketwo(tp->cell, opts, rlps, n_rlps, tp); free(rlps); if ( cell == NULL ) return 0; @@ -1631,14 +2186,27 @@ void *taketwo_prepare(IndexingMethod *indm, UnitCell *cell) tp->cell = cell; tp->indm = *indm; + tp->serial_num = -1; + tp->xtal_num = 0; + tp->prevSols = NULL; + tp->numPrevs = 0; + tp->prevScores = NULL; + tp->membership = NULL; + tp->vec_count = 0; + tp->theory_vecs = NULL; + + gen_theoretical_vecs(cell, &tp->theory_vecs, &tp->vec_count); return tp; } - void taketwo_cleanup(IndexingPrivate *pp) { struct taketwo_private *tp = (struct taketwo_private *)pp; + + partial_taketwo_cleanup(tp); + free(tp->theory_vecs); + free(tp); } diff --git a/libcrystfel/src/xds.c b/libcrystfel/src/xds.c index 41580e41..b28dc93a 100644 --- a/libcrystfel/src/xds.c +++ b/libcrystfel/src/xds.c @@ -3,12 +3,12 @@ * * Invoke xds for crystal autoindexing * - * Copyright © 2013-2017 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2013-2018 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2013 Cornelius Gati * * Authors: - * 2010-2017 Thomas White <taw@physics.org> + * 2010-2018 Thomas White <taw@physics.org> * 2013 Cornelius Gati <cornelius.gati@cfel.de> * * This file is part of CrystFEL. @@ -54,7 +54,8 @@ #include "cell-utils.h" -#define XDS_VERBOSE 0 +/* Fake pixel size and camera length, both in metres */ +#define FAKE_PIXEL_SIZE (70e-6) #define FAKE_CLEN (0.1) @@ -66,142 +67,7 @@ struct xds_private }; -struct xds_data { - - /* Low-level stuff */ - int pty; - pid_t pid; - char *rbuffer; - int rbufpos; - int rbuflen; - - /* High-level stuff */ - int step; - int finished_ok; - UnitCell *target_cell; -}; - -static void xds_parseline(const char *line, struct image *image, - struct xds_data *xds) -{ -#if XDS_VERBOSE - char *copy; - int i; - - copy = strdup(line); - for ( i=0; i<strlen(copy); i++ ) { - if ( copy[i] == '\r' ) copy[i]='r'; - if ( copy[i] == '\n' ) copy[i]='\0'; - } - STATUS("XDS: %s\n", copy); - free(copy); -#endif -} - - -static int xds_readable(struct image *image, struct xds_data *xds) -{ - int rval; - int no_string = 0; - - rval = read(xds->pty, xds->rbuffer+xds->rbufpos, - xds->rbuflen-xds->rbufpos); - if ( (rval == -1) || (rval == 0) ) return 1; - - xds->rbufpos += rval; - assert(xds->rbufpos <= xds->rbuflen); - - while ( (!no_string) && (xds->rbufpos > 0) ) { - - int i; - int block_ready = 0; - - /* See if there's a full line in the buffer yet */ - for ( i=0; i<xds->rbufpos-1; i++ ) { - /* Means the last value looked at is rbufpos-2 */ - - if ( (xds->rbuffer[i] == '\r') - && (xds->rbuffer[i+1] == '\n') ) { - block_ready = 1; - break; - } - - } - - if ( block_ready ) { - - unsigned int new_rbuflen; - unsigned int endbit_length; - char *block_buffer = NULL; - - block_buffer = malloc(i+1); - memcpy(block_buffer, xds->rbuffer, i); - block_buffer[i] = '\0'; - - if ( block_buffer[0] == '\r' ) { - memmove(block_buffer, block_buffer+1, i); - } - - xds_parseline(block_buffer, image, xds); - free(block_buffer); - endbit_length = i+2; - - /* Now the block's been parsed, it should be - * forgotten about */ - memmove(xds->rbuffer, - xds->rbuffer + endbit_length, - xds->rbuflen - endbit_length); - - /* Subtract the number of bytes removed */ - xds->rbufpos = xds->rbufpos - - endbit_length; - new_rbuflen = xds->rbuflen - endbit_length; - if ( new_rbuflen == 0 ) new_rbuflen = 256; - xds->rbuffer = realloc(xds->rbuffer, - new_rbuflen); - xds->rbuflen = new_rbuflen; - - } else { - - if ( xds->rbufpos==xds->rbuflen ) { - - /* More buffer space is needed */ - xds->rbuffer = realloc( - xds->rbuffer, - xds->rbuflen + 256); - xds->rbuflen = xds->rbuflen + 256; - /* The new space gets used at the next - * read, shortly... */ - - } - no_string = 1; - - } - - } - - return 0; -} - - -static int check_cell(struct xds_private *xp, struct image *image, - UnitCell *cell) -{ - Crystal *cr; - - cr = crystal_new(); - if ( cr == NULL ) { - ERROR("Failed to allocate crystal.\n"); - return 0; - } - crystal_set_cell(cr, cell); - image_add_crystal(image, cr); - - return 1; -} - - -static int read_cell(struct image *image, struct xds_private *xp) +static int read_cell(struct image *image) { FILE * fh; float axstar, aystar, azstar; @@ -213,12 +79,10 @@ static int read_cell(struct image *image, struct xds_private *xp) char *rval, line[1024]; int r; UnitCell *cell; + Crystal *cr; fh = fopen("IDXREF.LP", "r"); - if ( fh == NULL ) { - ERROR("Couldn't open 'IDXREF.LP'\n"); - return 0; - } + if ( fh == NULL ) return 0; /* Not indexable */ do { rval = fgets(line, 1023, fh); @@ -236,30 +100,41 @@ static int read_cell(struct image *image, struct xds_private *xp) fclose(fh); return 0; } + + /* Get first vector */ rval = fgets(line, 1023, fh); if ( rval == NULL ) { fclose(fh); return 0; } - + if ( line[4] != '1' ) { + ERROR("No first vector from XDS.\n"); + return 0; + } memcpy(asx, line+7, 10); asx[10] = '\0'; memcpy(asy, line+17, 10); asy[10] = '\0'; memcpy(asz, line+27, 10); asz[10] = '\0'; + /* Get second vector */ rval = fgets(line, 1023, fh); if ( rval == NULL ) { fclose(fh); return 0; } - + if ( line[4] != '2' ) { + ERROR("No second vector from XDS.\n"); + return 0; + } memcpy(bsx, line+7, 10); bsx[10] = '\0'; memcpy(bsy, line+17, 10); bsy[10] = '\0'; memcpy(bsz, line+27, 10); bsz[10] = '\0'; + /* Get third vector */ rval = fgets(line, 1023, fh); fclose(fh); if ( rval == NULL ) return 0; - + if ( line[4] != '3' ) return 0; /* No error message this time + * - happens a lot */ memcpy(csx, line+7, 10); csx[10] = '\0'; memcpy(csy, line+17, 10); csy[10] = '\0'; memcpy(csz, line+27, 10); csz[10] = '\0'; @@ -284,9 +159,16 @@ static int read_cell(struct image *image, struct xds_private *xp) axstar*10e9, aystar*10e9, azstar*10e9, bxstar*10e9, bystar*10e9, bzstar*10e9, -cxstar*10e9, -cystar*10e9, -czstar*10e9); - r = check_cell(xp, image, cell); - return r; + cr = crystal_new(); + if ( cr == NULL ) { + ERROR("Failed to allocate crystal.\n"); + return 0; + } + crystal_set_cell(cr, cell); + image_add_crystal(image, cr); + + return 1; } @@ -322,8 +204,8 @@ static void write_spot(struct image *image) x = tan(ttx)*FAKE_CLEN; y = tan(tty)*FAKE_CLEN; - x = (x / 70e-6) + 1500; - y = (y / 70e-6) + 1500; + x = (x / FAKE_PIXEL_SIZE) + 1500; + y = (y / FAKE_PIXEL_SIZE) + 1500; fprintf(fh, "%10.2f %10.2f %10.2f %10.0f.\n", x, y, 0.5, f->intensity); @@ -450,8 +332,8 @@ static int write_inp(struct image *image, struct xds_private *xp) fprintf(fh, "NX= 3000\n"); fprintf(fh, "NY= 3000\n"); - fprintf(fh, "QX= 0.07\n"); - fprintf(fh, "QY= 0.07\n"); + fprintf(fh, "QX= %f\n", FAKE_PIXEL_SIZE*1e3); /* Pixel size in mm */ + fprintf(fh, "QY= %f\n", FAKE_PIXEL_SIZE*1e3); /* Pixel size in mm */ fprintf(fh, "INDEX_ORIGIN=0 0 0\n"); fprintf(fh, "DIRECTION_OF_DETECTOR_X-AXIS=1 0 0\n"); fprintf(fh, "DIRECTION_OF_DETECTOR_Y-AXIS=0 1 0\n"); @@ -474,46 +356,33 @@ static int write_inp(struct image *image, struct xds_private *xp) int run_xds(struct image *image, void *priv) { - unsigned int opts; int status; int rval; int n; - struct xds_data *xds; + pid_t pid; + int pty; struct xds_private *xp = (struct xds_private *)priv; - xds = malloc(sizeof(struct xds_data)); - if ( xds == NULL ) { - ERROR("Couldn't allocate memory for xds data.\n"); - return 0; - } - - xds->target_cell = xp->cell; - if ( write_inp(image, xp) ) { ERROR("Failed to write XDS.INP file for XDS.\n"); - free(xds); return 0; } n = image_feature_count(image->features); - if ( n < 25 ) { - free(xds); - return 0; - } + if ( n < 25 ) return 0; write_spot(image); /* Delete any old indexing result which may exist */ remove("IDXREF.LP"); - xds->pid = forkpty(&xds->pty, NULL, NULL, NULL); + pid = forkpty(&pty, NULL, NULL, NULL); - if ( xds->pid == -1 ) { + if ( pid == -1 ) { ERROR("Failed to fork for XDS\n"); - free(xds); return 0; } - if ( xds->pid == 0 ) { + if ( pid == 0 ) { /* Child process: invoke XDS */ struct termios t; @@ -528,66 +397,10 @@ int run_xds(struct image *image, void *priv) _exit(0); } + waitpid(pid, &status, 0); - xds->rbuffer = malloc(256); - xds->rbuflen = 256; - xds->rbufpos = 0; - - /* Set non-blocking */ - opts = fcntl(xds->pty, F_GETFL); - fcntl(xds->pty, F_SETFL, opts | O_NONBLOCK); - - //xds->step = 1; /* This starts the "initialisation" procedure */ - xds->finished_ok = 0; - - do { - - fd_set fds; - struct timeval tv; - int sval; - - FD_ZERO(&fds); - FD_SET(xds->pty, &fds); - - tv.tv_sec = 30; - tv.tv_usec = 0; - - sval = select(xds->pty+1, &fds, NULL, NULL, &tv); - - if ( sval == -1 ) { - - const int err = errno; - - switch ( err ) { - - case EINTR: - STATUS("Restarting select()\n"); - rval = 0; - break; - - default: - ERROR("select() failed: %s\n", strerror(err)); - rval = 1; - break; - - } - - } else if ( sval != 0 ) { - rval = xds_readable(image, xds); - } else { - ERROR("No response from XDS..\n"); - rval = 1; - } - - } while ( !rval ); - - close(xds->pty); - free(xds->rbuffer); - waitpid(xds->pid, &status, 0); - - rval = read_cell(image, xp); - - free(xds); + close(pty); + rval = read_cell(image); return rval; } diff --git a/relnotes-0.6.3 b/relnotes-0.6.3 deleted file mode 100644 index 11a1f93b..00000000 --- a/relnotes-0.6.3 +++ /dev/null @@ -1,211 +0,0 @@ -CrystFEL - Crystallography with a FEL -------------------------------------- - -Release notes for version 0.6.3 - -Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, - a research centre of the Helmholtz Association. - -Authors: - Thomas White <taw@physics.org> - Richard Kirian <rkirian@asu.edu> - Kenneth Beyerlein <kenneth.beyerlein@desy.de> - Andrew Aquila <andrew.aquila@cfel.de> - Andrew Martin <andrew.martin@desy.de> - Lorenzo Galli <lorenzo.galli@desy.de> - Chun Hong Yoon <chun.hong.yoon@desy.de> - Kenneth Beyerlein <kenneth.beyerlein@desy.de> - Karol Nass <karol.nass@desy.de> - Nadia Zatsepin <nadia.zatsepin@asu.edu> - Anton Barty <anton.barty@desy.de> - Cornelius Gati <cornelius.gati@desy.de> - Fedor Chervinskii <fedor.chervinskii@gmail.com> - Alexandra Tolstikova <alexandra.tolstikova@desy.de> - Wolfgang Brehm <wolfgang.brehm@gmail.com> - Valerio Mariani <valerio.mariani@desy.de> - Parker de Waal <Parker.deWaal@vai.org> - Takanori Nakane <nakane.t@gmail.com> - Keitaro Yamashita <k.yamashita@spring8.or.jp> - Oleksandr Yefanov <oleksandr.yefanov@cfel.de> - Steve Aplin <steve.aplin@desy.de> - Helen Ginn <helen@strubi.ox.ac.uk> - -CrystFEL is free software: you can redistribute it and/or modify it under the -terms of the GNU General Public License as published by the Free Software -Foundation, either version 3 of the License, or (at your option) any later -version. - -CrystFEL is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -CrystFEL. If not, see <http://www.gnu.org/licenses/>. - - -Overview --------- - -The most important new features in this version of CrystFEL are: - -- Support for CBF files as input - -- TakeTwo indexing algorithm - -- Improved peak detection and basic hitfinder functionality - -- Detector panels not perpendicular to X-ray beam, and "rail direction" - -These changes have sections below. In addition, there were many bug fixes and -other improvements. See the "ChangeLog" or the changes page on the CrystFEL -website for details. - - -Support for CBF files as input ------------------------------- - -Crystallographic Binary Files (CBF) can now be used as input for indexamajig -and hdfsee. To enable this functionality, you must have "CBFlib" installed. -Some distributions (eg Fedora) include CBFlib in their standard repositories - -simply install "cbflib-devel" or similar. Then compile and install CrystFEL as -usual. If CBFlib is installed somewhere non-standard, e.g. if you has to -install it manually, you will need to give the path with the "--with-cbflib-dir" -option to "./configure". - -After that, you can simply give the names of CBF files in the indexamajig input, -and everything should work as usual! There are some catches, for example you -cannot yet use a bad pixel mask with CBF files, but these will be fixed in a -future release. - - -TakeTwo indexing algorithm --------------------------- - -This version of CrystFEL includes the TakeTwo indexing algorithm by Helen Ginn. -TakeTwo is a new indexing algorithm, designed specifically for diffraction -snapshots and operating completely differently to conventional FFT-based methods. - -To use it, simply use "taketwo" as the indexing method, or add it to the list of -indexing methods you have already: - $ indexamajig --indexing=taketwo - $ indexamajig --indexing=mosflm,taketwo,dirax - -If you use TakeTwo, please take careful note of the citation reminder which will -be shown to you, and cite the following paper in any resulting publications: - Ginn et al., Acta Cryst. (2016). D72, 956-965 - -TakeTwo is considered experimental in this version of CrystFEL. We welcome any -feedback, and will use the information to improve its performance for future -versions. - - -Improved peak detection and basic hitfinder functionality ---------------------------------------------------------- - -"Peak finder 8" from Cheetah, also used in OnDA, has been incorporated into -CrystFEL. It uses a radially varying background level to find peaks, which -works very well with most diffraction data. - -Simply use "--peaks=peakfinder8", and set --threshold, --min-snr, ---min-pix-count and --local-bg-radius. See the manual for other options. - -Indexamajig can now skip over patterns which have a small number of peaks. To -use this, use the option "--min-peaks=N", where N is the minimum number of peaks -before indexamajig will proceed to indexing (and hopefully integration) with the -pattern. - -By combining the improved peak detection with the hitfinder functionality and -CBF reading, you can process data from several different types of detector, -without converting the files to HDF5 format or using any external program for -hit finding! - -If the hit rate is very low, you might want to make CrystFEL stream smaller by -excluding non-hits from it using "--no-non-hits-in-stream". Without this -option, standard information about the non-hits (e.g. photon energy and peaks -found) will be recorded. - - -Detector panels not perpendicular to X-ray beam, and "rail direction" ----------------------------------------------------------------------- - -CrystFEL now supports three-dimensional detector geometry. The "fs" and "ss" -vectors (which are the real-space directions of the fast scan and slow scan -directions in the image data) can now contain a z component as well as x and y. - -For many detectors, the "rail" direction, along which the detector moves when -the "camera length" (sample-detector distance) is adjusted, is not perpendicular -to the detector plane. The geometry file now allows you to specify this "rail" -direction, containing x, y and z components. See the crystfel_geometry manual -page for more details. - - -API changes ------------ - -The following changes have been made to the libcrystfel API. The biggest -changes are the switch from the old "hdfile" API (which is still supported), -to the new "imagefile" API, while allows multiple file types to be used. -The indexing system was also updated, and there were changes to the detector -geometry system, which no longer uses the "data slab" representation internally. - -New functions: - - crystal_{get,set}_det_shift() - - panel_number() - - get_detector_geometry_2() - - fill_in_adu() - - adjust_centering_for_rail() - - get_peaks_2() - - get_peaks_cxi_2() - - imagefile_open() - - imagefile_read() - - imagefile_read_simple() - - imagefile_get_hdfile() - - imagefile_get_type() - - imagefile_copy_fields() - - imagefile_close() - - {new,free}_imagefile_field_list() - - add_imagefile_field() - - get_indm_from_string() - - setup_indexing() - - index_pattern_2() - - search_peaks_peakfinder8() - - reflist_add_command_and_version() - - reflist_{add,get}_notes() - - write_chunk_2() - - pointgroup_warning() - -New structure definitions: - - struct image - - struct imagefile_field_list - -Removed functions: - - get_q() - - find_panel() - - find_panel_number() - - fill_in_values() - - partial_event_substitution() - - build_indexer_list() - - prepare_indexing() - -Changed function prototypes: - - in_bad_region() - - simple_geometry() - - reverse_2d_mapping() - - hdfile_set_image() - - image_add_feature() - - image_feature_closest() - - image_reflection_closest() - - index_pattern() - - cleanup_indexing() - - write_chunk() - -Changed structure definitions: - - struct panel: removed {min,max}_{fs,ss}, added adu_per_photon, - added fsz,ssz, rail_{x,y,z} and clen_for_centering - - struct detector: removed max_{fs,ss} - - struct imagefeature: replaced "pn" with "p" - - struct beam_params: "copyme" is not an imagefile_field_list - (previously: copy_hdf5_field) - - struct image: "copyme" is not an imagefile_field_list - (previously: copy_hdf5_field). width and height remoted. - diff --git a/relnotes-0.7.0 b/relnotes-0.7.0 new file mode 100644 index 00000000..186694a0 --- /dev/null +++ b/relnotes-0.7.0 @@ -0,0 +1,176 @@ +CrystFEL - Data processing for serial crystallography +----------------------------------------------------- + +Release notes for version 0.7.0 + +Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, + a research centre of the Helmholtz Association. + +Authors: + Thomas White <taw@physics.org> + Richard Kirian <rkirian@asu.edu> + Kenneth Beyerlein <kenneth.beyerlein@desy.de> + Andrew Aquila <andrew.aquila@cfel.de> + Andrew Martin <andrew.martin@desy.de> + Lorenzo Galli <lorenzo.galli@desy.de> + Chun Hong Yoon <chun.hong.yoon@desy.de> + Karol Nass <karol.nass@desy.de> + Nadia Zatsepin <nadia.zatsepin@asu.edu> + Anton Barty <anton.barty@desy.de> + Cornelius Gati <cornelius.gati@desy.de> + Fedor Chervinskii <fedor.chervinskii@gmail.com> + Alexandra Tolstikova <alexandra.tolstikova@desy.de> + Wolfgang Brehm <wolfgang.brehm@gmail.com> + Valerio Mariani <valerio.mariani@desy.de> + Parker de Waal <Parker.deWaal@vai.org> + Takanori Nakane <nakane.t@gmail.com> + Keitaro Yamashita <k.yamashita@spring8.or.jp> + Oleksandr Yefanov <oleksandr.yefanov@cfel.de> + Steve Aplin <steve.aplin@desy.de> + Helen Ginn <helen@strubi.ox.ac.uk> + Thomas Grant <tgrant@hwi.buffalo.edu> + Mamoru Suzuki <mamoru.suzuki@protein.osaka-u.ac.jp> + +CrystFEL is free software: you can redistribute it and/or modify it under the +terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +CrystFEL is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +CrystFEL. If not, see <http://www.gnu.org/licenses/>. + + +Overview +-------- + +The most important new features in this version of CrystFEL are: + +- New options system for indexamajig, removing most of the profusion of indexing + method flags "comb", "bad", "retry" and so on. Indexamajig can also now + automatically determine which indexing methods can be used if you don't tell + it which ones to use. In addition, the options for the Felix indexing method + have now been greatly simplified. + +- The spectrum-based partiality model and numerical post-refinement algorithm + from Ginn et al [Acta D71 2015 p1400] was incorporated. + +- The symmetry of the merged reflection list is now tracked through automatically, + so you no longer need to give the "-y" option to compare_hkl and check_hkl. + +These changes have sections below. In addition, there were many bug fixes and +other improvements. See the "ChangeLog" or the changes page on the CrystFEL +website for details. + + +New indexamajig options system +------------------------------ + +The new version of indexamajig will automatically determine which indexing +methods to use based on the prior information (e.g. if a target unit cell was +provided) and which indexing programs are available on the computer. To enable +the automatic determination, simply omit any "--indexing=" option to indexamajig. +Of course, you can override the automatic choice by specifying --indexing, just +as it worked before. + +To options for finer control of the indexing process, which in previous versions +were enabled or disabled by adding flags to the indexing methods +(e.g. "mosflm-noretry"), now apply to all indexing methods at once and are +selected using new options, listed below. + +The indexing method should contain only the method itself and prior information +modifiers ('cell' or 'latt'). This simplifies things greatly by removing the +very long indexing method descriptions ("mosflm-raw-latt-noretry-refine-nomulti" +and so on). + +- To disable prediction refinement ('norefine'), use --no-refine. +- To check cell axes only ('axes'), use --no-cell-combinations. +- To disable all unit cell checks ('raw'), use --no-check-cell. +- To disable peak alignment check ('bad'), use --no-check-peaks. +- To disable indexing retry ('noretry'), use --no-retry. +- To enable multi-lattice indexing by 'delete and retry', use --multi. + + +Spectrum-based partiality model and post-refinement +--------------------------------------------------- + +This version includes an implementation of the spectrum-based partiality model +described by Ginn et al., Acta D71 2015 p1400 and associated numerical post- +refinement algorithm. To use it, use "partialator --model=xsphere". The old +models (scsphere and scgaussian) have been removed. You will probably need to +experiment with changing the bandwidth and reflection radius, using +--force-bandwidth and --force-radius. Add --no-pr to disable post-refinement, +while still using the partiality estimates. To use post-refinement, omit --no-pr, +but you should also run indexamajig with "--overpredict". You may also need to +disable B factor scaling in partialator with "--no-Bscale". + +If you use post-refinement, the new scripts "plot-pr" and "plot-contourmap" can +be used to visualise the distributions of partialities and the contour maps of +residual against parameters such as crystal rotation angle. The data for these +graphs is in the "pr-logs" folder which will be created by partialator. + +Partiality modelling and post-refinement should still be considered experimental +when applied to real data. The success of post-refinement depends on many +factors, and only has a chance of working if there are no other problems with +the dataset (e.g. inaccurate detector geometry). Further improvements should +appear in future versions. + + +Automatic tracking of reflection list symmetry +---------------------------------------------- + +The point group symmetry used for merging a reflection list has been stored in +the reflection file (.hkl) for a long time. However, version 0.7.0 of CrystFEL +uses the stored symmetry when performing tasks such as calculating figures of +merit using check_hkl. You can simply omit the "-y" options which were needed +previously. Of course, you can override the symmetry by specifying -y as before. + + +API changes +----------- + +The following changes have been made to the libcrystfel API. + +New functions: + - asdf_probe() and _probe for all other indexing methods. + - compare_cells() + - crystal_get_cell_const() + - crystal_get_profile_radius() + - predict_to_res() + - calculate_partialities() + - update_predictions() + - is_cbf_file() + - get_indm_from_string_2() + - detect_indexing_methods() + - integrate_all_5() + - read_reflections_2() + - {get,set}_{khalf,kpred,exerr}() + - first_refl_const(), next_refl_const() + - open_stream_for_write_4() + - stream_has_old_indexers() + - stream_audit_info() + +Removed functions: + - find_intersections() + - find_intersections_to_res() + - update_partilities() + - write_reflections_to_file() + - read_reflections_from_file() + - {get,sh}_partial() + +Changed function prototypes: + - asdf_prepare() and _prepare for all other indexing methods. + - cell_new_from_cell() (input UnitCell is now const) + - crystal_copy() (input Crystal is now const) + - x_gradient(), y_gradient() + - remove_flagged_crystals() (now returns number of crystals removed) + - setup_indexing() + +Changed structure/enum definitions: + - PartialityModel: removed PMODEL_SCSPHERE and PMODEL_SCGAUSSIAN, + added PMODEL_XSPHERE + - gparam: added GPARAM_ANG1, GPARAM_ANG2, GPARAM_WAVELENGTH, GPARAM_EOL + - Added IndexingFlags diff --git a/scripts/create-mtz b/scripts/create-mtz index 62d94405..add9f093 100755 --- a/scripts/create-mtz +++ b/scripts/create-mtz @@ -22,10 +22,7 @@ if [ -e $OUTFILE ]; then fi fi -ex -c '/End of reflections/ -.,$d -w create-mtz.temp.hkl -q!' $1 +sed -n '/End\ of\ reflections/q;p' $1 > create-mtz.temp.hkl echo "Running 'f2mtz'..." f2mtz HKLIN create-mtz.temp.hkl HKLOUT $OUTFILE > out.html << EOF diff --git a/scripts/plot-contourmap b/scripts/plot-contourmap new file mode 100755 index 00000000..053a6be9 --- /dev/null +++ b/scripts/plot-contourmap @@ -0,0 +1,153 @@ +#!/usr/bin/env python + +import numpy as np +import matplotlib +import matplotlib.cm as cm +import matplotlib.mlab as mlab +import matplotlib.pyplot as plt +import sys +import fnmatch +import os +from matplotlib.widgets import RadioButtons,Button + +im=None +cnt=None +centre_marker=None + +def next_click(w): + global crystal + crystal += 20 + print("Crystal %i" % crystal) + update_graph() + + +def prev_click(w): + global crystal + crystal -= 20 + print("Crystal %i" % crystal) + update_graph() + + +def iteration_click(label): + global cycle + cycle = label.split(" ")[1].split("\n")[0] + update_graph() + + +def variable_click(label): + global varpair + varpair = label + update_graph() + + +def update_graph(): + + global im, cnt, centre_marker + + filename="pr-logs/grid-crystal%i-cycle%s-%s.dat" % (crystal,cycle,varpair) + print filename + + with open(filename, "r") as f: + + l = f.readline().split(None,3) + + min1 = float(l[0]) + max1 = float(l[1]) + cur1 = float(l[2]) + label1 = l[3] + + l = f.readline().split(None,3) + min2 = float(l[0]) + max2 = float(l[1]) + cur2 = float(l[2]) + label2 = l[3] + + extent = (min1, max1, min2, max2) + Z = np.loadtxt(fname=filename, ndmin=2, delimiter=" ", skiprows=2) + + ax.set_xlim([min1,max1]) + ax.set_ylim([min2,max2]) + + if not im: + im = ax.imshow(Z, interpolation='none', origin='lower', + cmap=cm.gray, extent=extent, aspect='auto') + im.autoscale() + else: + im.set_data(Z) + im.set_extent(extent) + im.autoscale() + + levels = np.arange(0.0, 1.6, 0.1) + + if cnt: + for coll in cnt.collections: + coll.remove() + + cnt = ax.contour(Z, levels, origin='lower', linewidths=1, extent=extent) + + ax.set_title(filename) + ax.set_xlabel(label1) + ax.set_ylabel(label2) + plt.flag() + + if centre_marker: + centre_marker.remove() + centre_marker, = plt.plot(cur1, cur2, 'bH', color='r') + + fig.canvas.draw() + +fig = plt.figure(figsize=(10,5)) +fig.subplots_adjust(left=0.05, bottom=0.05, right=0.70, top=0.95) + +# Find out what there is to plot +crystals = [] +cycles = [] +varpairs = [] +for file in os.listdir("pr-logs"): + if not fnmatch.fnmatch(file, "grid-*.dat"): + continue + sp = file.rstrip(".dat").split("-") + crystals.append(int(sp[1].lstrip("crystal"))) + cycles.append(sp[2].lstrip("cycle")) + varpairs.append(sp[3]+"-"+sp[4]) + +crystals = sorted(set(crystals)) +cycles = sorted(set(cycles)) +varpairs = sorted(set(varpairs)) + +crystal = crystals[0] +cycle = cycles[0] +varpair = varpairs[0] + +# Iteration selector +ax = plt.axes([0.75, 0.55, 0.2, 0.40], facecolor="lightgoldenrodyellow") +iterations = ["Iteration "+str(f) for f in cycles] +if iterations[0] != "Iteration 0": + print("Whoops, couldn't find iteration 0!") + sys.exit(1) +iterations[0] = "Iteration 0\n(initial scaling only)" +if iterations[len(iterations)-1] != "Iteration F": + print("Whoops, couldn't find final iteration!") +else: + iterations[len(iterations)-1] = "Iteration F\n(after final merge)" +iteration = RadioButtons(ax, iterations) +iteration.on_clicked(iteration_click) + +# Variable selector +ax = plt.axes([0.75, 0.20, 0.2, 0.30], facecolor="lightgoldenrodyellow") +variable = RadioButtons(ax, varpairs) +variable.on_clicked(variable_click) + +# Crystal selector +ax = plt.axes([0.75, 0.08, 0.20, 0.06]) +crystal_prev = Button(ax, "Previous crystal") +crystal_prev.on_clicked(prev_click) +ax = plt.axes([0.75, 0.01, 0.20, 0.06]) +crystal_next = Button(ax, "Next crystal") +crystal_next.on_clicked(next_click) + +ax = plt.axes([0.1, 0.1, 0.6, 0.8]) +update_graph() +plt.colorbar(im, shrink=0.8) + +plt.show() diff --git a/scripts/plot-pr b/scripts/plot-pr new file mode 100755 index 00000000..25427cae --- /dev/null +++ b/scripts/plot-pr @@ -0,0 +1,196 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Plot post-refinement results +# +# Copyright © 2017-2018 Deutsches Elektronen-Synchrotron DESY, +# a research centre of the Helmholtz Association. +# +# Author: +# 2017-2018 Thomas White <taw@physics.org> +# + +import sys +import matplotlib +import numpy as np +from matplotlib import pyplot as plt +from operator import truediv, itemgetter +from matplotlib.widgets import RadioButtons,SpanSelector,Button + +crystal=0 +resmin=0 +resmax=100e9 +inum="0" +first=True + +def update_graph(): + + global inum, resmin, resmax, first + + # Update pobs/pcalc scatter plot + xlim = pgraphE.get_xlim() + ylim = pgraphE.get_ylim() + pgraphE.cla() + pgraphE.scatter([x[0] for x in pgraph if x[2]==inum if x[3]<resmax if x[3]>resmin], + [x[1] for x in pgraph if x[2]==inum if x[3]<resmax if x[3]>resmin], marker="+") + pgraphE.grid(b=True, which='major', color='k', linestyle='--') + if not first: + pgraphE.set_xlim(xlim) + pgraphE.set_ylim(ylim) + pgraphE.set_xlabel("pcalc") + pgraphE.set_ylabel("pobs") + pgraphE.set_title("Observed and calculated partialities") + + # Update spectrum plot + pobsE.set_data([x[0] for x in specgraph if x[3]==inum if x[4]<resmax if x[4]>resmin], + [x[2] for x in specgraph if x[3]==inum if x[4]<resmax if x[4]>resmin]) + pcalcE.set_data([x[0] for x in specgraph if x[3]==inum if x[4]<resmax if x[4]>resmin], + [x[1] for x in specgraph if x[3]==inum if x[4]<resmax if x[4]>resmin]) + specgraphE.set_title("Spectrum plot for crystal %i" % crystal) + + first = False + plt.draw() + + +def iteration_click(label): + global inum + inum = label.split(" ")[1].split("\n")[0] + update_graph() + + +def resolutionchange(rmin,rmax): + global resmin,resmax + resmin = rmin + resmax = rmax + update_graph() + + +def read_spectrum(filename): + + specgraph = [] + + try: + fh = open(filename, 'r') + except IOError: + print("Failed to read "+filename) + raise IOError + + fline = fh.readline() + fline = fh.readline() # Ignore header (two lines) + + while True: + fline = fh.readline() + if not fline: + break + kval = float(fline.split()[0]) + res = float(fline.split()[1]) + pcalc = float(fline.split()[2]) + pobs = float(fline.split()[3]) + inum = fline.split()[4] + specgraph.append([kval,pcalc,pobs,inum,res]) + + return sorted(specgraph, key=itemgetter(0)) + + +def next_click(w): + global specgraph, crystal + crystal += 20 + specgraph = read_spectrum("pr-logs/specgraph-crystal%i.dat" % crystal) + update_graph() + + +def prev_click(w): + global specgraph, crystal + crystal -= 20 + specgraph = read_spectrum("pr-logs/specgraph-crystal%i.dat" % crystal) + update_graph() + + +def set_sensible_axes(w): + pgraphE.set_xlim([-0.1,1.1]) + pgraphE.set_ylim([-0.2,2.0]) + specgraphE.set_ylim([-5.0,5.0]) + plt.draw() + + +# Read the pgraph file +pgraph = [] + +try: + fh = open("pr-logs/pgraph.dat", 'r') +except IOError: + print("Failed to read "+filename) + raise IOError + +fline = fh.readline() # Ignore header + +while True: + fline = fh.readline() + if not fline: + break + res = float(fline.split()[4]) + Ipart = float(fline.split()[5]) + pc = float(fline.split()[6]) + po = float(fline.split()[7]) + inum = fline.split()[8] + pgraph.append([pc,po,inum,res,Ipart]) + + +# Read the spectrum file +specgraph = read_spectrum("pr-logs/specgraph-crystal%i.dat" % crystal) + +fig = plt.figure(figsize=(15,6)) +fig.subplots_adjust(left=0.05, right=0.65) + +# pobs/pcalc +pgraphE = fig.add_subplot(121) + +# Spectrum +specgraphE = fig.add_subplot(122) +pobsE = specgraphE.plot([0,1],[0,1],label="pobs")[0] +pcalcE = specgraphE.plot([0,1],[0,1],label="pcalc")[0] +specgraphE.set_xlabel("khalf / m^-1") +specgraphE.set_ylabel("Partiality") +specgraphE.grid(b=True, which='major', color='k', linestyle='--') +specgraphE.legend() + +# Iteration selector +ax = plt.axes([0.75, 0.50, 0.2, 0.40], facecolor="lightgoldenrodyellow") +iterations = sorted(set(["Iteration "+str(x[2]) for x in pgraph])) +if iterations[0] != "Iteration 0": + print("Whoops, couldn't find iteration 0!") + sys.exit(1) +iterations[0] = "Iteration 0\n(initial scaling only)" +if iterations[len(iterations)-1] != "Iteration F": + print("Whoops, couldn't find final iteration!") +else: + iterations[len(iterations)-1] = "Iteration F\n(after final merge)" +iteration = RadioButtons(ax, iterations) +iteration.on_clicked(iteration_click) +iteration.set_active(len(iterations)-1) + +# Resolution selector +ax = plt.axes([0.75, 0.25, 0.2, 0.15], facecolor="lightgoldenrodyellow") +ax.hist([x[3] for x in pgraph],weights=[x[4] for x in pgraph],bins=100) +ax.set_xlabel("Resolution / m^-1") +ax.set_ylabel("Frequency") +ax.set_title("Resolution selector") +resolution = SpanSelector(ax, resolutionchange, 'horizontal', span_stays=True, useblit=True) + +# Crystal switcher +ax = plt.axes([0.75, 0.10, 0.09, 0.05]) +crystal_prev = Button(ax, "Previous crystal") +crystal_prev.on_clicked(prev_click) +ax = plt.axes([0.86, 0.10, 0.09, 0.05]) +crystal_next = Button(ax, "Next crystal") +crystal_next.on_clicked(next_click) + +# Set sensible axes +ax = plt.axes([0.75, 0.01, 0.09, 0.05]) +sensible_axes = Button(ax, "Set sensible axes") +sensible_axes.on_clicked(set_sensible_axes) + +specgraphE.relim() +specgraphE.autoscale_view(True,True,True) + +plt.show() diff --git a/src/ambigator.c b/src/ambigator.c index ba7629ab..bbf1d06e 100644 --- a/src/ambigator.c +++ b/src/ambigator.c @@ -674,7 +674,8 @@ static void reindex_reflections(FILE *fh, FILE *ofh, int assignment, /* This is nasty, but means the output includes absolutely everything in the * input, even stuff ignored by read_chunk() */ static void write_reindexed_stream(const char *infile, const char *outfile, - int *assignments, SymOpList *amb) + int *assignments, SymOpList *amb, + int argc, char *argv[]) { FILE *fh; FILE *ofh; @@ -683,6 +684,7 @@ static void write_reindexed_stream(const char *infile, const char *outfile, int have_as = 0; int have_bs = 0; int have_cs = 0; + int done = 0; fh = fopen(infile, "r"); if ( fh == NULL ) { @@ -696,6 +698,38 @@ static void write_reindexed_stream(const char *infile, const char *outfile, return; } + /* Copy the header */ + do { + + char line[1024]; + char *rval; + + rval = fgets(line, 1023, fh); + if ( rval == NULL ) { + ERROR("Failed to read stream audit info.\n"); + return; + } + + if ( strncmp(line, "-----", 5) == 0 ) { + + done = 1; + + /* Add our own header */ + fprintf(ofh, "Re-indexed by ambigator "CRYSTFEL_VERSIONSTRING"\n"); + if ( argc > 0 ) { + for ( i=0; i<argc; i++ ) { + if ( i > 0 ) fprintf(ofh, " "); + fprintf(ofh, "%s", argv[i]); + } + fprintf(ofh, "\n"); + } + + } + + fputs(line, ofh); + + } while ( !done ); + i = 0; do { @@ -1359,7 +1393,8 @@ int main(int argc, char *argv[]) n_dif); if ( (outfile != NULL) && (amb != NULL) ) { - write_reindexed_stream(infile, outfile, assignments, amb); + write_reindexed_stream(infile, outfile, assignments, amb, + argc, argv); } else if ( outfile != NULL ) { ERROR("Can only write stream with known ambiguity operator.\n"); ERROR("Try again with -w or --operator.\n"); diff --git a/src/geoptimiser.c b/src/geoptimiser.c index f63ba81b..9b03d1b9 100644 --- a/src/geoptimiser.c +++ b/src/geoptimiser.c @@ -85,20 +85,25 @@ static void show_help(const char *s) " -c, --connected=<rg_coll> Rigid group collection for connected\n" " ASICs.\n" " --no-error-maps Do not generate error map PNGs.\n" +" --stretch-map Generate stretch map PNG (panels distance).\n" " -x, --min-num-peaks-per-pixel=<num> Minimum number of peaks per pixel.\n" " Default: 3. \n" +" -z, --max-num-peaks-per-pixel=<num> Maximum number of peaks per pixel.\n" +" Default is num_patterns / 10. \n" " -p, --min-num-pixels-per-conn-group=<num> Minimum number of useful pixels per\n" " connected group.\n" -" f Default: 100.\n" +" Default: 100.\n" " -l, --most-freq-clen Use only the most frequent camera\n" " length.\n" " -s, --individual-dist-offset Use a distance offset for each panel.\n" " Default: whole-detector offset.\n" " --no-stretch Do not optimize distance offset.\n" " Default: distance offset is optimized\n" +" --no-rotation Do not optimize rotation of connectedd groups.\n" +" Default: rotation is optimized\n" " -m --max-peak-dist=<num> Maximum distance between predicted and\n" " detected peaks (in pixels)\n" -" Default: half of minimal inter-Bragg distance\n" +" Default: half of minimal inter-Bragg distance\n" ); } @@ -108,11 +113,14 @@ struct geoptimiser_params char *outfile; char *geometry_filename; int min_num_peaks_per_pix; + int max_num_peaks_per_pix; int min_num_pix_per_conn_group; int only_best_distance; int nostretch; + int norotation; int individual_coffset; int error_maps; + int stretch_map; int enforce_cspad_layout; int no_cspad; double max_peak_dist; @@ -808,11 +816,13 @@ static int compute_pixel_displacements(struct image *images, int n_images, static int compute_avg_pix_displ(struct gpanel *gp, int idx, - int num_peaks_per_pixel) + int num_peaks_per_pixel, + int max_peaks_per_pixel) { int ret; - if ( gp->num_pix_displ[idx] >= num_peaks_per_pixel ) { + if ( gp->num_pix_displ[idx] >= num_peaks_per_pixel && + gp->num_pix_displ[idx] < max_peaks_per_pixel ) { ret = fill_avg_pixel_displ(gp, idx); if ( ret != 0 ) return ret; @@ -833,12 +843,14 @@ static int compute_avg_pix_displ(struct gpanel *gp, int idx, static int compute_avg_displacements(struct detector *det, struct rg_collection *connected, struct connected_data *conn_data, - struct gpanel *gpanels) + struct gpanel *gpanels, + struct geoptimiser_params *gparams) { int di, ip, ifs, iss; - int pix_index, ret; + int pix_index, ret, max_peaks; struct panel *p; + max_peaks = 0; for ( di=0; di<connected->n_rigid_groups; di++ ) { for ( ip=0; ip<connected->rigid_groups[di]->n_panels; ip++ ) { @@ -854,9 +866,12 @@ static int compute_avg_displacements(struct detector *det, for ( ifs=0; ifs<p->w; ifs++ ) { pix_index = ifs+p->w*iss; + if ( max_peaks < gp->num_pix_displ[pix_index] ) + max_peaks = gp->num_pix_displ[pix_index]; ret = compute_avg_pix_displ(gp, pix_index, - conn_data[di].num_peaks_per_pixel); + conn_data[di].num_peaks_per_pixel, + gparams->max_num_peaks_per_pix); if ( ret != 0 ) return ret; @@ -865,6 +880,7 @@ static int compute_avg_displacements(struct detector *det, } } + STATUS("Maximum peaks per pixel is: %d\n", max_peaks); return 0; } @@ -1085,20 +1101,23 @@ static void correct_rotation_and_stretch(struct rg_collection *connected, /* Calculate corner adjustment * NB All panels follow the first one */ - if ( ip > 0 && connected->rigid_groups[di]->n_panels == 2 && !gparams->no_cspad ) { + if ( ip > 0 ) { struct panel *p0; - double delta_x, delta_y, delta; + double dx_old, dy_old, dx_new, dy_new; p0 = connected->rigid_groups[di]->panels[0]; - delta_x = p->cnx - p0->cnx / cs; - delta_y = p->cny - p0->cny / cs; + dx_old = p->cnx - p0->cnx / cs; + dy_old = p->cny - p0->cny / cs; - delta = sqrt(delta_x*delta_x + delta_y*delta_y); + dx_new = dx_old*cos(conn_data[di].cang)- + dy_old*sin(conn_data[di].cang); + dy_new = dx_old*sin(conn_data[di].cang)+ + dy_old*cos(conn_data[di].cang); - new_cnx = p0->cnx + delta*p0->fsx; - new_cny = p0->cny + delta*p0->fsy; + new_cnx = p0->cnx + dx_new; + new_cny = p0->cny + dy_new; } else { @@ -1577,6 +1596,14 @@ static double compute_rotation_and_stretch(struct rg_collection *connected, free(stretches); } + if ( gparams->norotation ) { + STATUS("However, rotation will not be optimized, so the " + "rotation angle has been set to 0\n"); + for ( di=0; di<connected->n_rigid_groups; di++ ) { + conn_data[di].cang = 0.0; + } + } + stretch_cf = 1.0; printf("Computing overall stretch coefficient.\n"); @@ -1642,7 +1669,8 @@ static double compute_rotation_and_stretch(struct rg_collection *connected, #ifdef CAN_SAVE_TO_PNG static void draw_panel(struct image *image, cairo_t *cr, - cairo_matrix_t *basic_m, GdkPixbuf **pixbufs, int i) { + cairo_matrix_t *basic_m, GdkPixbuf **pixbufs, int i) +{ struct panel p = image->det->panels[i]; int w = gdk_pixbuf_get_width(pixbufs[i]); int h = gdk_pixbuf_get_height(pixbufs[i]); @@ -1816,8 +1844,119 @@ static int save_data_to_png(char *filename, struct detector *det, return 0; } + +static int save_stretch_to_png(char *filename, struct detector *det, + struct rg_collection *connected, + struct connected_data *conn_data) + +{ + struct image im; + int i, di, ip; + float min_str, max_str; + struct rectangle rect; + GdkPixbuf *col_scale; + cairo_t *cr; + + cairo_status_t r; + cairo_surface_t *surf; + + im.det = det; + im.bad = NULL; + im.dp = malloc(det->n_panels*sizeof(float *)); + if ( im.dp == NULL ) { + ERROR("Failed to allocate data\n"); + return 1; + } + + min_str = 10000; + max_str = 0; + for ( di=0; di<connected->n_rigid_groups; di++ ) { + if (conn_data[di].cstr > max_str) max_str = conn_data[di].cstr; + if (conn_data[di].cstr < min_str) min_str = conn_data[di].cstr; + } + + i = 0; + for ( di=0; di<connected->n_rigid_groups; di++ ) { + for ( ip=0; ip<connected->rigid_groups[di]->n_panels; ip++ ) { + + int fs, ss; + float val; + struct panel *p; + + p = connected->rigid_groups[di]->panels[ip]; + + im.dp[i] = calloc(p->w * p->h, sizeof(float)); + if ( im.dp[i] == NULL ) { + ERROR("Failed to allocate data\n"); + return 1; + } + + val = (conn_data[di].cstr - min_str) / (max_str - min_str); + val *= 10.0; /* render_panels sets this as max */ + + for ( ss=0; ss<p->h; ss++ ) { + for ( fs=0; fs<p->w; fs++ ) { + + im.dp[i][fs+p->w*ss] = val; + + } + } + i++; + } + } + + get_pixel_extents(im.det, &rect.min_x, &rect.min_y, &rect.max_x, + &rect.max_y); + + if (rect.min_x > 0.0) rect.min_x = 0.0; + if (rect.max_x < 0.0) rect.max_x = 0.0; + if (rect.min_y > 0.0) rect.min_y = 0.0; + if (rect.max_y < 0.0) rect.max_y = 0.0; + + rect.width = rect.max_x - rect.min_x; + rect.height = rect.max_y - rect.min_y; + + /* Add a thin border */ + rect.width += 2.0; + rect.height += 2.0; + surf = cairo_image_surface_create(CAIRO_FORMAT_ARGB32, rect.width + 20, + rect.height); + + draw_detector(surf, &im, rect); + + col_scale = render_get_colour_scale(20, rect.height, SCALE_GEOPTIMISER); + + cr = cairo_create(surf); + cairo_identity_matrix(cr); + cairo_translate(cr, rect.width, 0.0); + cairo_rectangle(cr, 0.0, 0.0, 20.0, rect.height); + gdk_cairo_set_source_pixbuf(cr, col_scale, 0.0, 0.0); + cairo_fill(cr); + cairo_destroy(cr); + + for ( i=0; i<det->n_panels; i++ ) { + free(im.dp[i]); + } + free(im.dp); + + r = cairo_surface_write_to_png(surf, filename); + if ( r != CAIRO_STATUS_SUCCESS ) return 1; + + return 0; +} + #else /* CAN_SAVE_TO_PNG */ +static int save_stretch_to_png(char *filename, struct detector *det, + struct rg_collection *connected, + struct connected_data *conn_data) +{ + ERROR("WARNING: geoptimiser was compiled without gdk-pixbuf and cairo " + "support. Error maps will not be saved.\n"); + return 0; /* Return "success" so that program continues */ +} + + static int save_data_to_png(char *filename, struct detector *det, struct gpanel *gpanels) { @@ -1902,6 +2041,87 @@ int check_and_enforce_cspad_dist(struct geoptimiser_params *gparams, } } + return num_errors_found; +} + + +int check_and_enforce_agipd_dist(struct geoptimiser_params *gparams, + struct detector *det) +{ + int np = 0; + int npp = 0; + int num_errors_found = 0; + + double dist_to_check = 66.0; + double tol = 0.1; + + for ( np=0; np<det->n_panels; np = np+8 ) { + + double dist2; + double cnx2, cny2; + + struct panel *ep = &det->panels[np]; + + for ( npp=0; npp<8; npp++ ) { + + struct panel *op = &det->panels[np+npp]; + + cnx2 = ep->cnx + npp*dist_to_check*ep->ssx; + cny2 = ep->cny + npp*dist_to_check*ep->ssy; + + dist2 = (( cnx2 - op->cnx )*( cnx2 - op->cnx ) + + ( cny2 - op->cny )*( cny2 - op->cny )); + + if ( dist2 > (tol*tol)) { + + num_errors_found += 1; + + STATUS("Warning: distance between panels %s and %s " + "is outside acceptable margins (Corners are " + "more than 0.2 pixels away: %3.2f).\n", ep->name, + op->name, sqrt(dist2)); + + if ( gparams->enforce_cspad_layout ) { + + double new_op_cx, new_op_cy; + + new_op_cx = ep->cnx + ep->ssx*npp*dist_to_check; + new_op_cy = ep->cny + ep->ssy*npp*dist_to_check; + + op->cnx = new_op_cx; + op->cny = new_op_cy; + + STATUS("Enforcing distance....\n"); + } + + } + + if ( ep->fsx != op->fsx || ep->ssx != op->ssx || + ep->fsy != op->fsy || ep->ssx != op->ssx ) { + + num_errors_found += 1; + + STATUS("Warning: relative orientation between panels " + "%s and %s is incorrect.\n", ep->name, op->name); + + if ( gparams->enforce_cspad_layout ) { + + STATUS("Enforcing relative orientation....\n"); + + op->fsx = ep->fsx; + op->ssx = ep->ssx; + op->fsy = ep->fsy; + op->ssy = ep->ssy; + + op->xfs = ep->xfs; + op->xss = ep->xss; + op->yfs = ep->yfs; + op->yss = ep->yss; + } + + } + } + } return num_errors_found; } @@ -2076,9 +2296,9 @@ int optimize_geometry(struct geoptimiser_params *gparams, double avg_res; double clen_to_use; - // for angles and stretch calculation use - // only pixels which are distco*size_panel - // away + /* for angles and stretch calculation use + * only pixels which are distco*size_panel + * away */ double dist_coeff_for_rot_str = 0.2; double total_error; @@ -2099,6 +2319,7 @@ int optimize_geometry(struct geoptimiser_params *gparams, "accurate estimation of position/orientation: %i\n", gparams->min_num_pix_per_conn_group); + /* CS-PAD */ if ( (det->n_panels == 64) && !gparams->no_cspad ) { int num_errors = 0; @@ -2142,6 +2363,51 @@ int optimize_geometry(struct geoptimiser_params *gparams, } } + /* AGIPD */ + if ( (det->n_panels == 128) && det->panels[0].res > 4999 && + det->panels[0].res < 5001 && !gparams->no_cspad ) { + + int num_errors = 0; + + STATUS("It looks like the detector is an AGIPD. " + "Checking relative distance and orientation of " + "connected ASICS.\n"); + STATUS("If the detector is not an AGIPD , please rerun the " + "program with the --no-agipd option.\n"); + + STATUS("Enforcing AGIPD layout...\n"); + num_errors = check_and_enforce_agipd_dist(gparams, det); + + if ( gparams->enforce_cspad_layout ) { + + int geom_wr; + + STATUS("Saving geometry with enforced AGIPD layout.\n" + "Please restart geometry optimization using the " + "optimized geometry from this run as input " + "geometry file.\n"); + geom_wr = write_detector_geometry_2( + gparams->geometry_filename, + gparams->outfile, det, + gparams->command_line, 1); + if ( geom_wr != 0 ) { + ERROR("Error in writing output geometry file.\n"); + return 1; + } + STATUS("All done!\n"); + return 0; + } + + if ( !gparams->enforce_cspad_layout && num_errors > 0 ) { + ERROR("Relative distance and orientation of connected " + "ASICS do not respect the AGIPD layout.\n" + "Geometry optimization cannot continue.\n" + "Please rerun the program with the " + "--enforce-agipd-layout option.\n"); + return 1; + } + } + images = read_patterns_from_stream(gparams->infile, det, &n_images); if ( (n_images < 1) || (images == NULL) ) { ERROR("Error reading stream file\n"); @@ -2164,6 +2430,14 @@ int optimize_geometry(struct geoptimiser_params *gparams, avg_cell); if ( clen_to_use < 0.0 ) return 1; + if ( gparams->max_num_peaks_per_pix == 0 && n_images > 100 ) { + gparams->max_num_peaks_per_pix = n_images / 10; + } else if ( gparams->max_num_peaks_per_pix == 0 ) { + gparams->max_num_peaks_per_pix = n_images; + } + STATUS("Maximum number of measurements for a pixel to be included in " + "the refinement: %i\n", gparams->max_num_peaks_per_pix); + gpanels = calloc(det->n_panels, sizeof(struct gpanel)); if ( gpanels == NULL ) { ERROR("Failed to allocate panels.\n"); @@ -2206,7 +2480,7 @@ int optimize_geometry(struct geoptimiser_params *gparams, adjust_min_peaks_per_conn(connected, gpanels, det, gparams, conn_data); - if ( compute_avg_displacements(det, connected, conn_data, gpanels) ) { + if ( compute_avg_displacements(det, connected, conn_data, gpanels, gparams) ) { free(conn_data); free(images); return 1; @@ -2235,25 +2509,53 @@ int optimize_geometry(struct geoptimiser_params *gparams, } - stretch_coeff = compute_rotation_and_stretch(connected, conn_data, - det, gpanels, + if ( !gparams->nostretch || !gparams->norotation ) { + + stretch_coeff = compute_rotation_and_stretch(connected, + conn_data, det, gpanels, dist_coeff_for_rot_str, gparams); - if ( stretch_coeff < 0.0 ) { - free(conn_data); - return 1; - } + if ( stretch_coeff < 0.0 ) { + free(conn_data); + return 1; + } - ret = compute_rot_stretch_for_empty_panels(quadrants, connected, - gparams->min_num_pix_per_conn_group, conn_data); - if ( ret ) { - free(conn_data); - return 1; - } + ret = compute_rot_stretch_for_empty_panels(quadrants, connected, + gparams->min_num_pix_per_conn_group, + conn_data); + if ( ret ) { + free(conn_data); + return 1; + } + + if ( gparams->stretch_map ) { + +#ifdef HAVE_SAVE_TO_PNG + + STATUS("Saving stretch map - for out-of-plane rotations.\n"); - correct_rotation_and_stretch(connected, det, gpanels, conn_data, - clen_to_use, stretch_coeff, - gparams); + ret = save_stretch_to_png("stretch_co.png", det, + connected, conn_data); + + if ( ret ) { + ERROR("Error while writing data to file.\n"); + free(conn_data); + return 1; + } + +#else /* HAVE_SAVE_TO_PNG */ + + STATUS("ERROR: geoptimiser was compiled without GTK and " + "cairo support.\n Stretch map will not be saved.\n"); + +#endif /* HAVE_SAVE_TO_PNG */ + + } + + correct_rotation_and_stretch(connected, det, gpanels, + conn_data, clen_to_use, + stretch_coeff, gparams); + } ret = compute_shift(connected, conn_data, det, gparams, gpanels); if ( ret != 0 ) { @@ -2341,13 +2643,16 @@ int main(int argc, char *argv[]) gparams->infile = NULL; gparams->geometry_filename = NULL; gparams->min_num_peaks_per_pix = 3; + gparams->max_num_peaks_per_pix = 0; gparams->min_num_pix_per_conn_group = 100; gparams->only_best_distance = 0; gparams->enforce_cspad_layout = 0; gparams->nostretch = 0; + gparams->norotation = 0; gparams->individual_coffset = 0; gparams->no_cspad = 0; gparams->error_maps = 1; + gparams->stretch_map = 0; gparams->max_peak_dist = 0.0; const struct option longopts[] = { @@ -2361,6 +2666,7 @@ int main(int argc, char *argv[]) {"quadrants", 1, NULL, 'q'}, {"connected", 1, NULL, 'c'}, {"min-num-peaks-per-pixel", 1, NULL, 'x'}, + {"max-num-peaks-per-pixel", 0, NULL, 'z'}, {"min-num-pixels-per-conn-group", 1, NULL, 'p'}, {"most-few-clen", 0, NULL, 'l'}, {"max-peak-dist", 1, NULL, 'm'}, @@ -2368,9 +2674,13 @@ int main(int argc, char *argv[]) /* Long-only options with no arguments */ {"no-stretch", 0, &gparams->nostretch, 1}, + {"no-rotation", 0, &gparams->norotation, 1}, {"no-error-maps", 0, &gparams->error_maps, 0}, + {"stretch-map", 0, &gparams->stretch_map, 1}, {"enforce-cspad-layout", 0, &gparams->enforce_cspad_layout, 1}, + {"enforce-agipd-layout", 0, &gparams->enforce_cspad_layout, 1}, {"no-cspad", 0, &gparams->no_cspad, 1}, + {"no-agipd", 0, &gparams->no_cspad, 1}, /* Long-only options with arguments */ {"min-num-peaks-per-panel", 1, NULL, 11}, @@ -2380,7 +2690,7 @@ int main(int argc, char *argv[]) }; /* Short options */ - while ((c = getopt_long(argc, argv, "ho:i:g:q:c:o:x:p:lsm:", + while ((c = getopt_long(argc, argv, "ho:i:g:q:c:o:x:z:p:lsm:", longopts, NULL)) != -1) { switch (c) { @@ -2425,6 +2735,10 @@ int main(int argc, char *argv[]) gparams->min_num_peaks_per_pix = atoi(optarg); break; + case 'z' : + gparams->max_num_peaks_per_pix = atoi(optarg); + break; + case 'p' : gparams->min_num_pix_per_conn_group = atoi(optarg); break; diff --git a/src/im-sandbox.c b/src/im-sandbox.c index e01fcb74..74467529 100644 --- a/src/im-sandbox.c +++ b/src/im-sandbox.c @@ -196,7 +196,7 @@ static struct filename_plus_event *get_pattern(FILE *fh, int config_basename, line = tmp; } - scan_check = sscanf(line, "%s %s", filename_buf, event_buf ); + scan_check = sscanf(line, "%s %s", filename_buf, event_buf); len = strlen(prefix)+strlen(filename_buf)+1; @@ -261,9 +261,15 @@ static struct filename_plus_event *get_pattern(FILE *fh, int config_basename, struct event *ev_to_add; ev_to_add = get_event_from_event_string(event_buf); - append_event_to_event_list(ev_list, ev_to_add); - free_event(ev_to_add); - event_index = 0; + if ( ev_to_add == NULL ) { + ERROR("Bad event descriptor: '%s'\n", + event_buf); + } else { + append_event_to_event_list(ev_list, + ev_to_add); + free_event(ev_to_add); + event_index = 0; + } } diff --git a/src/indexamajig.c b/src/indexamajig.c index bac1ee04..38d83d58 100644 --- a/src/indexamajig.c +++ b/src/indexamajig.c @@ -123,11 +123,11 @@ static void show_help(const char *s) " --no-check-cell Don't check lattice parameters against input cell\n" " --no-cell-combinations\n" " Don't use axis combinations when checking cell\n" -" --no-multi Don't repeat indexing to index multiple hits\n" +" --multi Repeat indexing to index multiple hits\n" " --no-retry Don't repeat indexing to increase indexing rate\n" " --no-refine Skip the prediction refinement step\n" -" --check-peaks Check that most of the peaks can be accounted for\n" -" by the indexing solution\n" +" --no-check-peaks Don't check that most of the peaks can be accounted\n" +" for by the indexing solution\n" "\n" " --taketwo-member-threshold\n" " Minimum number of members in network\n" @@ -238,8 +238,8 @@ int main(int argc, char *argv[]) int if_refine = 1; int if_nocomb = 0; int if_nocheck = 0; - int if_peaks = 0; - int if_multi = 1; + int if_peaks = 1; + int if_multi = 0; int if_retry = 1; int serial_start = 1; @@ -342,8 +342,11 @@ int main(int argc, char *argv[]) {"no-check-cell", 0, &if_nocheck, 1}, {"no-cell-check", 0, &if_nocheck, 1}, {"check-peaks", 0, &if_peaks, 1}, + {"no-check-peaks", 0, &if_peaks, 0}, {"no-retry", 0, &if_retry, 0}, + {"retry", 0, &if_retry, 1}, {"no-multi", 0, &if_multi, 0}, + {"multi", 0, &if_multi, 1}, {"overpredict", 0, &iargs.overpredict, 1}, /* Long-only options which don't actually do anything */ @@ -851,6 +854,8 @@ int main(int argc, char *argv[]) if ( have_push_res && !(iargs.int_meth & INTEGRATION_RESCUT) ) { ERROR("WARNING: You used --push-res, but not -rescut, " "therefore --push-res will have no effect.\n"); + ERROR("WARNING: Add --integration=rings-rescut or " + "--integration=prof2d-rescut.\n"); } /* Parse unit cell tolerance */ @@ -955,15 +960,6 @@ int main(int argc, char *argv[]) } - /* Open output stream */ - st = open_stream_for_write_3(outfile, geom_filename, iargs.cell, - argc, argv); - if ( st == NULL ) { - ERROR("Failed to open stream '%s'\n", outfile); - return 1; - } - free(outfile); - if ( indm_str == NULL ) { STATUS("No indexing methods specified. I will try to "); @@ -1035,10 +1031,19 @@ int main(int argc, char *argv[]) ERROR("Failed to set up indexing system\n"); return 1; } - free(indm_str); } + /* Open output stream */ + st = open_stream_for_write_4(outfile, geom_filename, iargs.cell, + argc, argv, indm_str); + if ( st == NULL ) { + ERROR("Failed to open stream '%s'\n", outfile); + return 1; + } + free(outfile); + free(indm_str); + gsl_set_error_handler_off(); create_sandbox(&iargs, n_proc, prefix, config_basename, fh, diff --git a/src/merge.c b/src/merge.c index 17e57c90..14460953 100644 --- a/src/merge.c +++ b/src/merge.c @@ -50,7 +50,7 @@ /* Minimum partiality of a reflection for it to be merged */ -#define MIN_PART_MERGE (0.05) +#define MIN_PART_MERGE (0.3) struct merge_queue_args @@ -59,7 +59,6 @@ struct merge_queue_args pthread_rwlock_t full_lock; Crystal **crystals; int n_started; - PartialityModel pmodel; double push_res; int use_weak; long long int n_reflections; @@ -159,7 +158,6 @@ static void run_merge_job(void *vwargs, int cookie) signed int h, k, l; double mean, sumweight, M2, temp, delta, R; double corr, res, w; - //double esd; if ( get_partiality(refl) < MIN_PART_MERGE ) continue; @@ -189,7 +187,7 @@ static void run_merge_job(void *vwargs, int cookie) } /* Total (multiplicative) correction factor */ - corr = exp(-G) * exp(B*res*res) * get_lorentz(refl) + corr = 1.0/G * exp(B*res*res) * get_lorentz(refl) / get_partiality(refl); if ( isnan(corr) ) { ERROR("NaN corr:\n"); @@ -200,7 +198,7 @@ static void run_merge_job(void *vwargs, int cookie) continue; } - //esd = get_esd_intensity(refl) * corr; + /* Reflections count less the more they have to be scaled up */ w = 1.0; /* Running mean and variance calculation */ @@ -229,7 +227,7 @@ static void finalise_merge_job(void *vqargs, void *vwargs) RefList *merge_intensities(Crystal **crystals, int n, int n_threads, - PartialityModel pmodel, int min_meas, + int min_meas, double push_res, int use_weak) { RefList *full; @@ -245,7 +243,6 @@ RefList *merge_intensities(Crystal **crystals, int n, int n_threads, qargs.full = full; qargs.n_started = 0; qargs.crystals = crystals; - qargs.pmodel = pmodel; qargs.push_res = push_res; qargs.use_weak = use_weak; qargs.n_reflections = 0; @@ -285,3 +282,120 @@ RefList *merge_intensities(Crystal **crystals, int n, int n_threads, reflist_free(full); return full2; } + + +double residual(Crystal *cr, const RefList *full, int free, + int *pn_used, const char *filename) +{ + Reflection *refl; + RefListIterator *iter; + int n_used = 0; + double num = 0.0; + double den = 0.0; + double G = crystal_get_osf(cr); + double B = crystal_get_Bfac(cr); + UnitCell *cell = crystal_get_cell(cr); + + for ( refl = first_refl(crystal_get_reflections(cr), &iter); + refl != NULL; + refl = next_refl(refl, iter) ) + { + double p, w, corr, res; + signed int h, k, l; + Reflection *match; + double I_full; + double int1, int2; + + if ( free && !get_flag(refl) ) continue; + + get_indices(refl, &h, &k, &l); + res = resolution(cell, h, k, l); + match = find_refl(full, h, k, l); + if ( match == NULL ) continue; + I_full = get_intensity(match); + + if ( get_redundancy(match) < 2 ) continue; + + p = get_partiality(refl); + //if ( p < 0.2 ) continue; + + corr = get_lorentz(refl) / (G * exp(-B*res*res)); + int1 = get_intensity(refl) * corr; + int2 = p*I_full; + w = p; + + num += fabs(int1 - int2) * w; + den += fabs(int1 + int2) * w/2.0; + + n_used++; + + } + + if ( pn_used != NULL ) *pn_used = n_used; + return num/(den*sqrt(2)); +} + + +double log_residual(Crystal *cr, const RefList *full, int free, + int *pn_used, const char *filename) +{ + double dev = 0.0; + double G, B; + Reflection *refl; + RefListIterator *iter; + int n_used = 0; + FILE *fh = NULL; + + G = crystal_get_osf(cr); + B = crystal_get_Bfac(cr); + if ( filename != NULL ) { + fh = fopen(filename, "a"); + if ( fh == NULL ) { + ERROR("Failed to open '%s'\n", filename); + } + } + + for ( refl = first_refl(crystal_get_reflections(cr), &iter); + refl != NULL; + refl = next_refl(refl, iter) ) + { + double p, L, s, w; + signed int h, k, l; + Reflection *match; + double esd, I_full, I_partial; + double fx; + + if ( free && !get_flag(refl) ) continue; + + get_indices(refl, &h, &k, &l); + match = find_refl(full, h, k, l); + if ( match == NULL ) continue; + + p = get_partiality(refl); + L = get_lorentz(refl); + I_partial = get_intensity(refl); + I_full = get_intensity(match); + esd = get_esd_intensity(refl); + s = resolution(crystal_get_cell(cr), h, k, l); + + if ( I_partial <= 3.0*esd ) continue; /* Also because of log */ + if ( get_redundancy(match) < 2 ) continue; + if ( I_full <= 0 ) continue; /* Because log */ + if ( p <= 0.0 ) continue; /* Because of log */ + + fx = log(G) - B*s*s + log(p) + log(I_full) - log(I_partial) - log(L); + w = 1.0; + dev += w*fx*fx; + + if ( fh != NULL ) { + fprintf(fh, "%4i %4i %4i %e %e\n", + h, k, l, s, dev); + } + + } + + if ( fh != NULL ) fclose(fh); + + if ( pn_used != NULL ) *pn_used = n_used; + return dev; +} diff --git a/src/merge.h b/src/merge.h index 1911584f..b33afdea 100644 --- a/src/merge.h +++ b/src/merge.h @@ -40,7 +40,12 @@ #include "geometry.h" extern RefList *merge_intensities(Crystal **crystals, int n, int n_threads, - PartialityModel pmodel, int min_meas, - double push_res, int use_weak); + int min_meas, double push_res, int use_weak); + +extern double residual(Crystal *cr, const RefList *full, int free, + int *pn_used, const char *filename); + +extern double log_residual(Crystal *cr, const RefList *full, int free, + int *pn_used, const char *filename); #endif /* MERGE */ diff --git a/src/partial_sim.c b/src/partial_sim.c index 5926043f..218780fd 100644 --- a/src/partial_sim.c +++ b/src/partial_sim.c @@ -385,7 +385,7 @@ static void run_job(void *vwargs, int cookie) reflections = predict_to_res(cr, largest_q(&wargs->image)); crystal_set_reflections(cr, reflections); - calculate_partialities(cr, PMODEL_SCSPHERE); + calculate_partialities(cr, PMODEL_XSPHERE); for ( i=0; i<NBINS; i++ ) { wargs->n_ref[i] = 0; @@ -861,7 +861,7 @@ int main(int argc, char *argv[]) STATUS(" Background: none (no image " "output)\n"); } - STATUS(" Partiality model: scsphere (hardcoded)\n"); + STATUS(" Partiality model: xsphere (hardcoded)\n"); STATUS(" Noise standard deviation: %.2f detector units\n", noise_stddev); if ( random_intensities ) { @@ -873,6 +873,7 @@ int main(int argc, char *argv[]) input_file, sym_str); } STATUS(" Max error in cell components: %.2f %%\n", cnoise); + STATUS("Scale factor standard deviation: %.2f\n", osf_stddev); if ( random_intensities ) { full = reflist_new(); diff --git a/src/partialator.c b/src/partialator.c index 2540521d..e152db7c 100644 --- a/src/partialator.c +++ b/src/partialator.c @@ -3,11 +3,11 @@ * * Scaling and post refinement for coherent nanocrystallography * - * Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2015 Thomas White <taw@physics.org> + * 2010-2018 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -40,6 +40,7 @@ #include <assert.h> #include <pthread.h> #include <gsl/gsl_errno.h> +#include <sys/stat.h> #include <image.h> #include <utils.h> @@ -189,7 +190,7 @@ static void write_split(Crystal **crystals, int n_crystals, const char *outfile, } snprintf(tmp, 1024, "%s1", outfile); split = merge_intensities(crystals1, n_crystals1, nthreads, - pmodel, min_measurements, push_res, 1); + min_measurements, push_res, 1); if ( split == NULL ) { ERROR("Not enough crystals for two way split!\n"); @@ -201,7 +202,7 @@ static void write_split(Crystal **crystals, int n_crystals, const char *outfile, reflist_free(split); snprintf(tmp, 1024, "%s2", outfile); split = merge_intensities(crystals2, n_crystals2, nthreads, - pmodel, min_measurements, push_res, 1); + min_measurements, push_res, 1); STATUS("and %s\n", tmp); write_reflist_2(tmp, split, sym); reflist_free(split); @@ -288,7 +289,7 @@ static void write_custom_split(struct custom_split *csplit, int dsn, STATUS("Writing dataset '%s' to %s (%i crystals)\n", csplit->dataset_names[dsn], tmp, n_crystalsn); split = merge_intensities(crystalsn, n_crystalsn, nthreads, - pmodel, min_measurements, push_res, 1); + min_measurements, push_res, 1); write_reflist_2(tmp, split, sym); reflist_free(split); @@ -315,6 +316,7 @@ static void show_help(const char *s) " --stop-after=<n> Stop after merging <n> crystals.\n" " -n, --iterations=<n> Run <n> cycles of scaling and post-refinement.\n" " --no-scale Disable scale factor (G, B) refinement.\n" +" --no-Bscale Disable B factor scaling.\n" " --no-pr Disable orientation/physics refinement.\n" " -m, --model=<model> Specify partiality model.\n" " --min-measurements=<n> Minimum number of measurements to require.\n" @@ -324,7 +326,12 @@ static void show_help(const char *s) " -j <n> Run <n> analyses in parallel.\n" " --no-free Disable cross-validation (testing only).\n" " --custom-split List of files for custom dataset splitting.\n" -" --max-rel-B Maximum allowable relative |B| factor.\n"); +" --max-rel-B Maximum allowable relative |B| factor.\n" +" --no-logs Do not write extensive log files.\n" +" -w <pg> Apparent point group for resolving ambiguities.\n" +" --operator=<op> Indexing ambiguity operator for resolving.\n" +" --force-bandwidth=<n> Set all bandwidths to <n> (fraction).\n" +" --force-radius=<n> Set all profile radii to <n> nm^-1.\n"); } @@ -578,7 +585,7 @@ static int set_initial_params(Crystal *cr, FILE *fh) } else { - crystal_set_osf(cr, 0.0); + crystal_set_osf(cr, 1.0); crystal_set_Bfac(cr, 0.0); } @@ -603,13 +610,22 @@ static void select_free_reflections(RefList *list, gsl_rng *rng) static void write_to_pgraph(FILE *fh, RefList *list, RefList *full, Crystal *cr, - int fr) + int fr, signed int inum) { Reflection *refl; RefListIterator *iter; double G = crystal_get_osf(cr); double B = crystal_get_Bfac(cr); UnitCell *cell = crystal_get_cell(cr); + char ins[5]; + + if ( inum >= 0 ) { + snprintf(ins, 4, "%i", inum); + } else { + ins[0] = 'F'; + ins[1] = '\0'; + } + for ( refl = first_refl(list, &iter); refl != NULL; @@ -622,6 +638,9 @@ static void write_to_pgraph(FILE *fh, RefList *list, RefList *full, Crystal *cr, if ( !get_flag(refl) ) continue; /* Not free-flagged */ + /* Strong reflections only */ + if ( get_intensity(refl) < 3.0*get_esd_intensity(refl) ) continue; + get_indices(refl, &h, &k, &l); res = resolution(cell, h, k, l); if ( 2.0*res > crystal_get_resolution_limit(cr) ) continue; @@ -629,46 +648,55 @@ static void write_to_pgraph(FILE *fh, RefList *list, RefList *full, Crystal *cr, match = find_refl(full, h, k, l); if ( match == NULL ) continue; + /* Don't calculate pobs if reference reflection is weak */ + if ( fabs(get_intensity(match)) / get_esd_intensity(match) < 3.0 ) continue; + /* Calculated partiality */ pcalc = get_partiality(refl); /* Observed partiality */ - corr = exp(-G) * exp(B*res*res) * get_lorentz(refl); + corr = G * exp(B*res*res) * get_lorentz(refl); Ipart = get_intensity(refl) * corr; pobs = Ipart / get_intensity(match); - fprintf(fh, "%5i %4i %4i %4i %8.4f %8.3f %8.3f\n", - fr, h, k, l, 2*res/1e9, pcalc, pobs); + fprintf(fh, "%5i %4i %4i %4i %e %e %8.3f %8.3f %s\n", + fr, h, k, l, 2*res, Ipart, pcalc, pobs, ins); } } static void write_pgraph(RefList *full, Crystal **crystals, int n_crystals, - int iter) + signed int iter, const char *suff) { FILE *fh; char tmp[256]; int i; - snprintf(tmp, 256, "pgraph-iter%i.dat", iter); + snprintf(tmp, 256, "pr-logs/pgraph%s.dat", suff); + + if ( iter == 0 ) { + fh = fopen(tmp, "w"); + } else { + fh = fopen(tmp, "a"); + } - fh = fopen(tmp, "w"); if ( fh == NULL ) { ERROR("Failed to open '%s'\n", tmp); return; } - fprintf(fh, " fr h k l 1/d(nm) pcalc pobs\n"); + if ( iter == 0 ) { + fprintf(fh, " Crystal h k l 1/d(m) Ipart pcalc pobs iteration\n"); + } for ( i=0; i<n_crystals; i++ ) { if ( crystal_get_user_flag(crystals[i]) != 0 ) continue; write_to_pgraph(fh, crystal_get_reflections(crystals[i]), full, - crystals[i], i); + crystals[i], i, iter); } fclose(fh); - } @@ -689,6 +717,7 @@ static void all_residuals(Crystal **crystals, int n_crystals, RefList *full, if ( crystal_get_user_flag(crystals[i]) ) continue; + /* Scaling should have been done right before calling this */ r = residual(crystals[i], full, 0, NULL, NULL); free_r = residual(crystals[i], full, 1, NULL, NULL); log_r = log_residual(crystals[i], full, 0, NULL, NULL); @@ -719,32 +748,82 @@ static void show_all_residuals(Crystal **crystals, int n_crystals, } -static void dump_parameters(const char *filename, Crystal **crystals, - int n_crystals) +struct log_qargs { - FILE *fh; - fh = fopen(filename, "w"); - if ( fh == NULL ) { - ERROR("Couldn't open partialator.params!\n"); - } else { - fprintf(fh, " cr OSF relB div" - " flag filename event\n"); - int i; - for ( i=0; i<n_crystals; i++ ) { - struct image *img; - char *evt_str; - img = crystal_get_image(crystals[i]); - evt_str = get_event_string(img->event); - fprintf(fh, "%4i %10.5f %10.2f %8.5e %-25s %s %s\n", - i, crystal_get_osf(crystals[i]), - crystal_get_Bfac(crystals[i])*1e20, - crystal_get_image(crystals[i])->div, - str_prflag(crystal_get_user_flag(crystals[i])), - img->filename, evt_str); - free(evt_str); - } - fclose(fh); - } + int iter; + int next; + Crystal **crystals; + int n_crystals; + RefList *full; + int scaleflags; +}; + + +struct log_args +{ + Crystal *cr; + RefList *full; + int scaleflags; + int iter; + int cnum; +}; + + +static void *get_log_task(void *vp) +{ + struct log_qargs *qargs = vp; + struct log_args *task; + + if ( qargs->next >= qargs->n_crystals ) return NULL; + + task = malloc(sizeof(struct log_args)); + if ( task == NULL ) return NULL; + + task->cr = qargs->crystals[qargs->next]; + task->full = qargs->full; + task->iter = qargs->iter; + task->cnum = qargs->next; + task->scaleflags = qargs->scaleflags; + + qargs->next += 20; + return task; +} + + +static void write_logs(void *vp, int cookie) +{ + struct log_args *args = vp; + write_specgraph(args->cr, args->full, args->iter, args->cnum); + write_gridscan(args->cr, args->full, args->iter, args->cnum, + args->scaleflags); + write_test_logs(args->cr, args->full, args->iter, args->cnum); +} + + +static void done_log(void *qargs, void *vp) +{ + struct log_args *task = vp; + free(task); +} + + +static void write_logs_parallel(Crystal **crystals, int n_crystals, + RefList *full, int iter, int n_threads, + int scaleflags) +{ + struct log_qargs qargs; + + qargs.iter = iter; + qargs.next = 0; + qargs.full = full; + qargs.crystals = crystals; + qargs.n_crystals = n_crystals; + qargs.scaleflags = scaleflags; + + STATUS("Writing logs...\n"); + run_threads(n_threads, write_logs, get_log_task, done_log, &qargs, + n_crystals/20, 0, 0, 0); + STATUS("Done.\n"); } @@ -755,9 +834,10 @@ int main(int argc, char *argv[]) char *outfile = NULL; char *sym_str = NULL; SymOpList *sym; + SymOpList *amb; + SymOpList *w_sym; int nthreads = 1; int i; - struct image *images; int n_iter = 10; RefList *full; int n_images = 0; @@ -765,11 +845,12 @@ int main(int argc, char *argv[]) int n_crystals_seen = 0; char cmdline[1024]; int no_scale = 0; + int no_Bscale = 0; int no_pr = 0; Stream *st; Crystal **crystals; char *pmodel_str = NULL; - PartialityModel pmodel = PMODEL_SCSPHERE; + PartialityModel pmodel = PMODEL_XSPHERE; int min_measurements = 2; char *rval; int polarisation = 1; @@ -785,6 +866,15 @@ int main(int argc, char *argv[]) char *csplit_fn = NULL; struct custom_split *csplit = NULL; double max_B = 1e-18; + char *rfile = NULL; + RefList *reference = NULL; + int no_logs = 0; + char *w_sym_str = NULL; + char *operator = NULL; + double force_bandwidth = -1.0; + double force_radius = -1.0; + char *audit_info; + int scaleflags = 0; /* Long options */ const struct option longopts[] = { @@ -805,10 +895,15 @@ int main(int argc, char *argv[]) {"push-res", 1, NULL, 5}, {"res-push", 1, NULL, 5}, /* compat */ {"custom-split", 1, NULL, 6}, - {"max-rel-B" , 1, NULL, 7}, - {"max-rel-b" , 1, NULL, 7}, /* compat */ + {"max-rel-B", 1, NULL, 7}, + {"max-rel-b", 1, NULL, 7}, /* compat */ + {"reference", 1, NULL, 8}, /* ssshhh! */ + {"operator", 1, NULL, 9}, + {"force-bandwidth", 1, NULL, 10}, + {"force-radius", 1, NULL, 11}, {"no-scale", 0, &no_scale, 1}, + {"no-Bscale", 0, &no_Bscale, 1}, {"no-pr", 0, &no_pr, 1}, {"no-polarisation", 0, &polarisation, 0}, {"no-polarization", 0, &polarisation, 0}, /* compat */ @@ -816,6 +911,7 @@ int main(int argc, char *argv[]) {"polarization", 0, &polarisation, 1}, /* compat */ {"no-free", 0, &no_free, 1}, {"output-every-cycle", 0, &output_everycycle, 1}, + {"no-logs", 0, &no_logs, 1}, {0, 0, NULL, 0} }; @@ -827,7 +923,7 @@ int main(int argc, char *argv[]) } /* Short options */ - while ((c = getopt_long(argc, argv, "hi:o:g:b:y:n:j:m:", + while ((c = getopt_long(argc, argv, "hi:o:g:b:y:n:j:m:w:", longopts, NULL)) != -1) { @@ -886,6 +982,10 @@ int main(int argc, char *argv[]) pmodel_str = strdup(optarg); break; + case 'w' : + w_sym_str = strdup(optarg); + break; + case 2 : errno = 0; min_measurements = strtod(optarg, &rval); @@ -932,6 +1032,33 @@ int main(int argc, char *argv[]) max_B = max_B * 1e-20; break; + case 8 : + rfile = strdup(optarg); + break; + + case 9 : + operator = strdup(optarg); + break; + + case 10 : + errno = 0; + force_bandwidth = strtod(optarg, &rval); + if ( *rval != '\0' ) { + ERROR("Invalid value for --force-bandwidth.\n"); + return 1; + } + break; + + case 11 : + errno = 0; + force_radius = strtod(optarg, &rval); + if ( *rval != '\0' ) { + ERROR("Invalid value for --force-radius.\n"); + return 1; + } + force_radius *= 1e9; + break; + case 0 : break; @@ -971,13 +1098,49 @@ int main(int argc, char *argv[]) free(sym_str); if ( sym == NULL ) return 1; + if ( (w_sym_str != NULL) && (operator != NULL) ) { + ERROR("Specify the apparent symmetry (-w) or the operator, " + "not both.\n"); + return 1; + } + + if ( w_sym_str == NULL ) { + w_sym = NULL; + amb = NULL; + } else { + pointgroup_warning(w_sym_str); + w_sym = get_pointgroup(w_sym_str); + free(w_sym_str); + if ( w_sym == NULL ) return 1; + amb = get_ambiguities(w_sym, sym); + if ( amb == NULL ) { + ERROR("Couldn't find ambiguity operator.\n"); + ERROR("Check that your values for -y and -w are " + "correct.\n"); + return 1; + } + + } + + if ( operator ) { + amb = parse_symmetry_operations(operator); + if ( amb == NULL ) return 1; + set_symmetry_name(amb, "Ambiguity"); + } + + if ( amb != NULL ) { + STATUS("Indexing ambiguity resolution enabled. " + "The ambiguity operation(s) are:\n"); + describe_symmetry(amb); + /* In contrast to ambigator, partialator can deal with multiple + * ambiguities at once */ + } + if ( pmodel_str != NULL ) { if ( strcmp(pmodel_str, "unity") == 0 ) { pmodel = PMODEL_UNITY; - } else if ( strcmp(pmodel_str, "scgaussian") == 0 ) { - pmodel = PMODEL_SCGAUSSIAN; - } else if ( strcmp(pmodel_str, "scsphere") == 0 ) { - pmodel = PMODEL_SCSPHERE; + } else if ( strcmp(pmodel_str, "xsphere") == 0 ) { + pmodel = PMODEL_XSPHERE; } else if ( strcmp(pmodel_str, "random") == 0 ) { pmodel = PMODEL_RANDOM; } else { @@ -988,7 +1151,25 @@ int main(int argc, char *argv[]) if ( (pmodel == PMODEL_UNITY) && !no_pr ) { no_pr = 1; - STATUS("Setting --no-pr because we are not modelling partialities (--model=unity).\n"); + STATUS("Setting --no-pr because we are not modelling " + "partialities (--model=unity).\n"); + } + + if ( no_Bscale ) { + scaleflags |= SCALE_NO_B; + } + + if ( !no_logs ) { + int r = mkdir("pr-logs", S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + if ( r ) { + if ( errno == EEXIST ) { + ERROR("WARNING: pr-logs folder already exists. " + "Beware of mixing old and new log files!\n"); + } else { + ERROR("Failed to create pr-logs folder.\n"); + return 1; + } + } } /* Read the custom split list (if applicable) */ @@ -1001,6 +1182,20 @@ int main(int argc, char *argv[]) free(csplit_fn); } + if ( rfile != NULL ) { + RefList *rread; + rread = read_reflections(rfile); + if ( rread == NULL ) { + ERROR("Failed to read reference reflections\n"); + return 1; + } + reference = asymmetric_indices(rread, sym); + reflist_free(rread); + ERROR("WARNING: Using an external reference.\n"); + ERROR("WARNING: If you publish a structure based on the result," + " expect to have to retract your paper!\n"); + } + gsl_set_error_handler_off(); rng = gsl_rng_alloc(gsl_rng_mt19937); @@ -1008,7 +1203,6 @@ int main(int argc, char *argv[]) n_images = 0; n_crystals = 0; n_crystals_seen = 0; - images = NULL; crystals = NULL; if ( sparams_fn != NULL ) { char line[1024]; @@ -1032,35 +1226,27 @@ int main(int argc, char *argv[]) RefList *as; int i; - struct image *images_new; - struct image *cur; + struct image cur; - images_new = realloc(images, (n_images+1)*sizeof(struct image)); - if ( images_new == NULL ) { - ERROR("Failed to allocate memory for image list.\n"); - return 1; - } - images = images_new; - cur = &images[n_images]; - - cur->div = NAN; - cur->bw = NAN; - cur->det = NULL; - if ( read_chunk_2(st, cur, STREAM_READ_REFLECTIONS - | STREAM_READ_UNITCELL) != 0 ) { + cur.div = NAN; + cur.bw = NAN; + cur.det = NULL; + if ( read_chunk_2(st, &cur, STREAM_READ_REFLECTIONS + | STREAM_READ_UNITCELL) != 0 ) { break; } - if ( isnan(cur->div) || isnan(cur->bw) ) { + if ( isnan(cur.div) || isnan(cur.bw) ) { ERROR("Chunk doesn't contain beam parameters.\n"); return 1; } - for ( i=0; i<cur->n_crystals; i++ ) { + for ( i=0; i<cur.n_crystals; i++ ) { Crystal *cr; Crystal **crystals_new; RefList *cr_refl; + struct image *image; n_crystals_seen++; if ( n_crystals_seen <= start_after ) continue; @@ -1073,11 +1259,19 @@ int main(int argc, char *argv[]) return 1; } crystals = crystals_new; - crystals[n_crystals] = cur->crystals[i]; + crystals[n_crystals] = cur.crystals[i]; cr = crystals[n_crystals]; - /* Image pointer will change due to later reallocs */ - crystal_set_image(cr, NULL); + image = malloc(sizeof(struct image)); + if ( image == NULL ) { + ERROR("Failed to allocatea memory for image.\n"); + return 1; + } + + crystal_set_image(cr, image); + *image = cur; + image->n_crystals = 1; + image->crystals = &crystals[n_crystals]; /* This is the raw list of reflections */ cr_refl = crystal_get_reflections(cr); @@ -1087,7 +1281,7 @@ int main(int argc, char *argv[]) if ( polarisation ) { polarisation_correction(cr_refl, crystal_get_cell(cr), - cur); + image); } if ( !no_free ) select_free_reflections(cr_refl, rng); @@ -1120,68 +1314,77 @@ int main(int argc, char *argv[]) display_progress(n_images, n_crystals); fprintf(stderr, "\n"); if ( sparams_fh != NULL ) fclose(sparams_fh); - + audit_info = stream_audit_info(st); close_stream(st); - /* Fill in image pointers */ - for ( i=0; i<n_images; i++ ) { - int j; - for ( j=0; j<images[i].n_crystals; j++ ) { - - Crystal *cryst; - - cryst = images[i].crystals[j]; - crystal_set_image(cryst, &images[i]); - - /* Now it's safe to do the following */ - update_predictions(cryst); - calculate_partialities(cryst, pmodel); - + STATUS("Initial partiality calculation...\n"); + for ( i=0; i<n_crystals; i++ ) { + Crystal *cr = crystals[i]; + if ( force_bandwidth > 0.0 ) { + crystal_get_image(cr)->bw = force_bandwidth; } + if ( force_radius > 0.0 ) { + crystal_set_profile_radius(cr, force_radius); + } + update_predictions(cr); + calculate_partialities(cr, pmodel); } if (csplit != NULL) check_csplit(crystals, n_crystals, csplit); /* Make a first pass at cutting out crap */ - STATUS("Checking patterns.\n"); + //STATUS("Early rejection...\n"); //early_rejection(crystals, n_crystals); - /* Initial rejection, figures of merit etc */ - full = merge_intensities(crystals, n_crystals, nthreads, pmodel, - min_measurements, push_res, 1); + /* Create reference data set if we don't already have one */ + if ( reference == NULL ) { + if ( !no_scale ) { + STATUS("Initial scaling...\n"); + scale_all(crystals, n_crystals, nthreads, scaleflags); + } + full = merge_intensities(crystals, n_crystals, nthreads, + min_measurements, push_res, 1); + } else { + full = reference; + } + + /* Check rejection and write figures of merit */ check_rejection(crystals, n_crystals, full, max_B); show_all_residuals(crystals, n_crystals, full); - write_pgraph(full, crystals, n_crystals, 0); + if ( !no_pr && !no_logs ) { + write_pgraph(full, crystals, n_crystals, 0, ""); + write_logs_parallel(crystals, n_crystals, full, 0, nthreads, + scaleflags); + } /* Iterate */ for ( i=0; i<n_iter; i++ ) { STATUS("Scaling and refinement cycle %i of %i\n", i+1, n_iter); - if ( !no_scale ) { - scale_all(crystals, n_crystals, nthreads, pmodel); - reflist_free(full); - full = merge_intensities(crystals, n_crystals, nthreads, - pmodel, min_measurements, - push_res, 1); + if ( !no_pr ) { + refine_all(crystals, n_crystals, full, nthreads, pmodel, + 0, i+1, no_logs, sym, amb, scaleflags); } - if ( !no_pr ) { - refine_all(crystals, n_crystals, full, nthreads, - pmodel); + /* Create new reference if needed */ + if ( reference == NULL ) { reflist_free(full); + if ( !no_scale ) { + scale_all(crystals, n_crystals, nthreads, + scaleflags); + } full = merge_intensities(crystals, n_crystals, nthreads, - pmodel, min_measurements, + min_measurements, push_res, 1); - } + } /* else full still equals reference */ check_rejection(crystals, n_crystals, full, max_B); - reflist_free(full); - full = merge_intensities(crystals, n_crystals, nthreads, - pmodel, min_measurements, - push_res, 1); show_all_residuals(crystals, n_crystals, full); - write_pgraph(full, crystals, n_crystals, i+1); + + if ( !no_logs ) { + write_pgraph(full, crystals, n_crystals, i+1, ""); + } if ( output_everycycle ) { @@ -1208,21 +1411,37 @@ int main(int argc, char *argv[]) } } - /* Dump parameters */ - snprintf(tmp, 1024, "iter%.2d_partialator.params", i+1); - dump_parameters(tmp, crystals, n_crystals); + } + } + /* Final merge */ + STATUS("Final merge...\n"); + if ( reference == NULL ) { + reflist_free(full); + if ( !no_scale ) { + scale_all(crystals, n_crystals, nthreads, scaleflags); } + full = merge_intensities(crystals, n_crystals, nthreads, + min_measurements, + push_res, 1); + } else { + full = merge_intensities(crystals, n_crystals, nthreads, + min_measurements, push_res, 1); } - full = merge_intensities(crystals, n_crystals, nthreads, pmodel, - min_measurements, push_res, 1); + /* Write final figures of merit (no rejection any more) */ show_all_residuals(crystals, n_crystals, full); - write_pgraph(full, crystals, n_crystals, n_iter+1); + if ( !no_pr && !no_logs ) { + write_pgraph(full, crystals, n_crystals, -1, ""); + write_logs_parallel(crystals, n_crystals, full, -1, nthreads, + scaleflags); + } /* Output results */ STATUS("Writing overall results to %s\n", outfile); reflist_add_command_and_version(full, argc, argv); + reflist_add_notes(full, "Audit information from stream:"); + reflist_add_notes(full, audit_info); write_reflist_2(outfile, full, sym); /* Output split results */ @@ -1238,9 +1457,6 @@ int main(int argc, char *argv[]) } } - /* Dump parameters */ - dump_parameters("partialator.params", crystals, n_crystals); - /* Clean up */ gsl_rng_free(rng); for ( i=0; i<n_crystals; i++ ) { @@ -1248,13 +1464,9 @@ int main(int argc, char *argv[]) crystal_free(crystals[i]); } reflist_free(full); - free(sym); + free_symoplist(sym); free(outfile); free(crystals); - for ( i=0; i<n_images; i++ ) { - free(images[i].filename); - } - free(images); free(infile); return 0; diff --git a/src/post-refinement.c b/src/post-refinement.c index bf30d299..bde9c02c 100644 --- a/src/post-refinement.c +++ b/src/post-refinement.c @@ -3,11 +3,11 @@ * * Post refinement * - * Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2015 Thomas White <taw@physics.org> + * 2010-2018 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -33,11 +33,8 @@ #include <stdlib.h> #include <assert.h> -#include <gsl/gsl_matrix.h> -#include <gsl/gsl_vector.h> -#include <gsl/gsl_linalg.h> -#include <gsl/gsl_eigen.h> -#include <gsl/gsl_blas.h> +#include <gsl/gsl_multimin.h> +#include <gsl/gsl_fit.h> #include "image.h" #include "post-refinement.h" @@ -46,15 +43,14 @@ #include "geometry.h" #include "cell.h" #include "cell-utils.h" +#include "reflist-utils.h" +#include "scaling.h" +#include "merge.h" -/* Maximum number of iterations of NLSq to do for each image per macrocycle. */ -#define MAX_CYCLES (10) - struct prdata { int refined; - int n_filtered; }; const char *str_prflag(enum prflag flag) @@ -79,625 +75,820 @@ const char *str_prflag(enum prflag flag) case PRFLAG_BIGB : return "B too big"; + case PRFLAG_SCALEBAD : + return "bad scaling"; + default : return "Unknown flag"; } } -/* Returns dp(gauss)/dr at "r" */ -static double gaussian_fraction_gradient(double r, double R) +static UnitCell *rotate_cell_xy(const UnitCell *cell, double ang1, double ang2) { - const double ng = 2.6; - const double sig = R/ng; - - /* If the Ewald sphere isn't within the profile, the gradient is zero */ - if ( r < -R ) return 0.0; - if ( r > +R ) return 0.0; - - return exp(-pow(r/sig, 2.0)/2.0) / (sig*sqrt(2.0*M_PI)); + UnitCell *o; + double asx, asy, asz; + double bsx, bsy, bsz; + double csx, csy, csz; + double xnew, ynew, znew; + + o = cell_new_from_cell(cell); + + cell_get_reciprocal(o, &asx, &asy, &asz, + &bsx, &bsy, &bsz, + &csx, &csy, &csz); + + /* "a" around x */ + xnew = asx; + ynew = asy*cos(ang1) + asz*sin(ang1); + znew = -asy*sin(ang1) + asz*cos(ang1); + asx = xnew; asy = ynew; asz = znew; + + /* "b" around x */ + xnew = bsx; + ynew = bsy*cos(ang1) + bsz*sin(ang1); + znew = -bsy*sin(ang1) + bsz*cos(ang1); + bsx = xnew; bsy = ynew; bsz = znew; + + /* "c" around x */ + xnew = csx; + ynew = csy*cos(ang1) + csz*sin(ang1); + znew = -csy*sin(ang1) + csz*cos(ang1); + csx = xnew; csy = ynew; csz = znew; + + /* "a" around y */ + xnew = asx*cos(ang2) + asz*sin(ang2); + ynew = asy; + znew = -asx*sin(ang2) + asz*cos(ang2); + asx = xnew; asy = ynew; asz = znew; + + /* "b" around y */ + xnew = bsx*cos(ang2) + bsz*sin(ang2); + ynew = bsy; + znew = -bsx*sin(ang2) + bsz*cos(ang2); + bsx = xnew; bsy = ynew; bsz = znew; + + /* "c" around y */ + xnew = csx*cos(ang2) + csz*sin(ang2); + ynew = csy; + znew = -csx*sin(ang2) + csz*cos(ang2); + csx = xnew; csy = ynew; csz = znew; + + cell_set_reciprocal(o, asx, asy, asz, bsx, bsy, bsz, csx, csy, csz); + + return o; } -/* Returns dp(sph)/dr at "r" */ -static double sphere_fraction_gradient(double r, double pr) +static const char *get_label(enum gparam p) { - double q, dpdq, dqdr; - - /* If the Ewald sphere isn't within the profile, the gradient is zero */ - if ( r < -pr ) return 0.0; - if ( r > +pr ) return 0.0; - - q = (r + pr)/(2.0*pr); - dpdq = 6.0*(q - q*q); - dqdr = 1.0 / (2.0*pr); - return dpdq * dqdr; + switch ( p ) { + case GPARAM_ANG1 : return "First angle (radians)"; + case GPARAM_ANG2 : return "Second angle (radians)"; + case GPARAM_R : return "Profile radius (m^-1)"; + case GPARAM_WAVELENGTH : return "Wavelength (Angstroms)"; + default : return "unknown"; + } } -/* Returns dp/dr at "r" */ -static double partiality_gradient(double r, double pr, - PartialityModel pmodel, - double rlow, double rhigh) +/* We set all the step sizes to 1, then scale them. + * This way, the size of the simplex stays meaningful and we possibly also + * avoid some roundoff errors */ +static double get_scale(enum gparam p) { - double A, D; - - D = rlow - rhigh; - - switch ( pmodel ) { + switch ( p ) { - default: - case PMODEL_UNITY: - return 0.0; + case GPARAM_ANG1 : return deg2rad(0.05); + case GPARAM_ANG2 : return deg2rad(0.05); + case GPARAM_R : return 0.0005e9; + case GPARAM_WAVELENGTH : return 0.001e-10; - case PMODEL_SCSPHERE: - A = sphere_fraction_gradient(r, pr)/D; - return 4.0*pr*A/3.0; - - case PMODEL_SCGAUSSIAN: - A = gaussian_fraction_gradient(r, pr)/D; - return 4.0*pr*A/3.0; + default : return 0.0; } } -static double sphere_fraction_rgradient(double r, double R) -{ - /* If the Ewald sphere isn't within the profile, the gradient is zero */ - if ( r < -R ) return 0.0; - if ( r > +R ) return 0.0; - - return -(3.0*r/(4.0*R*R)) * (1.0 - r*r/(R*R)); -} +struct rf_priv { + const Crystal *cr; + const RefList *full; + enum gparam rv[32]; + int verbose; + int serial; + gsl_vector *initial; + int scaleflags; + /* For freeing later */ + gsl_vector *vals; + gsl_vector *step; -static double gaussian_fraction_rgradient(double r, double R) -{ - const double ng = 2.6; - const double sig = R/ng; + /* So that it stays around until the end of minimisation */ + gsl_multimin_function f; +}; - /* If the Ewald sphere isn't within the profile, the gradient is zero */ - if ( r < -R ) return 0.0; - if ( r > +R ) return 0.0; - return -(ng*r/(sqrt(2.0*M_PI)*R*R))*exp(-r*r/(2.0*sig*sig)); +static double get_actual_val(const gsl_vector *v, const gsl_vector *initial, + enum gparam *rv, int i) +{ + return gsl_vector_get(v, i) * get_scale(rv[i]) + + gsl_vector_get(initial, i); } -static double volume_fraction_rgradient(double r, double pr, - PartialityModel pmodel) +static void apply_parameters(const gsl_vector *v, const gsl_vector *initial, + enum gparam *rv, Crystal *cr) { - switch ( pmodel ) - { - case PMODEL_UNITY : - return 1.0; - - case PMODEL_SCSPHERE : - return sphere_fraction_rgradient(r, pr); + int i; + double ang1, ang2, R, lambda; + UnitCell *cell; - case PMODEL_SCGAUSSIAN : - return gaussian_fraction_rgradient(r, pr); + /* Default parameters if not used in refinement */ + ang1 = 0.0; + ang2 = 0.0; + R = crystal_get_profile_radius(cr); + lambda = crystal_get_image(cr)->lambda; - default : - ERROR("No pmodel in volume_fraction_rgradient!\n"); - return 1.0; - } -} + for ( i=0; i<v->size; i++ ) { + double val; -static double volume_fraction(double rlow, double rhigh, double pr, - PartialityModel pmodel) -{ - switch ( pmodel ) - { - case PMODEL_UNITY : - return 1.0; + val = get_actual_val(v, initial, rv, i); - case PMODEL_SCSPHERE : - return sphere_fraction(rlow, rhigh, pr); + switch ( rv[i] ) { - case PMODEL_SCGAUSSIAN : - return gaussian_fraction(rlow, rhigh, pr); + case GPARAM_ANG1 : + ang1 = val; + break; - default : - ERROR("No pmodel in volume_fraction!\n"); - return 1.0; - } -} + case GPARAM_ANG2 : + ang2 = val; + break; + case GPARAM_R : + R = val; + break; -/* Return the gradient of "fx" wrt parameter 'k' given the current - * status of the crystal. */ -double gradient(Crystal *cr, int k, Reflection *refl, PartialityModel pmodel) -{ - double glow, ghigh; - double rlow, rhigh, p; - struct image *image = crystal_get_image(cr); - double R = crystal_get_profile_radius(cr); - double gr; + case GPARAM_WAVELENGTH : + lambda = val; + break; - get_partial(refl, &rlow, &rhigh, &p); + default : + ERROR("Don't understand parameter %i\n", rv[i]); + break; - if ( k == GPARAM_R ) { + } + } - double Rglow, Rghigh; - double D, psph; + cell = rotate_cell_xy(crystal_get_cell_const(cr), ang1, ang2); + crystal_set_cell(cr, cell); - D = rlow - rhigh; - psph = volume_fraction(rlow, rhigh, R, pmodel); + crystal_set_profile_radius(cr, R); + crystal_get_image(cr)->lambda = lambda; +} - Rglow = volume_fraction_rgradient(rlow, R, pmodel); - Rghigh = volume_fraction_rgradient(rhigh, R, pmodel); - gr = 4.0*psph/(3.0*D) + (4.0*R/(3.0*D))*(Rglow - Rghigh); - return gr; +static double residual_f(const gsl_vector *v, void *pp) +{ + struct rf_priv *pv = pp; + RefList *list; + struct image im; + Crystal *cr; + UnitCell *cell; + double res; + int i; + for ( i=0; i<v->size; i++ ) { + if ( gsl_vector_get(v, i) > 100.0 ) return GSL_NAN; } - /* Calculate the gradient of partiality wrt excitation error. */ - glow = partiality_gradient(rlow, R, pmodel, rlow, rhigh); - ghigh = partiality_gradient(rhigh, R, pmodel, rlow, rhigh); + cr = crystal_copy(pv->cr); + cell = cell_new_from_cell(crystal_get_cell(cr)); + if ( cell == NULL ) return GSL_NAN; + crystal_set_cell(cr, cell); + im = *crystal_get_image(cr); + crystal_set_image(cr, &im); + apply_parameters(v, pv->initial, pv->rv, cr); + + if ( fabs(crystal_get_profile_radius(cr)) > 5e9 ) { + crystal_free(cr); + if ( pv->verbose ) STATUS("radius > 5e9\n"); + return GSL_NAN; + } - if ( k == GPARAM_DIV ) { + /* Can happen with grid scans and certain --force-radius values */ + if ( fabs(crystal_get_profile_radius(cr)) < 0.0000001e9 ) { + crystal_free(cr); + if ( pv->verbose ) STATUS("radius very small\n"); + return GSL_NAN; + } - double D, psph, ds; - signed int hs, ks, ls; + if ( im.lambda <= 0.0 ) { + crystal_free(cr); + if ( pv->verbose ) STATUS("lambda < 0\n"); + return GSL_NAN; + } - D = rlow - rhigh; - psph = volume_fraction(rlow, rhigh, R, pmodel); - get_symmetric_indices(refl, &hs, &ks, &ls); - ds = 2.0 * resolution(crystal_get_cell(cr), hs, ks, ls); + list = copy_reflist(crystal_get_reflections(cr)); + crystal_set_reflections(cr, list); - gr = (ds/2.0)*(glow+ghigh) - 4.0*R*psph*ds/(3.0*D*D); - return gr; + update_predictions(cr); + calculate_partialities(cr, PMODEL_XSPHERE); + if ( scale_one_crystal(cr, pv->full, pv->scaleflags) ) { + crystal_free(cr); + if ( pv->verbose ) STATUS("Bad scaling\n"); + return GSL_NAN; } + res = residual(cr, pv->full, 0, NULL, NULL); + + cell_free(crystal_get_cell(cr)); + reflist_free(crystal_get_reflections(cr)); + crystal_free(cr); - gr = r_gradient(crystal_get_cell(cr), k, refl, image) * (glow-ghigh); - return gr; + return res; } -static void apply_cell_shift(UnitCell *cell, int k, double shift) +static double get_initial_param(Crystal *cr, enum gparam p) { - double asx, asy, asz; - double bsx, bsy, bsz; - double csx, csy, csz; - double as, bs, cs; + switch ( p ) { - cell_get_reciprocal(cell, &asx, &asy, &asz, - &bsx, &bsy, &bsz, - &csx, &csy, &csz); - as = modulus(asx, asy, asz); - bs = modulus(bsx, bsy, bsz); - cs = modulus(csx, csy, csz); + case GPARAM_ANG1 : return 0.0; + case GPARAM_ANG2 : return 0.0; + case GPARAM_R : return crystal_get_profile_radius(cr); + case GPARAM_WAVELENGTH : return crystal_get_image(cr)->lambda; - /* Forbid any step which looks too large */ - switch ( k ) - { - case GPARAM_ASX : - case GPARAM_ASY : - case GPARAM_ASZ : - if ( fabs(shift) > 0.1*as ) return; - break; - - case GPARAM_BSX : - case GPARAM_BSY : - case GPARAM_BSZ : - if ( fabs(shift) > 0.1*bs ) return; - break; - - case GPARAM_CSX : - case GPARAM_CSY : - case GPARAM_CSZ : - if ( fabs(shift) > 0.1*cs ) return; - break; - } - - switch ( k ) - { - case GPARAM_ASX : asx += shift; break; - case GPARAM_ASY : asy += shift; break; - case GPARAM_ASZ : asz += shift; break; - case GPARAM_BSX : bsx += shift; break; - case GPARAM_BSY : bsy += shift; break; - case GPARAM_BSZ : bsz += shift; break; - case GPARAM_CSX : csx += shift; break; - case GPARAM_CSY : csy += shift; break; - case GPARAM_CSZ : csz += shift; break; - } - - cell_set_reciprocal(cell, asx, asy, asz, - bsx, bsy, bsz, - csx, csy, csz); + default: return 0.0; + + } } -/* Apply the given shift to the 'k'th parameter of 'image'. */ -static void apply_shift(Crystal *cr, int k, double shift) +static int check_angle_shifts(gsl_vector *cur, const gsl_vector *initial, + enum gparam *rv, struct rf_priv *residual_f_priv) { - double t; - struct image *image = crystal_get_image(cr); - - switch ( k ) { - - case GPARAM_DIV : - if ( shift > 0.1*image->div ) return; - image->div += shift; - if ( image->div < 0.0 ) image->div = 0.0; - break; - - case GPARAM_R : - t = crystal_get_profile_radius(cr); - if ( shift > 0.1*t ) return; - t += shift; - crystal_set_profile_radius(cr, t); - break; - - case GPARAM_ASX : - case GPARAM_ASY : - case GPARAM_ASZ : - case GPARAM_BSX : - case GPARAM_BSY : - case GPARAM_BSZ : - case GPARAM_CSX : - case GPARAM_CSY : - case GPARAM_CSZ : - apply_cell_shift(crystal_get_cell(cr), k, shift); - break; + int i = 0; + double ang = 0.0; - default : - ERROR("No shift defined for parameter %i\n", k); - abort(); + while ( rv[i] != GPARAM_EOL ) { + if ( (rv[i] == GPARAM_ANG1) || (rv[i] == GPARAM_ANG2) ) { + ang += fabs(get_actual_val(cur, initial, rv, i)); + } + rv++; + } + if ( rad2deg(ang) > 5.0 ) { + ERROR("More than 5 degrees total rotation!\n"); + residual_f_priv->verbose = 1; + double res = residual_f(cur, residual_f_priv); + STATUS("residual after rotation = %e\n", res); + residual_f_priv->verbose = 2; + res = residual_f(initial, residual_f_priv); + STATUS("residual before rotation = %e\n", res); + return 1; } + return 0; } -/* Perform one cycle of post refinement on 'image' against 'full' */ -static double pr_iterate(Crystal *cr, const RefList *full, - PartialityModel pmodel, - int *n_filtered, int verbose) +static RefList *reindex_reflections(RefList *input, SymOpList *amb, + SymOpList *sym, int idx) { - gsl_matrix *M; - gsl_vector *v; - gsl_vector *shifts; - int param; + RefList *n; Reflection *refl; RefListIterator *iter; - RefList *reflections; - double max_shift; - int nref = 0; - int num_params = 0; - enum gparam rv[32]; - double G, B; - - if ( n_filtered != NULL ) *n_filtered = 0; - - rv[num_params++] = GPARAM_ASX; - rv[num_params++] = GPARAM_ASY; - rv[num_params++] = GPARAM_ASZ; - rv[num_params++] = GPARAM_BSX; - rv[num_params++] = GPARAM_BSY; - rv[num_params++] = GPARAM_BSZ; - rv[num_params++] = GPARAM_CSX; - rv[num_params++] = GPARAM_CSY; - rv[num_params++] = GPARAM_CSZ; -// rv[num_params++] = GPARAM_R; -// rv[num_params++] = GPARAM_DIV; - - M = gsl_matrix_calloc(num_params, num_params); - v = gsl_vector_calloc(num_params); - - reflections = crystal_get_reflections(cr); - G = crystal_get_osf(cr); - B = crystal_get_Bfac(cr); - - /* Post-refinement terms */ - for ( refl = first_refl(reflections, &iter); + + n = reflist_new(); + + for ( refl = first_refl(input, &iter); refl != NULL; refl = next_refl(refl, iter) ) { - signed int ha, ka, la; - double I_full, delta_I, esd; - double w; - double I_partial; - int k; - double p, L, s; - double fx; - Reflection *match; - double gradients[num_params]; + signed int h, k, l; + Reflection *rn; - if ( get_flag(refl) ) continue; + get_indices(refl, &h, &k, &l); + get_equiv(amb, NULL, idx, h, k, l, &h, &k, &l); + get_asymm(sym, h, k, l, &h, &k, &l); + rn = add_refl(n, h, k, l); + copy_data(rn, refl); + + get_symmetric_indices(rn, &h, &k, &l); + get_equiv(amb, NULL, idx, h, k, l, &h, &k, &l); + set_symmetric_indices(rn, h, k, l); + } - get_indices(refl, &ha, &ka, &la); - match = find_refl(full, ha, ka, la); - if ( match == NULL ) continue; - I_full = get_intensity(match); + return n; +} - if ( get_redundancy(match) < 2 ) continue; - p = get_partiality(refl); - L = get_lorentz(refl); - I_partial = get_intensity(refl); - esd = get_esd_intensity(refl); - s = resolution(crystal_get_cell(cr), ha, ka, la); +static void reindex_cell(UnitCell *cell, SymOpList *amb, int idx) +{ + signed int h, k, l; + struct rvec na, nb, nc; + struct rvec as, bs, cs; + + cell_get_reciprocal(cell, &as.u, &as.v, &as.w, + &bs.u, &bs.v, &bs.w, + &cs.u, &cs.v, &cs.w); + + get_equiv(amb, NULL, idx, 1, 0, 0, &h, &k, &l); + na.u = as.u*h + bs.u*k + cs.u*l; + na.v = as.v*h + bs.v*k + cs.v*l; + na.w = as.w*h + bs.w*k + cs.w*l; + + get_equiv(amb, NULL, idx, 0, 1, 0, &h, &k, &l); + nb.u = as.u*h + bs.u*k + cs.u*l; + nb.v = as.v*h + bs.v*k + cs.v*l; + nb.w = as.w*h + bs.w*k + cs.w*l; + + get_equiv(amb, NULL, idx, 0, 0, 1, &h, &k, &l); + nc.u = as.u*h + bs.u*k + cs.u*l; + nc.v = as.v*h + bs.v*k + cs.v*l; + nc.w = as.w*h + bs.w*k + cs.w*l; + + cell_set_reciprocal(cell, na.u, na.v, na.w, + nb.u, nb.v, nb.w, + nc.u, nc.v, nc.w); +} - if ( I_partial < 3.0*esd ) continue; - /* Calculate the weight for this reflection */ - w = (s/1e9)*(s/1e9) / (esd*esd); +static void try_reindex(Crystal *crin, const RefList *full, + SymOpList *sym, SymOpList *amb, int scaleflags) +{ + RefList *list; + Crystal *cr; + UnitCell *cell; + double residual_original, residual_flipped; + int idx, n; - /* Calculate all gradients for this reflection */ - for ( k=0; k<num_params; k++ ) { - gradients[k] = gradient(cr, rv[k], refl, pmodel); - gradients[k] *= exp(G)*exp(-B*s*s)*I_full/L; - } + if ( sym == NULL || amb == NULL ) return; - for ( k=0; k<num_params; k++ ) { + if ( scale_one_crystal(crin, full, scaleflags) ) return; + residual_original = residual(crin, full, 0, NULL, NULL); - int g; - double v_c, v_curr; + cr = crystal_copy(crin); - for ( g=0; g<num_params; g++ ) { + n = num_equivs(amb, NULL); - double M_c, M_curr; + for ( idx=0; idx<n; idx++ ) { - /* Matrix is symmetric */ - if ( g > k ) continue; + cell = cell_new_from_cell(crystal_get_cell(crin)); + if ( cell == NULL ) return; + reindex_cell(cell, amb, idx); + crystal_set_cell(cr, cell); - M_c = w * gradients[g] * gradients[k]; + list = reindex_reflections(crystal_get_reflections(crin), + amb, sym, idx); + crystal_set_reflections(cr, list); - M_curr = gsl_matrix_get(M, k, g); - gsl_matrix_set(M, k, g, M_curr + M_c); - gsl_matrix_set(M, g, k, M_curr + M_c); + update_predictions(cr); + calculate_partialities(cr, PMODEL_XSPHERE); + + if ( scale_one_crystal(cr, full, scaleflags) ) return; + residual_flipped = residual(cr, full, 0, NULL, NULL); + + if ( residual_flipped < residual_original ) { + crystal_set_cell(crin, cell); + crystal_set_reflections(crin, list); + residual_original = residual_flipped; + } else { + cell_free(crystal_get_cell(cr)); + reflist_free(crystal_get_reflections(cr)); + } + } - } + crystal_free(cr); +} - fx = exp(G)*p*exp(-B*s*s)*I_full/L; - delta_I = I_partial - fx; - v_c = w * delta_I * gradients[k]; - v_curr = gsl_vector_get(v, k); - gsl_vector_set(v, k, v_curr + v_c); - } +void write_test_logs(Crystal *crystal, const RefList *full, + signed int cycle, int serial) +{ + FILE *fh; + struct image *image = crystal_get_image(crystal); + char tmp[256]; + char ins[5]; - nref++; - } - if ( verbose ) { - //STATUS("The original equation:\n"); - //show_matrix_eqn(M, v); - STATUS("%i reflections went into the equations.\n", nref); - } - - if ( nref < num_params ) { - crystal_set_user_flag(cr, PRFLAG_FEWREFL); - gsl_matrix_free(M); - gsl_vector_free(v); - return 0.0; - } - - max_shift = 0.0; - shifts = solve_svd(v, M, n_filtered, 0); - if ( shifts != NULL ) { - - for ( param=0; param<num_params; param++ ) { - double shift = gsl_vector_get(shifts, param); - if ( verbose ) STATUS("Shift %i: %e\n", param, shift); - if ( isnan(shift) ) { - //ERROR("NaN shift parameter %i (image ser %i), " - // "%i reflections.\n", rv[param], - // crystal_get_image(cr)->serial, - // nref); - } else { - apply_shift(cr, rv[param], shift); - } - if ( fabs(shift) > max_shift ) max_shift = fabs(shift); - } + snprintf(tmp, 256, "pr-logs/parameters-crystal%i.dat", serial); + if ( cycle == 0 ) { + fh = fopen(tmp, "w"); } else { - crystal_set_user_flag(cr, PRFLAG_SOLVEFAIL); + fh = fopen(tmp, "a"); } - gsl_matrix_free(M); - gsl_vector_free(v); - gsl_vector_free(shifts); + if ( fh == NULL ) { + ERROR("Failed to open '%s'\n", tmp); + return; + } + + if ( cycle == 0 ) { + fprintf(fh, "Image: %s %s\n", + image->filename, get_event_string(image->event)); + } + + if ( cycle >= 0 ) { + snprintf(ins, 4, "%i", cycle); + } else { + ins[0] = 'F'; + ins[1] = '\0'; + } - return max_shift; + fprintf(fh, "%s rlp_size = %e\n", ins, crystal_get_profile_radius(crystal)/1e10); + fprintf(fh, "%s mosaicity = %e\n", ins, crystal_get_mosaicity(crystal)); + fprintf(fh, "%s wavelength = %f A\n", ins, image->lambda*1e10); + fprintf(fh, "%s bandwidth = %f\n", ins, image->bw); + fprintf(fh, "%s my scale factor = %e\n", ins, crystal_get_osf(crystal)); + + double asx, asy, asz, bsx, bsy, bsz, csx, csy, csz; + cell_get_reciprocal(crystal_get_cell(crystal), &asx, &asy, &asz, + &bsx, &bsy, &bsz, + &csx, &csy, &csz); + fprintf(fh, "%s %f, %f, %f, %f, %f, %f, %f, %f, %f\n", ins, + asx/1e10, bsx/1e10, csx/1e10, + asy/1e10, bsy/1e10, csy/1e10, + asz/1e10, bsz/1e10, csz/1e10); + fclose(fh); } -double residual(Crystal *cr, const RefList *full, int free, - int *pn_used, const char *filename) +void write_specgraph(Crystal *crystal, const RefList *full, + signed int cycle, int serial) { - double dev = 0.0; - double G, B; + FILE *fh; + char tmp[256]; Reflection *refl; RefListIterator *iter; - FILE *fh = NULL; - int n_used = 0; + double G = crystal_get_osf(crystal); + double B = crystal_get_Bfac(crystal); + UnitCell *cell; + struct image *image = crystal_get_image(crystal); + char ins[5]; - if ( filename != NULL ) { - fh = fopen(filename, "a"); - if ( fh == NULL ) { - ERROR("Failed to open '%s'\n", filename); - } + snprintf(tmp, 256, "pr-logs/specgraph-crystal%i.dat", serial); + + if ( cycle == 0 ) { + fh = fopen(tmp, "w"); + } else { + fh = fopen(tmp, "a"); + } + + if ( fh == NULL ) { + ERROR("Failed to open '%s'\n", tmp); + return; + } + + if ( cycle == 0 ) { + fprintf(fh, "Image: %s %s\n", + image->filename, get_event_string(image->event)); + fprintf(fh, "khalf/m 1/d(m) pcalc pobs iteration h k l\n"); } - G = crystal_get_osf(cr); - B = crystal_get_Bfac(cr); + cell = crystal_get_cell(crystal); - for ( refl = first_refl(crystal_get_reflections(cr), &iter); + if ( cycle >= 0 ) { + snprintf(ins, 4, "%i", cycle); + } else { + ins[0] = 'F'; + ins[1] = '\0'; + } + + for ( refl = first_refl(crystal_get_reflections(crystal), &iter); refl != NULL; refl = next_refl(refl, iter) ) { - double p, L, s, w; + double corr, Ipart, Ifull, pobs, pcalc; + double res; signed int h, k, l; Reflection *match; - double esd, I_full, I_partial; - double fx, dc; - - if ( free && !get_flag(refl) ) continue; get_indices(refl, &h, &k, &l); + res = resolution(cell, h, k, l); + match = find_refl(full, h, k, l); if ( match == NULL ) continue; - I_full = get_intensity(match); - if ( get_redundancy(match) < 2 ) continue; + /* Don't calculate pobs if reference reflection is weak */ + if ( fabs(get_intensity(match)) / get_esd_intensity(match) < 3.0 ) continue; - p = get_partiality(refl); - L = get_lorentz(refl); - I_partial = get_intensity(refl); - esd = get_esd_intensity(refl); - s = resolution(crystal_get_cell(cr), h, k, l); + corr = G * exp(B*res*res) * get_lorentz(refl); + Ipart = get_intensity(refl) * corr; + Ifull = get_intensity(match); + pobs = Ipart / Ifull; + pcalc = get_partiality(refl); - if ( I_partial < 3.0*esd ) continue; + fprintf(fh, "%e %e %f %f %s %4i %4i %4i\n", + get_khalf(refl), 2.0*res, pcalc, pobs, ins, h, k, l); - fx = exp(G)*p*exp(-B*s*s)*I_full/L; - dc = I_partial - fx; - w = (s/1e9)*(s/1e9)/(esd*esd); - dev += w*dc*dc; - n_used++; - - if ( fh != NULL ) { - fprintf(fh, "%4i %4i %4i %e %e\n", - h, k, l, s, dev); - } } - if ( fh != NULL ) fclose(fh); - - if ( pn_used != NULL ) *pn_used = n_used; - return dev; + fclose(fh); } -static void write_residual_graph(Crystal *cr, const RefList *full) +static gsl_multimin_fminimizer *setup_minimiser(Crystal *cr, const RefList *full, + int verbose, int serial, + int scaleflags, + struct rf_priv *priv) { + gsl_multimin_fminimizer *min; + int n_params = 0; int i; - double asx, asy, asz; - double bsx, bsy, bsz; - double csx, csy, csz; - double a; - double step; - double orig_asx; - FILE *fh; - UnitCell *cell; - cell = crystal_get_cell(cr); + /* The parameters to be refined */ + priv->rv[n_params++] = GPARAM_ANG1; + priv->rv[n_params++] = GPARAM_ANG2; + priv->rv[n_params++] = GPARAM_R; + priv->rv[n_params++] = GPARAM_WAVELENGTH; + priv->rv[n_params] = GPARAM_EOL; /* End of list */ + + priv->cr = cr; + priv->full = full; + priv->verbose = verbose; + priv->serial = serial; + priv->scaleflags = scaleflags; + + priv->f.f = residual_f; + priv->f.n = n_params; + priv->f.params = priv; + + priv->initial = gsl_vector_alloc(n_params); + priv->vals = gsl_vector_alloc(n_params); + priv->step = gsl_vector_alloc(n_params); + + for ( i=0; i<n_params; i++ ) { + gsl_vector_set(priv->initial, i, get_initial_param(cr, priv->rv[i])); + gsl_vector_set(priv->vals, i, 0.0); + gsl_vector_set(priv->step, i, 1.0); + } - fh = fopen("residual-plot.dat", "w"); + min = gsl_multimin_fminimizer_alloc(gsl_multimin_fminimizer_nmsimplex2, + n_params); + gsl_multimin_fminimizer_set(min, &priv->f, priv->vals, priv->step); - cell_get_reciprocal(cell, &asx, &asy, &asz, - &bsx, &bsy, &bsz, - &csx, &csy, &csz); - a = modulus(asx, asy, asz); - orig_asx = asx; + return min; +} - step = a/100.0; - for ( i=-50; i<=50; i++ ) { +static void write_grid(Crystal *cr, const RefList *full, + signed int cycle, int serial, int scaleflags, + const enum gparam par1, const enum gparam par2, + const char *name) +{ + FILE *fh; + char fn[64]; + char ins[5]; + gsl_multimin_fminimizer *min; + struct rf_priv priv; + int idx1, idx2; + int i; - double res; - int n; - asx = orig_asx + (i*step); - cell_set_reciprocal(cell, asx, asy, asz, - bsx, bsy, bsz, - csx, csy, csz); - update_predictions(cr); - calculate_partialities(cr, PMODEL_SCSPHERE); - res = residual(cr, full, 0, &n, NULL); - fprintf(fh, "%i %e %e %i\n", i, asx, res, n); + min = setup_minimiser(cr, full, 0, serial, scaleflags, &priv); + + idx1 = 99; + idx2 = 99; + for ( i=0; i<priv.f.n; i++ ) { + if ( priv.rv[i] == par1 ) idx1 = i; + if ( priv.rv[i] == par2 ) idx2 = i; } + assert(idx1 != 99); + assert(idx2 != 99); - cell_set_reciprocal(cell, orig_asx, asy, asz, - bsx, bsy, bsz, - csx, csy, csz); - update_predictions(cr); - calculate_partialities(cr, PMODEL_SCSPHERE); + if ( cycle >= 0 ) { + snprintf(ins, 4, "%i", cycle); + } else { + ins[0] = 'F'; + ins[1] = '\0'; + } + + snprintf(fn, 63, "pr-logs/grid-crystal%i-cycle%s-%s.dat", + serial, ins, name); + fh = fopen(fn, "w"); + if ( fh != NULL ) { + double v1, v2; + fprintf(fh, "%e %e %e %s\n", + -5.0*get_scale(par1)+get_initial_param(cr, par1), + 5.0*get_scale(par1)+get_initial_param(cr, par1), + get_initial_param(cr, par1), + get_label(par1)); + fprintf(fh, "%e %e %e %s\n", + -5.0*get_scale(par2)+get_initial_param(cr, par2), + 5.0*get_scale(par2)+get_initial_param(cr, par2), + get_initial_param(cr, par2), + get_label(par2)); + for ( v2=-5.0; v2<=5.0; v2+=0.25 ) { + int first=1; + for ( v1=-5.0; v1<=5.0; v1+=0.25 ) { + double res; + gsl_vector_set(min->x, idx1, v1); + gsl_vector_set(min->x, idx2, v2); + res = residual_f(min->x, &priv); + if ( !first ) fprintf(fh, " "); + first = 0; + fprintf(fh, "%e", res); + } + fprintf(fh, "\n"); + } + } fclose(fh); + + gsl_multimin_fminimizer_free(min); + gsl_vector_free(priv.initial); + gsl_vector_free(priv.vals); + gsl_vector_free(priv.step); +} + + +void write_gridscan(Crystal *cr, const RefList *full, + signed int cycle, int serial, int scaleflags) +{ + write_grid(cr, full, cycle, serial, scaleflags, + GPARAM_ANG1, GPARAM_ANG2, "ang1-ang2"); + write_grid(cr, full, cycle, serial, scaleflags, + GPARAM_ANG1, GPARAM_WAVELENGTH, "ang1-wave"); + write_grid(cr, full, cycle, serial, scaleflags, + GPARAM_R, GPARAM_WAVELENGTH, "R-wave"); } static void do_pr_refine(Crystal *cr, const RefList *full, - PartialityModel pmodel, int verbose) + PartialityModel pmodel, int verbose, int serial, + int cycle, int write_logs, + SymOpList *sym, SymOpList *amb, int scaleflags) { - int i, done; - double old_dev; - UnitCell *cell = crystal_get_cell(cr); + gsl_multimin_fminimizer *min; + struct rf_priv priv; + int n_iter = 0; + int status; + double residual_start, residual_free_start; + FILE *fh = NULL; + + try_reindex(cr, full, sym, amb, scaleflags); + + if ( scale_one_crystal(cr, full, scaleflags | SCALE_VERBOSE_ERRORS) ) { + ERROR("Bad scaling at start of refinement.\n"); + return; + } + residual_start = residual(cr, full, 0, NULL, NULL); + residual_free_start = residual(cr, full, 1, NULL, NULL); + + if ( verbose ) { + STATUS("\nPR initial: dev = %10.5e, free dev = %10.5e\n", + residual_start, residual_free_start); + } - old_dev = residual(cr, full, 0, 0, NULL); + min = setup_minimiser(cr, full, verbose, serial, scaleflags, &priv); if ( verbose ) { - double asx, asy, asz; - double bsx, bsy, bsz; - double csx, csy, csz; - cell_get_reciprocal(cell, &asx, &asy, &asz, - &bsx, &bsy, &bsz, - &csx, &csy, &csz); - STATUS("Initial asx = %e\n", asx); - STATUS("PR initial dev = %10.5e, free dev = %10.5e\n", - old_dev, residual(cr, full, 1, NULL, NULL)); - } - - i = 0; - done = 0; - do { + double res = residual_f(min->x, &priv); + double size = gsl_multimin_fminimizer_size(min); + STATUS("At start: %f %f %f %f ----> %f %f %e %f residual = %e size %f\n", + gsl_vector_get(min->x, 0), + gsl_vector_get(min->x, 1), + gsl_vector_get(min->x, 2), + gsl_vector_get(min->x, 3), + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 0)), + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 1)), + get_actual_val(min->x, priv.initial, priv.rv, 2), + get_actual_val(min->x, priv.initial, priv.rv, 3)*1e10, + res, size); + } - double dev; + if ( write_logs ) { - pr_iterate(cr, full, pmodel, NULL, verbose); + char fn[64]; - update_predictions(cr); - calculate_partialities(cr, pmodel); + snprintf(fn, 63, "pr-logs/crystal%i-cycle%i.log", serial, cycle); + fh = fopen(fn, "w"); + if ( fh != NULL ) { + fprintf(fh, "iteration RtoReference CCtoReference nref " + "ang1 ang2 radius wavelength"); + double res = residual_f(min->x, &priv); + fprintf(fh, "%5i %10.8f %10.8f %5i %10.8f %10.8f %e %e\n", + n_iter, res, 0.0, 0, + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 0)), + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 1)), + get_actual_val(min->x, priv.initial, priv.rv, 2), + get_actual_val(min->x, priv.initial, priv.rv, 3)*1e10); + } - dev = residual(cr, full, 0, 0, NULL); - if ( fabs(dev - old_dev) < dev*0.0001 ) done = 1; + } + + do { + double res; + + n_iter++; + + status = gsl_multimin_fminimizer_iterate(min); + if ( status ) break; + + res = residual_f(min->x, &priv); + if ( isnan(res) ) { + status = GSL_ENOPROG; + break; + } if ( verbose ) { - STATUS("PR iter %2i: dev = %10.5e, free dev = %10.5e\n", - i+1, dev, residual(cr, full, 1, NULL, NULL)); + double res = residual_f(min->x, &priv); + double size = gsl_multimin_fminimizer_size(min); + STATUS("%f %f %f %f ----> %f %f %e %f residual = %e size %f\n", + gsl_vector_get(min->x, 0), + gsl_vector_get(min->x, 1), + gsl_vector_get(min->x, 2), + gsl_vector_get(min->x, 3), + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 0)), + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 1)), + get_actual_val(min->x, priv.initial, priv.rv, 2), + get_actual_val(min->x, priv.initial, priv.rv, 3)*1e10, + res, size); } - i++; - old_dev = dev; + if ( fh != NULL ) { + fprintf(fh, "%5i %10.8f %10.8f %5i %10.8f %10.8f %e %e\n", + n_iter, res, 0.0, 0, + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 0)), + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 1)), + get_actual_val(min->x, priv.initial, priv.rv, 2), + get_actual_val(min->x, priv.initial, priv.rv, 3)*1e10); + } + + status = gsl_multimin_test_size(min->size, 0.005); - } while ( i < 30 && !done ); + } while ( status == GSL_CONTINUE && n_iter < 1000 ); if ( verbose ) { - double asx, asy, asz; - double bsx, bsy, bsz; - double csx, csy, csz; - cell_get_reciprocal(cell, &asx, &asy, &asz, - &bsx, &bsy, &bsz, - &csx, &csy, &csz); - STATUS("Final asx = %e\n", asx); + STATUS("Done with refinement after %i iter\n", n_iter); + STATUS("status = %i (%s)\n", status, gsl_strerror(status)); } -} + if ( status == GSL_SUCCESS ) { -static struct prdata pr_refine(Crystal *cr, const RefList *full, - PartialityModel pmodel) -{ - int verbose = 0; - struct prdata prdata; + if ( check_angle_shifts(min->x, priv.initial, priv.rv, &priv) ) return; - prdata.refined = 0; - prdata.n_filtered = 0; + if ( verbose ) { - if ( verbose ) { - write_residual_graph(cr, full); + double res = residual_f(min->x, &priv); + double size = gsl_multimin_fminimizer_size(min); + STATUS("At end: %f %f %f %f ----> %f %f %e %f residual = %e size %f\n", + gsl_vector_get(min->x, 0), + gsl_vector_get(min->x, 1), + gsl_vector_get(min->x, 2), + gsl_vector_get(min->x, 3), + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 0)), + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 1)), + get_actual_val(min->x, priv.initial, priv.rv, 2), + get_actual_val(min->x, priv.initial, priv.rv, 3)*1e10, + res, size); + + } + + if ( fh != NULL ) { + double res = residual_f(min->x, &priv); + fprintf(fh, "%5i %10.8f %10.8f %5i %10.8f %10.8f %e %e\n", + n_iter, res, 0.0, 0, + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 0)), + rad2deg(get_actual_val(min->x, priv.initial, priv.rv, 1)), + get_actual_val(min->x, priv.initial, priv.rv, 2), + get_actual_val(min->x, priv.initial, priv.rv, 3)*1e10); + fclose(fh); + } + + /* Apply the final shifts */ + apply_parameters(min->x, priv.initial, priv.rv, cr); + update_predictions(cr); + calculate_partialities(cr, PMODEL_XSPHERE); + scale_one_crystal(cr, full, scaleflags); + + if ( verbose ) { + + STATUS("After applying final shifts:\n"); + STATUS("PR final: dev = %10.5e, free dev = %10.5e\n", + residual(cr, full, 0, NULL, NULL), + residual(cr, full, 1, NULL, NULL)); + STATUS("Final R = %e m^-1\n", crystal_get_profile_radius(cr)); + + } + + } else { + ERROR("Bad refinement: crystal %i\n", serial); } - do_pr_refine(cr, full, pmodel, verbose); + if ( write_logs ) { + write_gridscan(cr, full, cycle, serial, scaleflags); + write_specgraph(cr, full, cycle, serial); + write_test_logs(cr, full, cycle, serial); + } - if ( crystal_get_user_flag(cr) == 0 ) { - prdata.refined = 1; + if ( crystal_get_profile_radius(cr) > 5e9 ) { + ERROR("Very large radius: crystal %i\n", serial); } - return prdata; + gsl_multimin_fminimizer_free(min); + gsl_vector_free(priv.initial); + gsl_vector_free(priv.vals); + gsl_vector_free(priv.step); } @@ -706,7 +897,14 @@ struct refine_args RefList *full; Crystal *crystal; PartialityModel pmodel; + int serial; struct prdata prdata; + int verbose; + int cycle; + int no_logs; + SymOpList *sym; + SymOpList *amb; + int scaleflags; }; @@ -724,8 +922,18 @@ static void refine_image(void *task, int id) { struct refine_args *pargs = task; Crystal *cr = pargs->crystal; + int write_logs = 0; + + write_logs = !pargs->no_logs && (pargs->serial % 20 == 0); + pargs->prdata.refined = 0; + + do_pr_refine(cr, pargs->full, pargs->pmodel, pargs->verbose, + pargs->serial, pargs->cycle, write_logs, + pargs->sym, pargs->amb, pargs->scaleflags); - pargs->prdata = pr_refine(cr, pargs->full, pargs->pmodel); + if ( crystal_get_user_flag(cr) == 0 ) { + pargs->prdata.refined = 1; + } } @@ -738,6 +946,7 @@ static void *get_image(void *vqargs) memcpy(task, &qargs->task_defaults, sizeof(struct refine_args)); task->crystal = qargs->crystals[qargs->n_started]; + task->serial = qargs->n_started; qargs->n_started++; @@ -757,7 +966,9 @@ static void done_image(void *vqargs, void *task) void refine_all(Crystal **crystals, int n_crystals, - RefList *full, int nthreads, PartialityModel pmodel) + RefList *full, int nthreads, PartialityModel pmodel, + int verbose, int cycle, int no_logs, + SymOpList *sym, SymOpList *amb, int scaleflags) { struct refine_args task_defaults; struct queue_args qargs; @@ -766,7 +977,12 @@ void refine_all(Crystal **crystals, int n_crystals, task_defaults.crystal = NULL; task_defaults.pmodel = pmodel; task_defaults.prdata.refined = 0; - task_defaults.prdata.n_filtered = 0; + task_defaults.verbose = verbose; + task_defaults.cycle = cycle; + task_defaults.no_logs = no_logs; + task_defaults.sym = sym; + task_defaults.amb = amb; + task_defaults.scaleflags = scaleflags; qargs.task_defaults = task_defaults; qargs.n_started = 0; diff --git a/src/post-refinement.h b/src/post-refinement.h index 7f395013..71a6d7f3 100644 --- a/src/post-refinement.h +++ b/src/post-refinement.h @@ -3,11 +3,11 @@ * * Post refinement * - * Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2015 Thomas White <taw@physics.org> + * 2010-2018 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -41,6 +41,7 @@ #include "utils.h" #include "crystal.h" #include "geometry.h" +#include "symmetry.h" enum prflag @@ -51,19 +52,28 @@ enum prflag PRFLAG_EARLY = 18, PRFLAG_CC = 19, PRFLAG_BIGB = 20, + PRFLAG_SCALEBAD = 21, }; extern const char *str_prflag(enum prflag flag); extern void refine_all(Crystal **crystals, int n_crystals, - RefList *full, int nthreads, PartialityModel pmodel); + RefList *full, int nthreads, PartialityModel pmodel, + int verbose, int cycle, int no_logs, + SymOpList *sym, SymOpList *amb, int scaleflags); + +extern void write_gridscan(Crystal *cr, const RefList *full, + int cycle, int serial, int scaleflags); + +extern void write_specgraph(Crystal *crystal, const RefList *full, + signed int cycle, int serial); /* Exported so it can be poked by tests/pr_p_gradient_check */ extern double gradient(Crystal *cr, int k, Reflection *refl, PartialityModel pmodel); -extern double residual(Crystal *cr, const RefList *full, int free, - int *pn_used, const char *filename); +extern void write_test_logs(Crystal *crystal, const RefList *full, + signed int cycle, int serial); #endif /* POST_REFINEMENT_H */ diff --git a/src/process_hkl.c b/src/process_hkl.c index 5930c189..ab5b9af1 100644 --- a/src/process_hkl.c +++ b/src/process_hkl.c @@ -519,6 +519,7 @@ int main(int argc, char *argv[]) double push_res = +INFINITY; double min_cc = -INFINITY; int twopass = 0; + char *audit_info; /* Long options */ const struct option longopts[] = { @@ -822,11 +823,14 @@ int main(int argc, char *argv[]) hist_nbins); } + audit_info = stream_audit_info(st); + close_stream(st); + reflist_add_command_and_version(model, argc, argv); + reflist_add_notes(model, "Audit information from stream:"); + reflist_add_notes(model, audit_info); write_reflist_2(output, model, sym); - close_stream(st); - free_symoplist(sym); reflist_free(model); free(output); diff --git a/src/process_image.c b/src/process_image.c index 87089289..4b02e694 100644 --- a/src/process_image.c +++ b/src/process_image.c @@ -301,7 +301,7 @@ void process_image(const struct index_args *iargs, struct pattern_args *pargs, /* Integrate! */ time_accounts_set(taccs, TACC_INTEGRATION); sb_shared->pings[cookie]++; - integrate_all_5(&image, iargs->int_meth, PMODEL_SCSPHERE, + integrate_all_5(&image, iargs->int_meth, PMODEL_XSPHERE, iargs->push_res, iargs->ir_inn, iargs->ir_mid, iargs->ir_out, iargs->int_diag, iargs->int_diag_h, diff --git a/src/rejection.c b/src/rejection.c index 10f13f9c..9d41f9b4 100644 --- a/src/rejection.c +++ b/src/rejection.c @@ -137,7 +137,7 @@ static void check_cc(Crystal *cr, RefList *full) pcalc = get_partiality(refl); /* Observed partiality */ - corr = exp(-G) * exp(B*res*res) * get_lorentz(refl); + corr = G * exp(B*res*res) * get_lorentz(refl); Ipart = get_intensity(refl) * corr; pobs = Ipart / get_intensity(match); diff --git a/src/scaling.c b/src/scaling.c index 77a64384..cabc5952 100644 --- a/src/scaling.c +++ b/src/scaling.c @@ -3,11 +3,11 @@ * * Scaling * - * Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2015 Thomas White <taw@physics.org> + * 2010-2017 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -38,290 +38,22 @@ #include <gsl/gsl_linalg.h> #include <gsl/gsl_eigen.h> #include <gsl/gsl_blas.h> +#include <gsl/gsl_fit.h> +#include <gsl/gsl_statistics_double.h> #include "merge.h" #include "post-refinement.h" #include "symmetry.h" #include "cell.h" #include "cell-utils.h" - - -/* Maximum number of iterations of NLSq to do for each image per macrocycle. */ -#define MAX_CYCLES (10) - - -/* Apply the given shift to the 'k'th parameter of 'image'. */ -static void apply_shift(Crystal *cr, int k, double shift) -{ - double t; - - switch ( k ) { - - case GPARAM_BFAC : - t = crystal_get_Bfac(cr); - t += shift; - crystal_set_Bfac(cr, t); - break; - - case GPARAM_OSF : - t = crystal_get_osf(cr); - t += shift; - crystal_set_osf(cr, t); - break; - - default : - ERROR("No shift defined for parameter %i\n", k); - abort(); - - } -} - - -/* Perform one cycle of scaling of 'cr' against 'full' */ -static double scale_iterate(Crystal *cr, const RefList *full, - PartialityModel pmodel, int *nr) -{ - gsl_matrix *M; - gsl_vector *v; - gsl_vector *shifts; - int param; - Reflection *refl; - RefListIterator *iter; - RefList *reflections; - double max_shift; - int nref = 0; - int num_params = 0; - enum gparam rv[32]; - double G, B; - - *nr = 0; - - rv[num_params++] = GPARAM_OSF; - rv[num_params++] = GPARAM_BFAC; - - M = gsl_matrix_calloc(num_params, num_params); - v = gsl_vector_calloc(num_params); - - reflections = crystal_get_reflections(cr); - G = crystal_get_osf(cr); - B = crystal_get_Bfac(cr); - - /* Scaling terms */ - for ( refl = first_refl(reflections, &iter); - refl != NULL; - refl = next_refl(refl, iter) ) - { - signed int ha, ka, la; - double I_full, delta_I, esd; - double w; - double I_partial; - int k; - double p, L, s; - double fx; - Reflection *match; - double gradients[num_params]; - - /* If reflection is free-flagged, don't use it here */ - if ( get_flag(refl) ) continue; - - /* Find the full version */ - get_indices(refl, &ha, &ka, &la); - match = find_refl(full, ha, ka, la); - if ( match == NULL ) continue; - - /* Merged intensitty */ - I_full = get_intensity(match); - - /* Actual measurement of this reflection from this pattern */ - I_partial = get_intensity(refl); - esd = get_esd_intensity(refl); - p = get_partiality(refl); - - /* Scale only using strong reflections */ - if ( I_partial <= 3.0*esd ) continue; /* Also because of log */ - if ( get_redundancy(match) < 2 ) continue; - if ( I_full <= 0 ) continue; /* Because log */ - if ( p <= 0.0 ) continue; /* Because of log */ - - L = get_lorentz(refl); - s = resolution(crystal_get_cell(cr), ha, ka, la); - - /* Calculate the weight for this reflection */ - w = 1.0; - - /* Calculate all gradients for this reflection */ - for ( k=0; k<num_params; k++ ) { - - if ( rv[k] == GPARAM_OSF ) { - gradients[k] = 1.0; - } else if ( rv[k] == GPARAM_BFAC ) { - gradients[k] = -s*s; - } else { - ERROR("Unrecognised scaling gradient.\n"); - abort(); - } - } - - for ( k=0; k<num_params; k++ ) { - - int g; - double v_c, v_curr; - - for ( g=0; g<num_params; g++ ) { - - double M_c, M_curr; - - /* Matrix is symmetric */ - if ( g > k ) continue; - - M_c = w * gradients[g] * gradients[k]; - - M_curr = gsl_matrix_get(M, k, g); - gsl_matrix_set(M, k, g, M_curr + M_c); - gsl_matrix_set(M, g, k, M_curr + M_c); - - } - - fx = G + log(p) - log(L) - B*s*s + log(I_full); - delta_I = log(I_partial) - fx; - v_c = w * delta_I * gradients[k]; - v_curr = gsl_vector_get(v, k); - gsl_vector_set(v, k, v_curr + v_c); - - } - - nref++; - } - - *nr = nref; - - if ( nref < num_params ) { - crystal_set_user_flag(cr, PRFLAG_FEWREFL); - gsl_matrix_free(M); - gsl_vector_free(v); - return 0.0; - } - - max_shift = 0.0; - shifts = solve_svd(v, M, NULL, 0); - if ( shifts != NULL ) { - - for ( param=0; param<num_params; param++ ) { - double shift = gsl_vector_get(shifts, param); - apply_shift(cr, rv[param], shift); - if ( fabs(shift) > max_shift ) max_shift = fabs(shift); - } - - } else { - crystal_set_user_flag(cr, PRFLAG_SOLVEFAIL); - } - - gsl_matrix_free(M); - gsl_vector_free(v); - gsl_vector_free(shifts); - - return max_shift; -} - - -double log_residual(Crystal *cr, const RefList *full, int free, - int *pn_used, const char *filename) -{ - double dev = 0.0; - double G, B; - Reflection *refl; - RefListIterator *iter; - int n_used = 0; - FILE *fh = NULL; - - G = crystal_get_osf(cr); - B = crystal_get_Bfac(cr); - if ( filename != NULL ) { - fh = fopen(filename, "a"); - if ( fh == NULL ) { - ERROR("Failed to open '%s'\n", filename); - } - } - - for ( refl = first_refl(crystal_get_reflections(cr), &iter); - refl != NULL; - refl = next_refl(refl, iter) ) - { - double p, L, s, w; - signed int h, k, l; - Reflection *match; - double esd, I_full, I_partial; - double fx, dc; - - if ( free && !get_flag(refl) ) continue; - - get_indices(refl, &h, &k, &l); - match = find_refl(full, h, k, l); - if ( match == NULL ) continue; - - p = get_partiality(refl); - L = get_lorentz(refl); - I_partial = get_intensity(refl); - I_full = get_intensity(match); - esd = get_esd_intensity(refl); - s = resolution(crystal_get_cell(cr), h, k, l); - - if ( I_partial <= 3.0*esd ) continue; /* Also because of log */ - if ( get_redundancy(match) < 2 ) continue; - if ( I_full <= 0 ) continue; /* Because log */ - if ( p <= 0.0 ) continue; /* Because of log */ - - fx = G + log(p) - log(L) - B*s*s + log(I_full); - dc = log(I_partial) - fx; - w = 1.0; - dev += w*dc*dc; - - if ( fh != NULL ) { - fprintf(fh, "%4i %4i %4i %e %e\n", - h, k, l, s, dev); - } - - } - - if ( fh != NULL ) fclose(fh); - - if ( pn_used != NULL ) *pn_used = n_used; - return dev; -} - - -static void do_scale_refine(Crystal *cr, const RefList *full, - PartialityModel pmodel, int *nr) -{ - int i, done; - double old_dev; - - old_dev = log_residual(cr, full, 0, NULL, NULL); - - i = 0; - done = 0; - do { - - double dev; - - scale_iterate(cr, full, pmodel, nr); - - dev = log_residual(cr, full, 0, 0, NULL); - if ( fabs(dev - old_dev) < dev*0.01 ) done = 1; - - i++; - old_dev = dev; - - } while ( i < MAX_CYCLES && !done ); -} +#include "scaling.h" struct scale_args { RefList *full; Crystal *crystal; - PartialityModel pmodel; - int n_reflections; + int flags; }; @@ -331,7 +63,6 @@ struct queue_args int n_done; Crystal **crystals; int n_crystals; - long long int n_reflections; struct scale_args task_defaults; }; @@ -339,8 +70,7 @@ struct queue_args static void scale_crystal(void *task, int id) { struct scale_args *pargs = task; - do_scale_refine(pargs->crystal, pargs->full, pargs->pmodel, - &pargs->n_reflections); + scale_one_crystal(pargs->crystal, pargs->full, pargs->flags); } @@ -363,11 +93,7 @@ static void *get_crystal(void *vqargs) static void done_crystal(void *vqargs, void *task) { struct queue_args *qa = vqargs; - struct scale_args *wargs = task; - qa->n_done++; - qa->n_reflections += wargs->n_reflections; - progress_bar(qa->n_done, qa->n_crystals, "Scaling"); free(task); } @@ -381,14 +107,12 @@ static double total_log_r(Crystal **crystals, int n_crystals, RefList *full, int n = 0; for ( i=0; i<n_crystals; i++ ) { - double r; if ( crystal_get_user_flag(crystals[i]) ) continue; r = log_residual(crystals[i], full, 0, NULL, NULL); if ( isnan(r) ) continue; total += r; n++; - } if ( ninc != NULL ) *ninc = n; @@ -397,8 +121,7 @@ static double total_log_r(Crystal **crystals, int n_crystals, RefList *full, /* Perform iterative scaling, all the way to convergence */ -void scale_all(Crystal **crystals, int n_crystals, int nthreads, - PartialityModel pmodel) +void scale_all(Crystal **crystals, int n_crystals, int nthreads, int scaleflags) { struct scale_args task_defaults; struct queue_args qargs; @@ -406,7 +129,7 @@ void scale_all(Crystal **crystals, int n_crystals, int nthreads, int niter = 0; task_defaults.crystal = NULL; - task_defaults.pmodel = pmodel; + task_defaults.flags = scaleflags; qargs.task_defaults = task_defaults; qargs.n_crystals = n_crystals; @@ -422,18 +145,15 @@ void scale_all(Crystal **crystals, int n_crystals, int nthreads, double bef_res; full = merge_intensities(crystals, n_crystals, nthreads, - pmodel, 2, INFINITY, 0); + 2, INFINITY, 0); old_res = new_res; bef_res = total_log_r(crystals, n_crystals, full, NULL); qargs.task_defaults.full = full; qargs.n_started = 0; qargs.n_done = 0; - qargs.n_reflections = 0; run_threads(nthreads, scale_crystal, get_crystal, done_crystal, &qargs, n_crystals, 0, 0, 0); - STATUS("%lli reflections went into the scaling.\n", - qargs.n_reflections); new_res = total_log_r(crystals, n_crystals, full, &ninc); STATUS("Log residual went from %e to %e, %i crystals\n", @@ -456,3 +176,164 @@ void scale_all(Crystal **crystals, int n_crystals, int nthreads, ERROR("Too many iterations - giving up!\n"); } } + + +/* Calculates G and B, by which cr's reflections should be multiplied to fit reference */ +int scale_one_crystal(Crystal *cr, const RefList *listR, int flags) +{ + const Reflection *reflS; + RefListIterator *iter; + int max_n = 256; + int n = 0; + double *x; + double *y; + double *w; + int r; + double cov00, cov01, cov11, chisq; + int n_esdS = 0; + int n_esdR = 0; + int n_ihS = 0; + int n_ihR = 0; + int n_nanS = 0; + int n_nanR = 0; + int n_infS = 0; + int n_infR = 0; + int n_part = 0; + int n_nom = 0; + int n_red = 0; + RefList *listS = crystal_get_reflections(cr); + UnitCell *cell = crystal_get_cell(cr); + double G, B; + + assert(cell != NULL); + assert(listR != NULL); + assert(listS != NULL); + + x = malloc(max_n*sizeof(double)); + w = malloc(max_n*sizeof(double)); + y = malloc(max_n*sizeof(double)); + if ( (x==NULL) || (y==NULL) || (w==NULL) ) { + ERROR("Failed to allocate memory for scaling.\n"); + return 1; + } + + int nb = 0; + for ( reflS = first_refl_const(listS, &iter); + reflS != NULL; + reflS = next_refl_const(reflS, iter) ) + { + signed int h, k, l; + const Reflection *reflR; + double IhR, IhS, esdS, pS, LS; + double s; + + nb++; + + get_indices(reflS, &h, &k, &l); + reflR = find_refl(listR, h, k, l); + if ( reflR == NULL ) { + n_nom++; + continue; + } + + s = resolution(cell, h, k, l); + + IhR = get_intensity(reflR); + IhS = get_intensity(reflS); + esdS = get_esd_intensity(reflS); + pS = get_partiality(reflS); + LS = get_lorentz(reflS); + + /* Problem cases in approximate descending order of severity */ + if ( isnan(IhR) ) { n_nanR++; continue; } + if ( isinf(IhR) ) { n_infR++; continue; } + if ( isnan(IhS) ) { n_nanS++; continue; } + if ( isinf(IhS) ) { n_infS++; continue; } + if ( pS < 0.3 ) { n_part++; continue; } + if ( IhS <= 0.0 ) { n_ihS++; continue; } + if ( IhS <= 3.0*esdS ) { n_esdS++; continue; } + if ( IhR <= 0.0 ) { n_ihR++; continue; } + if ( get_redundancy(reflR) < 2 ) { n_red++; continue; } + + if ( n == max_n ) { + max_n *= 2; + x = realloc(x, max_n*sizeof(double)); + y = realloc(y, max_n*sizeof(double)); + w = realloc(w, max_n*sizeof(double)); + if ( (x==NULL) || (y==NULL) || (w==NULL) ) { + ERROR("Failed to allocate memory for scaling.\n"); + return 1; + } + } + + x[n] = s*s; + y[n] = log(LS) + log(IhS) -log(pS) - log(IhR); + w[n] = pS; + n++; + + } + + if ( n < 2 ) { + if ( flags & SCALE_VERBOSE_ERRORS ) { + ERROR("Not enough reflections for scaling (had %i, but %i remain)\n", nb, n); + if ( n_esdR ) ERROR("%i reference reflection esd\n", n_esdR); + if ( n_esdS ) ERROR("%i subject reflection esd\n", n_esdS); + if ( n_ihR ) ERROR("%i reference reflection intensity\n", n_ihR); + if ( n_red ) ERROR("%i reference reflection redundancy\n", n_red); + if ( n_ihS ) ERROR("%i subject reflection intensity\n", n_ihS); + if ( n_nanR ) ERROR("%i reference reflection nan\n", n_nanR); + if ( n_nanS ) ERROR("%i subject reflection nan\n", n_nanS); + if ( n_infR ) ERROR("%i reference reflection inf\n", n_infR); + if ( n_infS ) ERROR("%i subject reflection inf\n", n_infS); + if ( n_part ) ERROR("%i subject reflection partiality\n", n_part); + if ( n_nom ) ERROR("%i no match in reference list\n", n_nom); + } + free(x); + free(y); + free(w); + return 1; + } + + if ( flags & SCALE_NO_B ) { + G = gsl_stats_wmean(w, 1, y, 1, n); + B = 0.0; + r = 0; + } else { + r = gsl_fit_wlinear(x, 1, w, 1, y, 1, n, &G, &B, &cov00, &cov01, &cov11, &chisq); + } + + if ( r ) { + ERROR("Scaling failed.\n"); + free(x); + free(y); + free(w); + return 1; + } + + if ( isnan(G) ) { + + if ( flags & SCALE_VERBOSE_ERRORS ) { + ERROR("Scaling gave NaN (%i pairs)\n", n); + if ( n < 10 ) { + int i; + for ( i=0; i<n; i++ ) { + STATUS("%3i %e %e %e\n", i, x[i], y[i], w[i]); + } + } + } + + free(x); + free(y); + free(w); + return 1; + } + + crystal_set_osf(cr, exp(G)); + crystal_set_Bfac(cr, -B); + + free(x); + free(y); + free(w); + + return 0; +} diff --git a/src/scaling.h b/src/scaling.h index 534043d9..8c2e7ef2 100644 --- a/src/scaling.h +++ b/src/scaling.h @@ -3,11 +3,11 @@ * * Scaling * - * Copyright © 2012-2015 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * * Authors: - * 2010-2015 Thomas White <taw@physics.org> + * 2010-2018 Thomas White <taw@physics.org> * * This file is part of CrystFEL. * @@ -38,10 +38,16 @@ #include "crystal.h" #include "geometry.h" -extern double log_residual(Crystal *cr, const RefList *full, int free, - int *pn_used, const char *filename); +enum ScaleFlags +{ + SCALE_NONE = 0, + SCALE_NO_B = 1<<0, /* Don't use Debye-Waller part */ + SCALE_VERBOSE_ERRORS = 1<<1, +}; + +extern int scale_one_crystal(Crystal *cr, const RefList *reference, int flags); extern void scale_all(Crystal **crystals, int n_crystals, int nthreads, - PartialityModel pmodel); + int flags); #endif /* SCALING_H */ diff --git a/tests/.gitignore b/tests/.gitignore index 2afe2ec8..cac64f7b 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -13,3 +13,4 @@ ring_check prof2d_check ambi_check prediction_gradient_check +scaling_check diff --git a/tests/partialator_merge_check_1 b/tests/partialator_merge_check_1 index 2d888fd7..0123e391 100755 --- a/tests/partialator_merge_check_1 +++ b/tests/partialator_merge_check_1 @@ -44,14 +44,14 @@ EOF partialator -i partialator_merge_check_1.stream \ -o partialator_merge_check_1.hkl \ - --model=unity --iterations=0 --no-scale --no-polarisation + --model=unity --iterations=0 --no-scale --no-polarisation \ + --no-logs if [ $? -ne 0 ]; then exit 1 fi -ex -c '/End of reflections/ -.,$d -x' partialator_merge_check_1.hkl +sed -n '/End of reflections/q;p' partialator_merge_check_1.hkl > temp.hkl +mv temp.hkl partialator_merge_check_1.hkl diff partialator_merge_check_1.hkl partialator_merge_check_1_ans.hkl if [ $? -ne 0 ]; then exit 1 diff --git a/tests/partialator_merge_check_2 b/tests/partialator_merge_check_2 index 249421e1..e8aff651 100755 --- a/tests/partialator_merge_check_2 +++ b/tests/partialator_merge_check_2 @@ -49,14 +49,13 @@ EOF partialator -i partialator_merge_check_2.stream \ -o partialator_merge_check_2.hkl \ --model=unity --iterations=1 --no-polarisation \ - --no-free + --no-free --no-logs if [ $? -ne 0 ]; then exit 1 fi -ex -c '/End of reflections/ -.,$d -x' partialator_merge_check_2.hkl +sed -n '/End of reflections/q;p' partialator_merge_check_2.hkl > temp.hkl +mv temp.hkl partialator_merge_check_2.hkl diff partialator_merge_check_2.hkl partialator_merge_check_2_ans.hkl if [ $? -ne 0 ]; then exit 1 diff --git a/tests/partialator_merge_check_3 b/tests/partialator_merge_check_3 index 1f6acf35..b24b6d53 100755 --- a/tests/partialator_merge_check_3 +++ b/tests/partialator_merge_check_3 @@ -51,14 +51,13 @@ EOF partialator -i partialator_merge_check_3.stream \ -o partialator_merge_check_3.hkl \ --model=unity --iterations=1 -y 4 --no-polarisation \ - --no-free + --no-free --no-logs if [ $? -ne 0 ]; then exit 1 fi -ex -c '/End of reflections/ -.,$d -x' partialator_merge_check_3.hkl +sed -n '/End\ of\ reflections/q;p' partialator_merge_check_3.hkl > temp.hkl +mv temp.hkl partialator_merge_check_3.hkl diff partialator_merge_check_3.hkl partialator_merge_check_3_ans.hkl if [ $? -ne 0 ]; then exit 1 diff --git a/tests/pr_p_gradient_check.c b/tests/pr_p_gradient_check.c deleted file mode 100644 index 5322fcca..00000000 --- a/tests/pr_p_gradient_check.c +++ /dev/null @@ -1,529 +0,0 @@ -/* - * pr_p_gradient_check.c - * - * Check partiality gradients for post refinement - * - * Copyright © 2012-2014 Deutsches Elektronen-Synchrotron DESY, - * a research centre of the Helmholtz Association. - * - * Authors: - * 2012-2014 Thomas White <taw@physics.org> - * - * This file is part of CrystFEL. - * - * CrystFEL is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * CrystFEL is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. - * - */ - -#ifdef HAVE_CONFIG_H -#include <config.h> -#endif - - -#include <stdlib.h> -#include <stdio.h> -#include <gsl/gsl_statistics.h> -#include <getopt.h> - -#include <image.h> -#include <cell.h> -#include <cell-utils.h> -#include <geometry.h> -#include <reflist.h> -#include "../src/post-refinement.h" - - -static void scan_partialities(RefList *reflections, RefList *compare, - int *valid, long double *vals[3], int idx, - PartialityModel pmodel) -{ - int i; - Reflection *refl; - RefListIterator *iter; - - i = 0; - for ( refl = first_refl(reflections, &iter); - refl != NULL; - refl = next_refl(refl, iter) ) - { - signed int h, k, l; - Reflection *refl2; - double rlow, rhigh, p; - - get_indices(refl, &h, &k, &l); - refl2 = find_refl(compare, h, k, l); - if ( refl2 == NULL ) { - valid[i] = 0; - i++; - continue; - } - - get_partial(refl2, &rlow, &rhigh, &p); - vals[idx][i] = p; - if ( unlikely(p < 0.0) ) { - ERROR("Negative partiality! %3i %3i %3i %f\n", - h, k, l, p); - } - - i++; - } -} - - -static UnitCell *new_shifted_cell(UnitCell *input, int k, double shift) -{ - UnitCell *cell; - double asx, asy, asz; - double bsx, bsy, bsz; - double csx, csy, csz; - - cell = cell_new(); - cell_get_reciprocal(input, &asx, &asy, &asz, &bsx, &bsy, &bsz, - &csx, &csy, &csz); - switch ( k ) - { - case GPARAM_ASX : asx += shift; break; - case GPARAM_ASY : asy += shift; break; - case GPARAM_ASZ : asz += shift; break; - case GPARAM_BSX : bsx += shift; break; - case GPARAM_BSY : bsy += shift; break; - case GPARAM_BSZ : bsz += shift; break; - case GPARAM_CSX : csx += shift; break; - case GPARAM_CSY : csy += shift; break; - case GPARAM_CSZ : csz += shift; break; - } - cell_set_reciprocal(cell, asx, asy, asz, bsx, bsy, bsz, csx, csy, csz); - - return cell; -} - - -static void shift_parameter(struct image *image, int k, double shift) -{ - switch ( k ) - { - case GPARAM_DIV : image->div += shift; break; - } -} - - -static Crystal *new_shifted_crystal(Crystal *cr, int refine, double incr_val) -{ - Crystal *cr_new; - double r; - UnitCell *cell; - - cr_new = crystal_copy(cr); - if ( cr_new == NULL ) { - ERROR("Failed to allocate crystal.\n"); - return NULL; - } - - crystal_set_image(cr_new, crystal_get_image(cr)); - r = crystal_get_profile_radius(cr_new); - - switch ( refine ) { - - case GPARAM_ASX : - case GPARAM_ASY : - case GPARAM_ASZ : - case GPARAM_BSX : - case GPARAM_BSY : - case GPARAM_BSZ : - case GPARAM_CSX : - case GPARAM_CSY : - case GPARAM_CSZ : - cell = new_shifted_cell(crystal_get_cell(cr), refine, - incr_val); - crystal_set_cell(cr_new, cell); - break; - - case GPARAM_R : - cell = cell_new_from_cell(crystal_get_cell(cr)); - crystal_set_cell(cr_new, cell); - crystal_set_profile_radius(cr_new, r + incr_val); - break; - - default : - ERROR("Can't shift %i\n", refine); - break; - - } - - return cr_new; -} - - -static void calc_either_side(Crystal *cr, double incr_val, - int *valid, long double *vals[3], int refine, - PartialityModel pmodel) -{ - RefList *compare; - struct image *image = crystal_get_image(cr); - - if ( (refine != GPARAM_DIV) ) { - - Crystal *cr_new; - - /* Crystal properties */ - cr_new = new_shifted_crystal(cr, refine, -incr_val); - compare = predict_to_res(cr_new, largest_q(image)); - crystal_set_reflections(cr_new, compare); - calculate_partialities(cr_new, pmodel); - scan_partialities(crystal_get_reflections(cr), compare, valid, - vals, 0, pmodel); - cell_free(crystal_get_cell(cr_new)); - crystal_free(cr_new); - reflist_free(compare); - - cr_new = new_shifted_crystal(cr, refine, +incr_val); - compare = predict_to_res(cr_new, largest_q(image)); - crystal_set_reflections(cr_new, compare); - calculate_partialities(cr_new, pmodel); - scan_partialities(crystal_get_reflections(cr), compare, valid, - vals, 2, pmodel); - cell_free(crystal_get_cell(cr_new)); - crystal_free(cr_new); - reflist_free(compare); - - } else { - - struct image im_moved; - Crystal *cr_new = crystal_copy(cr); - crystal_set_image(cr_new, &im_moved); - - /* "Image" properties */ - im_moved = *image; - shift_parameter(&im_moved, refine, -incr_val); - compare = predict_to_res(cr_new, largest_q(&im_moved)); - crystal_set_reflections(cr_new, compare); - calculate_partialities(cr_new, pmodel); - scan_partialities(crystal_get_reflections(cr), compare, - valid, vals, 0, pmodel); - reflist_free(compare); - - im_moved = *image; - shift_parameter(&im_moved, refine, +incr_val); - compare = predict_to_res(cr_new, largest_q(&im_moved)); - crystal_set_reflections(cr_new, compare); - calculate_partialities(cr_new, pmodel); - scan_partialities(crystal_get_reflections(cr), compare, - valid, vals, 2, pmodel); - reflist_free(compare); - - } -} - - -static double test_gradients(Crystal *cr, double incr_val, int refine, - const char *str, const char *file, - PartialityModel pmodel, int quiet, int plot) -{ - Reflection *refl; - RefListIterator *iter; - long double *vals[3]; - int i; - int *valid; - int nref; - int n_good, n_invalid, n_small, n_nan, n_bad; - RefList *reflections; - FILE *fh = NULL; - int ntot = 0; - double total = 0.0; - char tmp[32]; - double *vec1; - double *vec2; - int n_line; - double cc; - - reflections = predict_to_res(cr, largest_q(crystal_get_image(cr))); - crystal_set_reflections(cr, reflections); - - nref = num_reflections(reflections); - if ( nref < 10 ) { - ERROR("Too few reflections found. Failing test by default.\n"); - return 0.0; - } - - vals[0] = malloc(nref*sizeof(long double)); - vals[1] = malloc(nref*sizeof(long double)); - vals[2] = malloc(nref*sizeof(long double)); - if ( (vals[0] == NULL) || (vals[1] == NULL) || (vals[2] == NULL) ) { - ERROR("Couldn't allocate memory.\n"); - return 0.0; - } - - valid = malloc(nref*sizeof(int)); - if ( valid == NULL ) { - ERROR("Couldn't allocate memory.\n"); - return 0.0; - } - for ( i=0; i<nref; i++ ) valid[i] = 1; - - scan_partialities(reflections, reflections, valid, vals, 1, pmodel); - - calc_either_side(cr, incr_val, valid, vals, refine, pmodel); - - if ( plot ) { - snprintf(tmp, 32, "gradient-test-%s.dat", file); - fh = fopen(tmp, "w"); - } - - vec1 = malloc(nref*sizeof(double)); - vec2 = malloc(nref*sizeof(double)); - if ( (vec1 == NULL) || (vec2 == NULL) ) { - ERROR("Couldn't allocate memory.\n"); - return 0.0; - } - - n_invalid = 0; n_good = 0; - n_nan = 0; n_small = 0; n_bad = 0; n_line = 0; - i = 0; - for ( refl = first_refl(reflections, &iter); - refl != NULL; - refl = next_refl(refl, iter) ) - { - - long double grad1, grad2, grad; - double cgrad; - signed int h, k, l; - - get_indices(refl, &h, &k, &l); - - if ( !valid[i] ) { - n_invalid++; - i++; - } else { - - double r1, r2, p; - - grad1 = (vals[1][i] - vals[0][i]) / incr_val; - grad2 = (vals[2][i] - vals[1][i]) / incr_val; - grad = (grad1 + grad2) / 2.0; - i++; - - cgrad = gradient(cr, refine, refl, pmodel); - - get_partial(refl, &r1, &r2, &p); - - if ( isnan(cgrad) ) { - n_nan++; - continue; - } - - if ( plot ) { - fprintf(fh, "%e %Le\n", cgrad, grad); - } - - vec1[n_line] = cgrad; - vec2[n_line] = grad; - n_line++; - - if ( (fabsl(cgrad) < 5e-8) && (fabsl(grad) < 5e-8) ) { - n_small++; - continue; - } - - total += fabsl(cgrad - grad); - ntot++; - - if ( !within_tolerance(grad, cgrad, 5.0) - || !within_tolerance(cgrad, grad, 5.0) ) - { - - if ( !quiet ) { - STATUS("!- %s %3i %3i %3i" - " %10.2Le %10.2e ratio = %5.2Lf" - " %10.2e %10.2e\n", - str, h, k, l, grad, cgrad, - cgrad/grad, r1, r2); - } - n_bad++; - - } else { - - //STATUS("OK %s %3i %3i %3i" - // " %10.2Le %10.2e ratio = %5.2Lf" - // " %10.2e %10.2e\n", - // str, h, k, l, grad, cgrad, cgrad/grad, - // r1, r2); - - n_good++; - - } - - } - - } - - STATUS("%3s: %3i within 5%%, %3i outside, %3i nan, %3i invalid, " - "%3i small. ", str, n_good, n_bad, n_nan, n_invalid, n_small); - - if ( plot ) { - fclose(fh); - } - - cc = gsl_stats_correlation(vec1, 1, vec2, 1, n_line); - STATUS("CC = %+f\n", cc); - return cc; -} - - -int main(int argc, char *argv[]) -{ - struct image image; - const double incr_frac = 1.0/1000000.0; - double incr_val; - double ax, ay, az; - double bx, by, bz; - double cx, cy, cz; - UnitCell *cell; - Crystal *cr; - struct quaternion orientation; - int i; - int fail = 0; - int quiet = 0; - int plot = 0; - int c; - gsl_rng *rng; - - const struct option longopts[] = { - {"quiet", 0, &quiet, 1}, - {"plot", 0, &plot, 1}, - {0, 0, NULL, 0} - }; - - while ((c = getopt_long(argc, argv, "", longopts, NULL)) != -1) { - switch (c) { - - case 0 : - break; - - case '?' : - break; - - default : - ERROR("Unhandled option '%c'\n", c); - break; - - } - - } - - image.det = simple_geometry(&image, 1024, 1024); - image.det->panels[0].res = 13333.3; - image.det->panels[0].clen = 80e-3; - image.det->panels[0].coffset = 0.0; - - image.lambda = ph_en_to_lambda(eV_to_J(8000.0)); - image.div = 1e-3; - image.bw = 0.01; - image.filename = malloc(256); - - cr = crystal_new(); - if ( cr == NULL ) { - ERROR("Failed to allocate crystal.\n"); - return 1; - } - crystal_set_mosaicity(cr, 0.0); - crystal_set_profile_radius(cr, 0.005e9); - crystal_set_image(cr, &image); - - cell = cell_new_from_parameters(10.0e-9, 10.0e-9, 10.0e-9, - deg2rad(90.0), - deg2rad(90.0), - deg2rad(90.0)); - - rng = gsl_rng_alloc(gsl_rng_mt19937); - - for ( i=0; i<2; i++ ) { - - UnitCell *rot; - double val; - PartialityModel pmodel; - - if ( i == 0 ) { - pmodel = PMODEL_SCSPHERE; - STATUS("Testing SCSphere model:\n"); - } else if ( i == 1 ) { - pmodel = PMODEL_SCGAUSSIAN; - STATUS("Testing SCGaussian model.\n"); - } else { - ERROR("WTF?\n"); - return 1; - } - - orientation = random_quaternion(rng); - rot = cell_rotate(cell, orientation); - crystal_set_cell(cr, rot); - - cell_get_reciprocal(rot, - &ax, &ay, &az, &bx, &by, - &bz, &cx, &cy, &cz); - - incr_val = incr_frac * image.div; - val = test_gradients(cr, incr_val, GPARAM_DIV, "div", "div", - pmodel, quiet, plot); - if ( val < 0.99 ) fail = 1; - - incr_val = incr_frac * crystal_get_profile_radius(cr); - val = test_gradients(cr, incr_val, GPARAM_R, "R", "R", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - - incr_val = incr_frac * ax; - val = test_gradients(cr, incr_val, GPARAM_ASX, "ax*", "x", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - incr_val = incr_frac * bx; - val = test_gradients(cr, incr_val, GPARAM_BSX, "bx*", "x", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - incr_val = incr_frac * cx; - val = test_gradients(cr, incr_val, GPARAM_CSX, "cx*", "x", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - - incr_val = incr_frac * ay; - val = test_gradients(cr, incr_val, GPARAM_ASY, "ay*", "y", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - incr_val = incr_frac * by; - val = test_gradients(cr, incr_val, GPARAM_BSY, "by*", "y", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - incr_val = incr_frac * cy; - val = test_gradients(cr, incr_val, GPARAM_CSY, "cy*", "y", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - - incr_val = incr_frac * az; - val = test_gradients(cr, incr_val, GPARAM_ASZ, "az*", "z", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - incr_val = incr_frac * bz; - val = test_gradients(cr, incr_val, GPARAM_BSZ, "bz*", "z", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - incr_val = incr_frac * cz; - val = test_gradients(cr, incr_val, GPARAM_CSZ, "cz*", "z", pmodel, - quiet, plot); - if ( val < 0.99 ) fail = 1; - - } - - gsl_rng_free(rng); - - return fail; -} diff --git a/tests/prediction_gradient_check.c b/tests/prediction_gradient_check.c index 0ffc07ca..85d61ead 100644 --- a/tests/prediction_gradient_check.c +++ b/tests/prediction_gradient_check.c @@ -73,7 +73,6 @@ static void scan(RefList *reflections, RefList *compare, { signed int h, k, l; Reflection *refl2; - double rlow, rhigh, p; double fs, ss, xh, yh; struct panel *panel; @@ -85,7 +84,6 @@ static void scan(RefList *reflections, RefList *compare, continue; } - get_partial(refl2, &rlow, &rhigh, &p); get_detector_pos(refl2, &fs, &ss); panel = get_panel(refl2); twod_mapping(fs, ss, &xh, &yh, panel); @@ -93,7 +91,7 @@ static void scan(RefList *reflections, RefList *compare, switch ( checkrxy ) { case 0 : - vals[idx][i] = (rlow + rhigh)/2.0; + vals[idx][i] = get_exerr(refl2); break; case 1 : @@ -279,8 +277,6 @@ static double test_gradients(Crystal *cr, double incr_val, int refine, i++; } else { - double r1, r2, p; - grad1 = (vals[1][i] - vals[0][i]) / incr_val; grad2 = (vals[2][i] - vals[1][i]) / incr_val; grad = (grad1 + grad2) / 2.0; @@ -300,18 +296,14 @@ static double test_gradients(Crystal *cr, double incr_val, int refine, if ( checkrxy == 1 ) { cgrad = x_gradient(refine, refl, crystal_get_cell(cr), - &image->det->panels[0], - crystal_get_image(cr)->lambda); + &image->det->panels[0]); } else { cgrad = y_gradient(refine, refl, crystal_get_cell(cr), - &image->det->panels[0], - crystal_get_image(cr)->lambda); + &image->det->panels[0]); } } - get_partial(refl, &r1, &r2, &p); - if ( isnan(cgrad) ) { n_nan++; continue; @@ -338,11 +330,11 @@ static double test_gradients(Crystal *cr, double incr_val, int refine, { if ( !quiet ) { - STATUS("!- %s %3i %3i %3i" - " %10.2Le %10.2e ratio = %5.2Lf" - " %10.2e %10.2e\n", + STATUS("!- %s %3i %3i %3i " + "%10.2Le %10.2e " + "ratio = %5.2Lf\n", str, h, k, l, grad, cgrad, - cgrad/grad, r1, r2); + cgrad/grad); } n_bad++; diff --git a/tests/scaling_check.c b/tests/scaling_check.c new file mode 100644 index 00000000..96df9cf0 --- /dev/null +++ b/tests/scaling_check.c @@ -0,0 +1,138 @@ +/* + * scaling_check.c + * + * Check that scaling works + * + * Copyright © 2017-2018 Deutsches Elektronen-Synchrotron DESY, + * a research centre of the Helmholtz Association. + * + * Authors: + * 2017-2018 Thomas White <taw@physics.org> + * + * This file is part of CrystFEL. + * + * CrystFEL is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * CrystFEL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with CrystFEL. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + + +#include <stdlib.h> +#include <stdio.h> + +#include <reflist.h> +#include <cell-utils.h> + +#include "../src/scaling.h" + +int test_scaling(double G, double B, int scaleflags, int do_partials, + gsl_rng *rng) +{ + int i; + Crystal *cr; + RefList *list1; + RefList *list2; + int r; + UnitCell *cell; + + list1 = reflist_new(); + list2 = reflist_new(); + + cell = cell_new(); + cell_set_parameters(cell, 50e-10, 50e-10, 50e-10, + deg2rad(90), deg2rad(90), deg2rad(90)); + + for ( i=0; i<50; i++ ) { + + signed int h, k, l; + Reflection *refl1; + Reflection *refl2; + double intens, p, s, L; + + h = gsl_rng_uniform_int(rng, 20) - gsl_rng_uniform_int(rng, 40); + k = gsl_rng_uniform_int(rng, 20) - gsl_rng_uniform_int(rng, 40); + l = gsl_rng_uniform_int(rng, 20) - gsl_rng_uniform_int(rng, 40); + + refl1 = add_refl(list1, h, k, l); + refl2 = add_refl(list2, h, k, l); + intens = gsl_rng_uniform(rng); /* [0,1) */ + p = do_partials ? gsl_rng_uniform(rng) : 1.0; + L = gsl_rng_uniform(rng); + + s = resolution(cell, h, k, l); + + /* Reference */ + set_intensity(refl2, intens); + set_partiality(refl2, 1.0); + set_lorentz(refl2, 1.0); + set_redundancy(refl2, 2); + + /* Crystal */ + set_intensity(refl1, intens * G * exp(-B*s*s) * p / L); + set_partiality(refl1, p); + set_lorentz(refl1, L); + + } + + cr = crystal_new(); + crystal_set_reflections(cr, list1); + crystal_set_cell(cr, cell); + + crystal_set_osf(cr, 999.0); + crystal_set_Bfac(cr, 999.0); + + r = scale_one_crystal(cr, list2, scaleflags | SCALE_VERBOSE_ERRORS); + STATUS("Scaling result: %i, G = %8.4f, B = %8.4f A^2\n", r, + crystal_get_osf(cr), crystal_get_Bfac(cr)*1e20); + + if ( fabs(G - crystal_get_osf(cr)) > 0.001 ) r = 1; + if ( fabs(B - crystal_get_Bfac(cr)) > 0.001e-20 ) r = 1; + + reflist_free(list1); + reflist_free(list2); + cell_free(cell); + crystal_free(cr); + + if ( r ) { + STATUS(" (should be: G = %8.4f, B = %8.4f A^2), %s partials\n", + G, B*1e20, do_partials ? "with" : "no"); + } + + return r; +} + + +int main(int argc, char *argv[]) +{ + int fail = 0; + gsl_rng *rng; + + rng = gsl_rng_alloc(gsl_rng_mt19937); + + fail += test_scaling(2.0, 0.0, SCALE_NO_B, 0, rng); + fail += test_scaling(2.0, 0.0, SCALE_NONE, 0, rng); + fail += test_scaling(2.0, 10.0e-20, SCALE_NONE, 0, rng); + fail += test_scaling(5.0, 30.0e-20, SCALE_NONE, 0, rng); + fail += test_scaling(2.0, 0.0, SCALE_NO_B, 1, rng); + fail += test_scaling(2.0, 0.0, SCALE_NONE, 1, rng); + fail += test_scaling(2.0, 10.0e-20, SCALE_NONE, 1, rng); + fail += test_scaling(5.0, 30.0e-20, SCALE_NONE, 1, rng); + + gsl_rng_free(rng); + + return fail; +} |