diff options
| author | Thomas White <taw@physics.org> | 2018-05-24 14:58:25 +0200 |
|---|---|---|
| committer | Thomas White <taw@physics.org> | 2018-05-24 15:00:52 +0200 |
| commit | 3f8db4b1884a95f4f52e744cb1639df80208381e (patch) | |
| tree | eab374546d93ebf33f597e456c171d0fc4fffe9a /src | |
| parent | 2b9b4cb52775cbc2a7e458fae3a799e0a03f378d (diff) | |
indexamajig: Warn if a frame takes more than 10 minutes to process
Diffstat (limited to 'src')
| -rw-r--r-- | src/im-sandbox.c | 17 | ||||
| -rw-r--r-- | src/im-sandbox.h | 6 |
2 files changed, 19 insertions, 4 deletions
diff --git a/src/im-sandbox.c b/src/im-sandbox.c index 09928a82..91479414 100644 --- a/src/im-sandbox.c +++ b/src/im-sandbox.c @@ -3,13 +3,13 @@ * * Sandbox for indexing * - * Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * Copyright © 2012 Lorenzo Galli * * Authors: - * 2010-2017 Thomas White <taw@physics.org> + * 2010-2018 Thomas White <taw@physics.org> * 2014 Valerio Mariani * 2011 Richard Kirian * 2012 Lorenzo Galli @@ -147,6 +147,16 @@ static void check_hung_workers(struct sandbox *sb) stamp_response(sb, i); } + if ( tnow - sb->shared->time_last_start[i] > 600 ) { + if ( !sb->shared->warned_long_running[i] ) { + STATUS("Worker %i has been working on one " + "frame for more than 10 minutes.\n", i); + STATUS("Event ID is: %s\n", + sb->shared->last_ev[i]); + sb->shared->warned_long_running[i] = 1; + } + } + } } @@ -378,6 +388,7 @@ static void run_work(const struct index_args *iargs, Stream *st, } + sb->shared->time_last_start[cookie] = get_monotonic_seconds(); process_image(iargs, &pargs, st, cookie, tmpdir, ser, sb->shared, taccs); @@ -558,6 +569,8 @@ static void start_worker_process(struct sandbox *sb, int slot) sb->shared->pings[slot] = 0; sb->last_ping[slot] = 0; + sb->shared->time_last_start[slot] = get_monotonic_seconds(); + sb->shared->warned_long_running[slot] = 0; p = fork(); if ( p == -1 ) { diff --git a/src/im-sandbox.h b/src/im-sandbox.h index e937875f..06c9a33c 100644 --- a/src/im-sandbox.h +++ b/src/im-sandbox.h @@ -3,13 +3,13 @@ * * Sandbox for indexing * - * Copyright © 2012-2017 Deutsches Elektronen-Synchrotron DESY, + * Copyright © 2012-2018 Deutsches Elektronen-Synchrotron DESY, * a research centre of the Helmholtz Association. * Copyright © 2012 Richard Kirian * Copyright © 2012 Lorenzo Galli * * Authors: - * 2010-2017 Thomas White <taw@physics.org> + * 2010-2018 Thomas White <taw@physics.org> * 2011 Richard Kirian * 2012 Lorenzo Galli * 2012 Chunhong Yoon @@ -62,6 +62,8 @@ struct sb_shm int no_more; char last_ev[MAX_NUM_WORKERS][MAX_EV_LEN]; int pings[MAX_NUM_WORKERS]; + time_t time_last_start[MAX_NUM_WORKERS]; + int warned_long_running[MAX_NUM_WORKERS]; pthread_mutex_t totals_lock; int n_processed; |