2 * Copyright (C) 2016, 2017 "IoT.bzh"
3 * Author José Bollo <jose.bollo@iot.bzh>
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
25 #include <sys/syscall.h>
29 #include <sys/eventfd.h>
31 #include <systemd/sd-event.h>
32 #ifndef NO_JOBS_WATCHDOG
33 #include <systemd/sd-daemon.h>
37 #include "sig-monitor.h"
41 #define _alert_ "do you really want to remove monitoring?"
42 #define sig_monitor_init_timeouts() ((void)0)
43 #define sig_monitor_clean_timeouts() ((void)0)
44 #define sig_monitor(to,cb,arg) (cb(0,arg))
47 #define EVENT_TIMEOUT_TOP ((uint64_t)-1)
48 #define EVENT_TIMEOUT_CHILD ((uint64_t)10000)
50 /** Internal shortcut for callback */
51 typedef void (*job_cb_t)(int, void*);
53 /** Description of a pending job */
56 struct job *next; /**< link to the next job enqueued */
57 const void *group; /**< group of the request */
58 job_cb_t callback; /**< processing callback */
59 void *arg; /**< argument */
60 int timeout; /**< timeout in second for processing the request */
61 unsigned blocked: 1; /**< is an other request blocking this one ? */
62 unsigned dropped: 1; /**< is removed ? */
65 /** Description of handled event loops */
68 unsigned state; /**< encoded state */
69 int efd; /**< event notification */
70 struct sd_event *sdev; /**< the systemd event loop */
71 pthread_cond_t cond; /**< condition */
74 #define EVLOOP_STATE_WAIT 1U
75 #define EVLOOP_STATE_RUN 2U
76 #define EVLOOP_STATE_LOCK 4U
78 /** Description of threads */
81 struct thread *next; /**< next thread of the list */
82 struct thread *upper; /**< upper same thread */
83 struct job *job; /**< currently processed job */
84 pthread_t tid; /**< the thread id */
85 unsigned stop: 1; /**< stop requested */
86 unsigned waits: 1; /**< is waiting? */
90 * Description of synchonous callback
94 struct thread thread; /**< thread loop data */
96 void (*callback)(int, void*); /**< the synchronous callback */
97 void (*enter)(int signum, void *closure, struct jobloop *jobloop);
98 /**< the entering synchronous routine */
100 void *arg; /**< the argument of the callback */
104 /* synchronisation of threads */
105 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
106 static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
108 /* count allowed, started and running threads */
109 static int allowed = 0; /** allowed count of threads */
110 static int started = 0; /** started count of threads */
111 static int running = 0; /** running count of threads */
112 static int remains = 0; /** allowed count of waiting jobs */
114 /* list of threads */
115 static struct thread *threads;
116 static _Thread_local struct thread *current_thread;
117 static _Thread_local struct evloop *current_evloop;
119 /* queue of pending jobs */
120 static struct job *first_job;
121 static struct job *free_jobs;
124 static struct evloop evloop[1];
127 * Create a new job with the given parameters
128 * @param group the group of the job
129 * @param timeout the timeout of the job (0 if none)
130 * @param callback the function that achieves the job
131 * @param arg the argument of the callback
132 * @return the created job unblock or NULL when no more memory
134 static struct job *job_create(
142 /* try recyle existing job */
145 free_jobs = job->next;
147 /* allocation without blocking */
148 pthread_mutex_unlock(&mutex);
149 job = malloc(sizeof *job);
150 pthread_mutex_lock(&mutex);
156 /* initialises the job */
158 job->timeout = timeout;
159 job->callback = callback;
168 * Adds 'job' at the end of the list of jobs, marking it
169 * as blocked if an other job with the same group is pending.
170 * @param job the job to add
172 static void job_add(struct job *job)
175 struct job *ijob, **pjob;
181 /* search end and blockers */
185 if (group && ijob->group == group)
196 * Get the next job to process or NULL if none.
197 * @return the first job that isn't blocked or NULL
199 static inline struct job *job_get()
201 struct job *job = first_job;
202 while (job && job->blocked)
208 * Releases the processed 'job': removes it
209 * from the list of jobs and unblock the first
210 * pending job of the same group if any.
211 * @param job the job to release
213 static inline void job_release(struct job *job)
215 struct job *ijob, **pjob;
218 /* first unqueue the job */
221 while (ijob != job) {
227 /* then unblock jobs of the same group */
231 while (ijob && ijob->group != group)
237 /* recycle the job */
238 job->next = free_jobs;
243 * Monitored cancel callback for a job.
244 * This function is called by the monitor
245 * to cancel the job when the safe environment
247 * @param signum 0 on normal flow or the number
248 * of the signal that interrupted the normal
250 * @param arg the job to run
252 static void job_cancel(int signum, void *arg)
254 struct job *job = arg;
255 job->callback(SIGABRT, job->arg);
259 * Monitored normal callback for events.
260 * This function is called by the monitor
261 * to run the event loop when the safe environment
263 * @param signum 0 on normal flow or the number
264 * of the signal that interrupted the normal
266 * @param arg the events to run
268 static void evloop_run(int signum, void *arg)
272 struct evloop *el = arg;
276 rc = sd_event_prepare(se);
279 ERROR("sd_event_prepare returned an error (state: %d): %m", sd_event_get_state(se));
282 rc = sd_event_wait(se, (uint64_t)(int64_t)-1);
285 ERROR("sd_event_wait returned an error (state: %d): %m", sd_event_get_state(se));
288 __atomic_and_fetch(&el->state, ~(EVLOOP_STATE_WAIT), __ATOMIC_RELAXED);
291 rc = sd_event_dispatch(se);
294 ERROR("sd_event_dispatch returned an error (state: %d): %m", sd_event_get_state(se));
299 __atomic_and_fetch(&el->state, ~(EVLOOP_STATE_WAIT|EVLOOP_STATE_RUN), __ATOMIC_RELAXED);
304 * Main processing loop of threads processing jobs.
305 * The loop must be called with the mutex locked
306 * and it returns with the mutex locked.
307 * @param me the description of the thread to use
308 * TODO: how are timeout handled when reentering?
310 static void thread_run(volatile struct thread *me)
316 /* initialize description of itself and link it in the list */
317 me->tid = pthread_self();
320 me->upper = current_thread;
321 if (!current_thread) {
323 sig_monitor_init_timeouts();
326 threads = (struct thread*)me;
327 current_thread = (struct thread*)me;
329 /* loop until stopped */
331 /* release the event loop */
332 if (current_evloop) {
333 __atomic_sub_fetch(¤t_evloop->state, EVLOOP_STATE_LOCK, __ATOMIC_RELAXED);
334 current_evloop = NULL;
338 job = job_get(first_job);
340 /* prepare running the job */
341 remains++; /* increases count of job that can wait */
342 job->blocked = 1; /* mark job as blocked */
343 me->job = job; /* record the job (only for terminate) */
346 pthread_mutex_unlock(&mutex);
347 sig_monitor(job->timeout, job->callback, job->arg);
348 pthread_mutex_lock(&mutex);
350 /* release the run job */
353 /* no job, check events */
355 if (el->sdev && !__atomic_load_n(&el->state, __ATOMIC_RELAXED)) {
357 __atomic_store_n(&el->state, EVLOOP_STATE_LOCK|EVLOOP_STATE_RUN|EVLOOP_STATE_WAIT, __ATOMIC_RELAXED);
359 pthread_mutex_unlock(&mutex);
360 sig_monitor(0, evloop_run, el);
361 pthread_mutex_lock(&mutex);
363 /* no job and not events */
366 ERROR("Entering job deep sleep! Check your bindings.");
368 pthread_cond_wait(&cond, &mutex);
375 /* release the event loop */
376 if (current_evloop) {
377 __atomic_sub_fetch(¤t_evloop->state, EVLOOP_STATE_LOCK, __ATOMIC_RELAXED);
378 current_evloop = NULL;
381 /* unlink the current thread and cleanup */
386 current_thread = me->upper;
387 if (!current_thread) {
388 sig_monitor_clean_timeouts();
394 * Entry point for created threads.
395 * @param data not used
398 static void *thread_main(void *data)
402 pthread_mutex_lock(&mutex);
406 pthread_mutex_unlock(&mutex);
411 * Starts a new thread
412 * @return 0 in case of success or -1 in case of error
414 static int start_one_thread()
419 rc = pthread_create(&tid, NULL, thread_main, NULL);
422 WARNING("not able to start thread: %m");
429 * Queues a new asynchronous job represented by 'callback' and 'arg'
430 * for the 'group' and the 'timeout'.
431 * Jobs are queued FIFO and are possibly executed in parallel
432 * concurrently except for job of the same group that are
433 * executed sequentially in FIFO order.
434 * @param group The group of the job or NULL when no group.
435 * @param timeout The maximum execution time in seconds of the job
436 * or 0 for unlimited time.
437 * @param callback The function to execute for achieving the job.
438 * Its first parameter is either 0 on normal flow
439 * or the signal number that broke the normal flow.
440 * The remaining parameter is the parameter 'arg1'
442 * @param arg The second argument for 'callback'
443 * @return 0 in case of success or -1 in case of error
448 void (*callback)(int, void*),
455 pthread_mutex_lock(&mutex);
457 /* allocates the job */
458 job = job_create(group, timeout, callback, arg);
461 info = "out of memory";
465 /* check availability */
468 info = "too many jobs";
472 /* start a thread if needed */
473 if (running == started && started < allowed) {
474 /* all threads are busy and a new can be started */
475 rc = start_one_thread();
476 if (rc < 0 && started == 0) {
477 info = "can't start first thread";
486 /* signal an existing job */
487 pthread_cond_signal(&cond);
488 pthread_mutex_unlock(&mutex);
492 job->next = free_jobs;
495 ERROR("can't process job with threads: %s, %m", info);
496 pthread_mutex_unlock(&mutex);
501 * Internal helper function for 'jobs_enter'.
502 * @see jobs_enter, jobs_leave
504 static void enter_cb(int signum, void *closure)
506 struct sync *sync = closure;
507 sync->enter(signum, sync->arg, (void*)&sync->thread);
511 * Internal helper function for 'jobs_call'.
514 static void call_cb(int signum, void *closure)
516 struct sync *sync = closure;
517 sync->callback(signum, sync->arg);
518 jobs_leave((void*)&sync->thread);
522 * Internal helper for synchronous jobs. It enters
523 * a new thread loop for evaluating the given job
524 * as recorded by the couple 'sync_cb' and 'sync'.
525 * @see jobs_call, jobs_enter, jobs_leave
530 void (*sync_cb)(int signum, void *closure),
536 pthread_mutex_lock(&mutex);
538 /* allocates the job */
539 job = job_create(group, timeout, sync_cb, sync);
541 ERROR("out of memory");
543 pthread_mutex_unlock(&mutex);
550 /* run until stopped */
551 thread_run(&sync->thread);
552 pthread_mutex_unlock(&mutex);
557 * Enter a synchronisation point: activates the job given by 'callback'
558 * and 'closure' using 'group' and 'timeout' to control sequencing and
560 * @param group the group for sequencing jobs
561 * @param timeout the time in seconds allocated to the job
562 * @param callback the callback that will handle the job.
563 * it receives 3 parameters: 'signum' that will be 0
564 * on normal flow or the catched signal number in case
565 * of interrupted flow, the context 'closure' as given and
566 * a 'jobloop' reference that must be used when the job is
567 * terminated to unlock the current execution flow.
568 * @param arg the argument to the callback
569 * @return 0 on success or -1 in case of error
574 void (*callback)(int signum, void *closure, struct jobloop *jobloop),
580 sync.enter = callback;
582 return do_sync(group, timeout, enter_cb, &sync);
586 * Unlocks the execution flow designed by 'jobloop'.
587 * @param jobloop indication of the flow to unlock
588 * @return 0 in case of success of -1 on error
590 int jobs_leave(struct jobloop *jobloop)
594 pthread_mutex_lock(&mutex);
596 while (t && t != (struct thread*)jobloop)
603 pthread_cond_broadcast(&cond);
605 pthread_mutex_unlock(&mutex);
610 * Calls synchronously the job represented by 'callback' and 'arg1'
611 * for the 'group' and the 'timeout' and waits for its completion.
612 * @param group The group of the job or NULL when no group.
613 * @param timeout The maximum execution time in seconds of the job
614 * or 0 for unlimited time.
615 * @param callback The function to execute for achieving the job.
616 * Its first parameter is either 0 on normal flow
617 * or the signal number that broke the normal flow.
618 * The remaining parameter is the parameter 'arg1'
620 * @param arg The second argument for 'callback'
621 * @return 0 in case of success or -1 in case of error
626 void (*callback)(int, void*),
631 sync.callback = callback;
634 return do_sync(group, timeout, call_cb, &sync);
638 * Internal callback for evloop management.
639 * The effect of this function is hidden: it exits
640 * the waiting poll if any. Then it wakes up a thread
641 * awaiting the evloop using signal.
643 static int on_evloop_efd(sd_event_source *s, int fd, uint32_t revents, void *userdata)
646 struct evloop *evloop = userdata;
647 read(evloop->efd, &x, sizeof x);
648 pthread_mutex_lock(&mutex);
649 pthread_cond_broadcast(&evloop->cond);
650 pthread_mutex_unlock(&mutex);
655 * Gets a sd_event item for the current thread.
656 * @return a sd_event or NULL in case of error
658 static struct sd_event *get_sd_event_locked()
664 /* creates the evloop on need */
667 /* start the creation */
669 /* creates the eventfd for waking up polls */
670 el->efd = eventfd(0, EFD_CLOEXEC);
672 ERROR("can't make eventfd for events");
675 /* create the systemd event loop */
676 rc = sd_event_new(&el->sdev);
678 ERROR("can't make new event loop");
681 /* put the eventfd in the event loop */
682 rc = sd_event_add_io(el->sdev, NULL, el->efd, EPOLLIN, on_evloop_efd, el);
684 ERROR("can't register eventfd");
685 sd_event_unref(el->sdev);
694 /* attach the event loop to the current thread */
695 if (current_evloop != el) {
697 __atomic_sub_fetch(¤t_evloop->state, EVLOOP_STATE_LOCK, __ATOMIC_RELAXED);
699 __atomic_add_fetch(&el->state, EVLOOP_STATE_LOCK, __ATOMIC_RELAXED);
702 /* wait for a modifiable event loop */
703 while (__atomic_load_n(&el->state, __ATOMIC_RELAXED) & EVLOOP_STATE_WAIT) {
705 write(el->efd, &x, sizeof x);
706 pthread_cond_wait(&el->cond, &mutex);
713 * Gets a sd_event item for the current thread.
714 * @return a sd_event or NULL in case of error
716 struct sd_event *jobs_get_sd_event()
718 struct sd_event *result;
720 pthread_mutex_lock(&mutex);
721 result = get_sd_event_locked();
722 pthread_mutex_unlock(&mutex);
728 * Enter the jobs processing loop.
729 * @param allowed_count Maximum count of thread for jobs including this one
730 * @param start_count Count of thread to start now, must be lower.
731 * @param waiter_count Maximum count of jobs that can be waiting.
732 * @param start The start routine to activate (can't be NULL)
733 * @return 0 in case of success or -1 in case of error.
735 int jobs_start(int allowed_count, int start_count, int waiter_count, void (*start)(int signum, void* arg), void *arg)
741 assert(allowed_count >= 1);
742 assert(start_count >= 0);
743 assert(waiter_count > 0);
744 assert(start_count <= allowed_count);
747 pthread_mutex_lock(&mutex);
749 /* check whether already running */
750 if (current_thread || allowed) {
751 ERROR("thread already started");
757 if (sig_monitor_init() < 0) {
758 ERROR("failed to initialise signal handlers");
762 /* records the allowed count */
763 allowed = allowed_count;
766 remains = waiter_count;
768 #ifndef NO_JOBS_WATCHDOG
769 /* set the watchdog */
770 if (sd_watchdog_enabled(0, NULL))
771 sd_event_set_watchdog(get_sd_event_locked(), 1);
774 /* start at least one thread */
776 while ((launched + 1) < start_count) {
777 if (start_one_thread() != 0) {
778 ERROR("Not all threads can be started");
784 /* queue the start job */
785 job = job_create(NULL, 0, start, arg);
787 ERROR("out of memory");
798 pthread_mutex_unlock(&mutex);
803 * Terminate all the threads and cancel all pending jobs.
805 void jobs_terminate()
807 struct job *job, *head, *tail;
808 pthread_t me, *others;
815 /* request all threads to stop */
816 pthread_mutex_lock(&mutex);
819 /* count the number of threads */
823 if (!t->upper && !pthread_equal(t->tid, me))
828 /* fill the array of threads */
829 others = alloca(count * sizeof *others);
833 if (!t->upper && !pthread_equal(t->tid, me))
834 others[count++] = t->tid;
838 /* stops the threads */
845 /* wait the threads */
846 pthread_cond_broadcast(&cond);
847 pthread_mutex_unlock(&mutex);
849 pthread_join(others[--count], NULL);
850 pthread_mutex_lock(&mutex);
852 /* cancel pending jobs of other threads */
862 /* search if job is stacked for current */
864 while (t && t->job != job)
867 /* yes, relink it at end */
875 /* no cancel the job */
876 pthread_mutex_unlock(&mutex);
877 sig_monitor(0, job_cancel, job);
879 pthread_mutex_lock(&mutex);
882 pthread_mutex_unlock(&mutex);