2 * Copyright (C) 2016, 2017, 2018 "IoT.bzh"
3 * Author José Bollo <jose.bollo@iot.bzh>
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
20 #if defined(NO_JOBS_WATCHDOG)
21 # define HAS_WATCHDOG 0
23 # define HAS_WATCHDOG 1
32 #include <sys/syscall.h>
36 #include <sys/eventfd.h>
38 #include <systemd/sd-event.h>
41 #include <systemd/sd-daemon.h>
45 #include "sig-monitor.h"
48 #define EVENT_TIMEOUT_TOP ((uint64_t)-1)
49 #define EVENT_TIMEOUT_CHILD ((uint64_t)10000)
53 /** Internal shortcut for callback */
54 typedef void (*job_cb_t)(int, void*);
56 /** Description of a pending job */
59 struct job *next; /**< link to the next job enqueued */
60 const void *group; /**< group of the request */
61 job_cb_t callback; /**< processing callback */
62 void *arg; /**< argument */
63 int timeout; /**< timeout in second for processing the request */
64 unsigned blocked: 1; /**< is an other request blocking this one ? */
65 unsigned dropped: 1; /**< is removed ? */
68 /** Description of handled event loops */
71 unsigned state; /**< encoded state */
72 int efd; /**< event notification */
73 struct sd_event *sdev; /**< the systemd event loop */
74 struct fdev *fdev; /**< handling of events */
75 struct thread *holder; /**< holder of the evloop */
78 #define EVLOOP_STATE_WAIT 1U
79 #define EVLOOP_STATE_RUN 2U
81 /** Description of threads */
84 struct thread *next; /**< next thread of the list */
85 struct thread *upper; /**< upper same thread */
86 struct thread *nholder;/**< next holder for evloop */
87 pthread_cond_t *cwhold;/**< condition wait for holding */
88 struct job *job; /**< currently processed job */
89 pthread_t tid; /**< the thread id */
90 volatile unsigned stop: 1; /**< stop requested */
91 volatile unsigned waits: 1; /**< is waiting? */
95 * Description of synchronous callback
99 struct thread thread; /**< thread loop data */
101 void (*callback)(int, void*); /**< the synchronous callback */
102 void (*enter)(int signum, void *closure, struct jobloop *jobloop);
103 /**< the entering synchronous routine */
105 void *arg; /**< the argument of the callback */
109 /* synchronisation of threads */
110 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
111 static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
113 /* count allowed, started and running threads */
114 static int allowed = 0; /** allowed count of threads */
115 static int started = 0; /** started count of threads */
116 static int running = 0; /** running count of threads */
117 static int remains = 0; /** allowed count of waiting jobs */
119 /* list of threads */
120 static struct thread *threads;
121 static _Thread_local struct thread *current_thread;
123 /* queue of pending jobs */
124 static struct job *first_job;
125 static struct job *free_jobs;
128 static struct evloop evloop;
131 * Create a new job with the given parameters
132 * @param group the group of the job
133 * @param timeout the timeout of the job (0 if none)
134 * @param callback the function that achieves the job
135 * @param arg the argument of the callback
136 * @return the created job unblock or NULL when no more memory
138 static struct job *job_create(
146 /* try recyle existing job */
149 free_jobs = job->next;
151 /* allocation without blocking */
152 pthread_mutex_unlock(&mutex);
153 job = malloc(sizeof *job);
154 pthread_mutex_lock(&mutex);
156 ERROR("out of memory");
161 /* initialises the job */
163 job->timeout = timeout;
164 job->callback = callback;
173 * Adds 'job' at the end of the list of jobs, marking it
174 * as blocked if an other job with the same group is pending.
175 * @param job the job to add
177 static void job_add(struct job *job)
180 struct job *ijob, **pjob;
186 /* search end and blockers */
190 if (group && ijob->group == group)
202 * Get the next job to process or NULL if none.
203 * @return the first job that isn't blocked or NULL
205 static inline struct job *job_get()
207 struct job *job = first_job;
208 while (job && job->blocked)
216 * Releases the processed 'job': removes it
217 * from the list of jobs and unblock the first
218 * pending job of the same group if any.
219 * @param job the job to release
221 static inline void job_release(struct job *job)
223 struct job *ijob, **pjob;
226 /* first unqueue the job */
229 while (ijob != job) {
235 /* then unblock jobs of the same group */
239 while (ijob && ijob->group != group)
245 /* recycle the job */
246 job->next = free_jobs;
251 * Monitored cancel callback for a job.
252 * This function is called by the monitor
253 * to cancel the job when the safe environment
255 * @param signum 0 on normal flow or the number
256 * of the signal that interrupted the normal
258 * @param arg the job to run
260 static void job_cancel(int signum, void *arg)
262 struct job *job = arg;
263 job->callback(SIGABRT, job->arg);
267 * Monitored normal callback for events.
268 * This function is called by the monitor
269 * to run the event loop when the safe environment
271 * @param signum 0 on normal flow or the number
272 * of the signal that interrupted the normal
274 * @param arg the events to run
276 static void evloop_run(int signum, void *arg)
283 rc = sd_event_prepare(se);
286 CRITICAL("sd_event_prepare returned an error (state: %d): %m", sd_event_get_state(se));
290 rc = sd_event_wait(se, (uint64_t)(int64_t)-1);
293 ERROR("sd_event_wait returned an error (state: %d): %m", sd_event_get_state(se));
296 evloop.state = EVLOOP_STATE_RUN;
298 rc = sd_event_dispatch(se);
301 ERROR("sd_event_dispatch returned an error (state: %d): %m", sd_event_get_state(se));
309 * Internal callback for evloop management.
310 * The effect of this function is hidden: it exits
311 * the waiting poll if any.
313 static void evloop_on_efd_event()
316 read(evloop.efd, &x, sizeof x);
320 * wakeup the event loop if needed by sending
323 static void evloop_wakeup()
327 if (evloop.state & EVLOOP_STATE_WAIT) {
329 write(evloop.efd, &x, sizeof x);
334 * Release the currently held event loop
336 static void evloop_release()
338 struct thread *nh, *ct = current_thread;
340 if (ct && evloop.holder == ct) {
344 pthread_cond_signal(nh->cwhold);
349 * get the eventloop for the current thread
351 static int evloop_get()
353 struct thread *ct = current_thread;
356 return evloop.holder == ct;
367 * acquire the eventloop for the current thread
369 static void evloop_acquire()
371 struct thread **pwait, *ct;
374 /* try to get the evloop */
376 /* failed, init waiting state */
380 pthread_cond_init(&cond, NULL);
382 /* queue current thread in holder list */
383 pwait = &evloop.holder;
385 pwait = &(*pwait)->nholder;
388 /* wake up the evloop */
391 /* wait to acquire the evloop */
392 pthread_cond_wait(&cond, &mutex);
393 pthread_cond_destroy(&cond);
399 * @param me the description of the thread to enter
401 static void thread_enter(volatile struct thread *me)
404 /* initialize description of itself and link it in the list */
405 me->tid = pthread_self();
408 me->upper = current_thread;
410 threads = (struct thread*)me;
411 current_thread = (struct thread*)me;
416 * @param me the description of the thread to leave
418 static void thread_leave()
420 struct thread **prv, *me;
422 /* unlink the current thread and cleanup */
429 current_thread = me->upper;
433 * Main processing loop of internal threads with processing jobs.
434 * The loop must be called with the mutex locked
435 * and it returns with the mutex locked.
436 * @param me the description of the thread to use
437 * TODO: how are timeout handled when reentering?
439 static void thread_run_internal(volatile struct thread *me)
446 /* loop until stopped */
448 /* release the current event loop */
454 /* prepare running the job */
455 job->blocked = 1; /* mark job as blocked */
456 me->job = job; /* record the job (only for terminate) */
459 pthread_mutex_unlock(&mutex);
460 sig_monitor(job->timeout, job->callback, job->arg);
461 pthread_mutex_lock(&mutex);
463 /* release the run job */
465 /* no job, check event loop wait */
466 } else if (evloop_get()) {
467 if (evloop.state != 0) {
469 CRITICAL("Can't enter dispatch while in dispatch!");
473 evloop.state = EVLOOP_STATE_RUN|EVLOOP_STATE_WAIT;
474 pthread_mutex_unlock(&mutex);
475 sig_monitor(0, evloop_run, NULL);
476 pthread_mutex_lock(&mutex);
479 /* no job and no event loop */
482 ERROR("Entering job deep sleep! Check your bindings.");
484 pthread_cond_wait(&cond, &mutex);
495 * Main processing loop of external threads.
496 * The loop must be called with the mutex locked
497 * and it returns with the mutex locked.
498 * @param me the description of the thread to use
500 static void thread_run_external(volatile struct thread *me)
505 /* loop until stopped */
508 pthread_cond_wait(&cond, &mutex);
514 * Root for created threads.
516 static void thread_main()
522 sig_monitor_init_timeouts();
523 thread_run_internal(&me);
524 sig_monitor_clean_timeouts();
530 * Entry point for created threads.
531 * @param data not used
534 static void *thread_starter(void *data)
536 pthread_mutex_lock(&mutex);
538 pthread_mutex_unlock(&mutex);
543 * Starts a new thread
544 * @return 0 in case of success or -1 in case of error
546 static int start_one_thread()
551 rc = pthread_create(&tid, NULL, thread_starter, NULL);
554 WARNING("not able to start thread: %m");
561 * Queues a new asynchronous job represented by 'callback' and 'arg'
562 * for the 'group' and the 'timeout'.
563 * Jobs are queued FIFO and are possibly executed in parallel
564 * concurrently except for job of the same group that are
565 * executed sequentially in FIFO order.
566 * @param group The group of the job or NULL when no group.
567 * @param timeout The maximum execution time in seconds of the job
568 * or 0 for unlimited time.
569 * @param callback The function to execute for achieving the job.
570 * Its first parameter is either 0 on normal flow
571 * or the signal number that broke the normal flow.
572 * The remaining parameter is the parameter 'arg1'
574 * @param arg The second argument for 'callback'
575 * @return 0 in case of success or -1 in case of error
580 void (*callback)(int, void*),
586 pthread_mutex_lock(&mutex);
588 /* allocates the job */
589 job = job_create(group, timeout, callback, arg);
593 /* check availability */
595 ERROR("can't process job with threads: too many jobs");
600 /* start a thread if needed */
601 if (running == started && started < allowed) {
602 /* all threads are busy and a new can be started */
603 rc = start_one_thread();
604 if (rc < 0 && started == 0) {
605 ERROR("can't start initial thread: %m");
613 /* signal an existing job */
614 pthread_cond_signal(&cond);
615 pthread_mutex_unlock(&mutex);
619 job->next = free_jobs;
622 pthread_mutex_unlock(&mutex);
627 * Internal helper function for 'jobs_enter'.
628 * @see jobs_enter, jobs_leave
630 static void enter_cb(int signum, void *closure)
632 struct sync *sync = closure;
633 sync->enter(signum, sync->arg, (void*)&sync->thread);
637 * Internal helper function for 'jobs_call'.
640 static void call_cb(int signum, void *closure)
642 struct sync *sync = closure;
643 sync->callback(signum, sync->arg);
644 jobs_leave((void*)&sync->thread);
648 * Internal helper for synchronous jobs. It enters
649 * a new thread loop for evaluating the given job
650 * as recorded by the couple 'sync_cb' and 'sync'.
651 * @see jobs_call, jobs_enter, jobs_leave
656 void (*sync_cb)(int signum, void *closure),
662 pthread_mutex_lock(&mutex);
664 /* allocates the job */
665 job = job_create(group, timeout, sync_cb, sync);
667 pthread_mutex_unlock(&mutex);
674 /* run until stopped */
676 thread_run_internal(&sync->thread);
678 thread_run_external(&sync->thread);
679 pthread_mutex_unlock(&mutex);
684 * Enter a synchronisation point: activates the job given by 'callback'
685 * and 'closure' using 'group' and 'timeout' to control sequencing and
687 * @param group the group for sequencing jobs
688 * @param timeout the time in seconds allocated to the job
689 * @param callback the callback that will handle the job.
690 * it receives 3 parameters: 'signum' that will be 0
691 * on normal flow or the catched signal number in case
692 * of interrupted flow, the context 'closure' as given and
693 * a 'jobloop' reference that must be used when the job is
694 * terminated to unlock the current execution flow.
695 * @param closure the argument to the callback
696 * @return 0 on success or -1 in case of error
701 void (*callback)(int signum, void *closure, struct jobloop *jobloop),
707 sync.enter = callback;
709 return do_sync(group, timeout, enter_cb, &sync);
713 * Unlocks the execution flow designed by 'jobloop'.
714 * @param jobloop indication of the flow to unlock
715 * @return 0 in case of success of -1 on error
717 int jobs_leave(struct jobloop *jobloop)
721 pthread_mutex_lock(&mutex);
723 while (t && t != (struct thread*)jobloop)
730 pthread_cond_broadcast(&cond);
734 pthread_mutex_unlock(&mutex);
739 * Calls synchronously the job represented by 'callback' and 'arg1'
740 * for the 'group' and the 'timeout' and waits for its completion.
741 * @param group The group of the job or NULL when no group.
742 * @param timeout The maximum execution time in seconds of the job
743 * or 0 for unlimited time.
744 * @param callback The function to execute for achieving the job.
745 * Its first parameter is either 0 on normal flow
746 * or the signal number that broke the normal flow.
747 * The remaining parameter is the parameter 'arg1'
749 * @param arg The second argument for 'callback'
750 * @return 0 in case of success or -1 in case of error
755 void (*callback)(int, void*),
760 sync.callback = callback;
763 return do_sync(group, timeout, call_cb, &sync);
767 * Internal callback for evloop management.
768 * The effect of this function is hidden: it exits
769 * the waiting poll if any. Then it wakes up a thread
770 * awaiting the evloop using signal.
772 static int on_evloop_efd(sd_event_source *s, int fd, uint32_t revents, void *userdata)
774 evloop_on_efd_event();
779 * Gets a sd_event item for the current thread.
780 * @return a sd_event or NULL in case of error
782 static struct sd_event *get_sd_event_locked()
786 /* creates the evloop on need */
788 /* start the creation */
790 /* creates the eventfd for waking up polls */
791 evloop.efd = eventfd(0, EFD_CLOEXEC|EFD_SEMAPHORE);
792 if (evloop.efd < 0) {
793 ERROR("can't make eventfd for events");
796 /* create the systemd event loop */
797 rc = sd_event_new(&evloop.sdev);
799 ERROR("can't make new event loop");
802 /* put the eventfd in the event loop */
803 rc = sd_event_add_io(evloop.sdev, NULL, evloop.efd, EPOLLIN, on_evloop_efd, NULL);
805 ERROR("can't register eventfd");
806 sd_event_unref(evloop.sdev);
815 /* acquire the event loop */
822 * Gets a sd_event item for the current thread.
823 * @return a sd_event or NULL in case of error
825 struct sd_event *jobs_get_sd_event()
827 struct sd_event *result;
830 /* ensure an existing thread environment */
831 if (!current_thread) {
832 memset(<, 0, sizeof lt);
833 current_thread = <
837 pthread_mutex_lock(&mutex);
838 result = get_sd_event_locked();
839 pthread_mutex_unlock(&mutex);
841 /* release the faked thread environment if needed */
842 if (current_thread == <) {
844 * Releasing it is needed because there is no way to guess
845 * when it has to be released really. But here is where it is
846 * hazardous: if the caller modifies the eventloop when it
847 * is waiting, there is no way to make the change effective.
848 * A workaround to achieve that goal is for the caller to
849 * require the event loop a second time after having modified it.
851 NOTICE("Requiring sd_event loop out of binder callbacks is hazardous!");
852 if (verbose_wants(Log_Level_Info))
853 sig_monitor_dumpstack();
855 current_thread = NULL;
862 * Enter the jobs processing loop.
863 * @param allowed_count Maximum count of thread for jobs including this one
864 * @param start_count Count of thread to start now, must be lower.
865 * @param waiter_count Maximum count of jobs that can be waiting.
866 * @param start The start routine to activate (can't be NULL)
867 * @return 0 in case of success or -1 in case of error.
869 int jobs_start(int allowed_count, int start_count, int waiter_count, void (*start)(int signum, void* arg), void *arg)
874 assert(allowed_count >= 1);
875 assert(start_count >= 0);
876 assert(waiter_count > 0);
877 assert(start_count <= allowed_count);
880 pthread_mutex_lock(&mutex);
882 /* check whether already running */
883 if (current_thread || allowed) {
884 ERROR("thread already started");
889 /* records the allowed count */
890 allowed = allowed_count;
893 remains = waiter_count;
896 /* set the watchdog */
897 if (sd_watchdog_enabled(0, NULL))
898 sd_event_set_watchdog(get_sd_event_locked(), 1);
901 /* start at least one thread: the current one */
903 while (launched < start_count) {
904 if (start_one_thread() != 0) {
905 ERROR("Not all threads can be started");
911 /* queue the start job */
912 job = job_create(NULL, 0, start, arg);
921 pthread_mutex_unlock(&mutex);
926 * Terminate all the threads and cancel all pending jobs.
928 void jobs_terminate()
930 struct job *job, *head, *tail;
931 pthread_t me, *others;
938 /* request all threads to stop */
939 pthread_mutex_lock(&mutex);
942 /* count the number of threads */
946 if (!t->upper && !pthread_equal(t->tid, me))
951 /* fill the array of threads */
952 others = alloca(count * sizeof *others);
956 if (!t->upper && !pthread_equal(t->tid, me))
957 others[count++] = t->tid;
961 /* stops the threads */
968 /* wait the threads */
969 pthread_cond_broadcast(&cond);
970 pthread_mutex_unlock(&mutex);
972 pthread_join(others[--count], NULL);
973 pthread_mutex_lock(&mutex);
975 /* cancel pending jobs of other threads */
985 /* search if job is stacked for current */
987 while (t && t->job != job)
990 /* yes, relink it at end */
998 /* no cancel the job */
999 pthread_mutex_unlock(&mutex);
1000 sig_monitor(0, job_cancel, job);
1002 pthread_mutex_lock(&mutex);
1005 pthread_mutex_unlock(&mutex);