diff options
Diffstat (limited to 'module/spl/spl-condvar.c')
-rw-r--r-- | module/spl/spl-condvar.c | 461 |
1 files changed, 0 insertions, 461 deletions
diff --git a/module/spl/spl-condvar.c b/module/spl/spl-condvar.c deleted file mode 100644 index 3cc33da62..000000000 --- a/module/spl/spl-condvar.c +++ /dev/null @@ -1,461 +0,0 @@ -/* - * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. - * Copyright (C) 2007 The Regents of the University of California. - * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). - * Written by Brian Behlendorf <[email protected]>. - * UCRL-CODE-235197 - * - * This file is part of the SPL, Solaris Porting Layer. - * For details, see <http://zfsonlinux.org/>. - * - * The SPL is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * The SPL is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with the SPL. If not, see <http://www.gnu.org/licenses/>. - * - * Solaris Porting Layer (SPL) Credential Implementation. - */ - -#include <sys/condvar.h> -#include <sys/time.h> -#include <sys/sysmacros.h> -#include <linux/hrtimer.h> -#include <linux/compiler_compat.h> -#include <linux/mod_compat.h> - -#include <linux/sched.h> - -#ifdef HAVE_SCHED_SIGNAL_HEADER -#include <linux/sched/signal.h> -#endif - -#define MAX_HRTIMEOUT_SLACK_US 1000 -unsigned int spl_schedule_hrtimeout_slack_us = 0; - -static int -param_set_hrtimeout_slack(const char *buf, zfs_kernel_param_t *kp) -{ - unsigned long val; - int error; - - error = kstrtoul(buf, 0, &val); - if (error) - return (error); - - if (val > MAX_HRTIMEOUT_SLACK_US) - return (-EINVAL); - - error = param_set_uint(buf, kp); - if (error < 0) - return (error); - - return (0); -} - -module_param_call(spl_schedule_hrtimeout_slack_us, param_set_hrtimeout_slack, - param_get_uint, &spl_schedule_hrtimeout_slack_us, 0644); -MODULE_PARM_DESC(spl_schedule_hrtimeout_slack_us, - "schedule_hrtimeout_range() delta/slack value in us, default(0)"); - -void -__cv_init(kcondvar_t *cvp, char *name, kcv_type_t type, void *arg) -{ - ASSERT(cvp); - ASSERT(name == NULL); - ASSERT(type == CV_DEFAULT); - ASSERT(arg == NULL); - - cvp->cv_magic = CV_MAGIC; - init_waitqueue_head(&cvp->cv_event); - init_waitqueue_head(&cvp->cv_destroy); - atomic_set(&cvp->cv_waiters, 0); - atomic_set(&cvp->cv_refs, 1); - cvp->cv_mutex = NULL; -} -EXPORT_SYMBOL(__cv_init); - -static int -cv_destroy_wakeup(kcondvar_t *cvp) -{ - if (!atomic_read(&cvp->cv_waiters) && !atomic_read(&cvp->cv_refs)) { - ASSERT(cvp->cv_mutex == NULL); - ASSERT(!waitqueue_active(&cvp->cv_event)); - return (1); - } - - return (0); -} - -void -__cv_destroy(kcondvar_t *cvp) -{ - ASSERT(cvp); - ASSERT(cvp->cv_magic == CV_MAGIC); - - cvp->cv_magic = CV_DESTROY; - atomic_dec(&cvp->cv_refs); - - /* Block until all waiters are woken and references dropped. */ - while (cv_destroy_wakeup(cvp) == 0) - wait_event_timeout(cvp->cv_destroy, cv_destroy_wakeup(cvp), 1); - - ASSERT3P(cvp->cv_mutex, ==, NULL); - ASSERT3S(atomic_read(&cvp->cv_refs), ==, 0); - ASSERT3S(atomic_read(&cvp->cv_waiters), ==, 0); - ASSERT3S(waitqueue_active(&cvp->cv_event), ==, 0); -} -EXPORT_SYMBOL(__cv_destroy); - -static void -cv_wait_common(kcondvar_t *cvp, kmutex_t *mp, int state, int io) -{ - DEFINE_WAIT(wait); - kmutex_t *m; - - ASSERT(cvp); - ASSERT(mp); - ASSERT(cvp->cv_magic == CV_MAGIC); - ASSERT(mutex_owned(mp)); - atomic_inc(&cvp->cv_refs); - - m = READ_ONCE(cvp->cv_mutex); - if (!m) - m = xchg(&cvp->cv_mutex, mp); - /* Ensure the same mutex is used by all callers */ - ASSERT(m == NULL || m == mp); - - prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); - atomic_inc(&cvp->cv_waiters); - - /* - * Mutex should be dropped after prepare_to_wait() this - * ensures we're linked in to the waiters list and avoids the - * race where 'cvp->cv_waiters > 0' but the list is empty. - */ - mutex_exit(mp); - if (io) - io_schedule(); - else - schedule(); - - /* No more waiters a different mutex could be used */ - if (atomic_dec_and_test(&cvp->cv_waiters)) { - /* - * This is set without any lock, so it's racy. But this is - * just for debug anyway, so make it best-effort - */ - cvp->cv_mutex = NULL; - wake_up(&cvp->cv_destroy); - } - - finish_wait(&cvp->cv_event, &wait); - atomic_dec(&cvp->cv_refs); - - /* - * Hold mutex after we release the cvp, otherwise we could dead lock - * with a thread holding the mutex and call cv_destroy. - */ - mutex_enter(mp); -} - -void -__cv_wait(kcondvar_t *cvp, kmutex_t *mp) -{ - cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 0); -} -EXPORT_SYMBOL(__cv_wait); - -void -__cv_wait_io(kcondvar_t *cvp, kmutex_t *mp) -{ - cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 1); -} -EXPORT_SYMBOL(__cv_wait_io); - -int -__cv_wait_io_sig(kcondvar_t *cvp, kmutex_t *mp) -{ - cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 1); - - return (signal_pending(current) ? 0 : 1); -} -EXPORT_SYMBOL(__cv_wait_io_sig); - -int -__cv_wait_sig(kcondvar_t *cvp, kmutex_t *mp) -{ - cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 0); - - return (signal_pending(current) ? 0 : 1); -} -EXPORT_SYMBOL(__cv_wait_sig); - -#if defined(HAVE_IO_SCHEDULE_TIMEOUT) -#define spl_io_schedule_timeout(t) io_schedule_timeout(t) -#else - -struct spl_task_timer { - struct timer_list timer; - struct task_struct *task; -}; - -static void -__cv_wakeup(spl_timer_list_t t) -{ - struct timer_list *tmr = (struct timer_list *)t; - struct spl_task_timer *task_timer = from_timer(task_timer, tmr, timer); - - wake_up_process(task_timer->task); -} - -static long -spl_io_schedule_timeout(long time_left) -{ - long expire_time = jiffies + time_left; - struct spl_task_timer task_timer; - struct timer_list *timer = &task_timer.timer; - - task_timer.task = current; - - timer_setup(timer, __cv_wakeup, 0); - - timer->expires = expire_time; - add_timer(timer); - - io_schedule(); - - del_timer_sync(timer); - - time_left = expire_time - jiffies; - - return (time_left < 0 ? 0 : time_left); -} -#endif - -/* - * 'expire_time' argument is an absolute wall clock time in jiffies. - * Return value is time left (expire_time - now) or -1 if timeout occurred. - */ -static clock_t -__cv_timedwait_common(kcondvar_t *cvp, kmutex_t *mp, clock_t expire_time, - int state, int io) -{ - DEFINE_WAIT(wait); - kmutex_t *m; - clock_t time_left; - - ASSERT(cvp); - ASSERT(mp); - ASSERT(cvp->cv_magic == CV_MAGIC); - ASSERT(mutex_owned(mp)); - - /* XXX - Does not handle jiffie wrap properly */ - time_left = expire_time - jiffies; - if (time_left <= 0) - return (-1); - - atomic_inc(&cvp->cv_refs); - m = READ_ONCE(cvp->cv_mutex); - if (!m) - m = xchg(&cvp->cv_mutex, mp); - /* Ensure the same mutex is used by all callers */ - ASSERT(m == NULL || m == mp); - - prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); - atomic_inc(&cvp->cv_waiters); - - /* - * Mutex should be dropped after prepare_to_wait() this - * ensures we're linked in to the waiters list and avoids the - * race where 'cvp->cv_waiters > 0' but the list is empty. - */ - mutex_exit(mp); - if (io) - time_left = spl_io_schedule_timeout(time_left); - else - time_left = schedule_timeout(time_left); - - /* No more waiters a different mutex could be used */ - if (atomic_dec_and_test(&cvp->cv_waiters)) { - /* - * This is set without any lock, so it's racy. But this is - * just for debug anyway, so make it best-effort - */ - cvp->cv_mutex = NULL; - wake_up(&cvp->cv_destroy); - } - - finish_wait(&cvp->cv_event, &wait); - atomic_dec(&cvp->cv_refs); - - /* - * Hold mutex after we release the cvp, otherwise we could dead lock - * with a thread holding the mutex and call cv_destroy. - */ - mutex_enter(mp); - return (time_left > 0 ? time_left : -1); -} - -clock_t -__cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) -{ - return (__cv_timedwait_common(cvp, mp, exp_time, - TASK_UNINTERRUPTIBLE, 0)); -} -EXPORT_SYMBOL(__cv_timedwait); - -clock_t -__cv_timedwait_io(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) -{ - return (__cv_timedwait_common(cvp, mp, exp_time, - TASK_UNINTERRUPTIBLE, 1)); -} -EXPORT_SYMBOL(__cv_timedwait_io); - -clock_t -__cv_timedwait_sig(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) -{ - return (__cv_timedwait_common(cvp, mp, exp_time, - TASK_INTERRUPTIBLE, 0)); -} -EXPORT_SYMBOL(__cv_timedwait_sig); - -/* - * 'expire_time' argument is an absolute clock time in nanoseconds. - * Return value is time left (expire_time - now) or -1 if timeout occurred. - */ -static clock_t -__cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t expire_time, - hrtime_t res, int state) -{ - DEFINE_WAIT(wait); - kmutex_t *m; - hrtime_t time_left; - ktime_t ktime_left; - u64 slack = 0; - - ASSERT(cvp); - ASSERT(mp); - ASSERT(cvp->cv_magic == CV_MAGIC); - ASSERT(mutex_owned(mp)); - - time_left = expire_time - gethrtime(); - if (time_left <= 0) - return (-1); - - atomic_inc(&cvp->cv_refs); - m = READ_ONCE(cvp->cv_mutex); - if (!m) - m = xchg(&cvp->cv_mutex, mp); - /* Ensure the same mutex is used by all callers */ - ASSERT(m == NULL || m == mp); - - prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); - atomic_inc(&cvp->cv_waiters); - - /* - * Mutex should be dropped after prepare_to_wait() this - * ensures we're linked in to the waiters list and avoids the - * race where 'cvp->cv_waiters > 0' but the list is empty. - */ - mutex_exit(mp); - - ktime_left = ktime_set(0, time_left); - slack = MIN(MAX(res, spl_schedule_hrtimeout_slack_us * NSEC_PER_USEC), - MAX_HRTIMEOUT_SLACK_US * NSEC_PER_USEC); - schedule_hrtimeout_range(&ktime_left, slack, HRTIMER_MODE_REL); - - /* No more waiters a different mutex could be used */ - if (atomic_dec_and_test(&cvp->cv_waiters)) { - /* - * This is set without any lock, so it's racy. But this is - * just for debug anyway, so make it best-effort - */ - cvp->cv_mutex = NULL; - wake_up(&cvp->cv_destroy); - } - - finish_wait(&cvp->cv_event, &wait); - atomic_dec(&cvp->cv_refs); - - mutex_enter(mp); - time_left = expire_time - gethrtime(); - return (time_left > 0 ? NSEC_TO_TICK(time_left) : -1); -} - -/* - * Compatibility wrapper for the cv_timedwait_hires() Illumos interface. - */ -static clock_t -cv_timedwait_hires_common(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, - hrtime_t res, int flag, int state) -{ - if (!(flag & CALLOUT_FLAG_ABSOLUTE)) - tim += gethrtime(); - - return (__cv_timedwait_hires(cvp, mp, tim, res, state)); -} - -clock_t -cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, hrtime_t res, - int flag) -{ - return (cv_timedwait_hires_common(cvp, mp, tim, res, flag, - TASK_UNINTERRUPTIBLE)); -} -EXPORT_SYMBOL(cv_timedwait_hires); - -clock_t -cv_timedwait_sig_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, - hrtime_t res, int flag) -{ - return (cv_timedwait_hires_common(cvp, mp, tim, res, flag, - TASK_INTERRUPTIBLE)); -} -EXPORT_SYMBOL(cv_timedwait_sig_hires); - -void -__cv_signal(kcondvar_t *cvp) -{ - ASSERT(cvp); - ASSERT(cvp->cv_magic == CV_MAGIC); - atomic_inc(&cvp->cv_refs); - - /* - * All waiters are added with WQ_FLAG_EXCLUSIVE so only one - * waiter will be set runnable with each call to wake_up(). - * Additionally wake_up() holds a spin_lock associated with - * the wait queue to ensure we don't race waking up processes. - */ - if (atomic_read(&cvp->cv_waiters) > 0) - wake_up(&cvp->cv_event); - - atomic_dec(&cvp->cv_refs); -} -EXPORT_SYMBOL(__cv_signal); - -void -__cv_broadcast(kcondvar_t *cvp) -{ - ASSERT(cvp); - ASSERT(cvp->cv_magic == CV_MAGIC); - atomic_inc(&cvp->cv_refs); - - /* - * Wake_up_all() will wake up all waiters even those which - * have the WQ_FLAG_EXCLUSIVE flag set. - */ - if (atomic_read(&cvp->cv_waiters) > 0) - wake_up_all(&cvp->cv_event); - - atomic_dec(&cvp->cv_refs); -} -EXPORT_SYMBOL(__cv_broadcast); |