summaryrefslogtreecommitdiffstats
path: root/sys/kern/kern_event.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/kern/kern_event.c')
-rw-r--r--sys/kern/kern_event.c162
1 files changed, 90 insertions, 72 deletions
diff --git a/sys/kern/kern_event.c b/sys/kern/kern_event.c
index fba163f..2c99803 100644
--- a/sys/kern/kern_event.c
+++ b/sys/kern/kern_event.c
@@ -47,7 +47,6 @@ __FBSDID("$FreeBSD$");
#include <sys/fcntl.h>
#include <sys/kthread.h>
#include <sys/selinfo.h>
-#include <sys/stdatomic.h>
#include <sys/queue.h>
#include <sys/event.h>
#include <sys/eventvar.h>
@@ -66,6 +65,7 @@ __FBSDID("$FreeBSD$");
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
+#include <machine/atomic.h>
#include <vm/uma.h>
@@ -184,7 +184,7 @@ static struct filterops user_filtops = {
};
static uma_zone_t knote_zone;
-static atomic_uint kq_ncallouts = ATOMIC_VAR_INIT(0);
+static unsigned int kq_ncallouts = 0;
static unsigned int kq_calloutmax = 4 * 1024;
SYSCTL_UINT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW,
&kq_calloutmax, 0, "Maximum number of callouts allocated for kqueue");
@@ -556,53 +556,84 @@ knote_fork(struct knlist *list, int pid)
#define NOTE_TIMER_PRECMASK (NOTE_SECONDS|NOTE_MSECONDS|NOTE_USECONDS| \
NOTE_NSECONDS)
-static __inline sbintime_t
+static sbintime_t
timer2sbintime(intptr_t data, int flags)
{
- sbintime_t modifier;
+ /*
+ * Macros for converting to the fractional second portion of an
+ * sbintime_t using 64bit multiplication to improve precision.
+ */
+#define NS_TO_SBT(ns) (((ns) * (((uint64_t)1 << 63) / 500000000)) >> 32)
+#define US_TO_SBT(us) (((us) * (((uint64_t)1 << 63) / 500000)) >> 32)
+#define MS_TO_SBT(ms) (((ms) * (((uint64_t)1 << 63) / 500)) >> 32)
switch (flags & NOTE_TIMER_PRECMASK) {
case NOTE_SECONDS:
- modifier = SBT_1S;
- break;
+#ifdef __LP64__
+ if (data > (SBT_MAX / SBT_1S))
+ return (SBT_MAX);
+#endif
+ return ((sbintime_t)data << 32);
case NOTE_MSECONDS: /* FALLTHROUGH */
case 0:
- modifier = SBT_1MS;
- break;
+ if (data >= 1000) {
+ int64_t secs = data / 1000;
+#ifdef __LP64__
+ if (secs > (SBT_MAX / SBT_1S))
+ return (SBT_MAX);
+#endif
+ return (secs << 32 | MS_TO_SBT(data % 1000));
+ }
+ return MS_TO_SBT(data);
case NOTE_USECONDS:
- modifier = SBT_1US;
- break;
+ if (data >= 1000000) {
+ int64_t secs = data / 1000000;
+#ifdef __LP64__
+ if (secs > (SBT_MAX / SBT_1S))
+ return (SBT_MAX);
+#endif
+ return (secs << 32 | US_TO_SBT(data % 1000000));
+ }
+ return US_TO_SBT(data);
case NOTE_NSECONDS:
- modifier = SBT_1NS;
- break;
- default:
- return (-1);
- }
-
+ if (data >= 1000000000) {
+ int64_t secs = data / 1000000000;
#ifdef __LP64__
- if (data > SBT_MAX / modifier)
- return (SBT_MAX);
+ if (secs > (SBT_MAX / SBT_1S))
+ return (SBT_MAX);
#endif
- return (modifier * data);
+ return (secs << 32 | US_TO_SBT(data % 1000000000));
+ }
+ return (NS_TO_SBT(data));
+ default:
+ break;
+ }
+ return (-1);
}
+struct kq_timer_cb_data {
+ struct callout c;
+ sbintime_t next; /* next timer event fires at */
+ sbintime_t to; /* precalculated timer period */
+};
+
static void
filt_timerexpire(void *knx)
{
- struct callout *calloutp;
struct knote *kn;
+ struct kq_timer_cb_data *kc;
kn = knx;
kn->kn_data++;
KNOTE_ACTIVATE(kn, 0); /* XXX - handle locking */
- if ((kn->kn_flags & EV_ONESHOT) != EV_ONESHOT) {
- calloutp = (struct callout *)kn->kn_hook;
- *kn->kn_ptr.p_nexttime += timer2sbintime(kn->kn_sdata,
- kn->kn_sfflags);
- callout_reset_sbt_on(calloutp, *kn->kn_ptr.p_nexttime, 0,
- filt_timerexpire, kn, PCPU_GET(cpuid), C_ABSOLUTE);
- }
+ if ((kn->kn_flags & EV_ONESHOT) != 0)
+ return;
+
+ kc = kn->kn_ptr.p_v;
+ kc->next += kc->to;
+ callout_reset_sbt_on(&kc->c, kc->next, 0, filt_timerexpire, kn,
+ PCPU_GET(cpuid), C_ABSOLUTE);
}
/*
@@ -611,39 +642,36 @@ filt_timerexpire(void *knx)
static int
filt_timerattach(struct knote *kn)
{
- struct callout *calloutp;
+ struct kq_timer_cb_data *kc;
sbintime_t to;
unsigned int ncallouts;
- if ((intptr_t)kn->kn_sdata < 0)
+ if (kn->kn_sdata < 0)
return (EINVAL);
- if ((intptr_t)kn->kn_sdata == 0 && (kn->kn_flags & EV_ONESHOT) == 0)
+ if (kn->kn_sdata == 0 && (kn->kn_flags & EV_ONESHOT) == 0)
kn->kn_sdata = 1;
/* Only precision unit are supported in flags so far */
- if (kn->kn_sfflags & ~NOTE_TIMER_PRECMASK)
+ if ((kn->kn_sfflags & ~NOTE_TIMER_PRECMASK) != 0)
return (EINVAL);
to = timer2sbintime(kn->kn_sdata, kn->kn_sfflags);
if (to < 0)
return (EINVAL);
- ncallouts = atomic_load_explicit(&kq_ncallouts, memory_order_relaxed);
do {
+ ncallouts = kq_ncallouts;
if (ncallouts >= kq_calloutmax)
return (ENOMEM);
- } while (!atomic_compare_exchange_weak_explicit(&kq_ncallouts,
- &ncallouts, ncallouts + 1, memory_order_relaxed,
- memory_order_relaxed));
+ } while (!atomic_cmpset_int(&kq_ncallouts, ncallouts, ncallouts + 1));
kn->kn_flags |= EV_CLEAR; /* automatically set */
kn->kn_status &= ~KN_DETACHED; /* knlist_add clears it */
- kn->kn_ptr.p_nexttime = malloc(sizeof(sbintime_t), M_KQUEUE, M_WAITOK);
- calloutp = malloc(sizeof(*calloutp), M_KQUEUE, M_WAITOK);
- callout_init(calloutp, CALLOUT_MPSAFE);
- kn->kn_hook = calloutp;
- *kn->kn_ptr.p_nexttime = to + sbinuptime();
- callout_reset_sbt_on(calloutp, *kn->kn_ptr.p_nexttime, 0,
- filt_timerexpire, kn, PCPU_GET(cpuid), C_ABSOLUTE);
+ kn->kn_ptr.p_v = kc = malloc(sizeof(*kc), M_KQUEUE, M_WAITOK);
+ callout_init(&kc->c, 1);
+ kc->next = to + sbinuptime();
+ kc->to = to;
+ callout_reset_sbt_on(&kc->c, kc->next, 0, filt_timerexpire, kn,
+ PCPU_GET(cpuid), C_ABSOLUTE);
return (0);
}
@@ -651,14 +679,13 @@ filt_timerattach(struct knote *kn)
static void
filt_timerdetach(struct knote *kn)
{
- struct callout *calloutp;
+ struct kq_timer_cb_data *kc;
unsigned int old;
- calloutp = (struct callout *)kn->kn_hook;
- callout_drain(calloutp);
- free(calloutp, M_KQUEUE);
- free(kn->kn_ptr.p_nexttime, M_KQUEUE);
- old = atomic_fetch_sub_explicit(&kq_ncallouts, 1, memory_order_relaxed);
+ kc = kn->kn_ptr.p_v;
+ callout_drain(&kc->c);
+ free(kc, M_KQUEUE);
+ old = atomic_fetchadd_int(&kq_ncallouts, -1);
KASSERT(old > 0, ("Number of callouts cannot become negative"));
kn->kn_status |= KN_DETACHED; /* knlist_remove sets it */
}
@@ -1912,6 +1939,7 @@ knote(struct knlist *list, long hint, int lockflags)
struct kqueue *kq;
struct knote *kn, *tkn;
int error;
+ bool own_influx;
if (list == NULL)
return;
@@ -1942,11 +1970,14 @@ knote(struct knlist *list, long hint, int lockflags)
*/
KQ_UNLOCK(kq);
} else if ((lockflags & KNF_NOKQLOCK) != 0) {
- kn->kn_status |= KN_INFLUX;
+ own_influx = (kn->kn_status & KN_INFLUX) == 0;
+ if (own_influx)
+ kn->kn_status |= KN_INFLUX;
KQ_UNLOCK(kq);
error = kn->kn_fop->f_event(kn, hint);
KQ_LOCK(kq);
- kn->kn_status &= ~KN_INFLUX;
+ if (own_influx)
+ kn->kn_status &= ~KN_INFLUX;
if (error)
KNOTE_ACTIVATE(kn, 1);
KQ_UNLOCK_FLUX(kq);
@@ -2031,12 +2062,12 @@ knlist_empty(struct knlist *knl)
{
KNL_ASSERT_LOCKED(knl);
- return SLIST_EMPTY(&knl->kl_list);
+ return (SLIST_EMPTY(&knl->kl_list));
}
-static struct mtx knlist_lock;
+static struct mtx knlist_lock;
MTX_SYSINIT(knlist_lock, &knlist_lock, "knlist lock for lockless objects",
- MTX_DEF);
+ MTX_DEF);
static void knlist_mtx_lock(void *arg);
static void knlist_mtx_unlock(void *arg);
@@ -2146,17 +2177,8 @@ void
knlist_destroy(struct knlist *knl)
{
-#ifdef INVARIANTS
- /*
- * if we run across this error, we need to find the offending
- * driver and have it call knlist_clear or knlist_delete.
- */
- if (!SLIST_EMPTY(&knl->kl_list))
- printf("WARNING: destroying knlist w/ knotes on it!\n");
-#endif
-
- knl->kl_lockarg = knl->kl_lock = knl->kl_unlock = NULL;
- SLIST_INIT(&knl->kl_list);
+ KASSERT(KNLIST_EMPTY(knl),
+ ("destroying knlist %p with knotes on it", knl));
}
/*
@@ -2275,17 +2297,15 @@ knote_attach(struct knote *kn, struct kqueue *kq)
if (kn->kn_fop->f_isfd) {
if (kn->kn_id >= kq->kq_knlistsize)
- return ENOMEM;
+ return (ENOMEM);
list = &kq->kq_knlist[kn->kn_id];
} else {
if (kq->kq_knhash == NULL)
- return ENOMEM;
+ return (ENOMEM);
list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
}
-
SLIST_INSERT_HEAD(list, kn, kn_link);
-
- return 0;
+ return (0);
}
/*
@@ -2394,11 +2414,9 @@ kqfd_register(int fd, struct kevent *kev, struct thread *td, int waitok)
goto noacquire;
error = kqueue_register(kq, kev, td, waitok);
-
kqueue_release(kq, 0);
noacquire:
fdrop(fp, td);
-
- return error;
+ return (error);
}
OpenPOWER on IntegriCloud