Implement the per-open file data for the cdev.

The patch does not change the cdevsw KBI. Management of the data is
provided by the functions
int	devfs_set_cdevpriv(void *priv, cdevpriv_dtr_t dtr);
int	devfs_get_cdevpriv(void **datap);
void	devfs_clear_cdevpriv(void);
All of the functions are supposed to be called from the cdevsw method
contexts.

- devfs_set_cdevpriv assigns the priv as private data for the file
  descriptor which is used to initiate currently performed driver
  operation. dtr is the function that will be called when either the
  last refernce to the file goes away, the device is destroyed  or
  devfs_clear_cdevpriv is called.
- devfs_get_cdevpriv is the obvious accessor.
- devfs_clear_cdevpriv allows to clear the private data for the still
  open file.

Implementation keeps the driver-supplied pointers in the struct
cdev_privdata, that is referenced both from the struct file and struct
cdev, and cannot outlive any of the referee.

Man pages will be provided after the KPI stabilizes.

Reviewed by:	jhb
Useful suggestions from:	jeff, antoine
Debugging help and tested by:	pho
MFC after:	1 month

7 files changed, 144 insertions, 3 deletions

diff --git a/sys/fs/devfs/devfs_int.h b/sys/fs/devfs/devfs_int.h
index 3848ab4..df0ad56 100644
--- a/sys/fs/devfs/devfs_int.h
+++ b/sys/fs/devfs/devfs_int.h
@@ -39,6 +39,13 @@
 
 struct devfs_dirent;
 
+struct cdev_privdata {
+	struct file		*cdpd_fp;
+	void			*cdpd_data;
+	void			(*cdpd_dtr)(void *);
+	LIST_ENTRY(cdev_privdata) cdpd_list;
+};
+
 struct cdev_priv {
 	struct cdev		cdp_c;
 	TAILQ_ENTRY(cdev_priv)	cdp_list;
@@ -57,17 +64,21 @@ struct cdev_priv {
 	TAILQ_ENTRY(cdev_priv)	cdp_dtr_list;
 	void			(*cdp_dtr_cb)(void *);
 	void			*cdp_dtr_cb_arg;
+
+	LIST_HEAD(, cdev_privdata) cdp_fdpriv;
 };
 
 struct cdev *devfs_alloc(void);
 void devfs_free(struct cdev *);
 void devfs_create(struct cdev *dev);
 void devfs_destroy(struct cdev *dev);
+void devfs_destroy_cdevpriv(struct cdev_privdata *p);
 
 extern struct unrhdr *devfs_inos;
 extern struct mtx devmtx;
 extern struct mtx devfs_de_interlock;
 extern struct sx clone_drain_lock;
+extern struct mtx cdevpriv_mtx;
 extern TAILQ_HEAD(cdev_priv_list, cdev_priv) cdevp_list;
 
 #endif /* _KERNEL */
diff --git a/sys/fs/devfs/devfs_vnops.c b/sys/fs/devfs/devfs_vnops.c
index 21d8ba9..6c3f8fb 100644
--- a/sys/fs/devfs/devfs_vnops.c
+++ b/sys/fs/devfs/devfs_vnops.c
@@ -73,10 +73,14 @@ static struct fileops devfs_ops_f;
 
 #include <security/mac/mac_framework.h>
 
+static MALLOC_DEFINE(M_CDEVPDATA, "DEVFSP", "Metainfo for cdev-fp data");
+
 struct mtx	devfs_de_interlock;
 MTX_SYSINIT(devfs_de_interlock, &devfs_de_interlock, "devfs interlock", MTX_DEF);
 struct sx	clone_drain_lock;
 SX_SYSINIT(clone_drain_lock, &clone_drain_lock, "clone events drain lock");
+struct mtx	cdevpriv_mtx;
+MTX_SYSINIT(cdevpriv_mtx, &cdevpriv_mtx, "cdevpriv lock", MTX_DEF);
 
 static int
 devfs_fp_check(struct file *fp, struct cdev **devp, struct cdevsw **dswp)
@@ -92,9 +96,97 @@ devfs_fp_check(struct file *fp, struct cdev **devp, struct cdevsw **dswp)
 	    ("devfs: un-referenced struct cdev *(%s)", devtoname(*devp)));
 	if (*dswp == NULL)
 		return (ENXIO);
+	curthread->td_fpop = fp;
 	return (0);
 }
 
+int
+devfs_get_cdevpriv(void **datap)
+{
+	struct file *fp;
+	struct cdev_privdata *p;
+	int error;
+
+	fp = curthread->td_fpop;
+	if (fp == NULL)
+		return (EBADF);
+	mtx_lock(&cdevpriv_mtx);
+	p = fp->f_cdevpriv;
+	mtx_unlock(&cdevpriv_mtx);
+	if (p != NULL) {
+		error = 0;
+		*datap = p->cdpd_data;
+	} else
+		error = ENOENT;
+	return (error);
+}
+
+int
+devfs_set_cdevpriv(void *priv, cdevpriv_dtr_t priv_dtr)
+{
+	struct file *fp;
+	struct cdev_priv *cdp;
+	struct cdev_privdata *p;
+	int error;
+
+	fp = curthread->td_fpop;
+	if (fp == NULL)
+		return (ENOENT);
+	cdp = ((struct cdev *)fp->f_data)->si_priv;
+	p = malloc(sizeof(struct cdev_privdata), M_CDEVPDATA, M_WAITOK);
+	p->cdpd_data = priv;
+	p->cdpd_dtr = priv_dtr;
+	p->cdpd_fp = fp;
+	mtx_lock(&cdevpriv_mtx);
+	if (fp->f_cdevpriv == NULL) {
+		LIST_INSERT_HEAD(&cdp->cdp_fdpriv, p, cdpd_list);
+		fp->f_cdevpriv = p;
+		mtx_unlock(&cdevpriv_mtx);
+		error = 0;
+	} else {
+		mtx_unlock(&cdevpriv_mtx);
+		free(p, M_CDEVPDATA);
+		error = EBUSY;
+	}
+	return (error);
+}
+
+void
+devfs_destroy_cdevpriv(struct cdev_privdata *p)
+{
+
+	mtx_assert(&cdevpriv_mtx, MA_OWNED);
+	p->cdpd_fp->f_cdevpriv = NULL;
+	LIST_REMOVE(p, cdpd_list);
+	mtx_unlock(&cdevpriv_mtx);
+	(p->cdpd_dtr)(p->cdpd_data);
+	free(p, M_CDEVPDATA);
+}
+
+void
+devfs_fpdrop(struct file *fp)
+{
+	struct cdev_privdata *p;
+
+	mtx_lock(&cdevpriv_mtx);
+	if ((p = fp->f_cdevpriv) == NULL) {
+		mtx_unlock(&cdevpriv_mtx);
+		return;
+	}
+	devfs_destroy_cdevpriv(p);
+}
+
+void
+devfs_clear_cdevpriv(void)
+{
+	struct file *fp;
+
+	fp = curthread->td_fpop;
+	if (fp == NULL)
+		return;
+	devfs_fpdrop(fp);
+}
+
 /*
  * Construct the fully qualified path name relative to the mountpoint
  */
@@ -374,8 +466,12 @@ devfs_close(struct vop_close_args *ap)
 static int
 devfs_close_f(struct file *fp, struct thread *td)
 {
+	int error;
 
-	return (vnops.fo_close(fp, td));
+	curthread->td_fpop = fp;
+	error = vnops.fo_close(fp, td);
+	curthread->td_fpop = NULL;
+	return (error);
 }
 
 /* ARGSUSED */
@@ -472,6 +568,7 @@ devfs_ioctl_f(struct file *fp, u_long com, void *data, struct ucred *cred, struc
 
 	if (com == FIODTYPE) {
 		*(int *)data = dsw->d_flags & D_TYPEMASK;
+		td->td_fpop = NULL;
 		dev_relthread(dev);
 		return (0);
 	} else if (com == FIODGNAME) {
@@ -482,10 +579,12 @@ devfs_ioctl_f(struct file *fp, u_long com, void *data, struct ucred *cred, struc
 			error = EINVAL;
 		else
 			error = copyout(p, fgn->buf, i);
+		td->td_fpop = NULL;
 		dev_relthread(dev);
 		return (error);
 	}
 	error = dsw->d_ioctl(dev, com, data, fp->f_flag, td);
+	td->td_fpop = NULL;
 	dev_relthread(dev);
 	if (error == ENOIOCTL)
 		error = ENOTTY;
@@ -529,6 +628,7 @@ devfs_kqfilter_f(struct file *fp, struct knote *kn)
 	if (error)
 		return (error);
 	error = dsw->d_kqfilter(dev, kn);
+	curthread->td_fpop = NULL;
 	dev_relthread(dev);
 	return (error);
 }
@@ -766,10 +866,15 @@ devfs_open(struct vop_open_args *ap)
 
 	VOP_UNLOCK(vp, 0);
 
+	if (fp != NULL) {
+		td->td_fpop = fp;
+		fp->f_data = dev;
+	}
 	if (dsw->d_fdopen != NULL)
 		error = dsw->d_fdopen(dev, ap->a_mode, td, fp);
 	else
 		error = dsw->d_open(dev, ap->a_mode, S_IFCHR, td);
+	td->td_fpop = NULL;
 
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 
@@ -825,6 +930,7 @@ devfs_poll_f(struct file *fp, int events, struct ucred *cred, struct thread *td)
 	if (error)
 		return (error);
 	error = dsw->d_poll(dev, events, td);
+	curthread->td_fpop = NULL;
 	dev_relthread(dev);
 	return(error);
 }
@@ -862,6 +968,7 @@ devfs_read_f(struct file *fp, struct uio *uio, struct ucred *cred, int flags, st
 	error = dsw->d_read(dev, uio, ioflag);
 	if (uio->uio_resid != resid || (error == 0 && resid != 0))
 		vfs_timestamp(&dev->si_atime);
+	curthread->td_fpop = NULL;
 	dev_relthread(dev);
 
 	if ((flags & FOF_OFFSET) == 0)
@@ -1298,6 +1405,7 @@ devfs_write_f(struct file *fp, struct uio *uio, struct ucred *cred, int flags, s
 		vfs_timestamp(&dev->si_ctime);
 		dev->si_mtime = dev->si_ctime;
 	}
+	curthread->td_fpop = NULL;
 	dev_relthread(dev);
 
 	if ((flags & FOF_OFFSET) == 0)
diff --git a/sys/kern/kern_conf.c b/sys/kern/kern_conf.c
index f8e7a10..0e4aa22 100644
--- a/sys/kern/kern_conf.c
+++ b/sys/kern/kern_conf.c
@@ -839,6 +839,7 @@ static void
 destroy_devl(struct cdev *dev)
 {
 	struct cdevsw *csw;
+	struct cdev_privdata *p, *p1;
 
 	mtx_assert(&devmtx, MA_OWNED);
 	KASSERT(dev->si_flags & SI_NAMED,
@@ -880,9 +881,15 @@ destroy_devl(struct cdev *dev)
 		msleep(&csw, &devmtx, PRIBIO, "devdrn", hz / 10);
 	}
 
-	mtx_unlock(&devmtx);
+	dev_unlock();
 	notify_destroy(dev);
-	mtx_lock(&devmtx);
+	mtx_lock(&cdevpriv_mtx);
+	LIST_FOREACH_SAFE(p, &dev->si_priv->cdp_fdpriv, cdpd_list, p1) {
+		devfs_destroy_cdevpriv(p);
+		mtx_lock(&cdevpriv_mtx);
+	}
+	mtx_unlock(&cdevpriv_mtx);
+	dev_lock();
 
 	dev->si_drv1 = 0;
 	dev->si_drv2 = 0;
diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index bdc6d78..cbb7653 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -2221,6 +2221,12 @@ _fdrop(struct file *fp, struct thread *td)
 		panic("fdrop: count %d", fp->f_count);
 	if (fp->f_ops != &badfileops)
 		error = fo_close(fp, td);
+	/*
+	 * The f_cdevpriv cannot be assigned non-NULL value while we
+	 * are destroying the file.
+	 */
+	if (fp->f_cdevpriv != NULL)
+		devfs_fpdrop(fp);
 	atomic_subtract_int(&openfiles, 1);
 	crfree(fp->f_cred);
 	uma_zfree(file_zone, fp);
diff --git a/sys/sys/conf.h b/sys/sys/conf.h
index 91c2efc..bd6aaf0 100644
--- a/sys/sys/conf.h
+++ b/sys/sys/conf.h
@@ -281,6 +281,12 @@ int	unit2minor(int _unit);
 u_int	minor2unit(u_int _minor);
 void	setconf(void);
 
+typedef	void (*cdevpriv_dtr_t)(void *data);
+int	devfs_get_cdevpriv(void **datap);
+int	devfs_set_cdevpriv(void *priv, cdevpriv_dtr_t dtr);
+void	devfs_clear_cdevpriv(void);
+void	devfs_fpdrop(struct file *fp);	/* XXX This is not public KPI */
+
 #define		UID_ROOT	0
 #define		UID_BIN		3
 #define		UID_UUCP	66
diff --git a/sys/sys/file.h b/sys/sys/file.h
index 0281d21..e3fdd60 100644
--- a/sys/sys/file.h
+++ b/sys/sys/file.h
@@ -104,6 +104,7 @@ struct fileops {
  * Below is the list of locks that protects members in struct file.
  *
  * (f) protected with mtx_lock(mtx_pool_find(fp))
+ * (d) cdevpriv_mtx
  * none	not locked
  */
 
@@ -121,6 +122,7 @@ struct file {
 	 */
 	int		f_seqcount;	/* Count of sequential accesses. */
 	off_t		f_nextoff;	/* next expected read/write offset. */
+	struct cdev_privdata *f_cdevpriv; /* (d) Private data for the cdev. */
 	/*
 	 *  DFLAG_SEEKABLE specific fields
 	 */
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index c88bb92..fa10291 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -231,6 +231,7 @@ struct thread {
 	u_long		td_profil_addr;	/* (k) Temporary addr until AST. */
 	u_int		td_profil_ticks; /* (k) Temporary ticks until AST. */
 	char		td_name[MAXCOMLEN + 1];	/* (*) Thread name. */
+	struct file	*td_fpop;	/* (k) file referencing cdev under op */
 #define	td_endzero td_base_pri
 
 /* Copied during fork1() or thread_sched_upcall(). */