Streamline use of cdevpriv and correct some corner cases.

1) It is not useful to call "devfs_clear_cdevpriv()" from
"d_close" callbacks, hence for example read, write, ioctl and
so on might be sleeping at the time of "d_close" being called
and then then freed private data can still be accessed.
Examples: dtrace, linux_compat, ksyms (all fixed by this patch)

2) In sys/dev/drm* there are some cases in which memory will
be freed twice, if open fails, first by code in the open
routine, secondly by the cdevpriv destructor. Move registration
of the cdevpriv to the end of the drm open routines.

3) devfs_clear_cdevpriv() is not called if the "d_open" callback
registered cdevpriv data and the "d_open" callback function
returned an error. Fix this.

Discussed with:	phk
MFC after:	2 weeks

7 files changed, 20 insertions, 22 deletions

diff --git a/share/man/man9/devfs_set_cdevpriv.9 b/share/man/man9/devfs_set_cdevpriv.9
index a2b0279..736a007 100644
--- a/share/man/man9/devfs_set_cdevpriv.9
+++ b/share/man/man9/devfs_set_cdevpriv.9
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 8, 2008
+.Dd August 15, 2012
 .Dt DEVFS_CDEVPRIV 9
 .Os
 .Sh NAME
@@ -79,6 +79,10 @@ finished operating, the
 callback is called, with private data supplied
 .Va data
 argument.
+The
+.Fn devfs_clear_cdevpriv
+function will be also be called if the open callback
+function returns an error code.
 .Pp
 On the last filedescriptor close, system automatically arranges
 .Fn devfs_clear_cdevpriv
diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
index ed60dda..09c4149 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
@@ -15517,8 +15517,6 @@ dtrace_close(struct cdev *dev, int flags, int fmt __unused, struct thread *td)
 		kmem_free(state, 0);
 #if __FreeBSD_version < 800039
 		dev->si_drv1 = NULL;
-#else
-		devfs_clear_cdevpriv();
 #endif
 #endif
 	}
diff --git a/sys/dev/drm/drm_fops.c b/sys/dev/drm/drm_fops.c
index 3f743e0..814958e 100644
--- a/sys/dev/drm/drm_fops.c
+++ b/sys/dev/drm/drm_fops.c
@@ -57,12 +57,6 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 		return ENOMEM;
 	}
 
-	retcode = devfs_set_cdevpriv(priv, drm_close);
-	if (retcode != 0) {
-		free(priv, DRM_MEM_FILES);
-		return retcode;
-	}
-
 	DRM_LOCK();
 	priv->dev		= dev;
 	priv->uid		= p->td_ucred->cr_svuid;
@@ -76,7 +70,6 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 		/* shared code returns -errno */
 		retcode = -dev->driver->open(dev, priv);
 		if (retcode != 0) {
-			devfs_clear_cdevpriv();
 			free(priv, DRM_MEM_FILES);
 			DRM_UNLOCK();
 			return retcode;
@@ -89,7 +82,12 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 	TAILQ_INSERT_TAIL(&dev->files, priv, link);
 	DRM_UNLOCK();
 	kdev->si_drv1 = dev;
-	return 0;
+
+	retcode = devfs_set_cdevpriv(priv, drm_close);
+	if (retcode != 0)
+		drm_close(priv);
+
+	return (retcode);
 }
 
 
diff --git a/sys/dev/drm2/drm_fops.c b/sys/dev/drm2/drm_fops.c
index 0071783..2685ff7 100644
--- a/sys/dev/drm2/drm_fops.c
+++ b/sys/dev/drm2/drm_fops.c
@@ -57,12 +57,6 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 		return ENOMEM;
 	}
 
-	retcode = devfs_set_cdevpriv(priv, drm_close);
-	if (retcode != 0) {
-		free(priv, DRM_MEM_FILES);
-		return retcode;
-	}
-
 	DRM_LOCK(dev);
 	priv->dev		= dev;
 	priv->uid		= p->td_ucred->cr_svuid;
@@ -83,7 +77,6 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 		/* shared code returns -errno */
 		retcode = -dev->driver->open(dev, priv);
 		if (retcode != 0) {
-			devfs_clear_cdevpriv();
 			free(priv, DRM_MEM_FILES);
 			DRM_UNLOCK(dev);
 			return retcode;
@@ -96,7 +89,12 @@ int drm_open_helper(struct cdev *kdev, int flags, int fmt, DRM_STRUCTPROC *p,
 	TAILQ_INSERT_TAIL(&dev->files, priv, link);
 	DRM_UNLOCK(dev);
 	kdev->si_drv1 = dev;
-	return 0;
+
+	retcode = devfs_set_cdevpriv(priv, drm_close);
+	if (retcode != 0)
+		drm_close(priv);
+
+	return (retcode);
 }
 
 static bool
diff --git a/sys/dev/ksyms/ksyms.c b/sys/dev/ksyms/ksyms.c
index 49298bc..61729210 100644
--- a/sys/dev/ksyms/ksyms.c
+++ b/sys/dev/ksyms/ksyms.c
@@ -579,8 +579,6 @@ ksyms_close(struct cdev *dev, int flags __unused, int fmt __unused,
 	/* Unmap the buffer from the process address space. */
 	error = copyout_unmap(td, sc->sc_uaddr, sc->sc_usize);
 
-	devfs_clear_cdevpriv();
-
 	return (error);
 }
 
diff --git a/sys/fs/devfs/devfs_vnops.c b/sys/fs/devfs/devfs_vnops.c
index d3e6cf5..97a1bcf 100644
--- a/sys/fs/devfs/devfs_vnops.c
+++ b/sys/fs/devfs/devfs_vnops.c
@@ -1081,6 +1081,9 @@ devfs_open(struct vop_open_args *ap)
 		error = dsw->d_fdopen(dev, ap->a_mode, td, fp);
 	else
 		error = dsw->d_open(dev, ap->a_mode, S_IFCHR, td);
+	/* cleanup any cdevpriv upon error */
+	if (error != 0)
+		devfs_clear_cdevpriv();
 	td->td_fpop = fpop;
 
 	vn_lock(vp, vlocked | LK_RETRY);
diff --git a/sys/ofed/include/linux/linux_compat.c b/sys/ofed/include/linux/linux_compat.c
index 90737f2..fb3783a 100644
--- a/sys/ofed/include/linux/linux_compat.c
+++ b/sys/ofed/include/linux/linux_compat.c
@@ -263,7 +263,6 @@ linux_dev_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
 	if ((error = devfs_get_cdevpriv((void **)&filp)) != 0)
 		return (error);
 	filp->f_flags = file->f_flag;
-	devfs_clear_cdevpriv();
 
 	return (0);
 }