Fix 3-way deadlock that can happen because of ZFS and vnode lock

order reversal.

thread0 (vfs_fhtovp)	thread1 (vop_getattr)	thread2 (zfs_recv)
--------------------	---------------------	------------------
			vn_lock
rrw_enter_read
						rrw_enter_write (hangs)
			rrw_enter_read (hangs)
vn_lock (hangs)

Submitted by:	Attila Nagy <bra@fsn.hu>
MFC after:	3 days

2 files changed, 10 insertions, 7 deletions

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
index ad8165b..07c9b61 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
@@ -868,13 +868,15 @@ zfs_root(vfs_t *vfsp, int flags, vnode_t **vpp)
 	ZFS_ENTER_NOERROR(zfsvfs);
 
 	error = zfs_zget(zfsvfs, zfsvfs->z_root, &rootzp);
+
+	ZFS_EXIT(zfsvfs);
+
 	if (error == 0) {
 		*vpp = ZTOV(rootzp);
 		error = vn_lock(*vpp, flags);
 		(*vpp)->v_vflag |= VV_ROOT;
 	}
 
-	ZFS_EXIT(zfsvfs);
 	return (error);
 }
 
@@ -1143,13 +1145,13 @@ zfs_vget(vfs_t *vfsp, ino_t ino, int flags, vnode_t **vpp)
 		VN_RELE(ZTOV(zp));
 		err = EINVAL;
 	}
+	ZFS_EXIT(zfsvfs);
 	if (err != 0)
 		*vpp = NULL;
 	else {
 		*vpp = ZTOV(zp);
 		vn_lock(*vpp, flags);
 	}
-	ZFS_EXIT(zfsvfs);
 	return (err);
 }
 
@@ -1237,8 +1239,8 @@ zfs_fhtovp(vfs_t *vfsp, fid_t *fidp, vnode_t **vpp)
 		} else {
 			VN_HOLD(*vpp);
 		}
-		vn_lock(*vpp, LK_EXCLUSIVE | LK_RETRY);
 		ZFS_EXIT(zfsvfs);
+		vn_lock(*vpp, LK_EXCLUSIVE | LK_RETRY);
 		return (0);
 	}
 
@@ -1259,10 +1261,11 @@ zfs_fhtovp(vfs_t *vfsp, fid_t *fidp, vnode_t **vpp)
 		return (EINVAL);
 	}
 
+	ZFS_EXIT(zfsvfs);
+
 	*vpp = ZTOV(zp);
 	vn_lock(*vpp, LK_EXCLUSIVE | LK_RETRY);
 	vnode_create_vobject(*vpp, zp->z_phys->zp_size, curthread);
-	ZFS_EXIT(zfsvfs);
 	return (0);
 }
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
index 4f61f5f..59a58dd 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
@@ -1209,15 +1209,17 @@ zfs_lookup(vnode_t *dvp, char *nm, vnode_t **vpp, struct componentname *cnp,
 			ltype = VOP_ISLOCKED(dvp);
 			VOP_UNLOCK(dvp, 0);
 		}
+		ZFS_EXIT(zfsvfs);
 		error = vn_lock(*vpp, cnp->cn_lkflags);
 		if (cnp->cn_flags & ISDOTDOT)
 			vn_lock(dvp, ltype | LK_RETRY);
 		if (error != 0) {
 			VN_RELE(*vpp);
 			*vpp = NULL;
-			ZFS_EXIT(zfsvfs);
 			return (error);
 		}
+	} else {
+		ZFS_EXIT(zfsvfs);
 	}
 
 #ifdef FREEBSD_NAMECACHE
@@ -1237,8 +1239,6 @@ zfs_lookup(vnode_t *dvp, char *nm, vnode_t **vpp, struct componentname *cnp,
 	}
 #endif
 
-	ZFS_EXIT(zfsvfs);
-
 	return (error);
 }