Modify the experimental NFSv4 server so that it handles

crossing of server mount points properly. The functions
nfsvno_fillattr() and nfsv4_fillattr() were modified to
take the extra arguments that are the mount point, a flag
to indicate that it is a file system root and the mounted
on fileno. The mount point argument needs to be busy when
nfsvno_fillattr() is called, since the vp argument is not
locked.

Reviewed by:	kib
MFC after:	2 weeks

7 files changed, 129 insertions, 51 deletions

diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c
index 77b7a6a..5cdb968 100644
--- a/sys/fs/nfs/nfs_commonsubs.c
+++ b/sys/fs/nfs/nfs_commonsubs.c
@@ -1916,9 +1916,10 @@ nfsrv_mtostr(struct nfsrv_descript *nd, char *str, int siz)
  * Fill in the attributes as marked by the bitmap (V4).
  */
 APPLESTATIC int
-nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
-    struct vattr *vap, fhandle_t *fhp, int rderror, nfsattrbit_t *attrbitp,
-    struct ucred *cred, NFSPROC_T *p, int isdgram, int reterr)
+nfsv4_fillattr(struct nfsrv_descript *nd, struct mount *mp, vnode_t vp,
+    NFSACL_T *saclp, struct vattr *vap, fhandle_t *fhp, int rderror,
+    nfsattrbit_t *attrbitp, struct ucred *cred, NFSPROC_T *p, int isdgram,
+    int reterr, int at_root, uint64_t mounted_on_fileno)
 {
 	int bitpos, retnum = 0;
 	u_int32_t *tl;
@@ -1928,7 +1929,6 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 	nfsattrbit_t *retbitp = &retbits;
 	u_int32_t freenum, *retnump;
 	u_int64_t uquad;
-	long fid;
 	struct statfs fs;
 	struct nfsfsinfo fsinf;
 	struct timespec temptime;
@@ -1958,7 +1958,7 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 	 * Get the VFS_STATFS(), since some attributes need them.
 	 */
 	if (NFSISSETSTATFS_ATTRBIT(retbitp)) {
-		error = VFS_STATFS(vnode_mount(vp), &fs);
+		error = VFS_STATFS(mp, &fs);
 		if (error != 0) {
 			if (reterr) {
 				nd->nd_repstat = NFSERR_ACCES;
@@ -1974,12 +1974,12 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 	 */
 	if (NFSISSET_ATTRBIT(retbitp, NFSATTRBIT_ACLSUPPORT) &&
 	    (nfsrv_useacl == 0 || ((cred != NULL || p != NULL) &&
-		!NFSHASNFS4ACL(vnode_mount(vp))))) {
+		!NFSHASNFS4ACL(mp)))) {
 		NFSCLRBIT_ATTRBIT(retbitp, NFSATTRBIT_ACLSUPPORT);
 	}
 	if (NFSISSET_ATTRBIT(retbitp, NFSATTRBIT_ACL)) {
 		if (nfsrv_useacl == 0 || ((cred != NULL || p != NULL) &&
-		    !NFSHASNFS4ACL(vnode_mount(vp)))) {
+		    !NFSHASNFS4ACL(mp))) {
 			NFSCLRBIT_ATTRBIT(retbitp, NFSATTRBIT_ACL);
 		} else if (naclp != NULL) {
 			if (vn_lock(vp, LK_SHARED) == 0) {
@@ -2016,7 +2016,7 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 		case NFSATTRBIT_SUPPORTEDATTRS:
 			NFSSETSUPP_ATTRBIT(&attrbits);
 			if (nfsrv_useacl == 0 || ((cred != NULL || p != NULL)
-			    && !NFSHASNFS4ACL(vnode_mount(vp)))) {
+			    && !NFSHASNFS4ACL(mp))) {
 			    NFSCLRBIT_ATTRBIT(&attrbits,NFSATTRBIT_ACLSUPPORT);
 			    NFSCLRBIT_ATTRBIT(&attrbits,NFSATTRBIT_ACL);
 			}
@@ -2066,9 +2066,9 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 		case NFSATTRBIT_FSID:
 			NFSM_BUILD(tl, u_int32_t *, NFSX_V4FSID);
 			*tl++ = 0;
-			*tl++=txdr_unsigned(vfs_statfs(vnode_mount(vp))->f_fsid.val[0]);
+			*tl++ = txdr_unsigned(mp->mnt_stat.f_fsid.val[0]);
 			*tl++ = 0;
-			*tl=txdr_unsigned(vfs_statfs(vnode_mount(vp))->f_fsid.val[1]);
+			*tl = txdr_unsigned(mp->mnt_stat.f_fsid.val[1]);
 			retnum += NFSX_V4FSID;
 			break;
 		case NFSATTRBIT_UNIQUEHANDLES:
@@ -2142,7 +2142,7 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 			 */
 			savuid = p->p_cred->p_ruid;
 			p->p_cred->p_ruid = cred->cr_uid;
-			if (!VFS_QUOTACTL(vnode_mount(vp),QCMD(Q_GETQUOTA,USRQUOTA),
+			if (!VFS_QUOTACTL(mp, QCMD(Q_GETQUOTA,USRQUOTA),
 			    cred->cr_uid, (caddr_t)&dqb))
 			    freenum = min(dqb.dqb_isoftlimit-dqb.dqb_curinodes,
 				freenum);
@@ -2249,7 +2249,7 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 			 */
 			savuid = p->p_cred->p_ruid;
 			p->p_cred->p_ruid = cred->cr_uid;
-			if (!VFS_QUOTACTL(vnode_mount(vp),QCMD(Q_GETQUOTA,USRQUOTA),
+			if (!VFS_QUOTACTL(mp, QCMD(Q_GETQUOTA,USRQUOTA),
 			    cred->cr_uid, (caddr_t)&dqb))
 			    freenum = min(dqb.dqb_bhardlimit, freenum);
 			p->p_cred->p_ruid = savuid;
@@ -2273,7 +2273,7 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 			 */
 			savuid = p->p_cred->p_ruid;
 			p->p_cred->p_ruid = cred->cr_uid;
-			if (!VFS_QUOTACTL(vnode_mount(vp),QCMD(Q_GETQUOTA,USRQUOTA),
+			if (!VFS_QUOTACTL(mp, QCMD(Q_GETQUOTA,USRQUOTA),
 			    cred->cr_uid, (caddr_t)&dqb))
 			    freenum = min(dqb.dqb_bsoftlimit, freenum);
 			p->p_cred->p_ruid = savuid;
@@ -2294,7 +2294,7 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 			 */
 			savuid = p->p_cred->p_ruid;
 			p->p_cred->p_ruid = cred->cr_uid;
-			if (!VFS_QUOTACTL(vnode_mount(vp),QCMD(Q_GETQUOTA,USRQUOTA),
+			if (!VFS_QUOTACTL(mp, QCMD(Q_GETQUOTA,USRQUOTA),
 			    cred->cr_uid, (caddr_t)&dqb))
 			    freenum = dqb.dqb_curblocks;
 			p->p_cred->p_ruid = savuid;
@@ -2390,11 +2390,11 @@ nfsv4_fillattr(struct nfsrv_descript *nd, vnode_t vp, NFSACL_T *saclp,
 			break;
 		case NFSATTRBIT_MOUNTEDONFILEID:
 			NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
-			*tl++ = 0;
-			if (nfsrv_atroot(vp, &fid))
-				*tl = txdr_unsigned(fid);
+			if (at_root != 0)
+				uquad = mounted_on_fileno;
 			else
-				*tl = txdr_unsigned(vap->va_fileid);
+				uquad = (u_int64_t)vap->va_fileid;
+			txdr_hyper(uquad, tl);
 			retnum += NFSX_HYPER;
 			break;
 		default:
diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h
index b767cbf..8a1c79f 100644
--- a/sys/fs/nfs/nfs_var.h
+++ b/sys/fs/nfs/nfs_var.h
@@ -288,9 +288,9 @@ int nfsrv_mtofh(struct nfsrv_descript *, struct nfsrvfh *);
 int nfsrv_putattrbit(struct nfsrv_descript *, nfsattrbit_t *);
 void nfsrv_wcc(struct nfsrv_descript *, int, struct nfsvattr *, int,
     struct nfsvattr *);
-int nfsv4_fillattr(struct nfsrv_descript *, vnode_t, NFSACL_T *,
+int nfsv4_fillattr(struct nfsrv_descript *, struct mount *, vnode_t, NFSACL_T *,
     struct vattr *, fhandle_t *, int, nfsattrbit_t *,
-    struct ucred *, NFSPROC_T *, int, int);
+    struct ucred *, NFSPROC_T *, int, int, int, uint64_t);
 void nfsrv_fillattr(struct nfsrv_descript *, struct nfsvattr *);
 void nfsrv_adj(mbuf_t, int, int);
 void nfsrv_postopattr(struct nfsrv_descript *, int, struct nfsvattr *);
@@ -556,9 +556,9 @@ void nfsvno_open(struct nfsrv_descript *, struct nameidata *, nfsquad_t,
     struct nfsexstuff *, vnode_t *);
 void nfsvno_updfilerev(vnode_t, struct nfsvattr *, struct ucred *,
     NFSPROC_T *);
-int nfsvno_fillattr(struct nfsrv_descript *, vnode_t,
+int nfsvno_fillattr(struct nfsrv_descript *, struct mount *, vnode_t,
     struct nfsvattr *, fhandle_t *, int, nfsattrbit_t *,
-    struct ucred *, NFSPROC_T *, int, int);
+    struct ucred *, NFSPROC_T *, int, int, int, uint64_t);
 int nfsrv_sattr(struct nfsrv_descript *, struct nfsvattr *, nfsattrbit_t *,
     NFSACL_T *, NFSPROC_T *);
 int nfsv4_sattr(struct nfsrv_descript *, struct nfsvattr *, nfsattrbit_t *,
diff --git a/sys/fs/nfsclient/nfs_clport.c b/sys/fs/nfsclient/nfs_clport.c
index e9ebbde..4ba5656 100644
--- a/sys/fs/nfsclient/nfs_clport.c
+++ b/sys/fs/nfsclient/nfs_clport.c
@@ -803,8 +803,8 @@ nfscl_fillsattr(struct nfsrv_descript *nd, struct vattr *vap,
 			NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEACCESSSET);
 		if (vap->va_mtime.tv_sec != VNOVAL)
 			NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEMODIFYSET);
-		(void) nfsv4_fillattr(nd, vp, NULL, vap, NULL, 0, &attrbits,
-		    NULL, NULL, 0, 0);
+		(void) nfsv4_fillattr(nd, vp->v_mount, vp, NULL, vap, NULL, 0,
+		    &attrbits, NULL, NULL, 0, 0, 0, (uint64_t)0);
 		break;
 	};
 }
diff --git a/sys/fs/nfsclient/nfs_clrpcops.c b/sys/fs/nfsclient/nfs_clrpcops.c
index 37e1b81..271b430 100644
--- a/sys/fs/nfsclient/nfs_clrpcops.c
+++ b/sys/fs/nfsclient/nfs_clrpcops.c
@@ -4176,8 +4176,8 @@ nfsrpc_setaclrpc(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
 	nfsm_stateidtom(nd, stateidp, NFSSTATEID_PUTSTATEID);
 	NFSZERO_ATTRBIT(&attrbits);
 	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_ACL);
-	(void) nfsv4_fillattr(nd, vp, aclp, NULL, NULL, 0, &attrbits,
-	    NULL, NULL, 0, 0);
+	(void) nfsv4_fillattr(nd, vnode_mount(vp), vp, aclp, NULL, NULL, 0,
+	    &attrbits, NULL, NULL, 0, 0, 0, (uint64_t)0);
 	error = nfscl_request(nd, vp, p, cred, stuff);
 	if (error)
 		return (error);
diff --git a/sys/fs/nfsclient/nfs_clstate.c b/sys/fs/nfsclient/nfs_clstate.c
index 2281f32..0e1fea2 100644
--- a/sys/fs/nfsclient/nfs_clstate.c
+++ b/sys/fs/nfsclient/nfs_clstate.c
@@ -3061,8 +3061,9 @@ nfscl_docb(struct nfsrv_descript *nd, NFSPROC_T *p)
 					NFSSETBIT_ATTRBIT(&rattrbits,
 					    NFSATTRBIT_CHANGE);
 				}
-				(void) nfsv4_fillattr(nd, NULL, NULL, &va,
-				    NULL, 0, &rattrbits, NULL, NULL, 0, 0);
+				(void) nfsv4_fillattr(nd, NULL, NULL, NULL, &va,
+				    NULL, 0, &rattrbits, NULL, NULL, 0, 0, 0,
+				    (uint64_t)0);
 				if (!ret)
 					vrele(vp);
 			}
diff --git a/sys/fs/nfsserver/nfs_nfsdport.c b/sys/fs/nfsserver/nfs_nfsdport.c
index c97df89..b3dea70 100644
--- a/sys/fs/nfsserver/nfs_nfsdport.c
+++ b/sys/fs/nfsserver/nfs_nfsdport.c
@@ -70,15 +70,15 @@ static uint32_t nfsv4_sysid = 0;
 static int nfssvc_srvcall(struct thread *, struct nfssvc_args *,
     struct ucred *);
 
-static int enable_crossmntpt = 1;
+int nfsrv_enable_crossmntpt = 1;
 static int nfs_commit_blks;
 static int nfs_commit_miss;
 extern int nfsrv_issuedelegs;
 extern int nfsrv_dolocallocks;
 
 SYSCTL_DECL(_vfs_newnfs);
-SYSCTL_INT(_vfs_newnfs, OID_AUTO, mirrormnt, CTLFLAG_RW, &enable_crossmntpt,
-    0, "Enable nfsd to cross mount points");
+SYSCTL_INT(_vfs_newnfs, OID_AUTO, mirrormnt, CTLFLAG_RW,
+    &nfsrv_enable_crossmntpt, 0, "Enable nfsd to cross mount points");
 SYSCTL_INT(_vfs_newnfs, OID_AUTO, commit_blks, CTLFLAG_RW, &nfs_commit_blks,
     0, "");
 SYSCTL_INT(_vfs_newnfs, OID_AUTO, commit_miss, CTLFLAG_RW, &nfs_commit_miss,
@@ -308,12 +308,12 @@ nfsvno_namei(struct nfsrv_descript *nd, struct nameidata *ndp,
 			dp = rootvnode;
 			VREF(dp);
 		}
-	} else if ((enable_crossmntpt == 0 && NFSVNO_EXPORTED(exp)) ||
+	} else if ((nfsrv_enable_crossmntpt == 0 && NFSVNO_EXPORTED(exp)) ||
 	    (nd->nd_flag & ND_NFSV4) == 0) {
 		/*
 		 * Only cross mount points for NFSv4 when doing a
 		 * mount while traversing the file system above
-		 * the mount point, unless enable_crossmntpt is set.
+		 * the mount point, unless nfsrv_enable_crossmntpt is set.
 		 */
 		cnp->cn_flags |= NOCROSSMOUNT;
 		crossmnt = 0;
@@ -1394,14 +1394,15 @@ nfsvno_updfilerev(struct vnode *vp, struct nfsvattr *nvap,
  * Glue routine to nfsv4_fillattr().
  */
 int
-nfsvno_fillattr(struct nfsrv_descript *nd, struct vnode *vp,
+nfsvno_fillattr(struct nfsrv_descript *nd, struct mount *mp, struct vnode *vp,
     struct nfsvattr *nvap, fhandle_t *fhp, int rderror, nfsattrbit_t *attrbitp,
-    struct ucred *cred, struct thread *p, int isdgram, int reterr)
+    struct ucred *cred, struct thread *p, int isdgram, int reterr, int at_root,
+    uint64_t mounted_on_fileno)
 {
 	int error;
 
-	error = nfsv4_fillattr(nd, vp, NULL, &nvap->na_vattr, fhp, rderror,
-	    attrbitp, cred, p, isdgram, reterr);
+	error = nfsv4_fillattr(nd, mp, vp, NULL, &nvap->na_vattr, fhp, rderror,
+	    attrbitp, cred, p, isdgram, reterr, at_root, mounted_on_fileno);
 	return (error);
 }
 
@@ -1691,8 +1692,9 @@ nfsrvd_readdirplus(struct nfsrv_descript *nd, int isdgram,
 	struct uio io;
 	struct iovec iv;
 	struct componentname cn;
-	int not_zfs;
-	struct mount *mp;
+	int at_root, needs_unbusy, not_zfs;
+	struct mount *mp, *new_mp;
+	uint64_t mounted_on_fileno;
 
 	if (nd->nd_repstat) {
 		nfsrv_postopattr(nd, getret, &at);
@@ -1932,6 +1934,10 @@ again:
 			nvp = NULL;
 			refp = NULL;
 			r = 0;
+			at_root = 0;
+			needs_unbusy = 0;
+			new_mp = mp;
+			mounted_on_fileno = (uint64_t)dp->d_fileno;
 			if ((nd->nd_flag & ND_NFSV3) ||
 			    NFSNONZERO_ATTRBIT(&savbits)) {
 				if (nd->nd_flag & ND_NFSV4)
@@ -1983,6 +1989,29 @@ again:
 								    0);
 						}
 					}
+
+					/*
+					 * For NFSv4, check to see if nvp is
+					 * a mount point and get the mount
+					 * point vnode, as required.
+					 */
+					if (r == 0 &&
+					    nfsrv_enable_crossmntpt != 0 &&
+					    (nd->nd_flag & ND_NFSV4) != 0 &&
+					    nvp->v_type == VDIR &&
+					    nvp->v_mountedhere != NULL) {
+						new_mp = nvp->v_mountedhere;
+						r = vfs_busy(new_mp, 0);
+						vput(nvp);
+						nvp = NULL;
+						if (r == 0) {
+							r = VFS_ROOT(new_mp,
+							    LK_SHARED, &nvp);
+							needs_unbusy = 1;
+							if (r == 0)
+								at_root = 1;
+						}
+					}
 				}
 				if (!r) {
 				    if (refp == NULL &&
@@ -2001,6 +2030,8 @@ again:
 					    NFSATTRBIT_RDATTRERROR)) {
 						if (nvp != NULL)
 							vput(nvp);
+						if (needs_unbusy != 0)
+							vfs_unbusy(new_mp);
 						nd->nd_repstat = r;
 						break;
 					}
@@ -2039,21 +2070,27 @@ again:
 					if (nd->nd_repstat) {
 						if (nvp != NULL)
 							vrele(nvp);
+						if (needs_unbusy != 0)
+							vfs_unbusy(new_mp);
 						break;
 					}
 				} else if (r) {
-					dirlen += nfsvno_fillattr(nd, nvp, nvap,
-					    &nfh, r, &rderrbits, nd->nd_cred,
-					    p, isdgram, 0);
+					dirlen += nfsvno_fillattr(nd, new_mp,
+					    nvp, nvap, &nfh, r, &rderrbits,
+					    nd->nd_cred, p, isdgram, 0, at_root,
+					    mounted_on_fileno);
 				} else {
-					dirlen += nfsvno_fillattr(nd, nvp, nvap,
-					    &nfh, r, &attrbits, nd->nd_cred,
-					    p, isdgram, 0);
+					dirlen += nfsvno_fillattr(nd, new_mp,
+					    nvp, nvap, &nfh, r, &attrbits,
+					    nd->nd_cred, p, isdgram, 0, at_root,
+					    mounted_on_fileno);
 				}
 				if (nvp != NULL)
 					vrele(nvp);
 				dirlen += (3 * NFSX_UNSIGNED);
 			}
+			if (needs_unbusy != 0)
+				vfs_unbusy(new_mp);
 			if (dirlen <= cnt)
 				entrycnt++;
 		}
diff --git a/sys/fs/nfsserver/nfs_nfsdserv.c b/sys/fs/nfsserver/nfs_nfsdserv.c
index 09a82b7..27398ab 100644
--- a/sys/fs/nfsserver/nfs_nfsdserv.c
+++ b/sys/fs/nfsserver/nfs_nfsdserv.c
@@ -52,6 +52,7 @@ extern u_int32_t newnfs_false, newnfs_true;
 extern enum vtype nv34tov_type[8];
 extern struct timeval nfsboottime;
 extern int nfs_rootfhset;
+extern int nfsrv_enable_crossmntpt;
 #endif	/* !APPLEKEXT */
 
 /*
@@ -169,9 +170,13 @@ nfsrvd_getattr(struct nfsrv_descript *nd, int isdgram,
 {
 	struct nfsvattr nva;
 	fhandle_t fh;
-	int error = 0;
+	int at_root = 0, error = 0;
 	struct nfsreferral *refp;
 	nfsattrbit_t attrbits;
+	struct mount *mp;
+	struct vnode *tvp = NULL;
+	struct vattr va;
+	uint64_t mounted_on_fileno = 0;
 
 	if (nd->nd_repstat)
 		return (0);
@@ -207,11 +212,46 @@ nfsrvd_getattr(struct nfsrv_descript *nd, int isdgram,
 			if (!nd->nd_repstat)
 				nd->nd_repstat = nfsrv_checkgetattr(nd, vp,
 				    &nva, &attrbits, nd->nd_cred, p);
-			NFSVOPUNLOCK(vp, 0, p);
-			if (!nd->nd_repstat)
-				(void) nfsvno_fillattr(nd, vp, &nva, &fh,
-				    0, &attrbits, nd->nd_cred, p, isdgram, 1);
-			vrele(vp);
+			if (nd->nd_repstat == 0) {
+				mp = vp->v_mount;
+				if (nfsrv_enable_crossmntpt != 0 &&
+				    vp->v_type == VDIR &&
+				    (vp->v_vflag & VV_ROOT) != 0 &&
+				    vp != rootvnode) {
+					tvp = mp->mnt_vnodecovered;
+					VREF(tvp);
+					at_root = 1;
+				} else
+					at_root = 0;
+				vfs_ref(mp);
+				VOP_UNLOCK(vp, 0);
+				if (at_root != 0) {
+					if ((nd->nd_repstat =
+					     vn_lock(tvp, LK_SHARED)) == 0) {
+						nd->nd_repstat = VOP_GETATTR(
+						    tvp, &va, nd->nd_cred);
+						vput(tvp);
+					} else
+						vrele(tvp);
+					if (nd->nd_repstat == 0)
+						mounted_on_fileno = (uint64_t)
+						    va.va_fileid;
+					else
+						at_root = 0;
+				}
+				if (nd->nd_repstat == 0)
+					nd->nd_repstat = vfs_busy(mp, 0);
+				vfs_rel(mp);
+				if (nd->nd_repstat == 0) {
+					(void)nfsvno_fillattr(nd, mp, vp, &nva,
+					    &fh, 0, &attrbits, nd->nd_cred, p,
+					    isdgram, 1, at_root,
+					    mounted_on_fileno);
+					vfs_unbusy(mp);
+				}
+				vrele(vp);
+			} else
+				vput(vp);
 		} else {
 			nfsrv_fillattr(nd, &nva);
 			vput(vp);