diff options
Diffstat (limited to 'fs')
-rw-r--r-- | fs/9p/cache.c | 100 | ||||
-rw-r--r-- | fs/afs/cache.c | 150 | ||||
-rw-r--r-- | fs/afs/cell.c | 6 | ||||
-rw-r--r-- | fs/afs/file.c | 6 | ||||
-rw-r--r-- | fs/afs/inode.c | 49 | ||||
-rw-r--r-- | fs/afs/internal.h | 7 | ||||
-rw-r--r-- | fs/afs/volume.c | 6 | ||||
-rw-r--r-- | fs/cachefiles/interface.c | 61 | ||||
-rw-r--r-- | fs/cachefiles/internal.h | 2 | ||||
-rw-r--r-- | fs/cachefiles/main.c | 1 | ||||
-rw-r--r-- | fs/cachefiles/namei.c | 75 | ||||
-rw-r--r-- | fs/cachefiles/rdwr.c | 1 | ||||
-rw-r--r-- | fs/cachefiles/xattr.c | 8 | ||||
-rw-r--r-- | fs/ceph/cache.c | 113 | ||||
-rw-r--r-- | fs/cifs/cache.c | 168 | ||||
-rw-r--r-- | fs/cifs/fscache.c | 130 | ||||
-rw-r--r-- | fs/cifs/fscache.h | 13 | ||||
-rw-r--r-- | fs/fscache/cache.c | 2 | ||||
-rw-r--r-- | fs/fscache/cookie.c | 386 | ||||
-rw-r--r-- | fs/fscache/fsdef.c | 55 | ||||
-rw-r--r-- | fs/fscache/internal.h | 44 | ||||
-rw-r--r-- | fs/fscache/main.c | 1 | ||||
-rw-r--r-- | fs/fscache/netfs.c | 71 | ||||
-rw-r--r-- | fs/fscache/object-list.c | 28 | ||||
-rw-r--r-- | fs/fscache/object.c | 66 | ||||
-rw-r--r-- | fs/fscache/operation.c | 26 | ||||
-rw-r--r-- | fs/fscache/page.c | 84 | ||||
-rw-r--r-- | fs/fscache/stats.c | 1 | ||||
-rw-r--r-- | fs/nfs/fscache-index.c | 159 | ||||
-rw-r--r-- | fs/nfs/fscache.c | 89 | ||||
-rw-r--r-- | fs/nfs/fscache.h | 15 |
31 files changed, 986 insertions, 937 deletions
diff --git a/fs/9p/cache.c b/fs/9p/cache.c index 64c58eb2..9eb3470 100644 --- a/fs/9p/cache.c +++ b/fs/9p/cache.c @@ -55,42 +55,27 @@ int v9fs_random_cachetag(struct v9fs_session_info *v9ses) return scnprintf(v9ses->cachetag, CACHETAG_LEN, "%lu", jiffies); } -static uint16_t v9fs_cache_session_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - struct v9fs_session_info *v9ses; - uint16_t klen = 0; - - v9ses = (struct v9fs_session_info *)cookie_netfs_data; - p9_debug(P9_DEBUG_FSC, "session %p buf %p size %u\n", - v9ses, buffer, bufmax); - - if (v9ses->cachetag) - klen = strlen(v9ses->cachetag); - - if (klen > bufmax) - return 0; - - memcpy(buffer, v9ses->cachetag, klen); - p9_debug(P9_DEBUG_FSC, "cache session tag %s\n", v9ses->cachetag); - return klen; -} - const struct fscache_cookie_def v9fs_cache_session_index_def = { .name = "9P.session", .type = FSCACHE_COOKIE_TYPE_INDEX, - .get_key = v9fs_cache_session_get_key, }; void v9fs_cache_session_get_cookie(struct v9fs_session_info *v9ses) { /* If no cache session tag was specified, we generate a random one. */ - if (!v9ses->cachetag) - v9fs_random_cachetag(v9ses); + if (!v9ses->cachetag) { + if (v9fs_random_cachetag(v9ses) < 0) { + v9ses->fscache = NULL; + return; + } + } v9ses->fscache = fscache_acquire_cookie(v9fs_cache_netfs.primary_index, &v9fs_cache_session_index_def, - v9ses, true); + v9ses->cachetag, + strlen(v9ses->cachetag), + NULL, 0, + v9ses, 0, true); p9_debug(P9_DEBUG_FSC, "session %p get cookie %p\n", v9ses, v9ses->fscache); } @@ -99,45 +84,15 @@ void v9fs_cache_session_put_cookie(struct v9fs_session_info *v9ses) { p9_debug(P9_DEBUG_FSC, "session %p put cookie %p\n", v9ses, v9ses->fscache); - fscache_relinquish_cookie(v9ses->fscache, 0); + fscache_relinquish_cookie(v9ses->fscache, NULL, false); v9ses->fscache = NULL; } - -static uint16_t v9fs_cache_inode_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct v9fs_inode *v9inode = cookie_netfs_data; - memcpy(buffer, &v9inode->qid.path, sizeof(v9inode->qid.path)); - p9_debug(P9_DEBUG_FSC, "inode %p get key %llu\n", - &v9inode->vfs_inode, v9inode->qid.path); - return sizeof(v9inode->qid.path); -} - -static void v9fs_cache_inode_get_attr(const void *cookie_netfs_data, - uint64_t *size) -{ - const struct v9fs_inode *v9inode = cookie_netfs_data; - *size = i_size_read(&v9inode->vfs_inode); - - p9_debug(P9_DEBUG_FSC, "inode %p get attr %llu\n", - &v9inode->vfs_inode, *size); -} - -static uint16_t v9fs_cache_inode_get_aux(const void *cookie_netfs_data, - void *buffer, uint16_t buflen) -{ - const struct v9fs_inode *v9inode = cookie_netfs_data; - memcpy(buffer, &v9inode->qid.version, sizeof(v9inode->qid.version)); - p9_debug(P9_DEBUG_FSC, "inode %p get aux %u\n", - &v9inode->vfs_inode, v9inode->qid.version); - return sizeof(v9inode->qid.version); -} - static enum fscache_checkaux v9fs_cache_inode_check_aux(void *cookie_netfs_data, const void *buffer, - uint16_t buflen) + uint16_t buflen, + loff_t object_size) { const struct v9fs_inode *v9inode = cookie_netfs_data; @@ -154,9 +109,6 @@ fscache_checkaux v9fs_cache_inode_check_aux(void *cookie_netfs_data, const struct fscache_cookie_def v9fs_cache_inode_index_def = { .name = "9p.inode", .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .get_key = v9fs_cache_inode_get_key, - .get_attr = v9fs_cache_inode_get_attr, - .get_aux = v9fs_cache_inode_get_aux, .check_aux = v9fs_cache_inode_check_aux, }; @@ -175,7 +127,13 @@ void v9fs_cache_inode_get_cookie(struct inode *inode) v9ses = v9fs_inode2v9ses(inode); v9inode->fscache = fscache_acquire_cookie(v9ses->fscache, &v9fs_cache_inode_index_def, - v9inode, true); + &v9inode->qid.path, + sizeof(v9inode->qid.path), + &v9inode->qid.version, + sizeof(v9inode->qid.version), + v9inode, + i_size_read(&v9inode->vfs_inode), + true); p9_debug(P9_DEBUG_FSC, "inode %p get cookie %p\n", inode, v9inode->fscache); @@ -190,7 +148,8 @@ void v9fs_cache_inode_put_cookie(struct inode *inode) p9_debug(P9_DEBUG_FSC, "inode %p put cookie %p\n", inode, v9inode->fscache); - fscache_relinquish_cookie(v9inode->fscache, 0); + fscache_relinquish_cookie(v9inode->fscache, &v9inode->qid.version, + false); v9inode->fscache = NULL; } @@ -203,7 +162,7 @@ void v9fs_cache_inode_flush_cookie(struct inode *inode) p9_debug(P9_DEBUG_FSC, "inode %p flush cookie %p\n", inode, v9inode->fscache); - fscache_relinquish_cookie(v9inode->fscache, 1); + fscache_relinquish_cookie(v9inode->fscache, NULL, true); v9inode->fscache = NULL; } @@ -236,12 +195,18 @@ void v9fs_cache_inode_reset_cookie(struct inode *inode) old = v9inode->fscache; mutex_lock(&v9inode->fscache_lock); - fscache_relinquish_cookie(v9inode->fscache, 1); + fscache_relinquish_cookie(v9inode->fscache, NULL, true); v9ses = v9fs_inode2v9ses(inode); v9inode->fscache = fscache_acquire_cookie(v9ses->fscache, &v9fs_cache_inode_index_def, - v9inode, true); + &v9inode->qid.path, + sizeof(v9inode->qid.path), + &v9inode->qid.version, + sizeof(v9inode->qid.version), + v9inode, + i_size_read(&v9inode->vfs_inode), + true); p9_debug(P9_DEBUG_FSC, "inode %p revalidating cookie old %p new %p\n", inode, old, v9inode->fscache); @@ -367,7 +332,8 @@ void __v9fs_readpage_to_fscache(struct inode *inode, struct page *page) const struct v9fs_inode *v9inode = V9FS_I(inode); p9_debug(P9_DEBUG_FSC, "inode %p page %p\n", inode, page); - ret = fscache_write_page(v9inode->fscache, page, GFP_KERNEL); + ret = fscache_write_page(v9inode->fscache, page, + i_size_read(&v9inode->vfs_inode), GFP_KERNEL); p9_debug(P9_DEBUG_FSC, "ret = %d\n", ret); if (ret != 0) v9fs_uncache_page(inode, page); diff --git a/fs/afs/cache.c b/fs/afs/cache.c index f62ff71..b1c31ec 100644 --- a/fs/afs/cache.c +++ b/fs/afs/cache.c @@ -12,167 +12,39 @@ #include <linux/sched.h> #include "internal.h" -static uint16_t afs_cell_cache_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t buflen); -static uint16_t afs_volume_cache_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t buflen); - -static uint16_t afs_vnode_cache_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t buflen); -static void afs_vnode_cache_get_attr(const void *cookie_netfs_data, - uint64_t *size); -static uint16_t afs_vnode_cache_get_aux(const void *cookie_netfs_data, - void *buffer, uint16_t buflen); static enum fscache_checkaux afs_vnode_cache_check_aux(void *cookie_netfs_data, const void *buffer, - uint16_t buflen); + uint16_t buflen, + loff_t object_size); struct fscache_netfs afs_cache_netfs = { .name = "afs", - .version = 1, + .version = 2, }; struct fscache_cookie_def afs_cell_cache_index_def = { .name = "AFS.cell", .type = FSCACHE_COOKIE_TYPE_INDEX, - .get_key = afs_cell_cache_get_key, }; struct fscache_cookie_def afs_volume_cache_index_def = { .name = "AFS.volume", .type = FSCACHE_COOKIE_TYPE_INDEX, - .get_key = afs_volume_cache_get_key, }; struct fscache_cookie_def afs_vnode_cache_index_def = { - .name = "AFS.vnode", - .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .get_key = afs_vnode_cache_get_key, - .get_attr = afs_vnode_cache_get_attr, - .get_aux = afs_vnode_cache_get_aux, - .check_aux = afs_vnode_cache_check_aux, + .name = "AFS.vnode", + .type = FSCACHE_COOKIE_TYPE_DATAFILE, + .check_aux = afs_vnode_cache_check_aux, }; /* - * set the key for the index entry - */ -static uint16_t afs_cell_cache_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct afs_cell *cell = cookie_netfs_data; - uint16_t klen; - - _enter("%p,%p,%u", cell, buffer, bufmax); - - klen = strlen(cell->name); - if (klen > bufmax) - return 0; - - memcpy(buffer, cell->name, klen); - return klen; -} - -/*****************************************************************************/ -/* - * set the key for the volume index entry - */ -static uint16_t afs_volume_cache_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct afs_volume *volume = cookie_netfs_data; - struct { - u64 volid; - } __packed key; - - _enter("{%u},%p,%u", volume->type, buffer, bufmax); - - if (bufmax < sizeof(key)) - return 0; - - key.volid = volume->vid; - memcpy(buffer, &key, sizeof(key)); - return sizeof(key); -} - -/*****************************************************************************/ -/* - * set the key for the index entry - */ -static uint16_t afs_vnode_cache_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct afs_vnode *vnode = cookie_netfs_data; - struct { - u32 vnode_id[3]; - } __packed key; - - _enter("{%x,%x,%llx},%p,%u", - vnode->fid.vnode, vnode->fid.unique, vnode->status.data_version, - buffer, bufmax); - - /* Allow for a 96-bit key */ - memset(&key, 0, sizeof(key)); - key.vnode_id[0] = vnode->fid.vnode; - key.vnode_id[1] = 0; - key.vnode_id[2] = 0; - - if (sizeof(key) > bufmax) - return 0; - - memcpy(buffer, &key, sizeof(key)); - return sizeof(key); -} - -/* - * provide updated file attributes - */ -static void afs_vnode_cache_get_attr(const void *cookie_netfs_data, - uint64_t *size) -{ - const struct afs_vnode *vnode = cookie_netfs_data; - - _enter("{%x,%x,%llx},", - vnode->fid.vnode, vnode->fid.unique, - vnode->status.data_version); - - *size = vnode->status.size; -} - -struct afs_vnode_cache_aux { - u64 data_version; - u32 fid_unique; -} __packed; - -/* - * provide new auxiliary cache data - */ -static uint16_t afs_vnode_cache_get_aux(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct afs_vnode *vnode = cookie_netfs_data; - struct afs_vnode_cache_aux aux; - - _enter("{%x,%x,%Lx},%p,%u", - vnode->fid.vnode, vnode->fid.unique, vnode->status.data_version, - buffer, bufmax); - - memset(&aux, 0, sizeof(aux)); - aux.data_version = vnode->status.data_version; - aux.fid_unique = vnode->fid.unique; - - if (bufmax < sizeof(aux)) - return 0; - - memcpy(buffer, &aux, sizeof(aux)); - return sizeof(aux); -} - -/* * check that the auxiliary data indicates that the entry is still valid */ static enum fscache_checkaux afs_vnode_cache_check_aux(void *cookie_netfs_data, const void *buffer, - uint16_t buflen) + uint16_t buflen, + loff_t object_size) { struct afs_vnode *vnode = cookie_netfs_data; struct afs_vnode_cache_aux aux; @@ -189,12 +61,6 @@ static enum fscache_checkaux afs_vnode_cache_check_aux(void *cookie_netfs_data, return FSCACHE_CHECKAUX_OBSOLETE; } - if (vnode->fid.unique != aux.fid_unique) { - _leave(" = OBSOLETE [uniq %x != %x]", - aux.fid_unique, vnode->fid.unique); - return FSCACHE_CHECKAUX_OBSOLETE; - } - if (vnode->status.data_version != aux.data_version) { _leave(" = OBSOLETE [vers %llx != %llx]", aux.data_version, vnode->status.data_version); diff --git a/fs/afs/cell.c b/fs/afs/cell.c index 3d2c5e0..4235a05 100644 --- a/fs/afs/cell.c +++ b/fs/afs/cell.c @@ -522,7 +522,9 @@ static int afs_activate_cell(struct afs_net *net, struct afs_cell *cell) #ifdef CONFIG_AFS_FSCACHE cell->cache = fscache_acquire_cookie(afs_cache_netfs.primary_index, &afs_cell_cache_index_def, - cell, true); + cell->name, strlen(cell->name), + NULL, 0, + cell, 0, true); #endif ret = afs_proc_cell_setup(net, cell); if (ret < 0) @@ -547,7 +549,7 @@ static void afs_deactivate_cell(struct afs_net *net, struct afs_cell *cell) spin_unlock(&net->proc_cells_lock); #ifdef CONFIG_AFS_FSCACHE - fscache_relinquish_cookie(cell->cache, 0); + fscache_relinquish_cookie(cell->cache, NULL, false); cell->cache = NULL; #endif diff --git a/fs/afs/file.c b/fs/afs/file.c index a39192c..79e665a 100644 --- a/fs/afs/file.c +++ b/fs/afs/file.c @@ -339,7 +339,8 @@ int afs_page_filler(void *data, struct page *page) /* send the page to the cache */ #ifdef CONFIG_AFS_FSCACHE if (PageFsCache(page) && - fscache_write_page(vnode->cache, page, GFP_KERNEL) != 0) { + fscache_write_page(vnode->cache, page, vnode->status.size, + GFP_KERNEL) != 0) { fscache_uncache_page(vnode->cache, page); BUG_ON(PageFsCache(page)); } @@ -403,7 +404,8 @@ static void afs_readpages_page_done(struct afs_call *call, struct afs_read *req) /* send the page to the cache */ #ifdef CONFIG_AFS_FSCACHE if (PageFsCache(page) && - fscache_write_page(vnode->cache, page, GFP_KERNEL) != 0) { + fscache_write_page(vnode->cache, page, vnode->status.size, + GFP_KERNEL) != 0) { fscache_uncache_page(vnode->cache, page); BUG_ON(PageFsCache(page)); } diff --git a/fs/afs/inode.c b/fs/afs/inode.c index 6b39d02..65c5b1e 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -243,6 +243,33 @@ struct inode *afs_iget_pseudo_dir(struct super_block *sb, bool root) } /* + * Get a cache cookie for an inode. + */ +static void afs_get_inode_cache(struct afs_vnode *vnode) +{ +#ifdef CONFIG_AFS_FSCACHE + struct { + u32 vnode_id; + u32 unique; + u32 vnode_id_ext[2]; /* Allow for a 96-bit key */ + } __packed key; + struct afs_vnode_cache_aux aux; + + key.vnode_id = vnode->fid.vnode; + key.unique = vnode->fid.unique; + key.vnode_id_ext[0] = 0; + key.vnode_id_ext[1] = 0; + aux.data_version = vnode->status.data_version; + + vnode->cache = fscache_acquire_cookie(vnode->volume->cache, + &afs_vnode_cache_index_def, + &key, sizeof(key), + &aux, sizeof(aux), + vnode, vnode->status.size, true); +#endif +} + +/* * inode retrieval */ struct inode *afs_iget(struct super_block *sb, struct key *key, @@ -307,11 +334,7 @@ struct inode *afs_iget(struct super_block *sb, struct key *key, /* set up caching before mapping the status, as map-status reads the * first page of symlinks to see if they're really mountpoints */ inode->i_size = vnode->status.size; -#ifdef CONFIG_AFS_FSCACHE - vnode->cache = fscache_acquire_cookie(vnode->volume->cache, - &afs_vnode_cache_index_def, - vnode, true); -#endif + afs_get_inode_cache(vnode); ret = afs_inode_map_status(vnode, key); if (ret < 0) @@ -327,7 +350,7 @@ struct inode *afs_iget(struct super_block *sb, struct key *key, /* failure */ bad_inode: #ifdef CONFIG_AFS_FSCACHE - fscache_relinquish_cookie(vnode->cache, 0); + fscache_relinquish_cookie(vnode->cache, NULL, ret == -ENOENT); vnode->cache = NULL; #endif iget_failed(inode); @@ -343,6 +366,10 @@ void afs_zap_data(struct afs_vnode *vnode) { _enter("{%x:%u}", vnode->fid.vid, vnode->fid.vnode); +#ifdef CONFIG_AFS_FSCACHE + fscache_invalidate(vnode->cache); +#endif + /* nuke all the non-dirty pages that aren't locked, mapped or being * written back in a regular file and completely discard the pages in a * directory or symlink */ @@ -507,8 +534,14 @@ void afs_evict_inode(struct inode *inode) } #ifdef CONFIG_AFS_FSCACHE - fscache_relinquish_cookie(vnode->cache, 0); - vnode->cache = NULL; + { + struct afs_vnode_cache_aux aux; + + aux.data_version = vnode->status.data_version; + fscache_relinquish_cookie(vnode->cache, &aux, + test_bit(AFS_VNODE_DELETED, &vnode->flags)); + vnode->cache = NULL; + } #endif afs_put_permits(vnode->permit_cache); diff --git a/fs/afs/internal.h b/fs/afs/internal.h index 7221717..a6a1d75 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -559,6 +559,13 @@ struct afs_fs_cursor { #define AFS_FS_CURSOR_NO_VSLEEP 0x0020 /* Set to prevent sleep on VBUSY, VOFFLINE, ... */ }; +/* + * Cache auxiliary data. + */ +struct afs_vnode_cache_aux { + u64 data_version; +} __packed; + #include <trace/events/afs.h> /*****************************************************************************/ diff --git a/fs/afs/volume.c b/fs/afs/volume.c index b517a58..3037bd0 100644 --- a/fs/afs/volume.c +++ b/fs/afs/volume.c @@ -225,7 +225,9 @@ void afs_activate_volume(struct afs_volume *volume) #ifdef CONFIG_AFS_FSCACHE volume->cache = fscache_acquire_cookie(volume->cell->cache, &afs_volume_cache_index_def, - volume, true); + &volume->vid, sizeof(volume->vid), + NULL, 0, + volume, 0, true); #endif write_lock(&volume->cell->proc_lock); @@ -245,7 +247,7 @@ void afs_deactivate_volume(struct afs_volume *volume) write_unlock(&volume->cell->proc_lock); #ifdef CONFIG_AFS_FSCACHE - fscache_relinquish_cookie(volume->cache, + fscache_relinquish_cookie(volume->cache, NULL, test_bit(AFS_VOLUME_DELETED, &volume->flags)); volume->cache = NULL; #endif diff --git a/fs/cachefiles/interface.c b/fs/cachefiles/interface.c index e7f16a7..222bc5d 100644 --- a/fs/cachefiles/interface.c +++ b/fs/cachefiles/interface.c @@ -32,7 +32,7 @@ static struct fscache_object *cachefiles_alloc_object( struct cachefiles_cache *cache; struct cachefiles_xattr *auxdata; unsigned keylen, auxlen; - void *buffer; + void *buffer, *p; char *key; cache = container_of(_cache, struct cachefiles_cache, cache); @@ -65,8 +65,12 @@ static struct fscache_object *cachefiles_alloc_object( if (!buffer) goto nomem_buffer; - keylen = cookie->def->get_key(cookie->netfs_data, buffer + 2, 512); - ASSERTCMP(keylen, <, 512); + keylen = cookie->key_len; + if (keylen <= sizeof(cookie->inline_key)) + p = cookie->inline_key; + else + p = cookie->key; + memcpy(buffer + 2, p, keylen); *(uint16_t *)buffer = keylen; ((char *)buffer)[keylen + 2] = 0; @@ -80,15 +84,17 @@ static struct fscache_object *cachefiles_alloc_object( /* get hold of the auxiliary data and prepend the object type */ auxdata = buffer; - auxlen = 0; - if (cookie->def->get_aux) { - auxlen = cookie->def->get_aux(cookie->netfs_data, - auxdata->data, 511); - ASSERTCMP(auxlen, <, 511); + auxlen = cookie->aux_len; + if (auxlen) { + if (auxlen <= sizeof(cookie->inline_aux)) + p = cookie->inline_aux; + else + p = cookie->aux; + memcpy(auxdata->data, p, auxlen); } auxdata->len = auxlen + 1; - auxdata->type = cookie->def->type; + auxdata->type = cookie->type; lookup_data->auxdata = auxdata; lookup_data->key = key; @@ -177,10 +183,12 @@ static void cachefiles_lookup_complete(struct fscache_object *_object) * increment the usage count on an inode object (may fail if unmounting) */ static -struct fscache_object *cachefiles_grab_object(struct fscache_object *_object) +struct fscache_object *cachefiles_grab_object(struct fscache_object *_object, + enum fscache_obj_ref_trace why) { struct cachefiles_object *object = container_of(_object, struct cachefiles_object, fscache); + int u; _enter("{OBJ%x,%d}", _object->debug_id, atomic_read(&object->usage)); @@ -188,7 +196,9 @@ struct fscache_object *cachefiles_grab_object(struct fscache_object *_object) ASSERT((atomic_read(&object->usage) & 0xffff0000) != 0x6b6b0000); #endif - atomic_inc(&object->usage); + u = atomic_inc_return(&object->usage); + trace_cachefiles_ref(object, _object->cookie, + (enum cachefiles_obj_ref_trace)why, u); return &object->fscache; } @@ -202,6 +212,7 @@ static void cachefiles_update_object(struct fscache_object *_object) struct cachefiles_cache *cache; struct fscache_cookie *cookie; const struct cred *saved_cred; + const void *aux; unsigned auxlen; _enter("{OBJ%x}", _object->debug_id); @@ -216,26 +227,29 @@ static void cachefiles_update_object(struct fscache_object *_object) } cookie = object->fscache.cookie; + auxlen = cookie->aux_len; - if (!cookie->def->get_aux) { + if (!auxlen) { fscache_unuse_cookie(_object); _leave(" [no aux]"); return; } - auxdata = kmalloc(2 + 512 + 3, cachefiles_gfp); + auxdata = kmalloc(2 + auxlen + 3, cachefiles_gfp); if (!auxdata) { fscache_unuse_cookie(_object); _leave(" [nomem]"); return; } - auxlen = cookie->def->get_aux(cookie->netfs_data, auxdata->data, 511); + aux = (auxlen <= sizeof(cookie->inline_aux)) ? + cookie->inline_aux : cookie->aux; + + memcpy(auxdata->data, aux, auxlen); fscache_unuse_cookie(_object); - ASSERTCMP(auxlen, <, 511); auxdata->len = auxlen + 1; - auxdata->type = cookie->def->type; + auxdata->type = cookie->type; cachefiles_begin_secure(cache, &saved_cred); cachefiles_update_object_xattr(object, auxdata); @@ -309,10 +323,12 @@ static void cachefiles_drop_object(struct fscache_object *_object) /* * dispose of a reference to an object */ -static void cachefiles_put_object(struct fscache_object *_object) +static void cachefiles_put_object(struct fscache_object *_object, + enum fscache_obj_ref_trace why) { struct cachefiles_object *object; struct fscache_cache *cache; + int u; ASSERT(_object); @@ -328,7 +344,11 @@ static void cachefiles_put_object(struct fscache_object *_object) ASSERTIFCMP(object->fscache.parent, object->fscache.parent->n_children, >, 0); - if (atomic_dec_and_test(&object->usage)) { + u = atomic_dec_return(&object->usage); + trace_cachefiles_ref(object, _object->cookie, + (enum cachefiles_obj_ref_trace)why, u); + ASSERTCMP(u, !=, -1); + if (u == 0) { _debug("- kill object OBJ%x", object->fscache.debug_id); ASSERT(!test_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags)); @@ -421,7 +441,7 @@ static int cachefiles_attr_changed(struct fscache_object *_object) loff_t oi_size; int ret; - _object->cookie->def->get_attr(_object->cookie->netfs_data, &ni_size); + ni_size = _object->store_limit_l; _enter("{OBJ%x},[%llu]", _object->debug_id, (unsigned long long) ni_size); @@ -493,8 +513,7 @@ static void cachefiles_invalidate_object(struct fscache_operation *op) cache = container_of(object->fscache.cache, struct cachefiles_cache, cache); - op->object->cookie->def->get_attr(op->object->cookie->netfs_data, - &ni_size); + ni_size = op->object->store_limit_l; _enter("{OBJ%x},[%llu]", op->object->debug_id, (unsigned long long)ni_size); diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h index bb3a02c..d2f6f99 100644 --- a/fs/cachefiles/internal.h +++ b/fs/cachefiles/internal.h @@ -124,6 +124,8 @@ struct cachefiles_xattr { uint8_t data[]; }; +#include <trace/events/cachefiles.h> + /* * note change of state for daemon */ diff --git a/fs/cachefiles/main.c b/fs/cachefiles/main.c index 711f13d..f54d3f5 100644 --- a/fs/cachefiles/main.c +++ b/fs/cachefiles/main.c @@ -22,6 +22,7 @@ #include <linux/statfs.h> #include <linux/sysctl.h> #include <linux/miscdevice.h> +#define CREATE_TRACE_POINTS #include "internal.h" unsigned cachefiles_debug; diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index 3978b32..0daa1e3 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -30,11 +30,11 @@ */ static noinline void __cachefiles_printk_object(struct cachefiles_object *object, - const char *prefix, - u8 *keybuf) + const char *prefix) { struct fscache_cookie *cookie; - unsigned keylen, loop; + const u8 *k; + unsigned loop; pr_err("%sobject: OBJ%x\n", prefix, object->fscache.debug_id); pr_err("%sobjstate=%s fl=%lx wbusy=%x ev=%lx[%lx]\n", @@ -56,23 +56,16 @@ void __cachefiles_printk_object(struct cachefiles_object *object, object->fscache.cookie->parent, object->fscache.cookie->netfs_data, object->fscache.cookie->flags); - if (keybuf && cookie->def) - keylen = cookie->def->get_key(cookie->netfs_data, keybuf, - CACHEFILES_KEYBUF_SIZE); - else - keylen = 0; + pr_err("%skey=[%u] '", prefix, cookie->key_len); + k = (cookie->key_len <= sizeof(cookie->inline_key)) ? + cookie->inline_key : cookie->key; + for (loop = 0; loop < cookie->key_len; loop++) + pr_cont("%02x", k[loop]); + pr_cont("'\n"); } else { pr_err("%scookie=NULL\n", prefix); - keylen = 0; } spin_unlock(&object->fscache.lock); - - if (keylen) { - pr_err("%skey=[%u] '", prefix, keylen); - for (loop = 0; loop < keylen; loop++) - pr_cont("%02x", keybuf[loop]); - pr_cont("'\n"); - } } /* @@ -81,14 +74,10 @@ void __cachefiles_printk_object(struct cachefiles_object *object, static noinline void cachefiles_printk_object(struct cachefiles_object *object, struct cachefiles_object *xobject) { - u8 *keybuf; - - keybuf = kmalloc(CACHEFILES_KEYBUF_SIZE, GFP_NOIO); if (object) - __cachefiles_printk_object(object, "", keybuf); + __cachefiles_printk_object(object, ""); if (xobject) - __cachefiles_printk_object(xobject, "x", keybuf); - kfree(keybuf); + __cachefiles_printk_object(xobject, "x"); } /* @@ -120,6 +109,7 @@ static void cachefiles_mark_object_buried(struct cachefiles_cache *cache, } write_unlock(&cache->active_lock); + trace_cachefiles_mark_buried(NULL, dentry, why); _leave(" [no owner]"); return; @@ -130,6 +120,8 @@ found_dentry: object->fscache.state->name, dentry); + trace_cachefiles_mark_buried(object, dentry, why); + if (fscache_object_is_live(&object->fscache)) { pr_err("\n"); pr_err("Error: Can't preemptively bury live object\n"); @@ -158,13 +150,15 @@ static int cachefiles_mark_object_active(struct cachefiles_cache *cache, try_again: write_lock(&cache->active_lock); + dentry = object->dentry; + trace_cachefiles_mark_active(object, dentry); + if (test_and_set_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags)) { pr_err("Error: Object already active\n"); cachefiles_printk_object(object, NULL); BUG(); } - dentry = object->dentry; _p = &cache->active_nodes.rb_node; while (*_p) { _parent = *_p; @@ -191,6 +185,8 @@ try_again: /* an old object from a previous incarnation is hogging the slot - we * need to wait for it to be destroyed */ wait_for_old_object: + trace_cachefiles_wait_active(object, dentry, xobject); + if (fscache_object_is_live(&xobject->fscache)) { pr_err("\n"); pr_err("Error: Unexpected object collision\n"); @@ -248,12 +244,12 @@ wait_for_old_object: ASSERT(!test_bit(CACHEFILES_OBJECT_ACTIVE, &xobject->flags)); - cache->cache.ops->put_object(&xobject->fscache); + cache->cache.ops->put_object(&xobject->fscache, cachefiles_obj_put_wait_retry); goto try_again; requeue: clear_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags); - cache->cache.ops->put_object(&xobject->fscache); + cache->cache.ops->put_object(&xobject->fscache, cachefiles_obj_put_wait_timeo); _leave(" = -ETIMEDOUT"); return -ETIMEDOUT; } @@ -265,6 +261,11 @@ void cachefiles_mark_object_inactive(struct cachefiles_cache *cache, struct cachefiles_object *object, blkcnt_t i_blocks) { + struct dentry *dentry = object->dentry; + struct inode *inode = d_backing_inode(dentry); + + trace_cachefiles_mark_inactive(object, dentry, inode); + write_lock(&cache->active_lock); rb_erase(&object->active_node, &cache->active_nodes); clear_bit(CACHEFILES_OBJECT_ACTIVE, &object->flags); @@ -288,6 +289,7 @@ void cachefiles_mark_object_inactive(struct cachefiles_cache *cache, * - unlocks the directory mutex */ static int cachefiles_bury_object(struct cachefiles_cache *cache, + struct cachefiles_object *object, struct dentry *dir, struct dentry *rep, bool preemptive, @@ -312,6 +314,7 @@ static int cachefiles_bury_object(struct cachefiles_cache *cache, if (ret < 0) { cachefiles_io_error(cache, "Unlink security error"); } else { + trace_cachefiles_unlink(object, rep, why); ret = vfs_unlink(d_inode(dir), rep, NULL); if (preemptive) @@ -413,6 +416,7 @@ try_again: if (ret < 0) { cachefiles_io_error(cache, "Rename security error %d", ret); } else { + trace_cachefiles_rename(object, rep, grave, why); ret = vfs_rename(d_inode(dir), rep, d_inode(cache->graveyard), grave, NULL, 0); if (ret != 0 && ret != -ENOMEM) @@ -458,7 +462,7 @@ int cachefiles_delete_object(struct cachefiles_cache *cache, /* we need to check that our parent is _still_ our parent - it * may have been renamed */ if (dir == object->dentry->d_parent) { - ret = cachefiles_bury_object(cache, dir, + ret = cachefiles_bury_object(cache, object, dir, object->dentry, false, FSCACHE_OBJECT_WAS_RETIRED); } else { @@ -486,6 +490,7 @@ int cachefiles_walk_to_object(struct cachefiles_object *parent, { struct cachefiles_cache *cache; struct dentry *dir, *next = NULL; + struct inode *inode; struct path path; unsigned long start; const char *name; @@ -529,13 +534,17 @@ lookup_again: start = jiffies; next = lookup_one_len(name, dir, nlen); cachefiles_hist(cachefiles_lookup_histogram, start); - if (IS_ERR(next)) + if (IS_ERR(next)) { + trace_cachefiles_lookup(object, next, NULL); goto lookup_error; + } - _debug("next -> %p %s", next, d_backing_inode(next) ? "positive" : "negative"); + inode = d_backing_inode(next); + trace_cachefiles_lookup(object, next, inode); + _debug("next -> %p %s", next, inode ? "positive" : "negative"); if (!key) - object->new = !d_backing_inode(next); + object->new = !inode; /* if this element of the path doesn't exist, then the lookup phase * failed, and we can release any readers in the certain knowledge that @@ -558,6 +567,8 @@ lookup_again: start = jiffies; ret = vfs_mkdir(d_inode(dir), next, 0); cachefiles_hist(cachefiles_mkdir_histogram, start); + if (!key) + trace_cachefiles_mkdir(object, next, ret); if (ret < 0) goto create_error; @@ -587,6 +598,7 @@ lookup_again: start = jiffies; ret = vfs_create(d_inode(dir), next, S_IFREG, true); cachefiles_hist(cachefiles_create_histogram, start); + trace_cachefiles_create(object, next, ret); if (ret < 0) goto create_error; @@ -629,7 +641,8 @@ lookup_again: * mutex) */ object->dentry = NULL; - ret = cachefiles_bury_object(cache, dir, next, true, + ret = cachefiles_bury_object(cache, object, dir, next, + true, FSCACHE_OBJECT_IS_STALE); dput(next); next = NULL; @@ -955,7 +968,7 @@ int cachefiles_cull(struct cachefiles_cache *cache, struct dentry *dir, /* actually remove the victim (drops the dir mutex) */ _debug("bury"); - ret = cachefiles_bury_object(cache, dir, victim, false, + ret = cachefiles_bury_object(cache, NULL, dir, victim, false, FSCACHE_OBJECT_WAS_CULLED); if (ret < 0) goto error; diff --git a/fs/cachefiles/rdwr.c b/fs/cachefiles/rdwr.c index 883bc7b..5082c8a 100644 --- a/fs/cachefiles/rdwr.c +++ b/fs/cachefiles/rdwr.c @@ -952,6 +952,7 @@ error: * - cache withdrawal is prevented by the caller */ void cachefiles_uncache_page(struct fscache_object *_object, struct page *page) + __releases(&object->fscache.cookie->lock) { struct cachefiles_object *object; struct cachefiles_cache *cache; diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c index d31c1a72..0a29a00 100644 --- a/fs/cachefiles/xattr.c +++ b/fs/cachefiles/xattr.c @@ -113,6 +113,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object, /* attempt to install the cache metadata directly */ _debug("SET #%u", auxdata->len); + clear_bit(FSCACHE_COOKIE_AUX_UPDATED, &object->fscache.cookie->flags); ret = vfs_setxattr(dentry, cachefiles_xattr_cache, &auxdata->type, auxdata->len, XATTR_CREATE); @@ -141,6 +142,7 @@ int cachefiles_update_object_xattr(struct cachefiles_object *object, /* attempt to install the cache metadata directly */ _debug("SET #%u", auxdata->len); + clear_bit(FSCACHE_COOKIE_AUX_UPDATED, &object->fscache.cookie->flags); ret = vfs_setxattr(dentry, cachefiles_xattr_cache, &auxdata->type, auxdata->len, XATTR_REPLACE); @@ -180,7 +182,8 @@ int cachefiles_check_auxdata(struct cachefiles_object *object) goto error; xlen--; - validity = fscache_check_aux(&object->fscache, &auxbuf->data, xlen); + validity = fscache_check_aux(&object->fscache, &auxbuf->data, xlen, + i_size_read(d_backing_inode(dentry))); if (validity != FSCACHE_CHECKAUX_OKAY) goto error; @@ -249,7 +252,8 @@ int cachefiles_check_object_xattr(struct cachefiles_object *object, object->fscache.cookie->def->name, dlen); result = fscache_check_aux(&object->fscache, - &auxbuf->data, dlen); + &auxbuf->data, dlen, + i_size_read(d_backing_inode(dentry))); switch (result) { /* entry okay as is */ diff --git a/fs/ceph/cache.c b/fs/ceph/cache.c index a3ab265..33a211b 100644 --- a/fs/ceph/cache.c +++ b/fs/ceph/cache.c @@ -27,7 +27,6 @@ struct ceph_aux_inode { u64 version; struct timespec mtime; - loff_t size; }; struct fscache_netfs ceph_cache_netfs = { @@ -41,34 +40,15 @@ static LIST_HEAD(ceph_fscache_list); struct ceph_fscache_entry { struct list_head list; struct fscache_cookie *fscache; - struct ceph_fsid fsid; size_t uniq_len; + /* The following members must be last */ + struct ceph_fsid fsid; char uniquifier[0]; }; -static uint16_t ceph_fscache_session_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t maxbuf) -{ - const struct ceph_fs_client* fsc = cookie_netfs_data; - const char *fscache_uniq = fsc->mount_options->fscache_uniq; - uint16_t fsid_len, uniq_len; - - fsid_len = sizeof(fsc->client->fsid); - uniq_len = fscache_uniq ? strlen(fscache_uniq) : 0; - if (fsid_len + uniq_len > maxbuf) - return 0; - - memcpy(buffer, &fsc->client->fsid, fsid_len); - if (uniq_len) - memcpy(buffer + fsid_len, fscache_uniq, uniq_len); - - return fsid_len + uniq_len; -} - static const struct fscache_cookie_def ceph_fscache_fsid_object_def = { .name = "CEPH.fsid", .type = FSCACHE_COOKIE_TYPE_INDEX, - .get_key = ceph_fscache_session_get_key, }; int ceph_fscache_register(void) @@ -110,16 +90,19 @@ int ceph_fscache_register_fs(struct ceph_fs_client* fsc) goto out_unlock; } + memcpy(&ent->fsid, fsid, sizeof(*fsid)); + if (uniq_len > 0) { + memcpy(&ent->uniquifier, fscache_uniq, uniq_len); + ent->uniq_len = uniq_len; + } + fsc->fscache = fscache_acquire_cookie(ceph_cache_netfs.primary_index, &ceph_fscache_fsid_object_def, - fsc, true); + &ent->fsid, sizeof(ent->fsid) + uniq_len, + NULL, 0, + fsc, 0, true); if (fsc->fscache) { - memcpy(&ent->fsid, fsid, sizeof(*fsid)); - if (uniq_len > 0) { - memcpy(&ent->uniquifier, fscache_uniq, uniq_len); - ent->uniq_len = uniq_len; - } ent->fscache = fsc->fscache; list_add_tail(&ent->list, &ceph_fscache_list); } else { @@ -133,59 +116,21 @@ out_unlock: return err; } -static uint16_t ceph_fscache_inode_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t maxbuf) -{ - const struct ceph_inode_info* ci = cookie_netfs_data; - uint16_t klen; - - /* use ceph virtual inode (id + snapshot) */ - klen = sizeof(ci->i_vino); - if (klen > maxbuf) - return 0; - - memcpy(buffer, &ci->i_vino, klen); - return klen; -} - -static uint16_t ceph_fscache_inode_get_aux(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - struct ceph_aux_inode aux; - const struct ceph_inode_info* ci = cookie_netfs_data; - const struct inode* inode = &ci->vfs_inode; - - memset(&aux, 0, sizeof(aux)); - aux.version = ci->i_version; - aux.mtime = inode->i_mtime; - aux.size = i_size_read(inode); - - memcpy(buffer, &aux, sizeof(aux)); - - return sizeof(aux); -} - -static void ceph_fscache_inode_get_attr(const void *cookie_netfs_data, - uint64_t *size) -{ - const struct ceph_inode_info* ci = cookie_netfs_data; - *size = i_size_read(&ci->vfs_inode); -} - static enum fscache_checkaux ceph_fscache_inode_check_aux( - void *cookie_netfs_data, const void *data, uint16_t dlen) + void *cookie_netfs_data, const void *data, uint16_t dlen, + loff_t object_size) { struct ceph_aux_inode aux; struct ceph_inode_info* ci = cookie_netfs_data; struct inode* inode = &ci->vfs_inode; - if (dlen != sizeof(aux)) + if (dlen != sizeof(aux) || + i_size_read(inode) != object_size) return FSCACHE_CHECKAUX_OBSOLETE; memset(&aux, 0, sizeof(aux)); aux.version = ci->i_version; aux.mtime = inode->i_mtime; - aux.size = i_size_read(inode); if (memcmp(data, &aux, sizeof(aux)) != 0) return FSCACHE_CHECKAUX_OBSOLETE; @@ -197,9 +142,6 @@ static enum fscache_checkaux ceph_fscache_inode_check_aux( static const struct fscache_cookie_def ceph_fscache_inode_object_def = { .name = "CEPH.inode", .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .get_key = ceph_fscache_inode_get_key, - .get_attr = ceph_fscache_inode_get_attr, - .get_aux = ceph_fscache_inode_get_aux, .check_aux = ceph_fscache_inode_check_aux, }; @@ -207,6 +149,7 @@ void ceph_fscache_register_inode_cookie(struct inode *inode) { struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_fs_client *fsc = ceph_inode_to_client(inode); + struct ceph_aux_inode aux; /* No caching for filesystem */ if (!fsc->fscache) @@ -218,9 +161,14 @@ void ceph_fscache_register_inode_cookie(struct inode *inode) inode_lock_nested(inode, I_MUTEX_CHILD); if (!ci->fscache) { + memset(&aux, 0, sizeof(aux)); + aux.version = ci->i_version; + aux.mtime = inode->i_mtime; ci->fscache = fscache_acquire_cookie(fsc->fscache, - &ceph_fscache_inode_object_def, - ci, false); + &ceph_fscache_inode_object_def, + &ci->i_vino, sizeof(ci->i_vino), + &aux, sizeof(aux), + ci, i_size_read(inode), false); } inode_unlock(inode); } @@ -235,7 +183,7 @@ void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci) ci->fscache = NULL; fscache_uncache_all_inode_pages(cookie, &ci->vfs_inode); - fscache_relinquish_cookie(cookie, 0); + fscache_relinquish_cookie(cookie, &ci->i_vino, false); } static bool ceph_fscache_can_enable(void *data) @@ -254,11 +202,11 @@ void ceph_fscache_file_set_cookie(struct inode *inode, struct file *filp) if (inode_is_open_for_write(inode)) { dout("fscache_file_set_cookie %p %p disabling cache\n", inode, filp); - fscache_disable_cookie(ci->fscache, false); + fscache_disable_cookie(ci->fscache, &ci->i_vino, false); fscache_uncache_all_inode_pages(ci->fscache, inode); } else { - fscache_enable_cookie(ci->fscache, ceph_fscache_can_enable, - inode); + fscache_enable_cookie(ci->fscache, &ci->i_vino, i_size_read(inode), + ceph_fscache_can_enable, inode); if (fscache_cookie_enabled(ci->fscache)) { dout("fscache_file_set_cookie %p %p enabling cache\n", inode, filp); @@ -351,7 +299,8 @@ void ceph_readpage_to_fscache(struct inode *inode, struct page *page) if (!cache_valid(ci)) return; - ret = fscache_write_page(ci->fscache, page, GFP_KERNEL); + ret = fscache_write_page(ci->fscache, page, i_size_read(inode), + GFP_KERNEL); if (ret) fscache_uncache_page(ci->fscache, page); } @@ -385,7 +334,7 @@ void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc) WARN_ON_ONCE(!found); mutex_unlock(&ceph_fscache_lock); - __fscache_relinquish_cookie(fsc->fscache, 0); + __fscache_relinquish_cookie(fsc->fscache, NULL, false); } fsc->fscache = NULL; } @@ -402,7 +351,7 @@ void ceph_fscache_revalidate_cookie(struct ceph_inode_info *ci) * truncate while the caller holds CEPH_CAP_FILE_RD */ mutex_lock(&ci->i_truncate_mutex); if (!cache_valid(ci)) { - if (fscache_check_consistency(ci->fscache)) + if (fscache_check_consistency(ci->fscache, &ci->i_vino)) fscache_invalidate(ci->fscache); spin_lock(&ci->i_ceph_lock); ci->i_fscache_gen = ci->i_rdcache_gen; diff --git a/fs/cifs/cache.c b/fs/cifs/cache.c index 2c14020..edf5f408 100644 --- a/fs/cifs/cache.c +++ b/fs/cifs/cache.c @@ -46,67 +46,11 @@ void cifs_fscache_unregister(void) } /* - * Key layout of CIFS server cache index object - */ -struct cifs_server_key { - uint16_t family; /* address family */ - __be16 port; /* IP port */ - union { - struct in_addr ipv4_addr; - struct in6_addr ipv6_addr; - } addr[0]; -}; - -/* - * Server object keyed by {IPaddress,port,family} tuple - */ -static uint16_t cifs_server_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t maxbuf) -{ - const struct TCP_Server_Info *server = cookie_netfs_data; - const struct sockaddr *sa = (struct sockaddr *) &server->dstaddr; - const struct sockaddr_in *addr = (struct sockaddr_in *) sa; - const struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *) sa; - struct cifs_server_key *key = buffer; - uint16_t key_len = sizeof(struct cifs_server_key); - - memset(key, 0, key_len); - - /* - * Should not be a problem as sin_family/sin6_family overlays - * sa_family field - */ - switch (sa->sa_family) { - case AF_INET: - key->family = sa->sa_family; - key->port = addr->sin_port; - key->addr[0].ipv4_addr = addr->sin_addr; - key_len += sizeof(key->addr[0].ipv4_addr); - break; - - case AF_INET6: - key->family = sa->sa_family; - key->port = addr6->sin6_port; - key->addr[0].ipv6_addr = addr6->sin6_addr; - key_len += sizeof(key->addr[0].ipv6_addr); - break; - - default: - cifs_dbg(VFS, "Unknown network family '%d'\n", sa->sa_family); - key_len = 0; - break; - } - - return key_len; -} - -/* * Server object for FS-Cache */ const struct fscache_cookie_def cifs_fscache_server_index_def = { .name = "CIFS.server", .type = FSCACHE_COOKIE_TYPE_INDEX, - .get_key = cifs_server_get_key, }; /* @@ -116,7 +60,7 @@ struct cifs_fscache_super_auxdata { u64 resource_id; /* unique server resource id */ }; -static char *extract_sharename(const char *treename) +char *extract_sharename(const char *treename) { const char *src; char *delim, *dst; @@ -140,56 +84,11 @@ static char *extract_sharename(const char *treename) return dst; } -/* - * Superblock object currently keyed by share name - */ -static uint16_t cifs_super_get_key(const void *cookie_netfs_data, void *buffer, - uint16_t maxbuf) -{ - const struct cifs_tcon *tcon = cookie_netfs_data; - char *sharename; - uint16_t len; - - sharename = extract_sharename(tcon->treeName); - if (IS_ERR(sharename)) { - cifs_dbg(FYI, "%s: couldn't extract sharename\n", __func__); - sharename = NULL; - return 0; - } - - len = strlen(sharename); - if (len > maxbuf) - return 0; - - memcpy(buffer, sharename, len); - - kfree(sharename); - - return len; -} - -static uint16_t -cifs_fscache_super_get_aux(const void *cookie_netfs_data, void *buffer, - uint16_t maxbuf) -{ - struct cifs_fscache_super_auxdata auxdata; - const struct cifs_tcon *tcon = cookie_netfs_data; - - memset(&auxdata, 0, sizeof(auxdata)); - auxdata.resource_id = tcon->resource_id; - - if (maxbuf > sizeof(auxdata)) - maxbuf = sizeof(auxdata); - - memcpy(buffer, &auxdata, maxbuf); - - return maxbuf; -} - static enum fscache_checkaux cifs_fscache_super_check_aux(void *cookie_netfs_data, const void *data, - uint16_t datalen) + uint16_t datalen, + loff_t object_size) { struct cifs_fscache_super_auxdata auxdata; const struct cifs_tcon *tcon = cookie_netfs_data; @@ -212,68 +111,14 @@ fscache_checkaux cifs_fscache_super_check_aux(void *cookie_netfs_data, const struct fscache_cookie_def cifs_fscache_super_index_def = { .name = "CIFS.super", .type = FSCACHE_COOKIE_TYPE_INDEX, - .get_key = cifs_super_get_key, - .get_aux = cifs_fscache_super_get_aux, .check_aux = cifs_fscache_super_check_aux, }; -/* - * Auxiliary data attached to CIFS inode within the cache - */ -struct cifs_fscache_inode_auxdata { - struct timespec last_write_time; - struct timespec last_change_time; - u64 eof; -}; - -static uint16_t cifs_fscache_inode_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t maxbuf) -{ - const struct cifsInodeInfo *cifsi = cookie_netfs_data; - uint16_t keylen; - - /* use the UniqueId as the key */ - keylen = sizeof(cifsi->uniqueid); - if (keylen > maxbuf) - keylen = 0; - else - memcpy(buffer, &cifsi->uniqueid, keylen); - - return keylen; -} - -static void -cifs_fscache_inode_get_attr(const void *cookie_netfs_data, uint64_t *size) -{ - const struct cifsInodeInfo *cifsi = cookie_netfs_data; - - *size = cifsi->vfs_inode.i_size; -} - -static uint16_t -cifs_fscache_inode_get_aux(const void *cookie_netfs_data, void *buffer, - uint16_t maxbuf) -{ - struct cifs_fscache_inode_auxdata auxdata; - const struct cifsInodeInfo *cifsi = cookie_netfs_data; - - memset(&auxdata, 0, sizeof(auxdata)); - auxdata.eof = cifsi->server_eof; - auxdata.last_write_time = cifsi->vfs_inode.i_mtime; - auxdata.last_change_time = cifsi->vfs_inode.i_ctime; - - if (maxbuf > sizeof(auxdata)) - maxbuf = sizeof(auxdata); - - memcpy(buffer, &auxdata, maxbuf); - - return maxbuf; -} - static enum fscache_checkaux cifs_fscache_inode_check_aux(void *cookie_netfs_data, const void *data, - uint16_t datalen) + uint16_t datalen, + loff_t object_size) { struct cifs_fscache_inode_auxdata auxdata; struct cifsInodeInfo *cifsi = cookie_netfs_data; @@ -295,8 +140,5 @@ fscache_checkaux cifs_fscache_inode_check_aux(void *cookie_netfs_data, const struct fscache_cookie_def cifs_fscache_inode_object_def = { .name = "CIFS.uniqueid", .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .get_key = cifs_fscache_inode_get_key, - .get_attr = cifs_fscache_inode_get_attr, - .get_aux = cifs_fscache_inode_get_aux, .check_aux = cifs_fscache_inode_check_aux, }; diff --git a/fs/cifs/fscache.c b/fs/cifs/fscache.c index 8d4b7bc..25d3f66 100644 --- a/fs/cifs/fscache.c +++ b/fs/cifs/fscache.c @@ -23,11 +23,63 @@ #include "cifs_debug.h" #include "cifs_fs_sb.h" +/* + * Key layout of CIFS server cache index object + */ +struct cifs_server_key { + struct { + uint16_t family; /* address family */ + __be16 port; /* IP port */ + } hdr; + union { + struct in_addr ipv4_addr; + struct in6_addr ipv6_addr; + }; +} __packed; + +/* + * Get a cookie for a server object keyed by {IPaddress,port,family} tuple + */ void cifs_fscache_get_client_cookie(struct TCP_Server_Info *server) { + const struct sockaddr *sa = (struct sockaddr *) &server->dstaddr; + const struct sockaddr_in *addr = (struct sockaddr_in *) sa; + const struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *) sa; + struct cifs_server_key key; + uint16_t key_len = sizeof(key.hdr); + + memset(&key, 0, sizeof(key)); + + /* + * Should not be a problem as sin_family/sin6_family overlays + * sa_family field + */ + key.hdr.family = sa->sa_family; + switch (sa->sa_family) { + case AF_INET: + key.hdr.port = addr->sin_port; + key.ipv4_addr = addr->sin_addr; + key_len += sizeof(key.ipv4_addr); + break; + + case AF_INET6: + key.hdr.port = addr6->sin6_port; + key.ipv6_addr = addr6->sin6_addr; + key_len += sizeof(key.ipv6_addr); + break; + + default: + cifs_dbg(VFS, "Unknown network family '%d'\n", sa->sa_family); + server->fscache = NULL; + return; + } + server->fscache = fscache_acquire_cookie(cifs_fscache_netfs.primary_index, - &cifs_fscache_server_index_def, server, true); + &cifs_fscache_server_index_def, + &key, key_len, + NULL, 0, + server, 0, true); cifs_dbg(FYI, "%s: (0x%p/0x%p)\n", __func__, server, server->fscache); } @@ -36,17 +88,29 @@ void cifs_fscache_release_client_cookie(struct TCP_Server_Info *server) { cifs_dbg(FYI, "%s: (0x%p/0x%p)\n", __func__, server, server->fscache); - fscache_relinquish_cookie(server->fscache, 0); + fscache_relinquish_cookie(server->fscache, NULL, false); server->fscache = NULL; } void cifs_fscache_get_super_cookie(struct cifs_tcon *tcon) { struct TCP_Server_Info *server = tcon->ses->server; + char *sharename; + + sharename = extract_sharename(tcon->treeName); + if (IS_ERR(sharename)) { + cifs_dbg(FYI, "%s: couldn't extract sharename\n", __func__); + tcon->fscache = NULL; + return; + } tcon->fscache = fscache_acquire_cookie(server->fscache, - &cifs_fscache_super_index_def, tcon, true); + &cifs_fscache_super_index_def, + sharename, strlen(sharename), + &tcon->resource_id, sizeof(tcon->resource_id), + tcon, 0, true); + kfree(sharename); cifs_dbg(FYI, "%s: (0x%p/0x%p)\n", __func__, server->fscache, tcon->fscache); } @@ -54,10 +118,28 @@ void cifs_fscache_get_super_cookie(struct cifs_tcon *tcon) void cifs_fscache_release_super_cookie(struct cifs_tcon *tcon) { cifs_dbg(FYI, "%s: (0x%p)\n", __func__, tcon->fscache); - fscache_relinquish_cookie(tcon->fscache, 0); + fscache_relinquish_cookie(tcon->fscache, &tcon->resource_id, false); tcon->fscache = NULL; } +static void cifs_fscache_acquire_inode_cookie(struct cifsInodeInfo *cifsi, + struct cifs_tcon *tcon) +{ + struct cifs_fscache_inode_auxdata auxdata; + + memset(&auxdata, 0, sizeof(auxdata)); + auxdata.eof = cifsi->server_eof; + auxdata.last_write_time = cifsi->vfs_inode.i_mtime; + auxdata.last_change_time = cifsi->vfs_inode.i_ctime; + + cifsi->fscache = + fscache_acquire_cookie(tcon->fscache, + &cifs_fscache_inode_object_def, + &cifsi->uniqueid, sizeof(cifsi->uniqueid), + &auxdata, sizeof(auxdata), + cifsi, cifsi->vfs_inode.i_size, true); +} + static void cifs_fscache_enable_inode_cookie(struct inode *inode) { struct cifsInodeInfo *cifsi = CIFS_I(inode); @@ -67,21 +149,28 @@ static void cifs_fscache_enable_inode_cookie(struct inode *inode) if (cifsi->fscache) return; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_FSCACHE) { - cifsi->fscache = fscache_acquire_cookie(tcon->fscache, - &cifs_fscache_inode_object_def, cifsi, true); - cifs_dbg(FYI, "%s: got FH cookie (0x%p/0x%p)\n", - __func__, tcon->fscache, cifsi->fscache); - } + if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_FSCACHE)) + return; + + cifs_fscache_acquire_inode_cookie(cifsi, tcon); + + cifs_dbg(FYI, "%s: got FH cookie (0x%p/0x%p)\n", + __func__, tcon->fscache, cifsi->fscache); } void cifs_fscache_release_inode_cookie(struct inode *inode) { + struct cifs_fscache_inode_auxdata auxdata; struct cifsInodeInfo *cifsi = CIFS_I(inode); if (cifsi->fscache) { + memset(&auxdata, 0, sizeof(auxdata)); + auxdata.eof = cifsi->server_eof; + auxdata.last_write_time = cifsi->vfs_inode.i_mtime; + auxdata.last_change_time = cifsi->vfs_inode.i_ctime; + cifs_dbg(FYI, "%s: (0x%p)\n", __func__, cifsi->fscache); - fscache_relinquish_cookie(cifsi->fscache, 0); + fscache_relinquish_cookie(cifsi->fscache, &auxdata, false); cifsi->fscache = NULL; } } @@ -93,7 +182,7 @@ static void cifs_fscache_disable_inode_cookie(struct inode *inode) if (cifsi->fscache) { cifs_dbg(FYI, "%s: (0x%p)\n", __func__, cifsi->fscache); fscache_uncache_all_inode_pages(cifsi->fscache, inode); - fscache_relinquish_cookie(cifsi->fscache, 1); + fscache_relinquish_cookie(cifsi->fscache, NULL, true); cifsi->fscache = NULL; } } @@ -110,16 +199,14 @@ void cifs_fscache_reset_inode_cookie(struct inode *inode) { struct cifsInodeInfo *cifsi = CIFS_I(inode); struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); struct fscache_cookie *old = cifsi->fscache; if (cifsi->fscache) { /* retire the current fscache cache and get a new one */ - fscache_relinquish_cookie(cifsi->fscache, 1); + fscache_relinquish_cookie(cifsi->fscache, NULL, true); - cifsi->fscache = fscache_acquire_cookie( - cifs_sb_master_tcon(cifs_sb)->fscache, - &cifs_fscache_inode_object_def, - cifsi, true); + cifs_fscache_acquire_inode_cookie(cifsi, tcon); cifs_dbg(FYI, "%s: new cookie 0x%p oldcookie 0x%p\n", __func__, cifsi->fscache, old); } @@ -214,13 +301,15 @@ int __cifs_readpages_from_fscache(struct inode *inode, void __cifs_readpage_to_fscache(struct inode *inode, struct page *page) { + struct cifsInodeInfo *cifsi = CIFS_I(inode); int ret; cifs_dbg(FYI, "%s: (fsc: %p, p: %p, i: %p)\n", - __func__, CIFS_I(inode)->fscache, page, inode); - ret = fscache_write_page(CIFS_I(inode)->fscache, page, GFP_KERNEL); + __func__, cifsi->fscache, page, inode); + ret = fscache_write_page(cifsi->fscache, page, + cifsi->vfs_inode.i_size, GFP_KERNEL); if (ret != 0) - fscache_uncache_page(CIFS_I(inode)->fscache, page); + fscache_uncache_page(cifsi->fscache, page); } void __cifs_fscache_readpages_cancel(struct inode *inode, struct list_head *pages) @@ -239,4 +328,3 @@ void __cifs_fscache_invalidate_page(struct page *page, struct inode *inode) fscache_wait_on_page_write(cookie, page); fscache_uncache_page(cookie, page); } - diff --git a/fs/cifs/fscache.h b/fs/cifs/fscache.h index 24794b6..c7e3ac2 100644 --- a/fs/cifs/fscache.h +++ b/fs/cifs/fscache.h @@ -27,6 +27,18 @@ #ifdef CONFIG_CIFS_FSCACHE +/* + * Auxiliary data attached to CIFS inode within the cache + */ +struct cifs_fscache_inode_auxdata { + struct timespec last_write_time; + struct timespec last_change_time; + u64 eof; +}; + +/* + * cache.c + */ extern struct fscache_netfs cifs_fscache_netfs; extern const struct fscache_cookie_def cifs_fscache_server_index_def; extern const struct fscache_cookie_def cifs_fscache_super_index_def; @@ -34,6 +46,7 @@ extern const struct fscache_cookie_def cifs_fscache_inode_object_def; extern int cifs_fscache_register(void); extern void cifs_fscache_unregister(void); +extern char *extract_sharename(const char *); /* * fscache.c diff --git a/fs/fscache/cache.c b/fs/fscache/cache.c index 56cce7f..c184c5a 100644 --- a/fs/fscache/cache.c +++ b/fs/fscache/cache.c @@ -125,7 +125,7 @@ struct fscache_cache *fscache_select_cache_for_object( } /* the parent is unbacked */ - if (cookie->def->type != FSCACHE_COOKIE_TYPE_INDEX) { + if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { /* cookie not an index and is unbacked */ spin_unlock(&cookie->lock); _leave(" = NULL [cookie ub,ni]"); diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c index d705125..7dc55b9 100644 --- a/fs/fscache/cookie.c +++ b/fs/fscache/cookie.c @@ -21,12 +21,54 @@ struct kmem_cache *fscache_cookie_jar; static atomic_t fscache_object_debug_id = ATOMIC_INIT(0); -static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie); +#define fscache_cookie_hash_shift 15 +static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; + +static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie, + loff_t object_size); static int fscache_alloc_object(struct fscache_cache *cache, struct fscache_cookie *cookie); static int fscache_attach_object(struct fscache_cookie *cookie, struct fscache_object *object); +static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) +{ + struct hlist_node *object; + const u8 *k; + unsigned loop; + + pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n", + prefix, cookie, cookie->parent, cookie->flags, + atomic_read(&cookie->n_children), + atomic_read(&cookie->n_active)); + pr_err("%c-cookie d=%p n=%p\n", + prefix, cookie->def, cookie->netfs_data); + + object = READ_ONCE(cookie->backing_objects.first); + if (object) + pr_err("%c-cookie o=%p\n", + prefix, hlist_entry(object, struct fscache_object, cookie_link)); + + pr_err("%c-key=[%u] '", prefix, cookie->key_len); + k = (cookie->key_len <= sizeof(cookie->inline_key)) ? + cookie->inline_key : cookie->key; + for (loop = 0; loop < cookie->key_len; loop++) + pr_cont("%02x", k[loop]); + pr_cont("'\n"); +} + +void fscache_free_cookie(struct fscache_cookie *cookie) +{ + if (cookie) { + BUG_ON(!hlist_empty(&cookie->backing_objects)); + if (cookie->aux_len > sizeof(cookie->inline_aux)) + kfree(cookie->aux); + if (cookie->key_len > sizeof(cookie->inline_key)) + kfree(cookie->key); + kmem_cache_free(fscache_cookie_jar, cookie); + } +} + /* * initialise an cookie jar slab element prior to any use */ @@ -41,6 +83,170 @@ void fscache_cookie_init_once(void *_cookie) } /* + * Set the index key in a cookie. The cookie struct has space for a 12-byte + * key plus length and hash, but if that's not big enough, it's instead a + * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then + * the key data. + */ +static int fscache_set_key(struct fscache_cookie *cookie, + const void *index_key, size_t index_key_len) +{ + unsigned long long h; + u32 *buf; + int i; + + cookie->key_len = index_key_len; + + if (index_key_len > sizeof(cookie->inline_key)) { + buf = kzalloc(index_key_len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + cookie->key = buf; + } else { + buf = (u32 *)cookie->inline_key; + buf[0] = 0; + buf[1] = 0; + buf[2] = 0; + } + + memcpy(buf, index_key, index_key_len); + + /* Calculate a hash and combine this with the length in the first word + * or first half word + */ + h = (unsigned long)cookie->parent; + h += index_key_len + cookie->type; + for (i = 0; i < (index_key_len + sizeof(u32) - 1) / sizeof(u32); i++) + h += buf[i]; + + cookie->key_hash = h ^ (h >> 32); + return 0; +} + +static long fscache_compare_cookie(const struct fscache_cookie *a, + const struct fscache_cookie *b) +{ + const void *ka, *kb; + + if (a->key_hash != b->key_hash) + return (long)a->key_hash - (long)b->key_hash; + if (a->parent != b->parent) + return (long)a->parent - (long)b->parent; + if (a->key_len != b->key_len) + return (long)a->key_len - (long)b->key_len; + if (a->type != b->type) + return (long)a->type - (long)b->type; + + if (a->key_len <= sizeof(a->inline_key)) { + ka = &a->inline_key; + kb = &b->inline_key; + } else { + ka = a->key; + kb = b->key; + } + return memcmp(ka, kb, a->key_len); +} + +/* + * Allocate a cookie. + */ +struct fscache_cookie *fscache_alloc_cookie( + struct fscache_cookie *parent, + const struct fscache_cookie_def *def, + const void *index_key, size_t index_key_len, + const void *aux_data, size_t aux_data_len, + void *netfs_data, + loff_t object_size) +{ + struct fscache_cookie *cookie; + + /* allocate and initialise a cookie */ + cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL); + if (!cookie) + return NULL; + + cookie->key_len = index_key_len; + cookie->aux_len = aux_data_len; + + if (fscache_set_key(cookie, index_key, index_key_len) < 0) + goto nomem; + + if (cookie->aux_len <= sizeof(cookie->inline_aux)) { + memcpy(cookie->inline_aux, aux_data, cookie->aux_len); + } else { + cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL); + if (!cookie->aux) + goto nomem; + } + + atomic_set(&cookie->usage, 1); + atomic_set(&cookie->n_children, 0); + + /* We keep the active count elevated until relinquishment to prevent an + * attempt to wake up every time the object operations queue quiesces. + */ + atomic_set(&cookie->n_active, 1); + + cookie->def = def; + cookie->parent = parent; + cookie->netfs_data = netfs_data; + cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET); + cookie->type = def->type; + + /* radix tree insertion won't use the preallocation pool unless it's + * told it may not wait */ + INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM); + return cookie; + +nomem: + fscache_free_cookie(cookie); + return NULL; +} + +/* + * Attempt to insert the new cookie into the hash. If there's a collision, we + * return the old cookie if it's not in use and an error otherwise. + */ +struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate) +{ + struct fscache_cookie *cursor; + struct hlist_bl_head *h; + struct hlist_bl_node *p; + unsigned int bucket; + + bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); + h = &fscache_cookie_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_compare_cookie(candidate, cursor) == 0) + goto collision; + } + + __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags); + fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent); + atomic_inc(&candidate->parent->n_children); + hlist_bl_add_head(&candidate->hash_link, h); + hlist_bl_unlock(h); + return candidate; + +collision: + if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) { + trace_fscache_cookie(cursor, fscache_cookie_collision, + atomic_read(&cursor->usage)); + pr_err("Duplicate cookie detected\n"); + fscache_print_cookie(cursor, 'O'); + fscache_print_cookie(candidate, 'N'); + hlist_bl_unlock(h); + return NULL; + } + + fscache_cookie_get(cursor, fscache_cookie_get_reacquire); + hlist_bl_unlock(h); + return cursor; +} + +/* * request a cookie to represent an object (index, datafile, xattr, etc) * - parent specifies the parent object * - the top level index cookie for each netfs is stored in the fscache_netfs @@ -58,10 +264,13 @@ void fscache_cookie_init_once(void *_cookie) struct fscache_cookie *__fscache_acquire_cookie( struct fscache_cookie *parent, const struct fscache_cookie_def *def, + const void *index_key, size_t index_key_len, + const void *aux_data, size_t aux_data_len, void *netfs_data, + loff_t object_size, bool enable) { - struct fscache_cookie *cookie; + struct fscache_cookie *candidate, *cookie; BUG_ON(!def); @@ -69,6 +278,13 @@ struct fscache_cookie *__fscache_acquire_cookie( parent ? (char *) parent->def->name : "<no-parent>", def->name, netfs_data, enable); + if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) + return NULL; + if (!aux_data || !aux_data_len) { + aux_data = NULL; + aux_data_len = 0; + } + fscache_stat(&fscache_n_acquires); /* if there's no parent cookie, then we don't create one here either */ @@ -79,41 +295,31 @@ struct fscache_cookie *__fscache_acquire_cookie( } /* validate the definition */ - BUG_ON(!def->get_key); BUG_ON(!def->name[0]); BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX && - parent->def->type != FSCACHE_COOKIE_TYPE_INDEX); + parent->type != FSCACHE_COOKIE_TYPE_INDEX); - /* allocate and initialise a cookie */ - cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL); - if (!cookie) { + candidate = fscache_alloc_cookie(parent, def, + index_key, index_key_len, + aux_data, aux_data_len, + netfs_data, object_size); + if (!candidate) { fscache_stat(&fscache_n_acquires_oom); _leave(" [ENOMEM]"); return NULL; } - atomic_set(&cookie->usage, 1); - atomic_set(&cookie->n_children, 0); - - /* We keep the active count elevated until relinquishment to prevent an - * attempt to wake up every time the object operations queue quiesces. - */ - atomic_set(&cookie->n_active, 1); - - atomic_inc(&parent->usage); - atomic_inc(&parent->n_children); + cookie = fscache_hash_cookie(candidate); + if (!cookie) { + trace_fscache_cookie(candidate, fscache_cookie_discard, 1); + goto out; + } - cookie->def = def; - cookie->parent = parent; - cookie->netfs_data = netfs_data; - cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET); + if (cookie == candidate) + candidate = NULL; - /* radix tree insertion won't use the preallocation pool unless it's - * told it may not wait */ - INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM); - - switch (cookie->def->type) { + switch (cookie->type) { case FSCACHE_COOKIE_TYPE_INDEX: fscache_stat(&fscache_n_cookie_index); break; @@ -125,16 +331,19 @@ struct fscache_cookie *__fscache_acquire_cookie( break; } + trace_fscache_acquire(cookie); + if (enable) { /* if the object is an index then we need do nothing more here * - we create indices on disk when we need them as an index * may exist in multiple caches */ - if (cookie->def->type != FSCACHE_COOKIE_TYPE_INDEX) { - if (fscache_acquire_non_index_cookie(cookie) == 0) { + if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { + if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) { set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); } else { atomic_dec(&parent->n_children); - __fscache_cookie_put(cookie); + fscache_cookie_put(cookie, + fscache_cookie_put_acquire_nobufs); fscache_stat(&fscache_n_acquires_nobufs); _leave(" = NULL"); return NULL; @@ -145,7 +354,9 @@ struct fscache_cookie *__fscache_acquire_cookie( } fscache_stat(&fscache_n_acquires_ok); - _leave(" = %p", cookie); + +out: + fscache_free_cookie(candidate); return cookie; } EXPORT_SYMBOL(__fscache_acquire_cookie); @@ -154,24 +365,30 @@ EXPORT_SYMBOL(__fscache_acquire_cookie); * Enable a cookie to permit it to accept new operations. */ void __fscache_enable_cookie(struct fscache_cookie *cookie, + const void *aux_data, + loff_t object_size, bool (*can_enable)(void *data), void *data) { _enter("%p", cookie); + trace_fscache_enable(cookie); + wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK, TASK_UNINTERRUPTIBLE); + fscache_update_aux(cookie, aux_data); + if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags)) goto out_unlock; if (can_enable && !can_enable(data)) { /* The netfs decided it didn't want to enable after all */ - } else if (cookie->def->type != FSCACHE_COOKIE_TYPE_INDEX) { + } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { /* Wait for outstanding disablement to complete */ __fscache_wait_on_invalidate(cookie); - if (fscache_acquire_non_index_cookie(cookie) == 0) + if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); } else { set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); @@ -188,11 +405,11 @@ EXPORT_SYMBOL(__fscache_enable_cookie); * - this must make sure the index chain is instantiated and instantiate the * object representation too */ -static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie) +static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie, + loff_t object_size) { struct fscache_object *object; struct fscache_cache *cache; - uint64_t i_size; int ret; _enter(""); @@ -231,9 +448,6 @@ static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie) return ret; } - /* pass on how big the object we're caching is supposed to be */ - cookie->def->get_attr(cookie->netfs_data, &i_size); - spin_lock(&cookie->lock); if (hlist_empty(&cookie->backing_objects)) { spin_unlock(&cookie->lock); @@ -243,7 +457,7 @@ static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie) object = hlist_entry(cookie->backing_objects.first, struct fscache_object, cookie_link); - fscache_set_store_limit(object, i_size); + fscache_set_store_limit(object, object_size); /* initiate the process of looking up all the objects in the chain * (done by fscache_initialise_object()) */ @@ -318,7 +532,7 @@ static int fscache_alloc_object(struct fscache_cache *cache, * attached to the cookie */ if (fscache_attach_object(cookie, object) < 0) { fscache_stat(&fscache_n_cop_put_object); - cache->ops->put_object(object); + cache->ops->put_object(object, fscache_obj_put_attach_fail); fscache_stat_d(&fscache_n_cop_put_object); } @@ -338,7 +552,7 @@ object_already_extant: error_put: fscache_stat(&fscache_n_cop_put_object); - cache->ops->put_object(object); + cache->ops->put_object(object, fscache_obj_put_alloc_fail); fscache_stat_d(&fscache_n_cop_put_object); error: _leave(" = %d", ret); @@ -398,7 +612,7 @@ static int fscache_attach_object(struct fscache_cookie *cookie, /* attach to the cookie */ object->cookie = cookie; - atomic_inc(&cookie->usage); + fscache_cookie_get(cookie, fscache_cookie_get_attach_object); hlist_add_head(&object->cookie_link, &cookie->backing_objects); fscache_objlist_add(object); @@ -426,10 +640,7 @@ void __fscache_invalidate(struct fscache_cookie *cookie) * there, and if it's doing that, it may as well just retire the * cookie. */ - ASSERTCMP(cookie->def->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); - - /* We will be updating the cookie too. */ - BUG_ON(!cookie->def->get_aux); + ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); /* If there's an object, we tell the object state machine to handle the * invalidation on our behalf, otherwise there's nothing to do. @@ -473,7 +684,7 @@ EXPORT_SYMBOL(__fscache_wait_on_invalidate); /* * update the index entries backing a cookie */ -void __fscache_update_cookie(struct fscache_cookie *cookie) +void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data) { struct fscache_object *object; @@ -487,10 +698,10 @@ void __fscache_update_cookie(struct fscache_cookie *cookie) _enter("{%s}", cookie->def->name); - BUG_ON(!cookie->def->get_aux); - spin_lock(&cookie->lock); + fscache_update_aux(cookie, aux_data); + if (fscache_cookie_enabled(cookie)) { /* update the index entry on disk in each cache backing this * cookie. @@ -509,13 +720,17 @@ EXPORT_SYMBOL(__fscache_update_cookie); /* * Disable a cookie to stop it from accepting new requests from the netfs. */ -void __fscache_disable_cookie(struct fscache_cookie *cookie, bool invalidate) +void __fscache_disable_cookie(struct fscache_cookie *cookie, + const void *aux_data, + bool invalidate) { struct fscache_object *object; bool awaken = false; _enter("%p,%u", cookie, invalidate); + trace_fscache_disable(cookie); + ASSERTCMP(atomic_read(&cookie->n_active), >, 0); if (atomic_read(&cookie->n_children) != 0) { @@ -526,6 +741,9 @@ void __fscache_disable_cookie(struct fscache_cookie *cookie, bool invalidate) wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK, TASK_UNINTERRUPTIBLE); + + fscache_update_aux(cookie, aux_data); + if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags)) goto out_unlock_enable; @@ -563,7 +781,7 @@ void __fscache_disable_cookie(struct fscache_cookie *cookie, bool invalidate) } /* Make sure any pending writes are cancelled. */ - if (cookie->def->type != FSCACHE_COOKIE_TYPE_INDEX) + if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) fscache_invalidate_writes(cookie); /* Reset the cookie state if it wasn't relinquished */ @@ -585,7 +803,9 @@ EXPORT_SYMBOL(__fscache_disable_cookie); * - all dependents of this cookie must have already been unregistered * (indices/files/pages) */ -void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) +void __fscache_relinquish_cookie(struct fscache_cookie *cookie, + const void *aux_data, + bool retire) { fscache_stat(&fscache_n_relinquishes); if (retire) @@ -601,10 +821,13 @@ void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) cookie, cookie->def->name, cookie->netfs_data, atomic_read(&cookie->n_active), retire); + trace_fscache_relinquish(cookie, retire); + /* No further netfs-accessing operations on this cookie permitted */ - set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags); + if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) + BUG(); - __fscache_disable_cookie(cookie, retire); + __fscache_disable_cookie(cookie, aux_data, retire); /* Clear pointers back to the netfs */ cookie->netfs_data = NULL; @@ -619,35 +842,54 @@ void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) /* Dispose of the netfs's link to the cookie */ ASSERTCMP(atomic_read(&cookie->usage), >, 0); - fscache_cookie_put(cookie); + fscache_cookie_put(cookie, fscache_cookie_put_relinquish); _leave(""); } EXPORT_SYMBOL(__fscache_relinquish_cookie); /* - * destroy a cookie + * Remove a cookie from the hash table. */ -void __fscache_cookie_put(struct fscache_cookie *cookie) +static void fscache_unhash_cookie(struct fscache_cookie *cookie) +{ + struct hlist_bl_head *h; + unsigned int bucket; + + bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); + h = &fscache_cookie_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_del(&cookie->hash_link); + hlist_bl_unlock(h); +} + +/* + * Drop a reference to a cookie. + */ +void fscache_cookie_put(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) { struct fscache_cookie *parent; + int usage; _enter("%p", cookie); - for (;;) { - _debug("FREE COOKIE %p", cookie); - parent = cookie->parent; - BUG_ON(!hlist_empty(&cookie->backing_objects)); - kmem_cache_free(fscache_cookie_jar, cookie); + do { + usage = atomic_dec_return(&cookie->usage); + trace_fscache_cookie(cookie, where, usage); - if (!parent) - break; + if (usage > 0) + return; + BUG_ON(usage < 0); + + parent = cookie->parent; + fscache_unhash_cookie(cookie); + fscache_free_cookie(cookie); cookie = parent; - BUG_ON(atomic_read(&cookie->usage) <= 0); - if (!atomic_dec_and_test(&cookie->usage)) - break; - } + where = fscache_cookie_put_parent; + } while (cookie); _leave(""); } @@ -657,7 +899,8 @@ void __fscache_cookie_put(struct fscache_cookie *cookie) * * NOTE: it only serves no-index type */ -int __fscache_check_consistency(struct fscache_cookie *cookie) +int __fscache_check_consistency(struct fscache_cookie *cookie, + const void *aux_data) { struct fscache_operation *op; struct fscache_object *object; @@ -666,7 +909,7 @@ int __fscache_check_consistency(struct fscache_cookie *cookie) _enter("%p,", cookie); - ASSERTCMP(cookie->def->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); + ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); if (fscache_wait_for_deferred_lookup(cookie) < 0) return -ERESTARTSYS; @@ -678,13 +921,16 @@ int __fscache_check_consistency(struct fscache_cookie *cookie) if (!op) return -ENOMEM; - fscache_operation_init(op, NULL, NULL, NULL); + fscache_operation_init(cookie, op, NULL, NULL, NULL); op->flags = FSCACHE_OP_MYTHREAD | (1 << FSCACHE_OP_WAITING) | (1 << FSCACHE_OP_UNUSE_COOKIE); + trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency); spin_lock(&cookie->lock); + fscache_update_aux(cookie, aux_data); + if (!fscache_cookie_enabled(cookie) || hlist_empty(&cookie->backing_objects)) goto inconsistent; diff --git a/fs/fscache/fsdef.c b/fs/fscache/fsdef.c index 5a117df..aa46e48 100644 --- a/fs/fscache/fsdef.c +++ b/fs/fscache/fsdef.c @@ -13,16 +13,11 @@ #include <linux/module.h> #include "internal.h" -static uint16_t fscache_fsdef_netfs_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax); - -static uint16_t fscache_fsdef_netfs_get_aux(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax); - static enum fscache_checkaux fscache_fsdef_netfs_check_aux(void *cookie_netfs_data, const void *data, - uint16_t datalen); + uint16_t datalen, + loff_t object_size); /* * The root index is owned by FS-Cache itself. @@ -60,6 +55,7 @@ struct fscache_cookie fscache_fsdef_index = { .backing_objects = HLIST_HEAD_INIT, .def = &fscache_fsdef_index_def, .flags = 1 << FSCACHE_COOKIE_ENABLED, + .type = FSCACHE_COOKIE_TYPE_INDEX, }; EXPORT_SYMBOL(fscache_fsdef_index); @@ -71,59 +67,18 @@ EXPORT_SYMBOL(fscache_fsdef_index); struct fscache_cookie_def fscache_fsdef_netfs_def = { .name = "FSDEF.netfs", .type = FSCACHE_COOKIE_TYPE_INDEX, - .get_key = fscache_fsdef_netfs_get_key, - .get_aux = fscache_fsdef_netfs_get_aux, .check_aux = fscache_fsdef_netfs_check_aux, }; /* - * get the key data for an FSDEF index record - this is the name of the netfs - * for which this entry is created - */ -static uint16_t fscache_fsdef_netfs_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct fscache_netfs *netfs = cookie_netfs_data; - unsigned klen; - - _enter("{%s.%u},", netfs->name, netfs->version); - - klen = strlen(netfs->name); - if (klen > bufmax) - return 0; - - memcpy(buffer, netfs->name, klen); - return klen; -} - -/* - * get the auxiliary data for an FSDEF index record - this is the index - * structure version number of the netfs for which this version is created - */ -static uint16_t fscache_fsdef_netfs_get_aux(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct fscache_netfs *netfs = cookie_netfs_data; - unsigned dlen; - - _enter("{%s.%u},", netfs->name, netfs->version); - - dlen = sizeof(uint32_t); - if (dlen > bufmax) - return 0; - - memcpy(buffer, &netfs->version, dlen); - return dlen; -} - -/* * check that the index structure version number stored in the auxiliary data * matches the one the netfs gave us */ static enum fscache_checkaux fscache_fsdef_netfs_check_aux( void *cookie_netfs_data, const void *data, - uint16_t datalen) + uint16_t datalen, + loff_t object_size) { struct fscache_netfs *netfs = cookie_netfs_data; uint32_t version; diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index 0ff4b49..500650f 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -29,6 +29,7 @@ #define pr_fmt(fmt) "FS-Cache: " fmt #include <linux/fscache-cache.h> +#include <trace/events/fscache.h> #include <linux/sched.h> #define FSCACHE_MIN_THREADS 4 @@ -48,8 +49,16 @@ extern struct fscache_cache *fscache_select_cache_for_object( */ extern struct kmem_cache *fscache_cookie_jar; +extern void fscache_free_cookie(struct fscache_cookie *); extern void fscache_cookie_init_once(void *); -extern void __fscache_cookie_put(struct fscache_cookie *); +extern struct fscache_cookie *fscache_alloc_cookie(struct fscache_cookie *, + const struct fscache_cookie_def *, + const void *, size_t, + const void *, size_t, + void *, loff_t); +extern struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *); +extern void fscache_cookie_put(struct fscache_cookie *, + enum fscache_cookie_trace); /* * fsdef.c @@ -311,14 +320,12 @@ static inline void fscache_raise_event(struct fscache_object *object, fscache_enqueue_object(object); } -/* - * drop a reference to a cookie - */ -static inline void fscache_cookie_put(struct fscache_cookie *cookie) +static inline void fscache_cookie_get(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) { - BUG_ON(atomic_read(&cookie->usage) <= 0); - if (atomic_dec_and_test(&cookie->usage)) - __fscache_cookie_put(cookie); + int usage = atomic_inc_return(&cookie->usage); + + trace_fscache_cookie(cookie, where, usage); } /* @@ -342,6 +349,27 @@ void fscache_put_context(struct fscache_cookie *cookie, void *context) cookie->def->put_context(cookie->netfs_data, context); } +/* + * Update the auxiliary data on a cookie. + */ +static inline +void fscache_update_aux(struct fscache_cookie *cookie, const void *aux_data) +{ + void *p; + + if (!aux_data) + return; + if (cookie->aux_len <= sizeof(cookie->inline_aux)) + p = cookie->inline_aux; + else + p = cookie->aux; + + if (memcmp(p, aux_data, cookie->aux_len) != 0) { + memcpy(p, aux_data, cookie->aux_len); + set_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags); + } +} + /*****************************************************************************/ /* * debug tracing diff --git a/fs/fscache/main.c b/fs/fscache/main.c index 249968d..7dce110 100644 --- a/fs/fscache/main.c +++ b/fs/fscache/main.c @@ -16,6 +16,7 @@ #include <linux/completion.h> #include <linux/slab.h> #include <linux/seq_file.h> +#define CREATE_TRACE_POINTS #include "internal.h" MODULE_DESCRIPTION("FS Cache Manager"); diff --git a/fs/fscache/netfs.c b/fs/fscache/netfs.c index a8aa00b..c2f6054 100644 --- a/fs/fscache/netfs.c +++ b/fs/fscache/netfs.c @@ -14,69 +14,51 @@ #include <linux/slab.h> #include "internal.h" -static LIST_HEAD(fscache_netfs_list); - /* * register a network filesystem for caching */ int __fscache_register_netfs(struct fscache_netfs *netfs) { - struct fscache_netfs *ptr; - struct fscache_cookie *cookie; - int ret; + struct fscache_cookie *candidate, *cookie; _enter("{%s}", netfs->name); - INIT_LIST_HEAD(&netfs->link); - /* allocate a cookie for the primary index */ - cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); - - if (!cookie) { + candidate = fscache_alloc_cookie(&fscache_fsdef_index, + &fscache_fsdef_netfs_def, + netfs->name, strlen(netfs->name), + &netfs->version, sizeof(netfs->version), + netfs, 0); + if (!candidate) { _leave(" = -ENOMEM"); return -ENOMEM; } - /* initialise the primary index cookie */ - atomic_set(&cookie->usage, 1); - atomic_set(&cookie->n_children, 0); - atomic_set(&cookie->n_active, 1); - - cookie->def = &fscache_fsdef_netfs_def; - cookie->parent = &fscache_fsdef_index; - cookie->netfs_data = netfs; - cookie->flags = 1 << FSCACHE_COOKIE_ENABLED; - - spin_lock_init(&cookie->lock); - spin_lock_init(&cookie->stores_lock); - INIT_HLIST_HEAD(&cookie->backing_objects); + candidate->flags = 1 << FSCACHE_COOKIE_ENABLED; /* check the netfs type is not already present */ - down_write(&fscache_addremove_sem); - - ret = -EEXIST; - list_for_each_entry(ptr, &fscache_netfs_list, link) { - if (strcmp(ptr->name, netfs->name) == 0) - goto already_registered; + cookie = fscache_hash_cookie(candidate); + if (!cookie) + goto already_registered; + if (cookie != candidate) { + trace_fscache_cookie(candidate, fscache_cookie_discard, 1); + fscache_free_cookie(candidate); } - atomic_inc(&cookie->parent->usage); + fscache_cookie_get(cookie->parent, fscache_cookie_get_register_netfs); atomic_inc(&cookie->parent->n_children); netfs->primary_index = cookie; - list_add(&netfs->link, &fscache_netfs_list); - ret = 0; pr_notice("Netfs '%s' registered for caching\n", netfs->name); + trace_fscache_netfs(netfs); + _leave(" = 0"); + return 0; already_registered: - up_write(&fscache_addremove_sem); - - if (ret < 0) - kmem_cache_free(fscache_cookie_jar, cookie); - - _leave(" = %d", ret); - return ret; + fscache_cookie_put(candidate, fscache_cookie_put_dup_netfs); + _leave(" = -EEXIST"); + return -EEXIST; } EXPORT_SYMBOL(__fscache_register_netfs); @@ -88,15 +70,8 @@ void __fscache_unregister_netfs(struct fscache_netfs *netfs) { _enter("{%s.%u}", netfs->name, netfs->version); - down_write(&fscache_addremove_sem); - - list_del(&netfs->link); - fscache_relinquish_cookie(netfs->primary_index, 0); - - up_write(&fscache_addremove_sem); - - pr_notice("Netfs '%s' unregistered from caching\n", - netfs->name); + fscache_relinquish_cookie(netfs->primary_index, NULL, false); + pr_notice("Netfs '%s' unregistered from caching\n", netfs->name); _leave(""); } diff --git a/fs/fscache/object-list.c b/fs/fscache/object-list.c index 0438d4c..43e6e28 100644 --- a/fs/fscache/object-list.c +++ b/fs/fscache/object-list.c @@ -36,8 +36,6 @@ struct fscache_objlist_data { #define FSCACHE_OBJLIST_CONFIG_NOEVENTS 0x00000800 /* show objects without no events */ #define FSCACHE_OBJLIST_CONFIG_WORK 0x00001000 /* show objects with work */ #define FSCACHE_OBJLIST_CONFIG_NOWORK 0x00002000 /* show objects without work */ - - u8 buf[512]; /* key and aux data buffer */ }; /* @@ -170,7 +168,7 @@ static int fscache_objlist_show(struct seq_file *m, void *v) struct fscache_cookie *cookie; unsigned long config = data->config; char _type[3], *type; - u8 *buf = data->buf, *p; + u8 *p; if ((unsigned long) v == 1) { seq_puts(m, "OBJECT PARENT STAT CHLDN OPS OOP IPR EX READS" @@ -254,7 +252,7 @@ static int fscache_objlist_show(struct seq_file *m, void *v) if (fscache_use_cookie(obj)) { uint16_t keylen = 0, auxlen = 0; - switch (cookie->def->type) { + switch (cookie->type) { case 0: type = "IX"; break; @@ -263,7 +261,7 @@ static int fscache_objlist_show(struct seq_file *m, void *v) break; default: snprintf(_type, sizeof(_type), "%02u", - cookie->def->type); + cookie->type); type = _type; break; } @@ -274,30 +272,30 @@ static int fscache_objlist_show(struct seq_file *m, void *v) cookie->flags, cookie->netfs_data); - if (cookie->def->get_key && - config & FSCACHE_OBJLIST_CONFIG_KEY) - keylen = cookie->def->get_key(cookie->netfs_data, - buf, 400); + if (config & FSCACHE_OBJLIST_CONFIG_KEY) + keylen = cookie->key_len; - if (cookie->def->get_aux && - config & FSCACHE_OBJLIST_CONFIG_AUX) - auxlen = cookie->def->get_aux(cookie->netfs_data, - buf + keylen, 512 - keylen); - fscache_unuse_cookie(obj); + if (config & FSCACHE_OBJLIST_CONFIG_AUX) + auxlen = cookie->aux_len; if (keylen > 0 || auxlen > 0) { seq_puts(m, " "); - for (p = buf; keylen > 0; keylen--) + p = keylen <= sizeof(cookie->inline_key) ? + cookie->inline_key : cookie->key; + for (; keylen > 0; keylen--) seq_printf(m, "%02x", *p++); if (auxlen > 0) { if (config & FSCACHE_OBJLIST_CONFIG_KEY) seq_puts(m, ", "); + p = auxlen <= sizeof(cookie->inline_aux) ? + cookie->inline_aux : cookie->aux; for (; auxlen > 0; auxlen--) seq_printf(m, "%02x", *p++); } } seq_puts(m, "\n"); + fscache_unuse_cookie(obj); } else { seq_puts(m, "<no_netfs>\n"); } diff --git a/fs/fscache/object.c b/fs/fscache/object.c index 7a182c8..1085ca1 100644 --- a/fs/fscache/object.c +++ b/fs/fscache/object.c @@ -138,10 +138,13 @@ static const struct fscache_transition fscache_osm_run_oob[] = { { 0, NULL } }; -static int fscache_get_object(struct fscache_object *); -static void fscache_put_object(struct fscache_object *); +static int fscache_get_object(struct fscache_object *, + enum fscache_obj_ref_trace); +static void fscache_put_object(struct fscache_object *, + enum fscache_obj_ref_trace); static bool fscache_enqueue_dependents(struct fscache_object *, int); static void fscache_dequeue_object(struct fscache_object *); +static void fscache_update_aux_data(struct fscache_object *); /* * we need to notify the parent when an op completes that we had outstanding @@ -170,6 +173,7 @@ static void fscache_object_sm_dispatcher(struct fscache_object *object) const struct fscache_transition *t; const struct fscache_state *state, *new_state; unsigned long events, event_mask; + bool oob; int event = -1; ASSERT(object != NULL); @@ -188,6 +192,7 @@ restart_masked: if (events & object->oob_event_mask) { _debug("{OBJ%x} oob %lx", object->debug_id, events & object->oob_event_mask); + oob = true; for (t = object->oob_table; t->events; t++) { if (events & t->events) { state = t->transit_to; @@ -199,6 +204,7 @@ restart_masked: } } } + oob = false; /* Wait states are just transition tables */ if (!state->work) { @@ -207,6 +213,8 @@ restart_masked: if (events & t->events) { new_state = t->transit_to; event = fls(events & t->events) - 1; + trace_fscache_osm(object, state, + true, false, event); clear_bit(event, &object->events); _debug("{OBJ%x} ev %d: %s -> %s", object->debug_id, event, @@ -226,6 +234,7 @@ restart_masked: execute_work_state: _debug("{OBJ%x} exec %s", object->debug_id, state->name); + trace_fscache_osm(object, state, false, oob, event); new_state = state->work(object, event); event = -1; if (new_state == NO_TRANSIT) { @@ -279,7 +288,7 @@ static void fscache_object_work_func(struct work_struct *work) start = jiffies; fscache_object_sm_dispatcher(object); fscache_hist(fscache_objs_histogram, start); - fscache_put_object(object); + fscache_put_object(object, fscache_obj_put_work); } /** @@ -397,7 +406,7 @@ static const struct fscache_state *fscache_initialise_object(struct fscache_obje fscache_stat(&fscache_n_cop_grab_object); success = false; if (fscache_object_is_live(parent) && - object->cache->ops->grab_object(object)) { + object->cache->ops->grab_object(object, fscache_obj_get_add_to_deps)) { list_add(&object->dep_link, &parent->dependents); success = true; } @@ -703,6 +712,11 @@ static const struct fscache_state *fscache_drop_object(struct fscache_object *ob ASSERT(cookie != NULL); ASSERT(!hlist_unhashed(&object->cookie_link)); + if (test_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags)) { + _debug("final update"); + fscache_update_aux_data(object); + } + /* Make sure the cookie no longer points here and that the netfs isn't * waiting for us. */ @@ -745,7 +759,7 @@ static const struct fscache_state *fscache_drop_object(struct fscache_object *ob } /* this just shifts the object release to the work processor */ - fscache_put_object(object); + fscache_put_object(object, fscache_obj_put_drop_obj); fscache_stat(&fscache_n_object_dead); _leave(""); @@ -755,12 +769,13 @@ static const struct fscache_state *fscache_drop_object(struct fscache_object *ob /* * get a ref on an object */ -static int fscache_get_object(struct fscache_object *object) +static int fscache_get_object(struct fscache_object *object, + enum fscache_obj_ref_trace why) { int ret; fscache_stat(&fscache_n_cop_grab_object); - ret = object->cache->ops->grab_object(object) ? 0 : -EAGAIN; + ret = object->cache->ops->grab_object(object, why) ? 0 : -EAGAIN; fscache_stat_d(&fscache_n_cop_grab_object); return ret; } @@ -768,10 +783,11 @@ static int fscache_get_object(struct fscache_object *object) /* * Discard a ref on an object */ -static void fscache_put_object(struct fscache_object *object) +static void fscache_put_object(struct fscache_object *object, + enum fscache_obj_ref_trace why) { fscache_stat(&fscache_n_cop_put_object); - object->cache->ops->put_object(object); + object->cache->ops->put_object(object, why); fscache_stat_d(&fscache_n_cop_put_object); } @@ -786,7 +802,7 @@ void fscache_object_destroy(struct fscache_object *object) fscache_objlist_remove(object); /* We can get rid of the cookie now */ - fscache_cookie_put(object->cookie); + fscache_cookie_put(object->cookie, fscache_cookie_put_object); object->cookie = NULL; } EXPORT_SYMBOL(fscache_object_destroy); @@ -798,7 +814,7 @@ void fscache_enqueue_object(struct fscache_object *object) { _enter("{OBJ%x}", object->debug_id); - if (fscache_get_object(object) >= 0) { + if (fscache_get_object(object, fscache_obj_get_queue) >= 0) { wait_queue_head_t *cong_wq = &get_cpu_var(fscache_object_cong_wait); @@ -806,7 +822,7 @@ void fscache_enqueue_object(struct fscache_object *object) if (fscache_object_congested()) wake_up(cong_wq); } else - fscache_put_object(object); + fscache_put_object(object, fscache_obj_put_queue); put_cpu_var(fscache_object_cong_wait); } @@ -866,7 +882,7 @@ static bool fscache_enqueue_dependents(struct fscache_object *object, int event) list_del_init(&dep->dep_link); fscache_raise_event(dep, event); - fscache_put_object(dep); + fscache_put_object(dep, fscache_obj_put_enq_dep); if (!list_empty(&object->dependents) && need_resched()) { ret = false; @@ -906,7 +922,8 @@ static void fscache_dequeue_object(struct fscache_object *object) * and creation). */ enum fscache_checkaux fscache_check_aux(struct fscache_object *object, - const void *data, uint16_t datalen) + const void *data, uint16_t datalen, + loff_t object_size) { enum fscache_checkaux result; @@ -916,7 +933,7 @@ enum fscache_checkaux fscache_check_aux(struct fscache_object *object, } result = object->cookie->def->check_aux(object->cookie->netfs_data, - data, datalen); + data, datalen, object_size); switch (result) { /* entry okay as is */ case FSCACHE_CHECKAUX_OKAY: @@ -972,11 +989,12 @@ static const struct fscache_state *_fscache_invalidate_object(struct fscache_obj if (!op) goto nomem; - fscache_operation_init(op, object->cache->ops->invalidate_object, + fscache_operation_init(cookie, op, object->cache->ops->invalidate_object, NULL, NULL); op->flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_EXCLUSIVE) | (1 << FSCACHE_OP_UNUSE_COOKIE); + trace_fscache_page_op(cookie, NULL, op, fscache_page_op_invalidate); spin_lock(&cookie->lock); if (fscache_submit_exclusive_op(object, op) < 0) @@ -1026,6 +1044,17 @@ static const struct fscache_state *fscache_invalidate_object(struct fscache_obje } /* + * Update auxiliary data. + */ +static void fscache_update_aux_data(struct fscache_object *object) +{ + fscache_stat(&fscache_n_updates_run); + fscache_stat(&fscache_n_cop_update_object); + object->cache->ops->update_object(object); + fscache_stat_d(&fscache_n_cop_update_object); +} + +/* * Asynchronously update an object. */ static const struct fscache_state *fscache_update_object(struct fscache_object *object, @@ -1033,10 +1062,7 @@ static const struct fscache_state *fscache_update_object(struct fscache_object * { _enter("{OBJ%x},%d", object->debug_id, event); - fscache_stat(&fscache_n_updates_run); - fscache_stat(&fscache_n_cop_update_object); - object->cache->ops->update_object(object); - fscache_stat_d(&fscache_n_cop_update_object); + fscache_update_aux_data(object); _leave(""); return transit_to(WAIT_FOR_CMD); diff --git a/fs/fscache/operation.c b/fs/fscache/operation.c index de67745..e30c597 100644 --- a/fs/fscache/operation.c +++ b/fs/fscache/operation.c @@ -32,7 +32,8 @@ static void fscache_operation_dummy_cancel(struct fscache_operation *op) * Do basic initialisation of an operation. The caller must still set flags, * object and processor if needed. */ -void fscache_operation_init(struct fscache_operation *op, +void fscache_operation_init(struct fscache_cookie *cookie, + struct fscache_operation *op, fscache_operation_processor_t processor, fscache_operation_cancel_t cancel, fscache_operation_release_t release) @@ -46,6 +47,7 @@ void fscache_operation_init(struct fscache_operation *op, op->release = release; INIT_LIST_HEAD(&op->pend_link); fscache_stat(&fscache_n_op_initialised); + trace_fscache_op(cookie, op, fscache_op_init); } EXPORT_SYMBOL(fscache_operation_init); @@ -59,6 +61,8 @@ EXPORT_SYMBOL(fscache_operation_init); */ void fscache_enqueue_operation(struct fscache_operation *op) { + struct fscache_cookie *cookie = op->object->cookie; + _enter("{OBJ%x OP%x,%u}", op->object->debug_id, op->debug_id, atomic_read(&op->usage)); @@ -71,12 +75,14 @@ void fscache_enqueue_operation(struct fscache_operation *op) fscache_stat(&fscache_n_op_enqueue); switch (op->flags & FSCACHE_OP_TYPE) { case FSCACHE_OP_ASYNC: + trace_fscache_op(cookie, op, fscache_op_enqueue_async); _debug("queue async"); atomic_inc(&op->usage); if (!queue_work(fscache_op_wq, &op->work)) fscache_put_operation(op); break; case FSCACHE_OP_MYTHREAD: + trace_fscache_op(cookie, op, fscache_op_enqueue_mythread); _debug("queue for caller's attention"); break; default: @@ -101,6 +107,8 @@ static void fscache_run_op(struct fscache_object *object, wake_up_bit(&op->flags, FSCACHE_OP_WAITING); if (op->processor) fscache_enqueue_operation(op); + else + trace_fscache_op(object->cookie, op, fscache_op_run); fscache_stat(&fscache_n_op_run); } @@ -155,6 +163,8 @@ int fscache_submit_exclusive_op(struct fscache_object *object, _enter("{OBJ%x OP%x},", object->debug_id, op->debug_id); + trace_fscache_op(object->cookie, op, fscache_op_submit_ex); + ASSERTCMP(op->state, ==, FSCACHE_OP_ST_INITIALISED); ASSERTCMP(atomic_read(&op->usage), >, 0); @@ -240,6 +250,8 @@ int fscache_submit_op(struct fscache_object *object, _enter("{OBJ%x OP%x},{%u}", object->debug_id, op->debug_id, atomic_read(&op->usage)); + trace_fscache_op(object->cookie, op, fscache_op_submit); + ASSERTCMP(op->state, ==, FSCACHE_OP_ST_INITIALISED); ASSERTCMP(atomic_read(&op->usage), >, 0); @@ -357,6 +369,8 @@ int fscache_cancel_op(struct fscache_operation *op, _enter("OBJ%x OP%x}", op->object->debug_id, op->debug_id); + trace_fscache_op(object->cookie, op, fscache_op_cancel); + ASSERTCMP(op->state, >=, FSCACHE_OP_ST_PENDING); ASSERTCMP(op->state, !=, FSCACHE_OP_ST_CANCELLED); ASSERTCMP(atomic_read(&op->usage), >, 0); @@ -419,6 +433,8 @@ void fscache_cancel_all_ops(struct fscache_object *object) fscache_stat(&fscache_n_op_cancelled); list_del_init(&op->pend_link); + trace_fscache_op(object->cookie, op, fscache_op_cancel_all); + ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING); op->cancel(op); op->state = FSCACHE_OP_ST_CANCELLED; @@ -454,9 +470,11 @@ void fscache_op_complete(struct fscache_operation *op, bool cancelled) spin_lock(&object->lock); if (!cancelled) { + trace_fscache_op(object->cookie, op, fscache_op_completed); op->state = FSCACHE_OP_ST_COMPLETE; } else { op->cancel(op); + trace_fscache_op(object->cookie, op, fscache_op_cancelled); op->state = FSCACHE_OP_ST_CANCELLED; } @@ -488,6 +506,8 @@ void fscache_put_operation(struct fscache_operation *op) if (!atomic_dec_and_test(&op->usage)) return; + trace_fscache_op(op->object ? op->object->cookie : NULL, op, fscache_op_put); + _debug("PUT OP"); ASSERTIFCMP(op->state != FSCACHE_OP_ST_INITIALISED && op->state != FSCACHE_OP_ST_COMPLETE, @@ -563,6 +583,8 @@ void fscache_operation_gc(struct work_struct *work) spin_unlock(&cache->op_gc_list_lock); object = op->object; + trace_fscache_op(object->cookie, op, fscache_op_gc); + spin_lock(&object->lock); _debug("GC DEFERRED REL OBJ%x OP%x", @@ -601,6 +623,8 @@ void fscache_op_work_func(struct work_struct *work) _enter("{OBJ%x OP%x,%d}", op->object->debug_id, op->debug_id, atomic_read(&op->usage)); + trace_fscache_op(op->object->cookie, op, fscache_op_work); + ASSERT(op->processor != NULL); start = jiffies; op->processor(op); diff --git a/fs/fscache/page.c b/fs/fscache/page.c index 961029e..111349f 100644 --- a/fs/fscache/page.c +++ b/fs/fscache/page.c @@ -27,6 +27,7 @@ bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page rcu_read_lock(); val = radix_tree_lookup(&cookie->stores, page->index); rcu_read_unlock(); + trace_fscache_check_page(cookie, page, val, 0); return val != NULL; } @@ -39,6 +40,8 @@ void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *pa { wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0); + trace_fscache_page(cookie, page, fscache_page_write_wait); + wait_event(*wq, !__fscache_check_page_write(cookie, page)); } EXPORT_SYMBOL(__fscache_wait_on_page_write); @@ -69,6 +72,8 @@ bool __fscache_maybe_release_page(struct fscache_cookie *cookie, _enter("%p,%p,%x", cookie, page, gfp); + trace_fscache_page(cookie, page, fscache_page_maybe_release); + try_again: rcu_read_lock(); val = radix_tree_lookup(&cookie->stores, page->index); @@ -101,6 +106,7 @@ try_again: } xpage = radix_tree_delete(&cookie->stores, page->index); + trace_fscache_page(cookie, page, fscache_page_radix_delete); spin_unlock(&cookie->stores_lock); if (xpage) { @@ -112,6 +118,7 @@ try_again: } wake_up_bit(&cookie->flags, 0); + trace_fscache_wake_cookie(cookie); if (xpage) put_page(xpage); __fscache_uncache_page(cookie, page); @@ -144,7 +151,7 @@ static void fscache_end_page_write(struct fscache_object *object, struct page *page) { struct fscache_cookie *cookie; - struct page *xpage = NULL; + struct page *xpage = NULL, *val; spin_lock(&object->lock); cookie = object->cookie; @@ -154,13 +161,24 @@ static void fscache_end_page_write(struct fscache_object *object, spin_lock(&cookie->stores_lock); radix_tree_tag_clear(&cookie->stores, page->index, FSCACHE_COOKIE_STORING_TAG); + trace_fscache_page(cookie, page, fscache_page_radix_clear_store); if (!radix_tree_tag_get(&cookie->stores, page->index, FSCACHE_COOKIE_PENDING_TAG)) { fscache_stat(&fscache_n_store_radix_deletes); xpage = radix_tree_delete(&cookie->stores, page->index); + trace_fscache_page(cookie, page, fscache_page_radix_delete); + trace_fscache_page(cookie, page, fscache_page_write_end); + + val = radix_tree_lookup(&cookie->stores, page->index); + trace_fscache_check_page(cookie, page, val, 1); + } else { + trace_fscache_page(cookie, page, fscache_page_write_end_pend); } spin_unlock(&cookie->stores_lock); wake_up_bit(&cookie->flags, 0); + trace_fscache_wake_cookie(cookie); + } else { + trace_fscache_page(cookie, page, fscache_page_write_end_noc); } spin_unlock(&object->lock); if (xpage) @@ -185,9 +203,11 @@ static void fscache_attr_changed_op(struct fscache_operation *op) fscache_stat_d(&fscache_n_cop_attr_changed); if (ret < 0) fscache_abort_object(object); + fscache_op_complete(op, ret < 0); + } else { + fscache_op_complete(op, true); } - fscache_op_complete(op, true); _leave(""); } @@ -213,7 +233,8 @@ int __fscache_attr_changed(struct fscache_cookie *cookie) return -ENOMEM; } - fscache_operation_init(op, fscache_attr_changed_op, NULL, NULL); + fscache_operation_init(cookie, op, fscache_attr_changed_op, NULL, NULL); + trace_fscache_page_op(cookie, NULL, op, fscache_page_op_attr_changed); op->flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_EXCLUSIVE) | (1 << FSCACHE_OP_UNUSE_COOKIE); @@ -297,7 +318,7 @@ static struct fscache_retrieval *fscache_alloc_retrieval( return NULL; } - fscache_operation_init(&op->op, NULL, + fscache_operation_init(cookie, &op->op, NULL, fscache_do_cancel_retrieval, fscache_release_retrieval_op); op->op.flags = FSCACHE_OP_MYTHREAD | @@ -368,6 +389,7 @@ int fscache_wait_for_operation_activation(struct fscache_object *object, fscache_stat(stat_op_waits); if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING, TASK_INTERRUPTIBLE) != 0) { + trace_fscache_op(object->cookie, op, fscache_op_signal); ret = fscache_cancel_op(op, false); if (ret == 0) return -ERESTARTSYS; @@ -389,6 +411,7 @@ check_if_dead: if (unlikely(fscache_object_is_dying(object) || fscache_cache_is_broken(object))) { enum fscache_operation_state state = op->state; + trace_fscache_op(object->cookie, op, fscache_op_signal); fscache_cancel_op(op, true); if (stat_object_dead) fscache_stat(stat_object_dead); @@ -443,6 +466,7 @@ int __fscache_read_or_alloc_page(struct fscache_cookie *cookie, return -ENOMEM; } atomic_set(&op->n_pages, 1); + trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_retr_one); spin_lock(&cookie->lock); @@ -571,6 +595,7 @@ int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie, if (!op) return -ENOMEM; atomic_set(&op->n_pages, *nr_pages); + trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi); spin_lock(&cookie->lock); @@ -682,6 +707,7 @@ int __fscache_alloc_page(struct fscache_cookie *cookie, if (!op) return -ENOMEM; atomic_set(&op->n_pages, 1); + trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_alloc_one); spin_lock(&cookie->lock); @@ -776,15 +802,17 @@ static void fscache_write_op(struct fscache_operation *_op) _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage)); +again: spin_lock(&object->lock); cookie = object->cookie; if (!fscache_object_is_active(object)) { - /* If we get here, then the on-disk cache object likely longer - * exists, so we should just cancel this write operation. + /* If we get here, then the on-disk cache object likely no + * longer exists, so we should just cancel this write + * operation. */ spin_unlock(&object->lock); - fscache_op_complete(&op->op, false); + fscache_op_complete(&op->op, true); _leave(" [inactive]"); return; } @@ -797,7 +825,7 @@ static void fscache_write_op(struct fscache_operation *_op) * cancel this write operation. */ spin_unlock(&object->lock); - fscache_op_complete(&op->op, false); + fscache_op_complete(&op->op, true); _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}", _op->flags, _op->state, object->state->short_name, object->flags); @@ -809,30 +837,33 @@ static void fscache_write_op(struct fscache_operation *_op) fscache_stat(&fscache_n_store_calls); /* find a page to store */ + results[0] = NULL; page = NULL; n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1, FSCACHE_COOKIE_PENDING_TAG); + trace_fscache_gang_lookup(cookie, &op->op, results, n, op->store_limit); if (n != 1) goto superseded; page = results[0]; _debug("gang %d [%lx]", n, page->index); - if (page->index >= op->store_limit) { - fscache_stat(&fscache_n_store_pages_over_limit); - goto superseded; - } radix_tree_tag_set(&cookie->stores, page->index, FSCACHE_COOKIE_STORING_TAG); radix_tree_tag_clear(&cookie->stores, page->index, FSCACHE_COOKIE_PENDING_TAG); + trace_fscache_page(cookie, page, fscache_page_radix_pend2store); spin_unlock(&cookie->stores_lock); spin_unlock(&object->lock); + if (page->index >= op->store_limit) + goto discard_page; + fscache_stat(&fscache_n_store_pages); fscache_stat(&fscache_n_cop_write_page); ret = object->cache->ops->write_page(op, page); fscache_stat_d(&fscache_n_cop_write_page); + trace_fscache_wrote_page(cookie, page, &op->op, ret); fscache_end_page_write(object, page); if (ret < 0) { fscache_abort_object(object); @@ -844,6 +875,12 @@ static void fscache_write_op(struct fscache_operation *_op) _leave(""); return; +discard_page: + fscache_stat(&fscache_n_store_pages_over_limit); + trace_fscache_wrote_page(cookie, page, &op->op, -ENOBUFS); + fscache_end_page_write(object, page); + goto again; + superseded: /* this writer is going away and there aren't any more things to * write */ @@ -851,7 +888,7 @@ superseded: spin_unlock(&cookie->stores_lock); clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); spin_unlock(&object->lock); - fscache_op_complete(&op->op, true); + fscache_op_complete(&op->op, false); _leave(""); } @@ -879,6 +916,8 @@ void fscache_invalidate_writes(struct fscache_cookie *cookie) for (i = n - 1; i >= 0; i--) { page = results[i]; radix_tree_delete(&cookie->stores, page->index); + trace_fscache_page(cookie, page, fscache_page_radix_delete); + trace_fscache_page(cookie, page, fscache_page_inval); } spin_unlock(&cookie->stores_lock); @@ -888,6 +927,7 @@ void fscache_invalidate_writes(struct fscache_cookie *cookie) } wake_up_bit(&cookie->flags, 0); + trace_fscache_wake_cookie(cookie); _leave(""); } @@ -923,6 +963,7 @@ void fscache_invalidate_writes(struct fscache_cookie *cookie) */ int __fscache_write_page(struct fscache_cookie *cookie, struct page *page, + loff_t object_size, gfp_t gfp) { struct fscache_storage *op; @@ -946,7 +987,7 @@ int __fscache_write_page(struct fscache_cookie *cookie, if (!op) goto nomem; - fscache_operation_init(&op->op, fscache_write_op, NULL, + fscache_operation_init(cookie, &op->op, fscache_write_op, NULL, fscache_release_write_op); op->op.flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_WAITING) | @@ -956,6 +997,8 @@ int __fscache_write_page(struct fscache_cookie *cookie, if (ret < 0) goto nomem_free; + trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_write_one); + ret = -ENOBUFS; spin_lock(&cookie->lock); @@ -967,9 +1010,15 @@ int __fscache_write_page(struct fscache_cookie *cookie, if (test_bit(FSCACHE_IOERROR, &object->cache->flags)) goto nobufs; + trace_fscache_page(cookie, page, fscache_page_write); + /* add the page to the pending-storage radix tree on the backing * object */ spin_lock(&object->lock); + + if (object->store_limit_l != object_size) + fscache_set_store_limit(object, object_size); + spin_lock(&cookie->stores_lock); _debug("store limit %llx", (unsigned long long) object->store_limit); @@ -982,8 +1031,10 @@ int __fscache_write_page(struct fscache_cookie *cookie, goto nobufs_unlock_obj; } + trace_fscache_page(cookie, page, fscache_page_radix_insert); radix_tree_tag_set(&cookie->stores, page->index, FSCACHE_COOKIE_PENDING_TAG); + trace_fscache_page(cookie, page, fscache_page_radix_set_pend); get_page(page); /* we only want one writer at a time, but we do need to queue new @@ -1026,6 +1077,7 @@ already_pending: submit_failed: spin_lock(&cookie->stores_lock); radix_tree_delete(&cookie->stores, page->index); + trace_fscache_page(cookie, page, fscache_page_radix_delete); spin_unlock(&cookie->stores_lock); wake_cookie = __fscache_unuse_cookie(cookie); put_page(page); @@ -1072,6 +1124,8 @@ void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page) if (!PageFsCache(page)) goto done; + trace_fscache_page(cookie, page, fscache_page_uncache); + /* get the object */ spin_lock(&cookie->lock); @@ -1120,6 +1174,8 @@ void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page) atomic_inc(&fscache_n_marks); #endif + trace_fscache_page(cookie, page, fscache_page_cached); + _debug("- mark %p{%lx}", page, page->index); if (TestSetPageFsCache(page)) { static bool once_only; diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index 7ac6e83..fcc8c2f 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -21,7 +21,6 @@ atomic_t fscache_n_op_pend; atomic_t fscache_n_op_run; atomic_t fscache_n_op_enqueue; -atomic_t fscache_n_op_requeue; atomic_t fscache_n_op_deferred_release; atomic_t fscache_n_op_initialised; atomic_t fscache_n_op_release; diff --git a/fs/nfs/fscache-index.c b/fs/nfs/fscache-index.c index 0ee4b93..1c5d8d3 100644 --- a/fs/nfs/fscache-index.c +++ b/fs/nfs/fscache-index.c @@ -50,59 +50,6 @@ void nfs_fscache_unregister(void) } /* - * Layout of the key for an NFS server cache object. - */ -struct nfs_server_key { - uint16_t nfsversion; /* NFS protocol version */ - uint16_t family; /* address family */ - uint16_t port; /* IP port */ - union { - struct in_addr ipv4_addr; /* IPv4 address */ - struct in6_addr ipv6_addr; /* IPv6 address */ - } addr[0]; -}; - -/* - * Generate a key to describe a server in the main NFS index - * - We return the length of the key, or 0 if we can't generate one - */ -static uint16_t nfs_server_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct nfs_client *clp = cookie_netfs_data; - const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) &clp->cl_addr; - const struct sockaddr_in *sin = (struct sockaddr_in *) &clp->cl_addr; - struct nfs_server_key *key = buffer; - uint16_t len = sizeof(struct nfs_server_key); - - memset(key, 0, len); - key->nfsversion = clp->rpc_ops->version; - key->family = clp->cl_addr.ss_family; - - switch (clp->cl_addr.ss_family) { - case AF_INET: - key->port = sin->sin_port; - key->addr[0].ipv4_addr = sin->sin_addr; - len += sizeof(key->addr[0].ipv4_addr); - break; - - case AF_INET6: - key->port = sin6->sin6_port; - key->addr[0].ipv6_addr = sin6->sin6_addr; - len += sizeof(key->addr[0].ipv6_addr); - break; - - default: - printk(KERN_WARNING "NFS: Unknown network family '%d'\n", - clp->cl_addr.ss_family); - len = 0; - break; - } - - return len; -} - -/* * Define the server object for FS-Cache. This is used to describe a server * object to fscache_acquire_cookie(). It is keyed by the NFS protocol and * server address parameters. @@ -110,33 +57,9 @@ static uint16_t nfs_server_get_key(const void *cookie_netfs_data, const struct fscache_cookie_def nfs_fscache_server_index_def = { .name = "NFS.server", .type = FSCACHE_COOKIE_TYPE_INDEX, - .get_key = nfs_server_get_key, }; /* - * Generate a key to describe a superblock key in the main NFS index - */ -static uint16_t nfs_super_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct nfs_fscache_key *key; - const struct nfs_server *nfss = cookie_netfs_data; - uint16_t len; - - key = nfss->fscache_key; - len = sizeof(key->key) + key->key.uniq_len; - if (len > bufmax) { - len = 0; - } else { - memcpy(buffer, &key->key, sizeof(key->key)); - memcpy(buffer + sizeof(key->key), - key->key.uniquifier, key->key.uniq_len); - } - - return len; -} - -/* * Define the superblock object for FS-Cache. This is used to describe a * superblock object to fscache_acquire_cookie(). It is keyed by all the NFS * parameters that might cause a separate superblock. @@ -144,84 +67,9 @@ static uint16_t nfs_super_get_key(const void *cookie_netfs_data, const struct fscache_cookie_def nfs_fscache_super_index_def = { .name = "NFS.super", .type = FSCACHE_COOKIE_TYPE_INDEX, - .get_key = nfs_super_get_key, }; /* - * Definition of the auxiliary data attached to NFS inode storage objects - * within the cache. - * - * The contents of this struct are recorded in the on-disk local cache in the - * auxiliary data attached to the data storage object backing an inode. This - * permits coherency to be managed when a new inode binds to an already extant - * cache object. - */ -struct nfs_fscache_inode_auxdata { - struct timespec mtime; - struct timespec ctime; - loff_t size; - u64 change_attr; -}; - -/* - * Generate a key to describe an NFS inode in an NFS server's index - */ -static uint16_t nfs_fscache_inode_get_key(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - const struct nfs_inode *nfsi = cookie_netfs_data; - uint16_t nsize; - - /* use the inode's NFS filehandle as the key */ - nsize = nfsi->fh.size; - memcpy(buffer, nfsi->fh.data, nsize); - return nsize; -} - -/* - * Get certain file attributes from the netfs data - * - This function can be absent for an index - * - Not permitted to return an error - * - The netfs data from the cookie being used as the source is presented - */ -static void nfs_fscache_inode_get_attr(const void *cookie_netfs_data, - uint64_t *size) -{ - const struct nfs_inode *nfsi = cookie_netfs_data; - - *size = nfsi->vfs_inode.i_size; -} - -/* - * Get the auxiliary data from netfs data - * - This function can be absent if the index carries no state data - * - Should store the auxiliary data in the buffer - * - Should return the amount of amount stored - * - Not permitted to return an error - * - The netfs data from the cookie being used as the source is presented - */ -static uint16_t nfs_fscache_inode_get_aux(const void *cookie_netfs_data, - void *buffer, uint16_t bufmax) -{ - struct nfs_fscache_inode_auxdata auxdata; - const struct nfs_inode *nfsi = cookie_netfs_data; - - memset(&auxdata, 0, sizeof(auxdata)); - auxdata.size = nfsi->vfs_inode.i_size; - auxdata.mtime = nfsi->vfs_inode.i_mtime; - auxdata.ctime = nfsi->vfs_inode.i_ctime; - - if (NFS_SERVER(&nfsi->vfs_inode)->nfs_client->rpc_ops->version == 4) - auxdata.change_attr = inode_peek_iversion_raw(&nfsi->vfs_inode); - - if (bufmax > sizeof(auxdata)) - bufmax = sizeof(auxdata); - - memcpy(buffer, &auxdata, bufmax); - return bufmax; -} - -/* * Consult the netfs about the state of an object * - This function can be absent if the index carries no state data * - The netfs data from the cookie being used as the target is @@ -230,7 +78,8 @@ static uint16_t nfs_fscache_inode_get_aux(const void *cookie_netfs_data, static enum fscache_checkaux nfs_fscache_inode_check_aux(void *cookie_netfs_data, const void *data, - uint16_t datalen) + uint16_t datalen, + loff_t object_size) { struct nfs_fscache_inode_auxdata auxdata; struct nfs_inode *nfsi = cookie_netfs_data; @@ -239,7 +88,6 @@ enum fscache_checkaux nfs_fscache_inode_check_aux(void *cookie_netfs_data, return FSCACHE_CHECKAUX_OBSOLETE; memset(&auxdata, 0, sizeof(auxdata)); - auxdata.size = nfsi->vfs_inode.i_size; auxdata.mtime = nfsi->vfs_inode.i_mtime; auxdata.ctime = nfsi->vfs_inode.i_ctime; @@ -288,9 +136,6 @@ static void nfs_fh_put_context(void *cookie_netfs_data, void *context) const struct fscache_cookie_def nfs_fscache_inode_object_def = { .name = "NFS.fh", .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .get_key = nfs_fscache_inode_get_key, - .get_attr = nfs_fscache_inode_get_attr, - .get_aux = nfs_fscache_inode_get_aux, .check_aux = nfs_fscache_inode_check_aux, .get_context = nfs_fh_get_context, .put_context = nfs_fh_put_context, diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c index d63bea8..b55fc79 100644 --- a/fs/nfs/fscache.c +++ b/fs/nfs/fscache.c @@ -18,6 +18,7 @@ #include <linux/in6.h> #include <linux/seq_file.h> #include <linux/slab.h> +#include <linux/iversion.h> #include "internal.h" #include "iostat.h" @@ -29,6 +30,21 @@ static struct rb_root nfs_fscache_keys = RB_ROOT; static DEFINE_SPINLOCK(nfs_fscache_keys_lock); /* + * Layout of the key for an NFS server cache object. + */ +struct nfs_server_key { + struct { + uint16_t nfsversion; /* NFS protocol version */ + uint16_t family; /* address family */ + __be16 port; /* IP port */ + } hdr; + union { + struct in_addr ipv4_addr; /* IPv4 address */ + struct in6_addr ipv6_addr; /* IPv6 address */ + }; +} __packed; + +/* * Get the per-client index cookie for an NFS client if the appropriate mount * flag was set * - We always try and get an index cookie for the client, but get filehandle @@ -36,10 +52,41 @@ static DEFINE_SPINLOCK(nfs_fscache_keys_lock); */ void nfs_fscache_get_client_cookie(struct nfs_client *clp) { + const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) &clp->cl_addr; + const struct sockaddr_in *sin = (struct sockaddr_in *) &clp->cl_addr; + struct nfs_server_key key; + uint16_t len = sizeof(key.hdr); + + memset(&key, 0, sizeof(key)); + key.hdr.nfsversion = clp->rpc_ops->version; + key.hdr.family = clp->cl_addr.ss_family; + + switch (clp->cl_addr.ss_family) { + case AF_INET: + key.hdr.port = sin->sin_port; + key.ipv4_addr = sin->sin_addr; + len += sizeof(key.ipv4_addr); + break; + + case AF_INET6: + key.hdr.port = sin6->sin6_port; + key.ipv6_addr = sin6->sin6_addr; + len += sizeof(key.ipv6_addr); + break; + + default: + printk(KERN_WARNING "NFS: Unknown network family '%d'\n", + clp->cl_addr.ss_family); + clp->fscache = NULL; + return; + } + /* create a cache index for looking up filehandles */ clp->fscache = fscache_acquire_cookie(nfs_fscache_netfs.primary_index, &nfs_fscache_server_index_def, - clp, true); + &key, len, + NULL, 0, + clp, 0, true); dfprintk(FSCACHE, "NFS: get client cookie (0x%p/0x%p)\n", clp, clp->fscache); } @@ -52,7 +99,7 @@ void nfs_fscache_release_client_cookie(struct nfs_client *clp) dfprintk(FSCACHE, "NFS: releasing client cookie (0x%p/0x%p)\n", clp, clp->fscache); - fscache_relinquish_cookie(clp->fscache, 0); + fscache_relinquish_cookie(clp->fscache, NULL, false); clp->fscache = NULL; } @@ -139,7 +186,9 @@ void nfs_fscache_get_super_cookie(struct super_block *sb, const char *uniq, int /* create a cache index for looking up filehandles */ nfss->fscache = fscache_acquire_cookie(nfss->nfs_client->fscache, &nfs_fscache_super_index_def, - nfss, true); + key, sizeof(*key) + ulen, + NULL, 0, + nfss, 0, true); dfprintk(FSCACHE, "NFS: get superblock cookie (0x%p/0x%p)\n", nfss, nfss->fscache); return; @@ -163,7 +212,7 @@ void nfs_fscache_release_super_cookie(struct super_block *sb) dfprintk(FSCACHE, "NFS: releasing superblock cookie (0x%p/0x%p)\n", nfss, nfss->fscache); - fscache_relinquish_cookie(nfss->fscache, 0); + fscache_relinquish_cookie(nfss->fscache, NULL, false); nfss->fscache = NULL; if (nfss->fscache_key) { @@ -180,14 +229,25 @@ void nfs_fscache_release_super_cookie(struct super_block *sb) */ void nfs_fscache_init_inode(struct inode *inode) { + struct nfs_fscache_inode_auxdata auxdata; struct nfs_inode *nfsi = NFS_I(inode); nfsi->fscache = NULL; if (!S_ISREG(inode->i_mode)) return; + + memset(&auxdata, 0, sizeof(auxdata)); + auxdata.mtime = nfsi->vfs_inode.i_mtime; + auxdata.ctime = nfsi->vfs_inode.i_ctime; + + if (NFS_SERVER(&nfsi->vfs_inode)->nfs_client->rpc_ops->version == 4) + auxdata.change_attr = inode_peek_iversion_raw(&nfsi->vfs_inode); + nfsi->fscache = fscache_acquire_cookie(NFS_SB(inode->i_sb)->fscache, &nfs_fscache_inode_object_def, - nfsi, false); + nfsi->fh.data, nfsi->fh.size, + &auxdata, sizeof(auxdata), + nfsi, nfsi->vfs_inode.i_size, false); } /* @@ -195,12 +255,16 @@ void nfs_fscache_init_inode(struct inode *inode) */ void nfs_fscache_clear_inode(struct inode *inode) { + struct nfs_fscache_inode_auxdata auxdata; struct nfs_inode *nfsi = NFS_I(inode); struct fscache_cookie *cookie = nfs_i_fscache(inode); dfprintk(FSCACHE, "NFS: clear cookie (0x%p/0x%p)\n", nfsi, cookie); - fscache_relinquish_cookie(cookie, false); + memset(&auxdata, 0, sizeof(auxdata)); + auxdata.mtime = nfsi->vfs_inode.i_mtime; + auxdata.ctime = nfsi->vfs_inode.i_ctime; + fscache_relinquish_cookie(cookie, &auxdata, false); nfsi->fscache = NULL; } @@ -232,20 +296,26 @@ static bool nfs_fscache_can_enable(void *data) */ void nfs_fscache_open_file(struct inode *inode, struct file *filp) { + struct nfs_fscache_inode_auxdata auxdata; struct nfs_inode *nfsi = NFS_I(inode); struct fscache_cookie *cookie = nfs_i_fscache(inode); if (!fscache_cookie_valid(cookie)) return; + memset(&auxdata, 0, sizeof(auxdata)); + auxdata.mtime = nfsi->vfs_inode.i_mtime; + auxdata.ctime = nfsi->vfs_inode.i_ctime; + if (inode_is_open_for_write(inode)) { dfprintk(FSCACHE, "NFS: nfsi 0x%p disabling cache\n", nfsi); clear_bit(NFS_INO_FSCACHE, &nfsi->flags); - fscache_disable_cookie(cookie, true); + fscache_disable_cookie(cookie, &auxdata, true); fscache_uncache_all_inode_pages(cookie, inode); } else { dfprintk(FSCACHE, "NFS: nfsi 0x%p enabling cache\n", nfsi); - fscache_enable_cookie(cookie, nfs_fscache_can_enable, inode); + fscache_enable_cookie(cookie, &auxdata, nfsi->vfs_inode.i_size, + nfs_fscache_can_enable, inode); if (fscache_cookie_enabled(cookie)) set_bit(NFS_INO_FSCACHE, &NFS_I(inode)->flags); } @@ -422,7 +492,8 @@ void __nfs_readpage_to_fscache(struct inode *inode, struct page *page, int sync) "NFS: readpage_to_fscache(fsc:%p/p:%p(i:%lx f:%lx)/%d)\n", nfs_i_fscache(inode), page, page->index, page->flags, sync); - ret = fscache_write_page(nfs_i_fscache(inode), page, GFP_KERNEL); + ret = fscache_write_page(nfs_i_fscache(inode), page, + inode->i_size, GFP_KERNEL); dfprintk(FSCACHE, "NFS: readpage_to_fscache: p:%p(i:%lu f:%lx) ret %d\n", page, page->index, page->flags, ret); diff --git a/fs/nfs/fscache.h b/fs/nfs/fscache.h index d7fe3e7..161ba2e 100644 --- a/fs/nfs/fscache.h +++ b/fs/nfs/fscache.h @@ -57,6 +57,21 @@ struct nfs_fscache_key { }; /* + * Definition of the auxiliary data attached to NFS inode storage objects + * within the cache. + * + * The contents of this struct are recorded in the on-disk local cache in the + * auxiliary data attached to the data storage object backing an inode. This + * permits coherency to be managed when a new inode binds to an already extant + * cache object. + */ +struct nfs_fscache_inode_auxdata { + struct timespec mtime; + struct timespec ctime; + u64 change_attr; +}; + +/* * fscache-index.c */ extern struct fscache_netfs nfs_fscache_netfs; |