diff options
Diffstat (limited to 'subversion/libsvn_fs_fs/caching.c')
-rw-r--r-- | subversion/libsvn_fs_fs/caching.c | 692 |
1 files changed, 692 insertions, 0 deletions
diff --git a/subversion/libsvn_fs_fs/caching.c b/subversion/libsvn_fs_fs/caching.c new file mode 100644 index 0000000..4af48b8 --- /dev/null +++ b/subversion/libsvn_fs_fs/caching.c @@ -0,0 +1,692 @@ +/* caching.c : in-memory caching + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + */ + +#include "fs.h" +#include "fs_fs.h" +#include "id.h" +#include "dag.h" +#include "tree.h" +#include "temp_serializer.h" +#include "../libsvn_fs/fs-loader.h" + +#include "svn_config.h" +#include "svn_cache_config.h" + +#include "svn_private_config.h" +#include "svn_hash.h" +#include "svn_pools.h" + +#include "private/svn_debug.h" +#include "private/svn_subr_private.h" + +/* Take the ORIGINAL string and replace all occurrences of ":" without + * limiting the key space. Allocate the result in POOL. + */ +static const char * +normalize_key_part(const char *original, + apr_pool_t *pool) +{ + apr_size_t i; + apr_size_t len = strlen(original); + svn_stringbuf_t *normalized = svn_stringbuf_create_ensure(len, pool); + + for (i = 0; i < len; ++i) + { + char c = original[i]; + switch (c) + { + case ':': svn_stringbuf_appendbytes(normalized, "%_", 2); + break; + case '%': svn_stringbuf_appendbytes(normalized, "%%", 2); + break; + default : svn_stringbuf_appendbyte(normalized, c); + } + } + + return normalized->data; +} + +/* Return a memcache in *MEMCACHE_P for FS if it's configured to use + memcached, or NULL otherwise. Also, sets *FAIL_STOP to a boolean + indicating whether cache errors should be returned to the caller or + just passed to the FS warning handler. + + *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set + according to FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix + to use. + + Use FS->pool for allocating the memcache and CACHE_NAMESPACE, and POOL + for temporary allocations. */ +static svn_error_t * +read_config(svn_memcache_t **memcache_p, + svn_boolean_t *fail_stop, + const char **cache_namespace, + svn_boolean_t *cache_txdeltas, + svn_boolean_t *cache_fulltexts, + svn_boolean_t *cache_revprops, + svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + SVN_ERR(svn_cache__make_memcache_from_config(memcache_p, ffd->config, + fs->pool)); + + /* No cache namespace by default. I.e. all FS instances share the + * cached data. If you specify different namespaces, the data will + * share / compete for the same cache memory but keys will not match + * across namespaces and, thus, cached data will not be shared between + * namespaces. + * + * Since the namespace will be concatenated with other elements to form + * the complete key prefix, we must make sure that the resulting string + * is unique and cannot be created by any other combination of elements. + */ + *cache_namespace + = normalize_key_part(svn_hash__get_cstring(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_NS, + ""), + pool); + + /* don't cache text deltas by default. + * Once we reconstructed the fulltexts from the deltas, + * these deltas are rarely re-used. Therefore, only tools + * like svnadmin will activate this to speed up operations + * dump and verify. + */ + *cache_txdeltas + = svn_hash__get_bool(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_DELTAS, + FALSE); + /* by default, cache fulltexts. + * Most SVN tools care about reconstructed file content. + * Thus, this is a reasonable default. + * SVN admin tools may set that to FALSE because fulltexts + * won't be re-used rendering the cache less effective + * by squeezing wanted data out. + */ + *cache_fulltexts + = svn_hash__get_bool(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS, + TRUE); + + /* don't cache revprops by default. + * Revprop caching significantly speeds up operations like + * svn ls -v. However, it requires synchronization that may + * not be available or efficient in the current server setup. + * + * If the caller chose option "2", enable revprop caching if + * the required API support is there to make it efficient. + */ + if (strcmp(svn_hash__get_cstring(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_REVPROPS, + ""), "2")) + *cache_revprops + = svn_hash__get_bool(fs->config, + SVN_FS_CONFIG_FSFS_CACHE_REVPROPS, + FALSE); + else + *cache_revprops = svn_named_atomic__is_efficient(); + + return svn_config_get_bool(ffd->config, fail_stop, + CONFIG_SECTION_CACHES, CONFIG_OPTION_FAIL_STOP, + FALSE); +} + + +/* Implements svn_cache__error_handler_t + * This variant clears the error after logging it. + */ +static svn_error_t * +warn_and_continue_on_cache_errors(svn_error_t *err, + void *baton, + apr_pool_t *pool) +{ + svn_fs_t *fs = baton; + (fs->warning)(fs->warning_baton, err); + svn_error_clear(err); + + return SVN_NO_ERROR; +} + +/* Implements svn_cache__error_handler_t + * This variant logs the error and passes it on to the callers. + */ +static svn_error_t * +warn_and_fail_on_cache_errors(svn_error_t *err, + void *baton, + apr_pool_t *pool) +{ + svn_fs_t *fs = baton; + (fs->warning)(fs->warning_baton, err); + return err; +} + +#ifdef SVN_DEBUG_CACHE_DUMP_STATS +/* Baton to be used for the dump_cache_statistics() pool cleanup function, */ +struct dump_cache_baton_t +{ + /* the pool about to be cleaned up. Will be used for temp. allocations. */ + apr_pool_t *pool; + + /* the cache to dump the statistics for */ + svn_cache__t *cache; +}; + +/* APR pool cleanup handler that will printf the statistics of the + cache referenced by the baton in BATON_VOID. */ +static apr_status_t +dump_cache_statistics(void *baton_void) +{ + struct dump_cache_baton_t *baton = baton_void; + + apr_status_t result = APR_SUCCESS; + svn_cache__info_t info; + svn_string_t *text_stats; + apr_array_header_t *lines; + int i; + + svn_error_t *err = svn_cache__get_info(baton->cache, + &info, + TRUE, + baton->pool); + + if (! err) + { + text_stats = svn_cache__format_info(&info, baton->pool); + lines = svn_cstring_split(text_stats->data, "\n", FALSE, baton->pool); + + for (i = 0; i < lines->nelts; ++i) + { + const char *line = APR_ARRAY_IDX(lines, i, const char *); +#ifdef SVN_DEBUG + SVN_DBG(("%s\n", line)); +#endif + } + } + + /* process error returns */ + if (err) + { + result = err->apr_err; + svn_error_clear(err); + } + + return result; +} +#endif /* SVN_DEBUG_CACHE_DUMP_STATS */ + +/* This function sets / registers the required callbacks for a given + * not transaction-specific CACHE object in FS, if CACHE is not NULL. + * + * All these svn_cache__t instances shall be handled uniformly. Unless + * ERROR_HANDLER is NULL, register it for the given CACHE in FS. + */ +static svn_error_t * +init_callbacks(svn_cache__t *cache, + svn_fs_t *fs, + svn_cache__error_handler_t error_handler, + apr_pool_t *pool) +{ + if (cache != NULL) + { +#ifdef SVN_DEBUG_CACHE_DUMP_STATS + + /* schedule printing the access statistics upon pool cleanup, + * i.e. end of FSFS session. + */ + struct dump_cache_baton_t *baton; + + baton = apr_palloc(pool, sizeof(*baton)); + baton->pool = pool; + baton->cache = cache; + + apr_pool_cleanup_register(pool, + baton, + dump_cache_statistics, + apr_pool_cleanup_null); +#endif + + if (error_handler) + SVN_ERR(svn_cache__set_error_handler(cache, + error_handler, + fs, + pool)); + + } + + return SVN_NO_ERROR; +} + +/* Sets *CACHE_P to cache instance based on provided options. + * Creates memcache if MEMCACHE is not NULL. Creates membuffer cache if + * MEMBUFFER is not NULL. Fallbacks to inprocess cache if MEMCACHE and + * MEMBUFFER are NULL and pages is non-zero. Sets *CACHE_P to NULL + * otherwise. + * + * Unless NO_HANDLER is true, register an error handler that reports errors + * as warnings to the FS warning callback. + * + * Cache is allocated in POOL. + * */ +static svn_error_t * +create_cache(svn_cache__t **cache_p, + svn_memcache_t *memcache, + svn_membuffer_t *membuffer, + apr_int64_t pages, + apr_int64_t items_per_page, + svn_cache__serialize_func_t serializer, + svn_cache__deserialize_func_t deserializer, + apr_ssize_t klen, + const char *prefix, + svn_fs_t *fs, + svn_boolean_t no_handler, + apr_pool_t *pool) +{ + svn_cache__error_handler_t error_handler = no_handler + ? NULL + : warn_and_fail_on_cache_errors; + + if (memcache) + { + SVN_ERR(svn_cache__create_memcache(cache_p, memcache, + serializer, deserializer, klen, + prefix, pool)); + error_handler = no_handler + ? NULL + : warn_and_continue_on_cache_errors; + } + else if (membuffer) + { + SVN_ERR(svn_cache__create_membuffer_cache( + cache_p, membuffer, serializer, deserializer, + klen, prefix, FALSE, pool)); + } + else if (pages) + { + SVN_ERR(svn_cache__create_inprocess( + cache_p, serializer, deserializer, klen, pages, + items_per_page, FALSE, prefix, pool)); + } + else + { + *cache_p = NULL; + } + + SVN_ERR(init_callbacks(*cache_p, fs, error_handler, pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_fs__initialize_caches(svn_fs_t *fs, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + const char *prefix = apr_pstrcat(pool, + "fsfs:", fs->uuid, + "/", normalize_key_part(fs->path, pool), + ":", + (char *)NULL); + svn_memcache_t *memcache; + svn_membuffer_t *membuffer; + svn_boolean_t no_handler; + svn_boolean_t cache_txdeltas; + svn_boolean_t cache_fulltexts; + svn_boolean_t cache_revprops; + const char *cache_namespace; + + /* Evaluating the cache configuration. */ + SVN_ERR(read_config(&memcache, + &no_handler, + &cache_namespace, + &cache_txdeltas, + &cache_fulltexts, + &cache_revprops, + fs, + pool)); + + prefix = apr_pstrcat(pool, "ns:", cache_namespace, ":", prefix, NULL); + + membuffer = svn_cache__get_global_membuffer_cache(); + + /* Make the cache for revision roots. For the vast majority of + * commands, this is only going to contain a few entries (svnadmin + * dump/verify is an exception here), so to reduce overhead let's + * try to keep it to just one page. I estimate each entry has about + * 72 bytes of overhead (svn_revnum_t key, svn_fs_id_t + + * id_private_t + 3 strings for value, and the cache_entry); the + * default pool size is 8192, so about a hundred should fit + * comfortably. */ + SVN_ERR(create_cache(&(ffd->rev_root_id_cache), + NULL, + membuffer, + 1, 100, + svn_fs_fs__serialize_id, + svn_fs_fs__deserialize_id, + sizeof(svn_revnum_t), + apr_pstrcat(pool, prefix, "RRI", (char *)NULL), + fs, + no_handler, + fs->pool)); + + /* Rough estimate: revision DAG nodes have size around 320 bytes, so + * let's put 16 on a page. */ + SVN_ERR(create_cache(&(ffd->rev_node_cache), + NULL, + membuffer, + 1024, 16, + svn_fs_fs__dag_serialize, + svn_fs_fs__dag_deserialize, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "DAG", (char *)NULL), + fs, + no_handler, + fs->pool)); + + /* 1st level DAG node cache */ + ffd->dag_node_cache = svn_fs_fs__create_dag_cache(pool); + + /* Very rough estimate: 1K per directory. */ + SVN_ERR(create_cache(&(ffd->dir_cache), + NULL, + membuffer, + 1024, 8, + svn_fs_fs__serialize_dir_entries, + svn_fs_fs__deserialize_dir_entries, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "DIR", (char *)NULL), + fs, + no_handler, + fs->pool)); + + /* Only 16 bytes per entry (a revision number + the corresponding offset). + Since we want ~8k pages, that means 512 entries per page. */ + SVN_ERR(create_cache(&(ffd->packed_offset_cache), + NULL, + membuffer, + 32, 1, + svn_fs_fs__serialize_manifest, + svn_fs_fs__deserialize_manifest, + sizeof(svn_revnum_t), + apr_pstrcat(pool, prefix, "PACK-MANIFEST", + (char *)NULL), + fs, + no_handler, + fs->pool)); + + /* initialize node revision cache, if caching has been enabled */ + SVN_ERR(create_cache(&(ffd->node_revision_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_node_revision, + svn_fs_fs__deserialize_node_revision, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "NODEREVS", (char *)NULL), + fs, + no_handler, + fs->pool)); + + /* initialize node change list cache, if caching has been enabled */ + SVN_ERR(create_cache(&(ffd->changes_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_changes, + svn_fs_fs__deserialize_changes, + sizeof(svn_revnum_t), + apr_pstrcat(pool, prefix, "CHANGES", (char *)NULL), + fs, + no_handler, + fs->pool)); + + /* if enabled, cache fulltext and other derived information */ + if (cache_fulltexts) + { + SVN_ERR(create_cache(&(ffd->fulltext_cache), + memcache, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "TEXT", (char *)NULL), + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->properties_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_properties, + svn_fs_fs__deserialize_properties, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "PROP", + (char *)NULL), + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->mergeinfo_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_mergeinfo, + svn_fs_fs__deserialize_mergeinfo, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "MERGEINFO", + (char *)NULL), + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->mergeinfo_existence_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "HAS_MERGEINFO", + (char *)NULL), + fs, + no_handler, + fs->pool)); + } + else + { + ffd->fulltext_cache = NULL; + ffd->properties_cache = NULL; + ffd->mergeinfo_cache = NULL; + ffd->mergeinfo_existence_cache = NULL; + } + + /* initialize revprop cache, if full-text caching has been enabled */ + if (cache_revprops) + { + SVN_ERR(create_cache(&(ffd->revprop_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_properties, + svn_fs_fs__deserialize_properties, + sizeof(pair_cache_key_t), + apr_pstrcat(pool, prefix, "REVPROP", + (char *)NULL), + fs, + no_handler, + fs->pool)); + } + else + { + ffd->revprop_cache = NULL; + } + + /* if enabled, cache text deltas and their combinations */ + if (cache_txdeltas) + { + SVN_ERR(create_cache(&(ffd->txdelta_window_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + svn_fs_fs__serialize_txdelta_window, + svn_fs_fs__deserialize_txdelta_window, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "TXDELTA_WINDOW", + (char *)NULL), + fs, + no_handler, + fs->pool)); + + SVN_ERR(create_cache(&(ffd->combined_window_cache), + NULL, + membuffer, + 0, 0, /* Do not use inprocess cache */ + /* Values are svn_stringbuf_t */ + NULL, NULL, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "COMBINED_WINDOW", + (char *)NULL), + fs, + no_handler, + fs->pool)); + } + else + { + ffd->txdelta_window_cache = NULL; + ffd->combined_window_cache = NULL; + } + + return SVN_NO_ERROR; +} + +/* Baton to be used for the remove_txn_cache() pool cleanup function, */ +struct txn_cleanup_baton_t +{ + /* the cache to reset */ + svn_cache__t *txn_cache; + + /* the position where to reset it */ + svn_cache__t **to_reset; +}; + +/* APR pool cleanup handler that will reset the cache pointer given in + BATON_VOID. */ +static apr_status_t +remove_txn_cache(void *baton_void) +{ + struct txn_cleanup_baton_t *baton = baton_void; + + /* be careful not to hurt performance by resetting newer txn's caches. */ + if (*baton->to_reset == baton->txn_cache) + { + /* This is equivalent to calling svn_fs_fs__reset_txn_caches(). */ + *baton->to_reset = NULL; + } + + return APR_SUCCESS; +} + +/* This function sets / registers the required callbacks for a given + * transaction-specific *CACHE object, if CACHE is not NULL and a no-op + * otherwise. In particular, it will ensure that *CACHE gets reset to NULL + * upon POOL destruction latest. + */ +static void +init_txn_callbacks(svn_cache__t **cache, + apr_pool_t *pool) +{ + if (*cache != NULL) + { + struct txn_cleanup_baton_t *baton; + + baton = apr_palloc(pool, sizeof(*baton)); + baton->txn_cache = *cache; + baton->to_reset = cache; + + apr_pool_cleanup_register(pool, + baton, + remove_txn_cache, + apr_pool_cleanup_null); + } +} + +svn_error_t * +svn_fs_fs__initialize_txn_caches(svn_fs_t *fs, + const char *txn_id, + apr_pool_t *pool) +{ + fs_fs_data_t *ffd = fs->fsap_data; + + /* Transaction content needs to be carefully prefixed to virtually + eliminate any chance for conflicts. The (repo, txn_id) pair + should be unique but if a transaction fails, it might be possible + to start a new transaction later that receives the same id. + Therefore, throw in a uuid as well - just to be sure. */ + const char *prefix = apr_pstrcat(pool, + "fsfs:", fs->uuid, + "/", fs->path, + ":", txn_id, + ":", svn_uuid_generate(pool), ":", + (char *)NULL); + + /* We don't support caching for concurrent transactions in the SAME + * FSFS session. Maybe, you forgot to clean POOL. */ + if (ffd->txn_dir_cache != NULL || ffd->concurrent_transactions) + { + ffd->txn_dir_cache = NULL; + ffd->concurrent_transactions = TRUE; + + return SVN_NO_ERROR; + } + + /* create a txn-local directory cache */ + SVN_ERR(create_cache(&ffd->txn_dir_cache, + NULL, + svn_cache__get_global_membuffer_cache(), + 1024, 8, + svn_fs_fs__serialize_dir_entries, + svn_fs_fs__deserialize_dir_entries, + APR_HASH_KEY_STRING, + apr_pstrcat(pool, prefix, "TXNDIR", + (char *)NULL), + fs, + TRUE, + pool)); + + /* reset the transaction-specific cache if the pool gets cleaned up. */ + init_txn_callbacks(&(ffd->txn_dir_cache), pool); + + return SVN_NO_ERROR; +} + +void +svn_fs_fs__reset_txn_caches(svn_fs_t *fs) +{ + /* we can always just reset the caches. This may degrade performance but + * can never cause in incorrect behavior. */ + + fs_fs_data_t *ffd = fs->fsap_data; + ffd->txn_dir_cache = NULL; +} |