This is an automated email from the git hooks/post-receive script.
firstyear pushed a commit to branch master
in repository 389-ds-base.
commit aa57d4bc32944718a10d1cb1c5dbd61be19d983a
Author: William Brown <firstyear(a)redhat.com>
Date: Tue Sep 5 14:15:31 2017 +1000
Ticket 48831 - Autotune dncache with entry cache.
Bug Description: Previously we only tuned db and entry cache. We
needed to tune the dncache to prevent over-access to id2entry.
Fix Description: Tune the dncache as 10% of the entrycache value
and entrycache now gets 90% of the allocation previously.
https://pagure.io/389-ds-base/issue/48831
Author: wibrown
Review by: mreynolds (Thanks!)
---
dirsrvtests/tests/suites/config/autotuning_test.py | 21 +++++++++++--------
ldap/servers/slapd/back-ldbm/back-ldbm.h | 3 ++-
ldap/servers/slapd/back-ldbm/start.c | 24 ++++++++++++++++------
3 files changed, 33 insertions(+), 15 deletions(-)
diff --git a/dirsrvtests/tests/suites/config/autotuning_test.py
b/dirsrvtests/tests/suites/config/autotuning_test.py
index e81270b..b099f81 100644
--- a/dirsrvtests/tests/suites/config/autotuning_test.py
+++ b/dirsrvtests/tests/suites/config/autotuning_test.py
@@ -37,7 +37,7 @@ def test_threads_basic(topo):
topo.standalone.config.set("nsslapd-threadnumber", "-1")
log.info("Assert nsslapd-threadnumber is equal to the documented expected
value")
- assert topo.standalone.config.get_attr_val("nsslapd-threadnumber") > 0
+ assert topo.standalone.config.get_attr_val_int("nsslapd-threadnumber") >
0
@pytest.mark.parametrize("invalid_value", ('-2', '0',
'invalid'))
@@ -71,18 +71,18 @@ def test_threads_back_from_manual_value(topo):
log.info("Set nsslapd-threadnumber: -1 to enable autotuning and save the new
value")
topo.standalone.config.set("nsslapd-threadnumber", "-1")
- autotuned_value =
topo.standalone.config.get_attr_val("nsslapd-threadnumber")
+ autotuned_value =
topo.standalone.config.get_attr_val_utf8("nsslapd-threadnumber")
log.info("Set nsslapd-threadnumber to the autotuned value decreased by 2")
new_value = str(int(autotuned_value) - 2)
topo.standalone.config.set("nsslapd-threadnumber", new_value)
- assert topo.standalone.config.get_attr_val("nsslapd-threadnumber") ==
new_value
+ assert topo.standalone.config.get_attr_val_utf8("nsslapd-threadnumber") ==
new_value
log.info("Set nsslapd-threadnumber: -1 to enable autotuning")
topo.standalone.config.set("nsslapd-threadnumber", "-1")
log.info("Assert nsslapd-threadnumber is back to the autotuned value")
- assert topo.standalone.config.get_attr_val("nsslapd-threadnumber") ==
autotuned_value
+ assert topo.standalone.config.get_attr_val_utf8("nsslapd-threadnumber") ==
autotuned_value
@pytest.mark.parametrize("autosize,autosize_split", (('', ''),
('', '0'), ('10', '40'), ('', '40'),
@@ -114,12 +114,14 @@ def test_cache_autosize_non_zero(topo, autosize, autosize_split):
dbcachesize_val = config_ldbm.get_attr_val('nsslapd-dbcachesize')
cachenensize_val = userroot_ldbm.get_attr_val('nsslapd-cachememsize')
+ dncachenensize_val = userroot_ldbm.get_attr_val('nsslapd-dncachememsize')
autosize_val = config_ldbm.get_attr_val('nsslapd-cache-autosize')
autosize_split_val =
config_ldbm.get_attr_val('nsslapd-cache-autosize-split')
log.info("Check nsslapd-dbcachesize and nsslapd-cachememsize before the
test")
log.info("nsslapd-dbcachesize == {}".format(dbcachesize_val))
log.info("nsslapd-cachememsize == {}".format(cachenensize_val))
+ log.info("nsslapd-dncachememsize == {}".format(dncachenensize_val))
log.info("nsslapd-cache-autosize == {}".format(autosize_val))
log.info("nsslapd-cache-autosize-split == {}".format(autosize_split_val))
@@ -153,18 +155,19 @@ def test_cache_autosize_non_zero(topo, autosize, autosize_split):
dbcachesize_val = config_ldbm.get_attr_val('nsslapd-dbcachesize')
cachenensize_val = userroot_ldbm.get_attr_val('nsslapd-cachememsize')
+ dncachenensize_val = userroot_ldbm.get_attr_val('nsslapd-dncachememsize')
autosize_val = config_ldbm.get_attr_val('nsslapd-cache-autosize')
autosize_split_val =
config_ldbm.get_attr_val('nsslapd-cache-autosize-split')
log.info("Check nsslapd-dbcachesize and nsslapd-cachememsize in the appropriate
range.")
log.info("nsslapd-dbcachesize == {}".format(dbcachesize_val))
log.info("nsslapd-cachememsize == {}".format(cachenensize_val))
+ log.info("nsslapd-dncachememsize == {}".format(dncachenensize_val))
log.info("nsslapd-cache-autosize == {}".format(autosize_val))
log.info("nsslapd-cache-autosize-split == {}".format(autosize_split_val))
assert int(dbcachesize_val) >= 512000
- assert int(dbcachesize_val) <= sys.maxint
assert int(cachenensize_val) >= 512000
- assert int(cachenensize_val) <= sys.maxint
+ assert int(dncachenensize_val) >= 512000
@pytest.mark.parametrize("autosize_split", ('0', '',
'40'))
@@ -196,6 +199,7 @@ def test_cache_autosize_basic_sane(topo, autosize_split):
for cachesize in ('0', '33333333'):
dbcachesize_val = config_ldbm.get_attr_val('nsslapd-dbcachesize')
cachenensize_val = userroot_ldbm.get_attr_val('nsslapd-cachememsize')
+ dncachenensize_val =
userroot_ldbm.get_attr_val('nsslapd-dncachememsize')
autosize_val = config_ldbm.get_attr_val('nsslapd-cache-autosize')
autosize_split_val =
config_ldbm.get_attr_val('nsslapd-cache-autosize-split')
@@ -223,18 +227,19 @@ def test_cache_autosize_basic_sane(topo, autosize_split):
dbcachesize_val = config_ldbm.get_attr_val('nsslapd-dbcachesize')
cachenensize_val = userroot_ldbm.get_attr_val('nsslapd-cachememsize')
+ dncachenensize_val =
userroot_ldbm.get_attr_val('nsslapd-dncachememsize')
autosize_val = config_ldbm.get_attr_val('nsslapd-cache-autosize')
autosize_split_val =
config_ldbm.get_attr_val('nsslapd-cache-autosize-split')
log.info("Check nsslapd-dbcachesize and nsslapd-cachememsize in the
appropriate range.")
log.info("nsslapd-dbcachesize == {}".format(dbcachesize_val))
log.info("nsslapd-cachememsize == {}".format(cachenensize_val))
+ log.info("nsslapd-dncachememsize == {}".format(dncachenensize_val))
log.info("nsslapd-cache-autosize == {}".format(autosize_val))
log.info("nsslapd-cache-autosize-split ==
{}".format(autosize_split_val))
assert int(dbcachesize_val) >= 512000
- assert int(dbcachesize_val) <= sys.maxint
assert int(cachenensize_val) >= 512000
- assert int(cachenensize_val) <= sys.maxint
+ assert int(dncachenensize_val) >= 512000
@pytest.mark.parametrize("invalid_value", ('-2', '102',
'invalid'))
diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h
b/ldap/servers/slapd/back-ldbm/back-ldbm.h
index e4d9d53..d2b25a7 100644
--- a/ldap/servers/slapd/back-ldbm/back-ldbm.h
+++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h
@@ -567,10 +567,11 @@ struct ldbminfo
* use for the libdb cache.
* the rest is split up among
* the instance entry caches */
- unsigned long li_cache_autosize_ec; /* new instances created while
+ uint64_t li_cache_autosize_ec; /* new instances created while
* the server is up, should
* use this as the entry cache
* size (0 = autosize off) */
+ uint64_t li_dncache_autosize_ec; /* Same as above, but dncache. */
size_t li_import_cachesize; /* size of the mpool for
* imports */
PRLock *li_dbcache_mutex;
diff --git a/ldap/servers/slapd/back-ldbm/start.c b/ldap/servers/slapd/back-ldbm/start.c
index fc70bf6..45ed091 100644
--- a/ldap/servers/slapd/back-ldbm/start.c
+++ b/ldap/servers/slapd/back-ldbm/start.c
@@ -34,9 +34,9 @@ ldbm_back_start_autotune(struct ldbminfo *li)
/* size_t is a platform unsigned int, IE uint64_t */
uint64_t total_cache_size = 0;
uint64_t entry_size = 0;
+ uint64_t dn_size = 0;
uint64_t zone_size = 0;
uint64_t import_size = 0;
- uint64_t cache_size = 0;
uint64_t db_size = 0;
/* For clamping the autotune value to a 64Mb boundary */
uint64_t clamp_div = 0;
@@ -151,6 +151,9 @@ ldbm_back_start_autotune(struct ldbminfo *li)
if (backend_count > 0) {
/* Number of entry cache pages per backend. */
entry_size = (zone_size - db_size) / backend_count;
+ /* Now split this into dn and entry */
+ dn_size = entry_size * 0.1;
+ entry_size = entry_size * 0.9;
/* Now, clamp this value to a 64mb boundary. */
/* Now divide the entry pages by this, and also mod. If mod != 0, we need
* to add 1 to the diveded number. This should give us:
@@ -167,6 +170,11 @@ ldbm_back_start_autotune(struct ldbminfo *li)
clamp_div = (entry_size / (64 * MEGABYTE)) + 1;
entry_size = clamp_div * (64 * MEGABYTE);
}
+ if (dn_size % (64 * MEGABYTE) != 0) {
+ /* If we want to clamp down, remove the "+1". This would change the
above from 510mb -> 448mb. */
+ clamp_div = (dn_size / (64 * MEGABYTE)) + 1;
+ dn_size = clamp_div * (64 * MEGABYTE);
+ }
}
slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "found %luk
physical memory\n", mi->system_total_bytes / 1024);
@@ -199,13 +207,15 @@ ldbm_back_start_autotune(struct ldbminfo *li)
/* apply the appropriate cache size if 0 */
if (backend_count > 0) {
li->li_cache_autosize_ec = entry_size;
+ li->li_dncache_autosize_ec = dn_size;
}
for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
inst = (ldbm_instance *)object_get_data(inst_obj);
- cache_size = (PRUint64)cache_get_max_size(&(inst->inst_cache));
+ uint64_t cache_size = (uint64_t)cache_get_max_size(&(inst->inst_cache));
+ uint64_t dncache_size =
(uint64_t)cache_get_max_size(&(inst->inst_dncache));
/* This is the point where we decide to apply or not.
* We have to check for the mincachesize as setting 0 resets
@@ -217,6 +227,11 @@ ldbm_back_start_autotune(struct ldbminfo *li)
cache_set_max_entries(&(inst->inst_cache), -1);
cache_set_max_size(&(inst->inst_cache), li->li_cache_autosize_ec,
CACHE_TYPE_ENTRY);
}
+ if (dncache_size == 0 || dncache_size == MINCACHESIZE || li->li_cache_autosize
> 0) {
+ slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "cache
autosizing: %s dn cache (%lu total): %luk\n", inst->inst_name, backend_count,
dn_size / 1024);
+ cache_set_max_entries(&(inst->inst_dncache), -1);
+ cache_set_max_size(&(inst->inst_dncache),
li->li_dncache_autosize_ec, CACHE_TYPE_DN);
+ }
/* Refresh this value now. */
cache_size = (PRUint64)cache_get_max_size(&(inst->inst_cache));
db_size = dblayer_get_id2entry_size(inst);
@@ -228,11 +243,8 @@ ldbm_back_start_autotune(struct ldbminfo *li)
"nsslapd-cachememsize.\n",
inst->inst_name, cache_size, db_size);
}
- /* We need to get each instances dncache size to add to the total */
- /* Else we can't properly check the cache allocations below */
- /* Trac 48831 exists to allow this to be auto-sized too ... */
- total_cache_size += (PRUint64)cache_get_max_size(&(inst->inst_dncache));
total_cache_size += cache_size;
+ total_cache_size += dncache_size;
}
/* autosizing importCache */
if (li->li_import_cache_autosize > 0) {
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.