This is an automated email from the git hooks/post-receive script.
firstyear pushed a commit to branch 389-ds-base-1.4.2
in repository 389-ds-base.
The following commit(s) were added to refs/heads/389-ds-base-1.4.2 by this push:
new 98fe0d6 Ticket 51072 - improve autotune defaults
98fe0d6 is described below
commit 98fe0d67834a74bf006620b1e90136b1cc5ac235
Author: William Brown <william(a)blackhats.net.au>
AuthorDate: Thu May 7 14:44:58 2020 +1000
Ticket 51072 - improve autotune defaults
Bug Description: we have learnt that the CPU autotuning is too aggresive, potentially
decreasing throughput due to overhead in context switching and lock contention, and
that our memory tuning is not aggressive enough, at only 10% of the system memory.
Additionally, in containers, we are able to have access to different memory limits
and reservations, so we can choose to be even more forward in our selection.
Fix Description: Change thread tuning to match the number of threads available on
the system. Change memory tuning to 25% of system memory by default. Finally add
an environment variable to containers allowing more aggressive tuning to be
set DS_MEMORY_PERCENTAGE. Later this could be set to a higher default value.
https://pagure.io/389-ds-base/issue/51072
Author: William Brown <william(a)blackhats.net.au>
Review by: mreynolds, vashirov, tbordaz (Thanks!)
---
ldap/servers/slapd/back-ldbm/db-bdb/bdb_config.c | 2 +-
ldap/servers/slapd/back-ldbm/db-bdb/bdb_misc.c | 2 +-
ldap/servers/slapd/util.c | 33 ++++--------------------
src/lib389/cli/dscontainer | 20 +++++++++++++-
4 files changed, 26 insertions(+), 31 deletions(-)
diff --git a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_config.c
b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_config.c
index 4289134..7c05d2f 100644
--- a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_config.c
+++ b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_config.c
@@ -1398,7 +1398,7 @@ static config_info bdb_config_param[] = {
{CONFIG_DB_DEBUG_CHECKPOINTING, CONFIG_TYPE_ONOFF, "off",
&bdb_config_db_debug_checkpointing_get, &bdb_config_db_debug_checkpointing_set,
0},
{CONFIG_DB_HOME_DIRECTORY, CONFIG_TYPE_STRING, "",
&bdb_config_db_home_directory_get, &bdb_config_db_home_directory_set, 0},
{CONFIG_IMPORT_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "-1",
&bdb_config_import_cache_autosize_get, &bdb_config_import_cache_autosize_set,
CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
- {CONFIG_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "10",
&bdb_config_cache_autosize_get, &bdb_config_cache_autosize_set,
CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
+ {CONFIG_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "25",
&bdb_config_cache_autosize_get, &bdb_config_cache_autosize_set,
CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
{CONFIG_CACHE_AUTOSIZE_SPLIT, CONFIG_TYPE_INT, "25",
&bdb_config_cache_autosize_split_get, &bdb_config_cache_autosize_split_set,
CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
{CONFIG_IMPORT_CACHESIZE, CONFIG_TYPE_UINT64, "16777216",
&bdb_config_import_cachesize_get, &bdb_config_import_cachesize_set,
CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
{CONFIG_BYPASS_FILTER_TEST, CONFIG_TYPE_STRING, "on",
&bdb_config_get_bypass_filter_test, &bdb_config_set_bypass_filter_test,
CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
diff --git a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_misc.c
b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_misc.c
index e502ed5..340e4a5 100644
--- a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_misc.c
+++ b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_misc.c
@@ -193,7 +193,7 @@ bdb_start_autotune(struct ldbminfo *li)
* sane defaults and populate these values, but it's only on first run.
*/
msg = "This can be corrected by altering the values of nsslapd-dbcachesize,
nsslapd-cachememsize and nsslapd-dncachememsize\n";
- autosize_percentage = 10;
+ autosize_percentage = 25;
} else {
/* In this case we really are setting the values each start up, so
* change the msg.
diff --git a/ldap/servers/slapd/util.c b/ldap/servers/slapd/util.c
index 5ef1cd9..f380ba8 100644
--- a/ldap/servers/slapd/util.c
+++ b/ldap/servers/slapd/util.c
@@ -1491,34 +1491,11 @@ util_get_hardware_threads(void)
long hw_threads = sysconf(_SC_NPROCESSORS_ONLN);
long threads = 0;
slapi_log_err(SLAPI_LOG_TRACE, "util_get_hardware_threads", "Detected
%lu hardware threads\n", threads);
- /*
- * Now we determine the number to run with based on threads. Initially, for
- * low processor counts we ramp up quickly, we plateau a little, then, we
- * at high numbers start to plateau and increase slowly.
- * Should be
- * 1 -> 16
- * 2 -> 16
- * 4 -> 24
- * 8 -> 32
- * 16 -> 48
- * 32 -> 64
- * 64 -> 96
- * 128 -> 192
- * 256 -> 384
- * 512 -> 512
- * 1024 -> 512
- * 2048 -> 512
- */
-
- if (hw_threads >= 0 && hw_threads < 4) {
- threads = 16;
- } else if (hw_threads >= 4 && hw_threads < 32) {
- threads = 16 + (hw_threads * 2);
- } else if (hw_threads >= 32 && hw_threads < 64) {
- threads = (hw_threads * 2);
- } else if (hw_threads >= 64 && hw_threads < 512) {
- /* Same as *1.5 */
- threads = (hw_threads * 2) - (hw_threads / 2);
+ if (hw_threads == 0) {
+ /* Error! */
+ threads = -1;
+ } else if (hw_threads < 512) {
+ threads = hw_threads;
} else {
/* Cap at 512 for now ... */
threads = 512;
diff --git a/src/lib389/cli/dscontainer b/src/lib389/cli/dscontainer
index 55d79d2..5671b5c 100755
--- a/src/lib389/cli/dscontainer
+++ b/src/lib389/cli/dscontainer
@@ -41,7 +41,14 @@ from lib389.instance.setup import SetupDs
from lib389.instance.options import General2Base, Slapd2Base
from lib389.passwd import password_generate
from lib389.paths import Paths
-from lib389._constants import DSRC_CONTAINER
+from lib389.config import LDBMConfig
+from lib389._constants import (
+ DSRC_CONTAINER,
+ CONTAINER_TLS_SERVER_KEY,
+ CONTAINER_TLS_SERVER_CERT,
+ CONTAINER_TLS_SERVER_CADIR,
+ CONTAINER_TLS_PWDFILE
+)
from lib389.idm.directorymanager import DirectoryManager
@@ -74,10 +81,21 @@ def _begin_environment_config():
# TODO: Should we reset cn=Directory Manager from env?
dm_pass = os.getenv("DS_DM_PASSWORD", None)
if dm_pass is not None:
+ log.debug("Setting Directory Manager Password ...")
dm = DirectoryManager(inst)
dm.change_password(dm_pass)
# TODO: Should we set replica id from env?
# TODO: Should we set replication agreements from env?
+ autotune_pct = os.getenv("DS_MEMORY_PERCENTAGE", None)
+ if autotune_pct is not None:
+ try:
+ autotune_pct = int(autotune_pct)
+ except:
+ log.error("Invalid DS_MEMORY_PERCENTAGE - resetting to system default
value")
+ autotune_pct = 0
+ log.debug("Setting LDBM Autotune Percentage to: %s", autotune_pct)
+ ldbmconfig = LDBMConfig(inst)
+ ldbmconfig.set("nsslapd-cache-autosize", str(autotune_pct))
inst.close()
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.