ldap/servers/plugins/replication/cl5.h | 3 - ldap/servers/plugins/replication/cl5_api.c | 17 ++++-- ldap/servers/plugins/replication/cl5_api.h | 63 ++++++++++++------------- ldap/servers/plugins/replication/cl5_config.c | 40 ++++++++++++--- ldap/servers/plugins/replication/cl5_init.c | 2 ldap/servers/plugins/replication/cl5_test.c | 2 ldap/servers/plugins/replication/repl_shared.h | 1 7 files changed, 80 insertions(+), 48 deletions(-)

New commits: commit 9f750277b5849fb9cc07030728147defac46c9c5 Author: Mark Reynolds mreynolds@redhat.com Date: Wed Dec 11 16:57:24 2013 -0500

Ticket 47617 - allow configuring changelog trim interval

Description: Add a new config setting for changelog trimming interval.

https://fedorahosted.org/389/ticket/47617

Reviewed by: rmeggins(Thanks!!)

diff --git a/ldap/servers/plugins/replication/cl5.h b/ldap/servers/plugins/replication/cl5.h index 9939d5b..9666e9a 100644 --- a/ldap/servers/plugins/replication/cl5.h +++ b/ldap/servers/plugins/replication/cl5.h @@ -56,7 +56,8 @@ typedef struct changelog5Config /* the changelog DB configuration parameters are defined as CL5DBConfig in cl5_api.h */ CL5DBConfig dbconfig; char *symmetricKey; - int compactInterval; + int compactInterval; + int trimInterval; }changelog5Config;

/* initializes changelog*/ diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c index 0fecf8c..4a9d81c 100644 --- a/ldap/servers/plugins/replication/cl5_api.c +++ b/ldap/servers/plugins/replication/cl5_api.c @@ -241,6 +241,7 @@ typedef struct cl5trim time_t maxAge; /* maximum entry age in seconds */ int maxEntries; /* maximum number of entries across all changelog files */ int compactInterval; /* interval to compact changelog db */ + int trimInterval; /* trimming interval */ PRLock* lock; /* controls access to trimming configuration */ } CL5Trim;

@@ -1177,12 +1178,13 @@ int cl5GetState () Description: sets changelog trimming parameters; changelog must be open. Parameters: maxEntries - maximum number of entries in the chnagelog (in all files); maxAge - maximum entry age; - compactInterval - interval to compact changelog db + compactInterval - interval to compact changelog db; + trimInterval - changelog trimming interval. Return: CL5_SUCCESS if successful; CL5_BAD_STATE if changelog is not open */ int -cl5ConfigTrimming (int maxEntries, const char *maxAge, int compactInterval) +cl5ConfigTrimming (int maxEntries, const char *maxAge, int compactInterval, int trimInterval) { if (s_cl5Desc.dbState == CL5_STATE_NONE) { @@ -1225,6 +1227,11 @@ cl5ConfigTrimming (int maxEntries, const char *maxAge, int compactInterval) s_cl5Desc.dbTrim.compactInterval = compactInterval; }

+ if (trimInterval != CL5_NUM_IGNORE) + { + s_cl5Desc.dbTrim.trimInterval = trimInterval; + } + PR_Unlock (s_cl5Desc.dbTrim.lock); _cl5RemoveThread (); @@ -3427,18 +3434,16 @@ static void _cl5TrimCleanup ()

static int _cl5TrimMain (void *param) { - PRIntervalTime interval; time_t timePrev = current_time (); time_t timeCompactPrev = current_time (); time_t timeNow;

PR_AtomicIncrement (&s_cl5Desc.threadCount); - interval = PR_SecondsToInterval(CHANGELOGDB_TRIM_INTERVAL);

while (s_cl5Desc.dbState != CL5_STATE_CLOSING) { timeNow = current_time (); - if (timeNow - timePrev >= CHANGELOGDB_TRIM_INTERVAL) + if (timeNow - timePrev >= s_cl5Desc.dbTrim.trimInterval) { /* time to trim */ timePrev = timeNow; @@ -3458,7 +3463,7 @@ static int _cl5TrimMain (void *param) } PR_Lock(s_cl5Desc.clLock); - PR_WaitCondVar(s_cl5Desc.clCvar, interval); + PR_WaitCondVar(s_cl5Desc.clCvar, PR_SecondsToInterval(s_cl5Desc.dbTrim.trimInterval)); PR_Unlock(s_cl5Desc.clLock); }

diff --git a/ldap/servers/plugins/replication/cl5_api.h b/ldap/servers/plugins/replication/cl5_api.h index 59d3bf2..ba9eb32 100644 --- a/ldap/servers/plugins/replication/cl5_api.h +++ b/ldap/servers/plugins/replication/cl5_api.h @@ -55,8 +55,8 @@ #define CL5_TYPE "Changelog5" /* changelog type */ #define VERSION_SIZE 127 /* size of the buffer to hold changelog version */ #define CL5_DEFAULT_CONFIG -1 /* value that indicates to changelog to use default */ -#define CL5_STR_IGNORE "-1" /* tels function to ignore this parameter */ -#define CL5_NUM_IGNORE -1 /* tels function to ignore this parameter */ +#define CL5_STR_IGNORE "-1" /* tells function to ignore this parameter */ +#define CL5_NUM_IGNORE -1 /* tells function to ignore this parameter */ #define CL5_STR_UNLIMITED "0" /* represent unlimited value (trimming ) */ #define CL5_NUM_UNLIMITED 0 /* represent unlimited value (trimming ) */

@@ -85,7 +85,7 @@ typedef struct cl5entry /* * For historical reasons, dbcachesize refers to number of bytes at the DB level, * whereas cachesize refers to number of entries at the changelog cache level (cachememsize is the - * one refering to number of bytes at the changelog cache level) + * one referring to number of bytes at the changelog cache level) */ #define CL5_DEFAULT_CONFIG_DB_DBCACHESIZE 10485760 /* 10M bytes */ #define CL5_DEFAULT_CONFIG_DB_DURABLE_TRANSACTIONS 1 @@ -178,10 +178,10 @@ void cl5Cleanup (); Parameters: dir - changelog dir config - db configuration parameters; currently not used openMode - open mode - Return: CL5_SUCCESS if successfull; + Return: CL5_SUCCESS if successful; CL5_BAD_DATA if invalid directory is passed; CL5_BAD_DBVERSION if dbversion file is missing or has unexpected data - CL5_SYSTEM_ERROR if NSPR error occured (during db directory creation); + CL5_SYSTEM_ERROR if NSPR error occurred (during db directory creation); CL5_MEMORY_ERROR if memory allocation fails; CL5_DB_ERROR if db initialization or open fails. */ @@ -218,9 +218,9 @@ int cl5DeleteDBSync (Object *replica); stored in the changelog for the replica. Parameters: r - replica for which the vector is requested ruv - contains a copy of the upper bound ruv if function is successful; - unchanged otherwise. It is responsobility pf the caller to free + unchanged otherwise. It is responsibility of the caller to free the ruv when it is no longer is in use - Return: CL5_SUCCESS if function is successfull + Return: CL5_SUCCESS if function is successful CL5_BAD_STATE if the changelog is not initialized; CL5_BAD_DATA - if NULL id is supplied CL5_NOTFOUND, if changelog file for replica is not found @@ -233,12 +233,12 @@ int cl5GetUpperBoundRUV (Replica *r, RUV **ruv); ldifFile - full path to ldif file to write replicas - optional list of replicas whose changes should be exported; if the list is NULL, entire changelog is exported. - Return: CL5_SUCCESS if function is successfull; + Return: CL5_SUCCESS if function is successful; CL5_BAD_DATA if invalid parameter is passed; CL5_BAD_STATE if changelog is not initialized; CL5_DB_ERROR if db api fails; CL5_SYSTEM_ERROR if NSPR call fails; - CL5_MEMORY_ERROR if memory allocation fials. + CL5_MEMORY_ERROR if memory allocation fails. */ int cl5ExportLDIF (const char *ldifFile, Object **replicas);

@@ -248,12 +248,12 @@ int cl5ExportLDIF (const char *ldifFile, Object **replicas); ldifFile - absolute path to the ldif file to import replicas - optional list of replicas whose data should be imported; if the list is NULL, all data in the file is imported. - Return: CL5_SUCCESS if function is successfull; + Return: CL5_SUCCESS if function is successful; CL5_BAD_DATA if invalid parameter is passed; - CL5_BAD_STATE if changelog is open or not inititalized; + CL5_BAD_STATE if changelog is open or not initialized; CL5_DB_ERROR if db api fails; CL5_SYSTEM_ERROR if NSPR call fails; - CL5_MEMORY_ERROR if memory allocation fials. + CL5_MEMORY_ERROR if memory allocation fails. */ int cl5ImportLDIF (const char *clDir, const char *ldifFile, Object **replicas);

@@ -269,21 +269,22 @@ int cl5GetState (); Description: sets changelog trimming parameters Parameters: maxEntries - maximum number of entries in the log; maxAge - maximum entry age; - compactInterval - interval to compact changelog db + compactInterval - interval to compact changelog db; + trimInterval - interval for changelog trimming. Return: CL5_SUCCESS if successful; CL5_BAD_STATE if changelog has not been open */ -int cl5ConfigTrimming (int maxEntries, const char *maxAge, int compactInterval); +int cl5ConfigTrimming (int maxEntries, const char *maxAge, int compactInterval, int trimInterval);

/* Name: cl5GetOperation - Description: retireves operation specified by its csn and databaseid + Description: retrieves operation specified by its csn and databaseid Parameters: op - must contain csn and databaseid; the rest of data is - filled if function is successfull - Return: CL5_SUCCESS if function is successfull; + filled if function is successful + Return: CL5_SUCCESS if function is successful; CL5_BAD_DATA if invalid op is passed; CL5_BAD_STATE if db has not been initialized; CL5_NOTFOUND if entry was not found; - CL5_DB_ERROR if any other db error occured; + CL5_DB_ERROR if any other db error occurred; CL5_BADFORMAT if db data format does not match entry format. */ int cl5GetOperation (Object *replica, slapi_operation_parameters *op); @@ -333,11 +334,11 @@ void cl5DestroyIterator (void *iterator); op - operation to write local - this is a non-replicated operation txn - the containing transaction - Return: CL5_SUCCESS if function is successfull; + Return: CL5_SUCCESS if function is successful; CL5_BAD_DATA if invalid op is passed; CL5_BAD_STATE if db has not been initialized; CL5_MEMORY_ERROR if memory allocation failed; - CL5_DB_ERROR if any other db error occured; + CL5_DB_ERROR if any other db error occurred; */ int cl5WriteOperationTxn(const char *repl_name, const char *repl_gen, const slapi_operation_parameters *op, PRBool local, void *txn); @@ -351,23 +352,23 @@ int cl5WriteOperationTxn(const char *repl_name, const char *repl_gen, is in progress (if the data is reloaded). !!! op - operation to write local - this is a non-replicated operation - Return: CL5_SUCCESS if function is successfull; + Return: CL5_SUCCESS if function is successful; CL5_BAD_DATA if invalid op is passed; CL5_BAD_STATE if db has not been initialized; CL5_MEMORY_ERROR if memory allocation failed; - CL5_DB_ERROR if any other db error occured; + CL5_DB_ERROR if any other db error occurred; */ int cl5WriteOperation(const char *repl_name, const char *repl_gen, const slapi_operation_parameters *op, PRBool local);

/* Name: cl5CreateReplayIterator - Description: creates an iterator that allows to retireve changes that should - to be sent to the consumer identified by ruv The iteration is peformed by + Description: creates an iterator that allows to retrieve changes that should + to be sent to the consumer identified by ruv The iteration is performed by repeated calls to cl5GetNextOperationToReplay. Parameters: replica - replica whose data we wish to iterate; ruv - consumer ruv; iterator - iterator to be passed to cl5GetNextOperationToReplay call - Return: CL5_SUCCESS, if function is successfull; + Return: CL5_SUCCESS, if function is successful; CL5_MISSING_DATA, if data that should be in the changelog is missing CL5_PURGED_DATA, if some data that consumer needs has been purged. Note that the iterator can be non null if the supplier contains @@ -375,7 +376,7 @@ int cl5WriteOperation(const char *repl_name, const char *repl_gen, CL5_NOTFOUND if the consumer is up to data with respect to the supplier CL5_BAD_DATA if invalid parameter is passed; CL5_BAD_STATE if db has not been open; - CL5_DB_ERROR if any other db error occured; + CL5_DB_ERROR if any other db error occurred; CL5_MEMORY_ERROR if memory allocation fails. */ int cl5CreateReplayIterator (Private_Repl_Protocol *prp, const RUV *ruv, @@ -390,11 +391,11 @@ int cl5CreateReplayIteratorEx (Private_Repl_Protocol *prp, const RUV *consumerRu is encoded in the iterator parameter that must be created by calling to cl5CreateIterator. Parameters: iterator - iterator that identifies next entry to retrieve; - op - operation retireved if function is successful - Return: CL5_SUCCESS if function is successfull; + op - operation retrieved if function is successful + Return: CL5_SUCCESS if function is successful; CL5_BAD_DATA if invalid parameter is passed; CL5_NOTFOUND if end of iteration list is reached - CL5_DB_ERROR if any other db error occured; + CL5_DB_ERROR if any other db error occurred; CL5_BADFORMAT if data in db is of unrecognized format; CL5_MEMORY_ERROR if memory allocation fails. */ @@ -402,8 +403,8 @@ int cl5GetNextOperationToReplay (CL5ReplayIterator *iterator, CL5Entry *entry);

/* Name: cl5DestroyReplayIterator - Description: destorys iterator - Parameters: iterator - iterator to destory + Description: destroys iterator + Parameters: iterator - iterator to destroy Return: none */ void cl5DestroyReplayIterator (CL5ReplayIterator **iterator); diff --git a/ldap/servers/plugins/replication/cl5_config.c b/ldap/servers/plugins/replication/cl5_config.c index a523a80..e168bbd 100644 --- a/ldap/servers/plugins/replication/cl5_config.c +++ b/ldap/servers/plugins/replication/cl5_config.c @@ -244,7 +244,7 @@ changelog5_config_add (Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter }

/* set trimming parameters */ - rc = cl5ConfigTrimming (config.maxEntries, config.maxAge, config.compactInterval); + rc = cl5ConfigTrimming (config.maxEntries, config.maxAge, config.compactInterval, config.trimInterval); if (rc != CL5_SUCCESS) { *returncode = 1; @@ -339,6 +339,7 @@ changelog5_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr config.compactInterval = CL5_NUM_IGNORE; slapi_ch_free_string(&config.maxAge); config.maxAge = slapi_ch_strdup(CL5_STR_IGNORE); + config.trimInterval = CL5_NUM_IGNORE;

slapi_pblock_get( pb, SLAPI_MODIFY_MODS, &mods ); for (i = 0; mods[i] != NULL; i++) @@ -411,6 +412,17 @@ changelog5_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr config.compactInterval = 0; } } + else if ( strcasecmp ( config_attr, CONFIG_CHANGELOG_TRIM_ATTRIBUTE ) == 0 ) + { + if (config_attr_value && config_attr_value[0] != '\0') + { + config.trimInterval = atoi (config_attr_value); + } + else + { + config.trimInterval = CHANGELOGDB_TRIM_INTERVAL; + } + } else if ( strcasecmp ( config_attr, CONFIG_CHANGELOG_SYMMETRIC_KEY ) == 0 ) { slapi_ch_free_string(&config.symmetricKey); @@ -438,6 +450,8 @@ changelog5_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr config.maxEntries = originalConfig->maxEntries; if (config.compactInterval == CL5_NUM_IGNORE) config.compactInterval = originalConfig->compactInterval; + if (config.trimInterval == CL5_NUM_IGNORE) + config.trimInterval = originalConfig->trimInterval; if (strcmp (config.maxAge, CL5_STR_IGNORE) == 0) { slapi_ch_free_string(&config.maxAge); if (originalConfig->maxAge) @@ -560,9 +574,10 @@ changelog5_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr

/* one of the changelog parameters is modified */ if (config.maxEntries != CL5_NUM_IGNORE || + config.trimInterval != CL5_NUM_IGNORE || strcmp (config.maxAge, CL5_STR_IGNORE) != 0) { - rc = cl5ConfigTrimming (config.maxEntries, config.maxAge, config.compactInterval); + rc = cl5ConfigTrimming (config.maxEntries, config.maxAge, config.compactInterval, config.trimInterval); if (rc != CL5_SUCCESS) { *returncode = 1; @@ -721,6 +736,7 @@ static changelog5Config * changelog5_dup_config(changelog5Config *config)

dup->maxEntries = config->maxEntries; dup->compactInterval = config->compactInterval; + dup->trimInterval = config->trimInterval;

dup->dbconfig.pageSize = config->dbconfig.pageSize; dup->dbconfig.fileMode = config->dbconfig.fileMode; @@ -741,13 +757,13 @@ static void changelog5_extract_config(Slapi_Entry* entry, changelog5Config *conf config->dir = slapi_entry_attr_get_charptr(entry,CONFIG_CHANGELOG_DIR_ATTRIBUTE); replace_bslash (config->dir);

- arg= slapi_entry_attr_get_charptr(entry,CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE); + arg = slapi_entry_attr_get_charptr(entry,CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE); if (arg) { config->maxEntries = atoi (arg); slapi_ch_free_string(&arg); } - arg= slapi_entry_attr_get_charptr(entry,CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE); + arg = slapi_entry_attr_get_charptr(entry,CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE); if (arg) { config->compactInterval = atoi (arg); @@ -757,13 +773,22 @@ static void changelog5_extract_config(Slapi_Entry* entry, changelog5Config *conf { config->compactInterval = CHANGELOGDB_COMPACT_INTERVAL; } - + arg = slapi_entry_attr_get_charptr(entry, CONFIG_CHANGELOG_TRIM_ATTRIBUTE); + if (arg) + { + config->trimInterval = atoi (arg); + slapi_ch_free_string(&arg); + } + else + { + config->trimInterval = CHANGELOGDB_TRIM_INTERVAL; + } config->maxAge = slapi_entry_attr_get_charptr(entry,CONFIG_CHANGELOG_MAXAGE_ATTRIBUTE);

/* * Read the Changelog Internal Configuration Parameters for the Changelog DB */ - arg= slapi_entry_attr_get_charptr(entry, CONFIG_CHANGELOG_MAX_CONCURRENT_WRITES); + arg = slapi_entry_attr_get_charptr(entry, CONFIG_CHANGELOG_MAX_CONCURRENT_WRITES); if (arg) { config->dbconfig.maxConcurrentWrites = atoi (arg); @@ -777,8 +802,7 @@ static void changelog5_extract_config(Slapi_Entry* entry, changelog5Config *conf /* * changelog encryption */ - arg = slapi_entry_attr_get_charptr(entry, - CONFIG_CHANGELOG_ENCRYPTION_ALGORITHM); + arg = slapi_entry_attr_get_charptr(entry, CONFIG_CHANGELOG_ENCRYPTION_ALGORITHM); if (arg) { config->dbconfig.encryptionAlgorithm = slapi_ch_strdup(arg); diff --git a/ldap/servers/plugins/replication/cl5_init.c b/ldap/servers/plugins/replication/cl5_init.c index 60e538d..5071432 100644 --- a/ldap/servers/plugins/replication/cl5_init.c +++ b/ldap/servers/plugins/replication/cl5_init.c @@ -88,7 +88,7 @@ int changelog5_init() }

/* set trimming parameters */ - rc = cl5ConfigTrimming (config.maxEntries, config.maxAge, config.compactInterval); + rc = cl5ConfigTrimming (config.maxEntries, config.maxAge, config.compactInterval, config.trimInterval); if (rc != CL5_SUCCESS) { slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, diff --git a/ldap/servers/plugins/replication/cl5_test.c b/ldap/servers/plugins/replication/cl5_test.c index 4bb83a7..8f15b2f 100644 --- a/ldap/servers/plugins/replication/cl5_test.c +++ b/ldap/servers/plugins/replication/cl5_test.c @@ -314,7 +314,7 @@ static void testTrimming () rc = populateChangelog (300, NULL);

if (rc == 0) - rc = cl5ConfigTrimming (300, "1d"); + rc = cl5ConfigTrimming (300, "1d", CHANGELOGDB_COMPACT_INTERVAL, CHANGELOGDB_TRIM_INTERVAL);

interval = PR_SecondsToInterval(300); /* 5 min is default trimming interval */ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, diff --git a/ldap/servers/plugins/replication/repl_shared.h b/ldap/servers/plugins/replication/repl_shared.h index f79a657..9a0d014 100644 --- a/ldap/servers/plugins/replication/repl_shared.h +++ b/ldap/servers/plugins/replication/repl_shared.h @@ -64,6 +64,7 @@ #define CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE "nsslapd-changelogmaxentries" #define CONFIG_CHANGELOG_MAXAGE_ATTRIBUTE "nsslapd-changelogmaxage" #define CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE "nsslapd-changelogcompactdb-interval" +#define CONFIG_CHANGELOG_TRIM_ATTRIBUTE "nsslapd-changelogtrim-interval" /* Changelog Internal Configuration Parameters -> Changelog Cache related */ #define CONFIG_CHANGELOG_MAX_CONCURRENT_WRITES "nsslapd-changelogmaxconcurrentwrites" #define CONFIG_CHANGELOG_ENCRYPTION_ALGORITHM "nsslapd-encryptionalgorithm"