Branch '389-ds-base-1.3.4' - ldap/admin
by Mark Reynolds
ldap/admin/src/scripts/repl-monitor.pl.in | 18 +++++-------------
1 file changed, 5 insertions(+), 13 deletions(-)
New commits:
commit be608a240b9ff3b99d9f644fc7bcf4110279b2fb
Author: Mark Reynolds <mreynolds(a)redhat.com>
Date: Wed Jun 29 10:06:03 2016 -0400
Ticket 47538 - repl-monitor.pl legend not properly sorted
Description: Custom color codes/lowmarks were not properly sorted
and led to the HTML legend being out of order. Also,
made the "unknown" lag time color code consistent.
https://fedorahosted.org/389/ticket/47538
Reviewed by: nhosoi & viktor(Thanks!)
(cherry picked from commit 14fcac3166a4925b080d7b3349f9664f42696588)
diff --git a/ldap/admin/src/scripts/repl-monitor.pl.in b/ldap/admin/src/scripts/repl-monitor.pl.in
index 88655ff..50b87be 100755
--- a/ldap/admin/src/scripts/repl-monitor.pl.in
+++ b/ldap/admin/src/scripts/repl-monitor.pl.in
@@ -365,7 +365,7 @@ sub read_cfg_file
$allcolors {5} = "#ffffcc"; #cream yellow
$allcolors {60} = "#ffcccc"; #pale pink
}
- @colorkeys = sort (keys (%allcolors));
+ @colorkeys = sort ({ $a <=> $b } keys (%allcolors));
return 0;
}
@@ -969,18 +969,10 @@ sub cacl_time_lag
$supplier_csn_str = &to_string_csn ($s_maxcsn);
$csn_str = &to_string_csn ($c_maxcsn);
- if ((!$s_maxcsn || $s_maxcsn eq "Unavailable") &&
- (!$c_maxcsn || $c_maxcsn eq "Unavailable")) {
+ if (!$s_maxcsn || $s_maxcsn eq "Unavailable" ||
+ !$c_maxcsn || $c_maxcsn eq "Unavailable") {
$lag_str = "?:??:??";
- $markcolor = "white"; # Both unknown
- }
- elsif ($s_maxcsn && (!$c_maxcsn || $c_maxcsn eq "Unavailable")) {
- $lag_str = "- ?:??:??";
- $markcolor = &get_color (36000); # assume consumer has big latency
- }
- elsif ((!$s_maxcsn || $s_maxcsn eq "Unavailable") && $c_maxcsn) {
- $lag_str = "+ ?:??:??";
- $markcolor = &get_color (1); # consumer is ahead of supplier
+ $markcolor = "white"; # True status unknown
}
elsif ($s_maxcsn le $c_maxcsn) {
$lag_str = "0:00:00";
@@ -1233,7 +1225,7 @@ sub get_color
$lag_minute /= 60;
my ($color) = $allcolors { $colorkeys[0] };
- foreach ( sort keys %allcolors) {
+ foreach ( sort { $a <=> $b } keys %allcolors) {
if ($lag_minute >= $_){
$color = $allcolors {$_};
}
7 years, 10 months
ldap/admin
by Mark Reynolds
ldap/admin/src/scripts/repl-monitor.pl.in | 18 +++++-------------
1 file changed, 5 insertions(+), 13 deletions(-)
New commits:
commit 14fcac3166a4925b080d7b3349f9664f42696588
Author: Mark Reynolds <mreynolds(a)redhat.com>
Date: Wed Jun 29 10:06:03 2016 -0400
Ticket 47538 - repl-monitor.pl legend not properly sorted
Description: Custom color codes/lowmarks were not properly sorted
and led to the HTML legend being out of order. Also,
made the "unknown" lag time color code consistent.
https://fedorahosted.org/389/ticket/47538
Reviewed by: nhosoi & viktor(Thanks!)
diff --git a/ldap/admin/src/scripts/repl-monitor.pl.in b/ldap/admin/src/scripts/repl-monitor.pl.in
index 248cf8b..aa7ab1e 100755
--- a/ldap/admin/src/scripts/repl-monitor.pl.in
+++ b/ldap/admin/src/scripts/repl-monitor.pl.in
@@ -364,7 +364,7 @@ sub read_cfg_file
$allcolors {5} = "#ffffcc"; #cream yellow
$allcolors {60} = "#ffcccc"; #pale pink
}
- @colorkeys = sort (keys (%allcolors));
+ @colorkeys = sort ({ $a <=> $b } keys (%allcolors));
return 0;
}
@@ -968,18 +968,10 @@ sub cacl_time_lag
$supplier_csn_str = &to_string_csn ($s_maxcsn);
$csn_str = &to_string_csn ($c_maxcsn);
- if ((!$s_maxcsn || $s_maxcsn eq "Unavailable") &&
- (!$c_maxcsn || $c_maxcsn eq "Unavailable")) {
+ if (!$s_maxcsn || $s_maxcsn eq "Unavailable" ||
+ !$c_maxcsn || $c_maxcsn eq "Unavailable") {
$lag_str = "?:??:??";
- $markcolor = "white"; # Both unknown
- }
- elsif ($s_maxcsn && (!$c_maxcsn || $c_maxcsn eq "Unavailable")) {
- $lag_str = "- ?:??:??";
- $markcolor = &get_color (36000); # assume consumer has big latency
- }
- elsif ((!$s_maxcsn || $s_maxcsn eq "Unavailable") && $c_maxcsn) {
- $lag_str = "+ ?:??:??";
- $markcolor = &get_color (1); # consumer is ahead of supplier
+ $markcolor = "white"; # True status unknown
}
elsif ($s_maxcsn le $c_maxcsn) {
$lag_str = "0:00:00";
@@ -1232,7 +1224,7 @@ sub get_color
$lag_minute /= 60;
my ($color) = $allcolors { $colorkeys[0] };
- foreach ( sort keys %allcolors) {
+ foreach ( sort { $a <=> $b } keys %allcolors) {
if ($lag_minute >= $_){
$color = $allcolors {$_};
}
7 years, 10 months
Branch '389-ds-base-1.3.4' - ldap/servers
by Noriko Hosoi
ldap/servers/plugins/replication/cl5_api.c | 171 ++------------
ldap/servers/plugins/replication/cl5_clcache.c | 292 +++++++++++++++----------
ldap/servers/plugins/replication/cl5_clcache.h | 2
3 files changed, 214 insertions(+), 251 deletions(-)
New commits:
commit ec15a75ccdba713e4d74dcd760e3244ba43b6191
Author: Ludwig Krispenz <lkrispen(a)redhat.com>
Date: Wed Jun 8 11:28:07 2016 +0200
Ticket 48766 - Replication changelog can incorrectly skip over updates
Bug Description:
The changelog iterator uses a buffer to load and send changes, when the buffer is empty
there were scenarios when the straing point for reloading the buffer was incorrectly set
and changes were skipped
Fix Description: reworked clcach buffer code following design at
http://www.port389.org/docs/389ds/design/changelog-processing-in-repl-sta...
https://fedorahosted.org/389/ticket/48766
Reviewed by: Mark and Thierry, thanks
(cherry picked from commit b08df71aa9eb18572f58e55e8d6b9ef7fe181773)
diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c
index ae23353..3adaf86 100644
--- a/ldap/servers/plugins/replication/cl5_api.c
+++ b/ldap/servers/plugins/replication/cl5_api.c
@@ -5489,18 +5489,13 @@ static int _cl5PositionCursorForReplay (ReplicaId consumerRID, const RUV *consum
{
CLC_Buffer *clcache = NULL;
CL5DBFile *file;
- int i;
- CSN **csns = NULL;
CSN *startCSN = NULL;
- CSN *minCSN = NULL;
char csnStr [CSN_STRSIZE];
int rc = CL5_SUCCESS;
Object *supplierRuvObj = NULL;
RUV *supplierRuv = NULL;
- PRBool newReplica;
PRBool haveChanges = PR_FALSE;
char *agmt_name;
- ReplicaId rid;
PR_ASSERT (consumerRuv && replica && fileObj && iterator);
csnStr[0] = '\0';
@@ -5528,111 +5523,32 @@ static int _cl5PositionCursorForReplay (ReplicaId consumerRID, const RUV *consum
ruv_dump (supplierRuv, agmt_name, NULL);
}
- /*
- * get the sorted list of SupplierMinCSN (if no ConsumerMaxCSN)
- * and ConsumerMaxCSN for those RIDs where consumer is not
- * up-to-date.
- */
- csns = cl5BuildCSNList (consumerRuv, supplierRuv);
- if (csns == NULL)
- {
- rc = CL5_NOTFOUND;
- goto done;
- }
- /* iterate over elements of consumer's (and/or supplier's) ruv */
- for (i = 0; csns[i]; i++)
- {
- CSN *consumerMaxCSN = NULL;
-
- rid = csn_get_replicaid(csns[i]);
-
- /*
- * Skip CSN that is originated from the consumer.
- * If RID==65535, the CSN is originated from a
- * legacy consumer. In this case the supplier
- * and the consumer may have the same RID.
- */
- if ((rid == consumerRID && rid != MAX_REPLICA_ID) || (is_cleaned_rid(rid)) )
- continue;
+ /* initialize the changelog buffer and do the initial load */
- startCSN = csns[i];
+ rc = clcache_get_buffer ( &clcache, file->db, consumerRID, consumerRuv, supplierRuv );
+ if ( rc != 0 ) goto done;
- rc = clcache_get_buffer ( &clcache, file->db, consumerRID, consumerRuv, supplierRuv );
- if ( rc != 0 ) goto done;
-
- /* This is the first loading of this iteration. For replicas
- * already known to the consumer, we exclude the last entry
- * sent to the consumer by using DB_NEXT. However, for
- * replicas new to the consumer, we include the first change
- * ever generated by that replica.
- */
- newReplica = ruv_get_largest_csn_for_replica (consumerRuv, rid, &consumerMaxCSN);
- csn_free(&consumerMaxCSN);
- rc = clcache_load_buffer (clcache, startCSN, (newReplica ? DB_SET : DB_NEXT));
-
- /* there is a special case which can occur just after migration - in this case,
- the consumer RUV will contain the last state of the supplier before migration,
- but the supplier will have an empty changelog, or the supplier changelog will
- not contain any entries within the consumer min and max CSN - also, since
- the purge RUV contains no CSNs, the changelog has never been purged
- ASSUMPTIONS - it is assumed that the supplier had no pending changes to send
- to any consumers; that is, we can assume that no changes were lost due to
- either changelog purging or database reload - bug# 603061 - richm(a)netscape.com
- */
- if ((rc == DB_NOTFOUND) && !ruv_has_csns(file->purgeRUV))
- {
- char mincsnStr[CSN_STRSIZE];
-
- /* use the supplier min csn for the buffer start csn - we know
- this csn is in our changelog */
- if ((RUV_SUCCESS == ruv_get_min_csn_ext(supplierRuv, &minCSN, 1 /* ignore cleaned rids */)) &&
- minCSN)
- { /* must now free startCSN */
- if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {
- csn_as_string(startCSN, PR_FALSE, csnStr);
- csn_as_string(minCSN, PR_FALSE, mincsnStr);
- slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl,
- "%s: CSN %s not found and no purging, probably a reinit\n",
- agmt_name, csnStr);
- slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl,
- "%s: Will try to use supplier min CSN %s to load changelog\n",
- agmt_name, mincsnStr);
- }
- startCSN = minCSN;
- rc = clcache_load_buffer (clcache, startCSN, DB_SET);
- }
- else
- {
- if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {
- csn_as_string(startCSN, PR_FALSE, csnStr);
- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
- "%s: CSN %s not found and no purging, probably a reinit\n",
- agmt_name, csnStr);
- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
- "%s: Could not get the min csn from the supplier RUV\n",
- agmt_name);
- }
- rc = CL5_RUV_ERROR;
- goto done;
- }
- }
+ rc = clcache_load_buffer (clcache, &startCSN);
if (rc == 0) {
- haveChanges = PR_TRUE;
- rc = CL5_SUCCESS;
- if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {
- csn_as_string(startCSN, PR_FALSE, csnStr);
- slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl,
- "%s: CSN %s found, position set for replay\n", agmt_name, csnStr);
- }
- if (startCSN != csns[i]) {
- csn_free(&startCSN);
- }
- break;
+ haveChanges = PR_TRUE;
+ rc = CL5_SUCCESS;
+ if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {
+ csn_as_string(startCSN, PR_FALSE, csnStr);
+ slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl,
+ "%s: CSN %s found, position set for replay\n", agmt_name, csnStr);
+ }
}
- else if (rc == DB_NOTFOUND) /* entry not found */
- {
+ else if (rc == DB_NOTFOUND) {
+ /* buffer not loaded.
+ * either because no changes have to be sent ==> startCSN is NULL
+ * or the calculated startCSN cannot be found in the changelog
+ */
+ if (startCSN == NULL) {
+ rc = CL5_NOTFOUND;
+ goto done;
+ }
/* check whether this csn should be present */
rc = _cl5CheckMissingCSN (startCSN, supplierRuv, file);
if (rc == CL5_MISSING_DATA) /* we should have had the change but we don't */
@@ -5650,17 +5566,6 @@ static int _cl5PositionCursorForReplay (ReplicaId consumerRID, const RUV *consum
"%s: CSN %s not found, we aren't as up to date, or we purged\n",
agmt_name, csnStr);
}
- if (startCSN != csns[i]) {
- csn_free(&startCSN);
- }
- if (rc == CL5_MISSING_DATA) /* we should have had the change but we don't */
- {
- break;
- }
- else /* we are not as up to date or we purged */
- {
- continue;
- }
}
else
{
@@ -5669,34 +5574,29 @@ static int _cl5PositionCursorForReplay (ReplicaId consumerRID, const RUV *consum
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
"%s: Failed to retrieve change with CSN %s; db error - %d %s\n",
agmt_name, csnStr, rc, db_strerror(rc));
- if (startCSN != csns[i]) {
- csn_free(&startCSN);
- }
rc = CL5_DB_ERROR;
- break;
- }
+ }
- } /* end for */
/* setup the iterator */
if (haveChanges)
{
- *iterator = (CL5ReplayIterator*) slapi_ch_calloc (1, sizeof (CL5ReplayIterator));
+ *iterator = (CL5ReplayIterator*) slapi_ch_calloc (1, sizeof (CL5ReplayIterator));
- if (*iterator == NULL)
- {
- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
+ if (*iterator == NULL)
+ {
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
"%s: _cl5PositionCursorForReplay: failed to allocate iterator\n", agmt_name);
- rc = CL5_MEMORY_ERROR;
- goto done;
- }
+ rc = CL5_MEMORY_ERROR;
+ goto done;
+ }
/* ONREPL - should we make a copy of both RUVs here ?*/
- (*iterator)->fileObj = fileObj;
- (*iterator)->clcache = clcache; clcache = NULL;
- (*iterator)->consumerRID = consumerRID;
- (*iterator)->consumerRuv = consumerRuv;
+ (*iterator)->fileObj = fileObj;
+ (*iterator)->clcache = clcache; clcache = NULL;
+ (*iterator)->consumerRID = consumerRID;
+ (*iterator)->consumerRuv = consumerRuv;
(*iterator)->supplierRuvObj = supplierRuvObj;
}
else if (rc == CL5_SUCCESS)
@@ -5706,11 +5606,8 @@ static int _cl5PositionCursorForReplay (ReplicaId consumerRID, const RUV *consum
}
done:
- if ( clcache )
- clcache_return_buffer ( &clcache );
-
- if (csns)
- cl5DestroyCSNList (&csns);
+ if ( clcache )
+ clcache_return_buffer ( &clcache );
if (rc != CL5_SUCCESS)
{
diff --git a/ldap/servers/plugins/replication/cl5_clcache.c b/ldap/servers/plugins/replication/cl5_clcache.c
index b53d7c0..2d3bb28 100644
--- a/ldap/servers/plugins/replication/cl5_clcache.c
+++ b/ldap/servers/plugins/replication/cl5_clcache.c
@@ -39,6 +39,7 @@
#define DEFAULT_CLC_BUFFER_COUNT_MAX 0
#define DEFAULT_CLC_BUFFER_PAGE_COUNT 32
#define DEFAULT_CLC_BUFFER_PAGE_SIZE 1024
+#define WORK_CLC_BUFFER_PAGE_SIZE 8*DEFAULT_CLC_BUFFER_PAGE_SIZE
enum {
CLC_STATE_READY = 0, /* ready to iterate */
@@ -56,8 +57,9 @@ struct csn_seq_ctrl_block {
ReplicaId rid; /* RID this block serves */
CSN *consumer_maxcsn; /* Don't send CSN <= this */
CSN *local_maxcsn; /* Don't send CSN > this */
- CSN *prev_local_maxcsn; /* */
- int state; /* CLC_STATE_* */
+ CSN *prev_local_maxcsn; /* Copy of last state at buffer loading */
+ CSN *local_mincsn; /* Used to determin anchor csn*/
+ int state; /* CLC_STATE_* */
};
/*
@@ -70,6 +72,8 @@ struct clc_buffer {
ReplicaId buf_consumer_rid; /* help checking threshold csn */
const RUV *buf_consumer_ruv; /* used to skip change */
const RUV *buf_local_ruv; /* used to refresh local_maxcsn */
+ int buf_ignoreConsumerRID; /* how to handle updates from consumer */
+ int buf_load_cnt; /* number of loads for session */
/*
* fields for retriving data from DB
@@ -90,7 +94,6 @@ struct clc_buffer {
int buf_max_cscbs;
/* fields for debugging stat */
- int buf_load_cnt; /* number of loads for session */
int buf_record_cnt; /* number of changes for session */
int buf_record_skipped; /* number of changes skipped */
int buf_skipped_new_rid; /* number of changes skipped due to new_rid */
@@ -133,7 +136,8 @@ struct clc_pool {
static struct clc_pool *_pool = NULL; /* process's buffer pool */
/* static prototypes */
-static int clcache_adjust_anchorcsn ( CLC_Buffer *buf );
+static int clcache_initial_anchorcsn ( CLC_Buffer *buf, int *flag );
+static int clcache_adjust_anchorcsn ( CLC_Buffer *buf, int *flag );
static void clcache_refresh_consumer_maxcsns ( CLC_Buffer *buf );
static int clcache_refresh_local_maxcsns ( CLC_Buffer *buf );
static int clcache_skip_change ( CLC_Buffer *buf );
@@ -251,8 +255,23 @@ clcache_get_buffer ( CLC_Buffer **buf, DB *db, ReplicaId consumer_rid, const RUV
}
if ( NULL != *buf ) {
+ CSN *c_csn = NULL;
+ CSN *l_csn = NULL;
(*buf)->buf_consumer_ruv = consumer_ruv;
(*buf)->buf_local_ruv = local_ruv;
+ (*buf)->buf_load_flag = DB_MULTIPLE_KEY;
+ ruv_get_largest_csn_for_replica (consumer_ruv, consumer_rid, &c_csn);
+ ruv_get_largest_csn_for_replica (local_ruv, consumer_rid, &l_csn);
+ if (l_csn && csn_compare(l_csn, c_csn) > 0) {
+ /* the supplier has updates for the consumer RID and
+ * these updates are newer than on the consumer
+ */
+ (*buf)->buf_ignoreConsumerRID = 0;
+ } else {
+ (*buf)->buf_ignoreConsumerRID = 1;
+ }
+ csn_free(&c_csn);
+ csn_free(&l_csn);
}
else {
slapi_log_error ( SLAPI_LOG_FATAL, get_thread_private_agmtname(),
@@ -305,36 +324,25 @@ clcache_return_buffer ( CLC_Buffer **buf )
* historic reason.
*/
int
-clcache_load_buffer ( CLC_Buffer *buf, CSN *anchorcsn, int flag )
+clcache_load_buffer ( CLC_Buffer *buf, CSN **anchorCSN )
{
int rc = 0;
+ int flag = DB_NEXT;
+ if (anchorCSN) *anchorCSN = NULL;
clcache_refresh_local_maxcsns ( buf );
- /* Set the loading key */
- if ( anchorcsn ) {
+ if (buf->buf_load_cnt == 0 ) {
clcache_refresh_consumer_maxcsns ( buf );
- buf->buf_load_flag = DB_MULTIPLE_KEY;
- csn_as_string ( anchorcsn, 0, (char*)buf->buf_key.data );
- slapi_log_error ( SLAPI_LOG_REPL, buf->buf_agmt_name,
- "session start: anchorcsn=%s\n", (char*)buf->buf_key.data );
- }
- else if ( csn_get_time(buf->buf_current_csn) == 0 ) {
- /* time == 0 means this csn has never been set */
- rc = DB_NOTFOUND;
- }
- else if ( clcache_adjust_anchorcsn ( buf ) != 0 ) {
- rc = DB_NOTFOUND;
- }
- else {
- csn_as_string ( buf->buf_current_csn, 0, (char*)buf->buf_key.data );
- slapi_log_error ( SLAPI_LOG_REPL, buf->buf_agmt_name,
- "load next: anchorcsn=%s\n", (char*)buf->buf_key.data );
+ rc = clcache_initial_anchorcsn ( buf, &flag );
+ } else {
+ rc = clcache_adjust_anchorcsn ( buf, &flag );
}
if ( rc == 0 ) {
buf->buf_state = CLC_STATE_READY;
+ if (anchorCSN) *anchorCSN = buf->buf_current_csn;
rc = clcache_load_buffer_bulk ( buf, flag );
/* Reset some flag variables */
@@ -344,21 +352,15 @@ clcache_load_buffer ( CLC_Buffer *buf, CSN *anchorcsn, int flag )
buf->buf_cscbs[i]->state = CLC_STATE_READY;
}
}
- else if ( anchorcsn ) {
- /* Report error only when the missing is persistent */
- if ( buf->buf_missing_csn && csn_compare (buf->buf_missing_csn, anchorcsn) == 0 ) {
- if (!buf->buf_prev_missing_csn || csn_compare (buf->buf_prev_missing_csn, anchorcsn)) {
- slapi_log_error ( SLAPI_LOG_FATAL, buf->buf_agmt_name,
- "Can't locate CSN %s in the changelog (DB rc=%d). If replication stops, the consumer may need to be reinitialized.\n",
- (char*)buf->buf_key.data, rc );
- csn_dup_or_init_by_csn (&buf->buf_prev_missing_csn, anchorcsn);
- }
- }
- else {
- csn_dup_or_init_by_csn (&buf->buf_missing_csn, anchorcsn);
- }
+ else {
+ slapi_log_error ( SLAPI_LOG_FATAL, buf->buf_agmt_name,
+ "Can't locate CSN %s in the changelog (DB rc=%d). If replication stops, the consumer may need to be reinitialized.\n",
+ (char*)buf->buf_key.data, rc );
}
+ } else if (rc == CLC_STATE_DONE) {
+ rc = DB_NOTFOUND;
}
+
if ( rc != 0 ) {
slapi_log_error ( SLAPI_LOG_REPL, buf->buf_agmt_name,
"clcache_load_buffer: rc=%d\n", rc );
@@ -483,7 +485,7 @@ clcache_get_next_change ( CLC_Buffer *buf, void **key, size_t *keylen, void **da
* We're done with the current buffer. Now load the next chunk.
*/
if ( NULL == *key && CLC_STATE_READY == buf->buf_state ) {
- rc = clcache_load_buffer ( buf, NULL, DB_NEXT );
+ rc = clcache_load_buffer ( buf, NULL );
if ( 0 == rc && buf->buf_record_ptr ) {
DB_MULTIPLE_KEY_NEXT ( buf->buf_record_ptr, &buf->buf_data,
*key, *keylen, *data, *datalen );
@@ -521,7 +523,6 @@ clcache_refresh_consumer_maxcsns ( CLC_Buffer *buf )
int i;
for ( i = 0; i < buf->buf_num_cscbs; i++ ) {
- csn_free(&buf->buf_cscbs[i]->consumer_maxcsn);
ruv_get_largest_csn_for_replica (
buf->buf_consumer_ruv,
buf->buf_cscbs[i]->rid,
@@ -538,14 +539,11 @@ clcache_refresh_local_maxcsn ( const ruv_enum_data *rid_data, void *data )
int i;
rid = csn_get_replicaid ( rid_data->csn );
-
- /*
- * No need to create cscb for consumer's RID.
- * If RID==65535, the CSN is originated from a
- * legacy consumer. In this case the supplier
- * and the consumer may have the same RID.
+ /* we do not handle updates originated at the consumer if not required
+ * and we ignore RID which have been cleaned
*/
- if ( rid == buf->buf_consumer_rid && rid != MAX_REPLICA_ID )
+ if ( (rid == buf->buf_consumer_rid && buf->buf_ignoreConsumerRID) ||
+ is_cleaned_rid(rid) )
return rc;
for ( i = 0; i < buf->buf_num_cscbs; i++ ) {
@@ -564,9 +562,20 @@ clcache_refresh_local_maxcsn ( const ruv_enum_data *rid_data, void *data )
}
buf->buf_cscbs[i]->rid = rid;
buf->buf_num_cscbs++;
+ /* this is the first time we have a local change for the RID
+ * we need to check what the consumer knows about it.
+ */
+ ruv_get_largest_csn_for_replica (
+ buf->buf_consumer_ruv,
+ buf->buf_cscbs[i]->rid,
+ &buf->buf_cscbs[i]->consumer_maxcsn );
}
+ if (buf->buf_cscbs[i]->local_maxcsn)
+ csn_dup_or_init_by_csn ( &buf->buf_cscbs[i]->prev_local_maxcsn, buf->buf_cscbs[i]->local_maxcsn );
+
csn_dup_or_init_by_csn ( &buf->buf_cscbs[i]->local_maxcsn, rid_data->csn );
+ csn_dup_or_init_by_csn ( &buf->buf_cscbs[i]->local_mincsn, rid_data->min_csn );
if ( buf->buf_cscbs[i]->consumer_maxcsn &&
csn_compare (buf->buf_cscbs[i]->consumer_maxcsn, rid_data->csn) >= 0 ) {
@@ -580,88 +589,147 @@ clcache_refresh_local_maxcsn ( const ruv_enum_data *rid_data, void *data )
static int
clcache_refresh_local_maxcsns ( CLC_Buffer *buf )
{
- int i;
- for ( i = 0; i < buf->buf_num_cscbs; i++ ) {
- csn_dup_or_init_by_csn ( &buf->buf_cscbs[i]->prev_local_maxcsn,
- buf->buf_cscbs[i]->local_maxcsn );
- }
return ruv_enumerate_elements ( buf->buf_local_ruv, clcache_refresh_local_maxcsn, buf );
}
/*
* Algorithm:
*
- * 1. Snapshot local RUVs;
- * 2. Load buffer;
- * 3. Send to the consumer only those CSNs that are covered
- * by the RUVs snapshot taken in the first step;
- * All CSNs that are covered by the RUVs snapshot taken in the
- * first step are guaranteed in consecutive order for the respected
- * RIDs because of the the CSN pending list control;
- * A CSN that is not covered by the RUVs snapshot may be out of order
- * since it is possible that a smaller CSN might not have committed
- * yet by the time the buffer was loaded.
- * 4. Determine anchorcsn for each RID:
- *
- * Case| Local vs. Buffer | New Local | Next
- * | MaxCSN MaxCSN | MaxCSN | Anchor-CSN
- * ----+-------------------+-----------+----------------
- * 1 | Cl >= Cb | * | Cb
- * 2 | Cl < Cb | Cl | Cb
- * 3 | Cl < Cb | Cl2 | Cl
- *
- * 5. Determine anchorcsn for next load:
+ * 1. Determine anchorcsn for each RID:
+ * 2. Determine anchorcsn for next load:
* Anchor-CSN = min { all Next-Anchor-CSN, Buffer-MaxCSN }
*/
static int
-clcache_adjust_anchorcsn ( CLC_Buffer *buf )
+clcache_initial_anchorcsn ( CLC_Buffer *buf, int *flag )
{
PRBool hasChange = PR_FALSE;
struct csn_seq_ctrl_block *cscb;
int i;
+ CSN *anchorcsn = NULL;
if ( buf->buf_state == CLC_STATE_READY ) {
for ( i = 0; i < buf->buf_num_cscbs; i++ ) {
+ CSN *rid_anchor = NULL;
+ int rid_flag = DB_NEXT;
cscb = buf->buf_cscbs[i];
- if ( cscb->state == CLC_STATE_UP_TO_DATE )
- continue;
+ if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {
+ char prevmax[CSN_STRSIZE];
+ char local[CSN_STRSIZE];
+ char curr[CSN_STRSIZE];
+ char conmaxcsn[CSN_STRSIZE];
+ csn_as_string(cscb->prev_local_maxcsn, 0, prevmax);
+ csn_as_string(cscb->local_maxcsn, 0, local);
+ csn_as_string(buf->buf_current_csn, 0, curr);
+ csn_as_string(cscb->consumer_maxcsn, 0, conmaxcsn);
+ slapi_log_error(SLAPI_LOG_REPL, "clcache_initial_anchorcsn" ,
+ "%s - (cscb %d - state %d) - csnPrevMax (%s) "
+ "csnMax (%s) csnBuf (%s) csnConsumerMax (%s)\n",
+ buf->buf_agmt_name, i, cscb->state, prevmax, local,
+ curr, conmaxcsn);
+ }
- /*
- * Case 3 unsafe ruv change: next buffer load should start
- * from where the maxcsn in the old ruv was. Since each
- * cscb has remembered the maxcsn sent to the consumer,
- * CSNs that may be loaded again could easily be skipped.
- */
- if ( cscb->prev_local_maxcsn &&
- csn_compare (cscb->prev_local_maxcsn, buf->buf_current_csn) < 0 &&
- csn_compare (cscb->local_maxcsn, cscb->prev_local_maxcsn) != 0 ) {
+ if (cscb->consumer_maxcsn == NULL) {
+ /* the consumer hasn't seen changes for this RID */
+ rid_anchor = cscb->local_mincsn;
+ rid_flag = DB_SET;
+ } else if ( csn_compare (cscb->local_maxcsn, cscb->consumer_maxcsn) > 0 ) {
+ rid_anchor = cscb->consumer_maxcsn;
+ }
+
+ if (rid_anchor && (anchorcsn == NULL ||
+ ( csn_compare(rid_anchor, anchorcsn) < 0))) {
+ anchorcsn = rid_anchor;
+ *flag = rid_flag;
hasChange = PR_TRUE;
- cscb->state = CLC_STATE_READY;
- csn_init_by_csn ( buf->buf_current_csn, cscb->prev_local_maxcsn );
- csn_as_string ( cscb->prev_local_maxcsn, 0, (char*)buf->buf_key.data );
- slapi_log_error ( SLAPI_LOG_REPL, buf->buf_agmt_name,
- "adjust anchor csn upon %s\n",
- ( cscb->state == CLC_STATE_CSN_GT_RUV ? "out of sequence csn" : "unsafe ruv change") );
- continue;
}
- /*
- * check if there are still changes to send for this RID
- * Assume we had compared the local maxcsn and the consumer
- * max csn before this function was called and hence the
- * cscb->state had been set accordingly.
- */
- if ( hasChange == PR_FALSE &&
- csn_compare (cscb->local_maxcsn, buf->buf_current_csn) > 0 ) {
+
+ }
+ }
+
+ if ( !hasChange ) {
+ buf->buf_state = CLC_STATE_DONE;
+ } else {
+ csn_init_by_csn(buf->buf_current_csn, anchorcsn);
+ csn_as_string(buf->buf_current_csn, 0, (char *)buf->buf_key.data);
+ slapi_log_error(SLAPI_LOG_REPL, "clcache_initial_anchorcsn",
+ "anchor is now: %s\n", (char *)buf->buf_key.data);
+ }
+
+ return buf->buf_state;
+}
+
+static int
+clcache_adjust_anchorcsn ( CLC_Buffer *buf, int *flag )
+{
+ PRBool hasChange = PR_FALSE;
+ struct csn_seq_ctrl_block *cscb;
+ int i;
+ CSN *anchorcsn = NULL;
+
+ if ( buf->buf_state == CLC_STATE_READY ) {
+ for ( i = 0; i < buf->buf_num_cscbs; i++ ) {
+ CSN *rid_anchor = NULL;
+ int rid_flag = DB_NEXT;
+ cscb = buf->buf_cscbs[i];
+
+ if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {
+ char prevmax[CSN_STRSIZE];
+ char local[CSN_STRSIZE];
+ char curr[CSN_STRSIZE];
+ char conmaxcsn[CSN_STRSIZE];
+ csn_as_string(cscb->prev_local_maxcsn, 0, prevmax);
+ csn_as_string(cscb->local_maxcsn, 0, local);
+ csn_as_string(buf->buf_current_csn, 0, curr);
+ csn_as_string(cscb->consumer_maxcsn, 0, conmaxcsn);
+ slapi_log_error(SLAPI_LOG_REPL, "clcache_adjust_anchorcsn" ,
+ "%s - (cscb %d - state %d) - csnPrevMax (%s) "
+ "csnMax (%s) csnBuf (%s) csnConsumerMax (%s)\n",
+ buf->buf_agmt_name, i, cscb->state, prevmax, local,
+ curr, conmaxcsn);
+ }
+
+ if (csn_compare (cscb->local_maxcsn, cscb->prev_local_maxcsn) == 0 ||
+ csn_compare (cscb->prev_local_maxcsn, buf->buf_current_csn) > 0 ) {
+ if (csn_compare (cscb->local_maxcsn, cscb->consumer_maxcsn) > 0 ) {
+ rid_anchor = buf->buf_current_csn;
+ }
+ } else {
+ /* prev local max csn < csnBuffer AND different from local maxcsn */
+ if (cscb->prev_local_maxcsn == NULL) {
+ if (cscb->consumer_maxcsn == NULL) {
+ /* the consumer hasn't seen changes for this RID */
+ rid_anchor = cscb->local_mincsn;
+ rid_flag = DB_SET;
+ } else if ( csn_compare (cscb->local_maxcsn, cscb->consumer_maxcsn) > 0 ) {
+ rid_anchor = cscb->consumer_maxcsn;
+ }
+ } else {
+ /* csnPrevMaxSup > 0 */
+ rid_anchor = cscb->consumer_maxcsn;
+ }
+ }
+
+ if (rid_anchor && (anchorcsn == NULL ||
+ ( csn_compare(rid_anchor, anchorcsn) < 0))) {
+ anchorcsn = rid_anchor;
+ *flag = rid_flag;
hasChange = PR_TRUE;
}
+
+
}
}
if ( !hasChange ) {
buf->buf_state = CLC_STATE_DONE;
+ } else {
+ csn_init_by_csn(buf->buf_current_csn, anchorcsn);
+ csn_as_string(buf->buf_current_csn, 0, (char *)buf->buf_key.data);
+ slapi_log_error(SLAPI_LOG_REPL, "clcache_adjust_anchorcsn",
+ "anchor is now: %s\n", (char *)buf->buf_key.data);
}
return buf->buf_state;
@@ -675,7 +743,6 @@ clcache_skip_change ( CLC_Buffer *buf )
int skip = 1;
int i;
char buf_cur_csn_str[CSN_STRSIZE];
- char oth_csn_str[CSN_STRSIZE];
do {
@@ -688,25 +755,14 @@ clcache_skip_change ( CLC_Buffer *buf )
* legacy consumer. In this case the supplier
* and the consumer may have the same RID.
*/
- if (rid == buf->buf_consumer_rid && rid != MAX_REPLICA_ID){
- CSN *cons_maxcsn = NULL;
-
- ruv_get_max_csn(buf->buf_consumer_ruv, &cons_maxcsn);
- if ( csn_compare ( buf->buf_current_csn, cons_maxcsn) > 0 ) {
- /*
- * The consumer must have been "restored" and needs this newer update.
- */
- skip = 0;
- } else if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {
+ if (rid == buf->buf_consumer_rid && buf->buf_ignoreConsumerRID){
+ if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {
csn_as_string(buf->buf_current_csn, 0, buf_cur_csn_str);
- csn_as_string(cons_maxcsn, 0, oth_csn_str);
slapi_log_error(SLAPI_LOG_REPL, buf->buf_agmt_name,
- "Skipping update because the changelog buffer current csn [%s] is "
- "less than or equal to the consumer max csn [%s]\n",
- buf_cur_csn_str, oth_csn_str);
+ "Skipping update because the consumer with Rid: [%d] is "
+ "ignored\n", rid);
buf->buf_skipped_csn_gt_cons_maxcsn++;
}
- csn_free(&cons_maxcsn);
break;
}
@@ -821,6 +877,7 @@ clcache_free_cscb ( struct csn_seq_ctrl_block ** cscb )
csn_free ( & (*cscb)->consumer_maxcsn );
csn_free ( & (*cscb)->local_maxcsn );
csn_free ( & (*cscb)->prev_local_maxcsn );
+ csn_free ( & (*cscb)->local_mincsn );
slapi_ch_free ( (void **) cscb );
}
@@ -1003,6 +1060,15 @@ clcache_cursor_get ( DBC *cursor, CLC_Buffer *buf, int flag )
{
int rc;
+ if (buf->buf_data.ulen > WORK_CLC_BUFFER_PAGE_SIZE) {
+ /*
+ * The buffer size had to be increased,
+ * reset it to a smaller working size,
+ * if not sufficient it will be increased again
+ */
+ buf->buf_data.ulen = WORK_CLC_BUFFER_PAGE_SIZE;
+ }
+
rc = cursor->c_get ( cursor,
& buf->buf_key,
& buf->buf_data,
diff --git a/ldap/servers/plugins/replication/cl5_clcache.h b/ldap/servers/plugins/replication/cl5_clcache.h
index 4c459ab..75b2817 100644
--- a/ldap/servers/plugins/replication/cl5_clcache.h
+++ b/ldap/servers/plugins/replication/cl5_clcache.h
@@ -23,7 +23,7 @@ typedef struct clc_buffer CLC_Buffer;
int clcache_init ( DB_ENV **dbenv );
void clcache_set_config ();
int clcache_get_buffer ( CLC_Buffer **buf, DB *db, ReplicaId consumer_rid, const RUV *consumer_ruv, const RUV *local_ruv );
-int clcache_load_buffer ( CLC_Buffer *buf, CSN *startCSN, int flag );
+int clcache_load_buffer ( CLC_Buffer *buf, CSN **anchorCSN );
void clcache_return_buffer ( CLC_Buffer **buf );
int clcache_get_next_change ( CLC_Buffer *buf, void **key, size_t *keylen, void **data, size_t *datalen, CSN **csn );
void clcache_destroy ();
7 years, 10 months
Branch '389-ds-base-1.3.4' - 2 commits - dirsrvtests/tests ldap/admin ldap/ldif ldap/servers Makefile.am Makefile.in
by Noriko Hosoi
Makefile.am | 3
Makefile.in | 3
dirsrvtests/tests/tickets/ticket48755_test.py | 61 ++++-
ldap/admin/src/scripts/91reindex.pl.in | 103 +++++++++
ldap/admin/src/scripts/91subtreereindex.pl | 8
ldap/admin/src/scripts/setup-ds.res.in | 1
ldap/ldif/template-dse.ldif.in | 1
ldap/servers/plugins/replication/repl5_replica_config.c | 9
ldap/servers/plugins/replication/repl5_tot_protocol.c | 169 +++++++++++-----
ldap/servers/slapd/back-ldbm/back-ldbm.h | 6
ldap/servers/slapd/back-ldbm/dblayer.c | 5
ldap/servers/slapd/back-ldbm/filterindex.c | 21 +
ldap/servers/slapd/back-ldbm/idl_new.c | 87 +++++++-
ldap/servers/slapd/back-ldbm/index.c | 22 +-
ldap/servers/slapd/back-ldbm/init.c | 2
ldap/servers/slapd/back-ldbm/ldbm_search.c | 33 ++-
ldap/servers/slapd/back-ldbm/misc.c | 1
ldap/servers/slapd/entry.c | 4
ldap/servers/slapd/slap.h | 4
ldap/servers/slapd/slapi-plugin.h | 8
ldap/servers/slapd/slapi-private.h | 1
21 files changed, 445 insertions(+), 107 deletions(-)
New commits:
commit 3c789a5170f1e599c5ea9b74da3b32221cd7a6c6
Author: Noriko Hosoi <nhosoi(a)redhat.com>
Date: Thu Jun 16 17:32:49 2016 -0700
Ticket #48755 - CI test: test case for ticket 48755
Description: moving an entry could make the online init fail
In addition to the existing test case, added password policy entries,
which are ldapsubentry.
(cherry picked from commit a20111b446a62c9a514ce23ca1a7eafabdfc904b)
diff --git a/dirsrvtests/tests/tickets/ticket48755_test.py b/dirsrvtests/tests/tickets/ticket48755_test.py
index e3b7b61..db4ae39 100644
--- a/dirsrvtests/tests/tickets/ticket48755_test.py
+++ b/dirsrvtests/tests/tickets/ticket48755_test.py
@@ -149,6 +149,44 @@ def add_user_entry(server, idx, myparent):
'cn': 'Test User%d' % idx,
'userpassword': 'password'})))
+def del_user_entry(server, idx, myparent):
+ name = 'tuser%d' % idx
+ dn = 'uid=%s,%s' % (name, myparent)
+ server.delete_s(dn)
+
+def add_ldapsubentry(server, myparent):
+ name = 'nsPwPolicyContainer'
+ container = 'cn=%s,%s' % (name, myparent)
+ server.add_s(Entry((container, {'objectclass': ['top', 'nsContainer'],
+ 'cn': '%s' % name})))
+
+ name = 'nsPwPolicyEntry'
+ pwpentry = 'cn=%s,%s' % (name, myparent)
+ pwpdn = 'cn="%s",%s' % (pwpentry, container)
+ server.add_s(Entry((pwpdn, {'objectclass': ['top', 'ldapsubentry', 'passwordpolicy'],
+ 'passwordStorageScheme': 'ssha',
+ 'passwordCheckSyntax': 'on',
+ 'passwordInHistory': '6',
+ 'passwordChange': 'on',
+ 'passwordMinAge': '0',
+ 'passwordExp': 'off',
+ 'passwordMustChange': 'off',
+ 'cn': '%s' % pwpentry})))
+
+ name = 'nsPwTemplateEntry'
+ tmplentry = 'cn=%s,%s' % (name, myparent)
+ tmpldn = 'cn="%s",%s' % (tmplentry, container)
+ server.add_s(Entry((tmpldn, {'objectclass': ['top', 'ldapsubentry', 'costemplate', 'extensibleObject'],
+ 'cosPriority': '1',
+ 'cn': '%s' % tmplentry})))
+
+ name = 'nsPwPolicy_CoS'
+ cos = 'cn=%s,%s' % (name, myparent)
+ server.add_s(Entry((cos, {'objectclass': ['top', 'ldapsubentry', 'cosPointerDefinition', 'cosSuperDefinition'],
+ 'costemplatedn': '%s' % tmpldn,
+ 'cosAttribute': 'pwdpolicysubentry default operational-default',
+ 'cn': '%s' % name})))
+
def test_ticket48755(topology):
log.info("Ticket 48755 - moving an entry could make the online init fail")
@@ -163,9 +201,14 @@ def test_ticket48755(topology):
parent0 = '%s,%s' % (ou0, DEFAULT_SUFFIX)
add_ou_entry(M1, idx, parent0)
+ add_ldapsubentry(M1, parent0)
+
parent00 = 'ou=OU%d,%s' % (idx, parent0)
for idx in range(0, 9):
add_user_entry(M1, idx, parent00)
+ if idx % 2 == 0:
+ log.info("Turning tuser%d into a tombstone entry" % idx)
+ del_user_entry(M1, idx, parent00)
log.info('%s => %s => %s => 10 USERS' % (DEFAULT_SUFFIX, parent0, parent00))
@@ -176,6 +219,8 @@ def test_ticket48755(topology):
parent1 = 'ou=OU%d,%s' % (idx, DEFAULT_SUFFIX)
add_ou_entry(M1, idx, parent1)
+ add_ldapsubentry(M1, parent1)
+
log.info("Moving %s to DIT_1" % parent00)
M1.rename_s(parent00, ou0, newsuperior=parent1, delold=1)
@@ -186,20 +231,14 @@ def test_ticket48755(topology):
parent001 = '%s,%s' % (ou0, parent01)
log.info("Moving USERS to %s" % parent0)
for idx in range(0, 9):
- name = 'tuser%d' % idx
- rdn = 'uid=%s' % name
- dn = 'uid=%s,%s' % (name, parent01)
- M1.rename_s(dn, rdn, newsuperior=parent001, delold=1)
+ if idx % 2 == 1:
+ name = 'tuser%d' % idx
+ rdn = 'uid=%s' % name
+ dn = 'uid=%s,%s' % (name, parent01)
+ M1.rename_s(dn, rdn, newsuperior=parent001, delold=1)
log.info('%s => %s => %s => %s => 10 USERS' % (DEFAULT_SUFFIX, parent1, parent01, parent001))
- log.info("Deleting 5 USERS to turn them into a tombstone entries")
- for idx in range(5, 9):
- name = 'tuser%d' % idx
- rdn = 'uid=%s' % name
- dn = 'uid=%s,%s' % (name, parent001)
- M1.delete_s(dn)
-
log.info("Run Consumer Initialization.")
global m1_m2_agmt
M1.startReplication_async(m1_m2_agmt)
commit cac1b0a6412ce232e4c24f486e1383ebe9b80a67
Author: Noriko Hosoi <nhosoi(a)redhat.com>
Date: Mon Jun 27 22:33:40 2016 -0700
Ticket #48755 - moving an entry could make the online init fail
Bug Description: Online init (aka Total update, bulk import) scans the
primary id2entry db in the order of ID. If Entry A is moved under a
new superior Entry B which was generated after Entry A, when Entry A
is sent to a consumer using online init, its parent entry does not
exist on the consumer and the online init fails.
Fix Description:
- Added a command BACK_INFO_IS_ENTRYRDN to slapi_back_get_info, which
returns the status of entryrdn switch maintained in the backend.
- If slapi_backend_get_info(BACK_INFO_IS_ENTRYRDN) returns true for
the replicated backend, repl5_tot_run searches the entry with the
filter:
(parentid>=1)
instead of:
(|(objectclass=ldapsubentry)(objectclass=nstombstone)(nsuniqueid=*))".
The filter MUST be this single range filter since the IDList created
for this bulk-import is not sorted by ID, but in the order of parentid.
- Added a new flag OP_FLAG_BULK_IMPORT to the operation. Setting the
flag implies to skip the internal operations such as eliminating
ldapsubentry from the search results or a filter check.
- In addition, idl_new_range_fetch is modified so that ...
* A range search for parentid ignores nsslapd-idlistscanlimit by
setting SLAPI_OP_RANGE_NO_ALLIDS as well as it skips sorting the
IDlist by ID by setting SLAPI_OP_RANGE_NO_IDL_SORT.
* In case SLAPI_OP_RANGE_NO_IDL_SORT is set, idl_new_range_fetch
checks whether the key (in this case parentid) is in the IDlist.
If it exists, the ID is appended. If it does not, the ID is in
the leftover list and appended when the parent ID is found in the
IDlist.
- Increased the version of rdn-format-# in DBVERSION to 3.
- Upgrade script 91reindex.pl.in is added which reindex the parentid
index file in the integer order if the version of rdn-format-# in
DBVERSION is less than 3.
- Notes: In case entryrdn is used (nsslapd-subtree-rename-switch: on),
all the type of entries including the tombstone entry and the ldap-
subentry are indexed with parentid.
- Another notes: This special treatment is enabled only when nsslapd-
subtree-rename-switch is on. If entrydn is used instead of entryrdn,
there is no possibility to establish the order that child comes first.
Plus, to replace nsslapd-subtree-rename-switch on with off, the main
db file id2entry.db needs to be recreated so that each entry stores
DN in it. It requires export then import, which corrects the order
of the entries and the order problem disappears.
https://fedorahosted.org/389/ticket/48755
Reviewed by mreynolds@redhat and wibrown(a)redhat.com (Thank you, Mark and William!!)
Also, thanks to lkrispen(a)redhat.com and tbordaz(a)redhat.com for their discussion!
(cherry picked from commit f50e8ec950980af70d892cc45eac07684d6721a5)
diff --git a/Makefile.am b/Makefile.am
index 8dcdb36..2d19a74 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -589,7 +589,8 @@ update_DATA = ldap/admin/src/scripts/exampleupdate.pl \
ldap/admin/src/scripts/50AES-pbe-plugin.ldif\
ldap/admin/src/scripts/50updateconfig.ldif \
ldap/admin/src/scripts/52updateAESplugin.pl \
- ldap/admin/src/scripts/dnaplugindepends.ldif
+ ldap/admin/src/scripts/dnaplugindepends.ldif \
+ ldap/admin/src/scripts/91reindex.pl
update_SCRIPTS = ldap/admin/src/scripts/exampleupdate.sh
diff --git a/Makefile.in b/Makefile.in
index a29509d..4298c57 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1997,7 +1997,8 @@ update_DATA = ldap/admin/src/scripts/exampleupdate.pl \
ldap/admin/src/scripts/50AES-pbe-plugin.ldif\
ldap/admin/src/scripts/50updateconfig.ldif \
ldap/admin/src/scripts/52updateAESplugin.pl \
- ldap/admin/src/scripts/dnaplugindepends.ldif
+ ldap/admin/src/scripts/dnaplugindepends.ldif \
+ ldap/admin/src/scripts/91reindex.pl
update_SCRIPTS = ldap/admin/src/scripts/exampleupdate.sh
diff --git a/ldap/admin/src/scripts/91reindex.pl.in b/ldap/admin/src/scripts/91reindex.pl.in
new file mode 100644
index 0000000..c861f64
--- /dev/null
+++ b/ldap/admin/src/scripts/91reindex.pl.in
@@ -0,0 +1,103 @@
+use Mozilla::LDAP::Conn;
+use Mozilla::LDAP::Utils qw(normalizeDN);
+use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn);
+use DSUpdate qw(isOffline);
+
+sub runinst {
+ my ($inf, $inst, $dseldif, $conn) = @_;
+ my $rc, @errs;
+
+ # List of index to be reindexed
+ my @toreindex = qw(parentid);
+ # rdn-format value. See $rdn_format set below.
+ # If equal to or greater than this value, no need to reindex.
+ # If it needs to be unconditionally reindexed, set 0.
+ my @rdnconditions = (4)
+
+ my $config = $conn->search("cn=config", "base", "(objectclass=*)");
+ if (!$config) {
+ push @errs, ['error_finding_config_entry', 'cn=config',
+ $conn->getErrorString()];
+ return @errs;
+ }
+
+ ($rc, @errs) = isOffline($inf, $inst, $conn);
+ if (!$rc) {
+ return @errs;
+ }
+
+ my $reindex = "@sbindir@/db2index -Z $inst";
+ my @errs;
+ my $instconf = $conn->search("cn=ldbm database,cn=plugins,cn=config", "onelevel", "(objectclass=*)");
+ if (!$instconf) {
+ push @errs, ['error_finding_config_entry', 'cn=*,cn=ldbm database,cn=plugins,cn=config', $conn->getErrorString()];
+ return @errs;
+ }
+
+ my $dbconf = $conn->search("cn=config,cn=ldbm database,cn=plugins,cn=config", "base", "(objectclass=*)");
+ if (!$dbconf) {
+ push @errs, ['error_finding_config_entry',
+ 'cn=config,cn=ldbm database,cn=plugins,cn=config',
+ $conn->getErrorString()];
+ return @errs;
+ }
+
+ # Get the value of nsslapd-subtree-rename-switch.
+ my $switch = $dbconf->getValues('nsslapd-subtree-rename-switch');
+ if ("" eq $switch) {
+ return (); # subtree-rename-switch does not exist; do nothing.
+ } elsif ("off" eq $switch || "OFF" eq $switch) {
+ return (); # subtree-rename-switch is OFF; do nothing.
+ }
+
+ my $dbdir = $dbconf->getValues('nsslapd-directory');
+ my $dbversion0 = $dbdir . "/DBVERSION";
+ my $rdn_format = 0;
+ my $dbversionstr = "";
+ if (!open(DBVERSION, "$dbversion0")) {
+ push @errs, ['error_opening_file', $dbversion0, $!];
+ return @errs;
+ } else {
+ while (<DBVERSION>) {
+ if ($_ =~ /rdn-format/) {
+ $rdn_format = 1;
+ $dbversionstr = $_;
+ if ($_ =~ /rdn-format-1/) {
+ $rdn_format = 2;
+ } elsif ($_ =~ /rdn-format-2/) {
+ $rdn_format = 3;
+ } elsif ($_ =~ /rdn-format-3/) {
+ $rdn_format = 4;
+ } elsif ($_ =~ /rdn-format-4/) {
+ $rdn_format = 5;
+ } elsif ($_ =~ /rdn-format-5/) {
+ $rdn_format = 6;
+ } elsif ($_ =~ /rdn-format-/) {
+ # assume greater than -5
+ $rdn_format = 7;
+ }
+ }
+ }
+ close DBVERSION;
+ }
+
+ while ($instconf) {
+ my $backend= $instconf->getValues('cn');
+ if (($backend eq "config") || ($backend eq "monitor")) {
+ goto NEXT;
+ }
+
+ for (my $idx = 0; $ <= $#toreindex; $idx++) {
+ if (0 == $rdnconditions[$idx] || $rdnconditions[$idx] > $rdn_format) {
+ my $rc = system("$reindex -n $backend -t $idx");
+ if ($rc) {
+ push @errs, ["error_reindexng", $idx, $backend, $rc];
+ }
+ }
+ }
+NEXT:
+ $instconf = $conn->nextEntry();
+ }
+
+ return @errs;
+}
diff --git a/ldap/admin/src/scripts/91subtreereindex.pl b/ldap/admin/src/scripts/91subtreereindex.pl
index a031cc1..c4b40a3 100644
--- a/ldap/admin/src/scripts/91subtreereindex.pl
+++ b/ldap/admin/src/scripts/91subtreereindex.pl
@@ -51,14 +51,18 @@ sub runinst {
if ($_ =~ /rdn-format-1/) {
$is_rdn_format = 2;
}
- if ($_ =~ /rdn-format-2/) {
+ elsif ($_ =~ /rdn-format-2/) {
$is_rdn_format = 3;
}
+ elsif ($_ =~ /rdn-format-/) {
+ # assume greater than -2
+ $is_rdn_format = 4;
+ }
}
}
close DBVERSION;
- if (3 == $is_rdn_format) {
+ if (3 <= $is_rdn_format) {
return (); # DB already has the new rdn format.
}
diff --git a/ldap/admin/src/scripts/setup-ds.res.in b/ldap/admin/src/scripts/setup-ds.res.in
index 7134e25..760db6f 100644
--- a/ldap/admin/src/scripts/setup-ds.res.in
+++ b/ldap/admin/src/scripts/setup-ds.res.in
@@ -208,3 +208,4 @@ error_opening_file = Opening file '%s' failed. Error: %s\n
error_format_error = '%s' has invalid format.\n
error_update_not_offline = Error: offline mode selected but the server [%s] is still running.\n
error_update_all = Failed to update all the Directory Server instances.\n
+error_reindexing = Failed to reindex '%s' in backend '%s'. Error: %s\n
diff --git a/ldap/ldif/template-dse.ldif.in b/ldap/ldif/template-dse.ldif.in
index 6acbfae..2988cb9 100644
--- a/ldap/ldif/template-dse.ldif.in
+++ b/ldap/ldif/template-dse.ldif.in
@@ -927,6 +927,7 @@ objectclass: nsIndex
cn: parentid
nssystemindex: true
nsindextype: eq
+nsmatchingrule: integerOrderingMatch
dn: cn=seeAlso,cn=default indexes, cn=config,cn=ldbm database,cn=plugins,cn=config
objectclass: top
diff --git a/ldap/servers/plugins/replication/repl5_replica_config.c b/ldap/servers/plugins/replication/repl5_replica_config.c
index 866a712..092f04e 100644
--- a/ldap/servers/plugins/replication/repl5_replica_config.c
+++ b/ldap/servers/plugins/replication/repl5_replica_config.c
@@ -405,12 +405,12 @@ replica_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry*
{
if (apply_mods)
replica_set_precise_purging(r, 0);
- }
+ }
else if (strcasecmp (config_attr, type_replicaReleaseTimeout) == 0 )
{
if (apply_mods)
replica_set_release_timeout(r, 0);
- }
+ }
else
{
*returncode = LDAP_UNWILLING_TO_PERFORM;
@@ -572,8 +572,7 @@ replica_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry*
{
if (apply_mods)
{
- if (apply_mods && config_attr_value[0])
- {
+ if (config_attr_value[0]) {
PRUint64 on_off = 0;
if (strcasecmp(config_attr_value, "on") == 0){
@@ -592,7 +591,7 @@ replica_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry*
break;
}
replica_set_precise_purging(r, on_off);
- } else if (apply_mods) {
+ } else {
replica_set_precise_purging(r, 0);
}
}
diff --git a/ldap/servers/plugins/replication/repl5_tot_protocol.c b/ldap/servers/plugins/replication/repl5_tot_protocol.c
index d0c4402..aac89bd 100644
--- a/ldap/servers/plugins/replication/repl5_tot_protocol.c
+++ b/ldap/servers/plugins/replication/repl5_tot_protocol.c
@@ -323,6 +323,10 @@ repl5_tot_run(Private_Repl_Protocol *prp)
int init_retry = 0;
Replica *replica;
ReplicaId rid = 0; /* Used to create the replica keep alive subentry */
+ Slapi_Entry *suffix = NULL;
+ char **instances = NULL;
+ Slapi_Backend *be = NULL;
+ int is_entryrdn = 0;
PR_ASSERT(NULL != prp);
@@ -354,21 +358,21 @@ retry:
*/
if (rc != ACQUIRE_SUCCESS)
{
- int optype, ldaprc, wait_retry;
- conn_get_error(prp->conn, &optype, &ldaprc);
- if (rc == ACQUIRE_TRANSIENT_ERROR && INIT_RETRY_MAX > init_retry++) {
- wait_retry = init_retry * INIT_RETRY_INT;
- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
- "acquire replica for total update, error: %d,"
+ int optype, ldaprc, wait_retry;
+ conn_get_error(prp->conn, &optype, &ldaprc);
+ if (rc == ACQUIRE_TRANSIENT_ERROR && INIT_RETRY_MAX > init_retry++) {
+ wait_retry = init_retry * INIT_RETRY_INT;
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "acquire replica for total update, error: %d,"
" retrying in %d seconds.\n",
- ldaprc, wait_retry);
- DS_Sleep(PR_SecondsToInterval(wait_retry));
- goto retry;
- } else {
- agmt_set_last_init_status(prp->agmt, ldaprc,
- prp->last_acquire_response_code, 0, NULL);
- goto done;
- }
+ ldaprc, wait_retry);
+ DS_Sleep(PR_SecondsToInterval(wait_retry));
+ goto retry;
+ } else {
+ agmt_set_last_init_status(prp->agmt, ldaprc,
+ prp->last_acquire_response_code, 0, NULL);
+ goto done;
+ }
}
else if (prp->terminate)
{
@@ -405,48 +409,121 @@ retry:
and that the order implies that perent entry is always ahead of the
child entry in the list. Otherwise, the consumer would not be
properly updated because bulk import at the moment skips orphand entries. */
- /* XXXggood above assumption may not be valid if orphaned entry moved???? */
+ /* XXXggood above assumption may not be valid if orphaned entry moved???? */
agmt_set_last_init_status(prp->agmt, 0, 0, 0, "Total update in progress");
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Beginning total update of replica "
- "\"%s\".\n", agmt_get_long_name(prp->agmt));
+ "\"%s\".\n", agmt_get_long_name(prp->agmt));
/* RMREPL - need to send schema here */
pb = slapi_pblock_new ();
- /* we need to provide managedsait control so that referral entries can
- be replicated */
- ctrls = (LDAPControl **)slapi_ch_calloc (3, sizeof (LDAPControl *));
- ctrls[0] = create_managedsait_control ();
- ctrls[1] = create_backend_control(area_sdn);
+ replica = (Replica*) object_get_data(prp->replica_object);
+ /*
+ * Get the info about the entryrdn vs. entrydn from the backend.
+ * If NOT is_entryrdn, its ancestor entries are always found prior to an entry.
+ */
+ rc = slapi_lookup_instance_name_by_suffix((char *)slapi_sdn_get_dn(area_sdn), NULL, &instances, 1);
+ if (rc || !instances) {
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "get the instance name for the suffix \"%s\".\n", slapi_sdn_get_dn(area_sdn));
+ goto done;
+ }
+ be = slapi_be_select_by_instance_name(instances[0]);
+ if (!be) {
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "get the instance for the suffix \"%s\".\n", slapi_sdn_get_dn(area_sdn));
+ goto done;
+ }
+ rc = slapi_back_get_info(be, BACK_INFO_IS_ENTRYRDN, (void **)&is_entryrdn);
+ if (is_entryrdn) {
+ /*
+ * Supporting entries out of order -- parent could have a larger id than its children.
+ * Entires are retireved sorted by parentid without the allid threshold.
+ */
+ /* Get suffix */
+ rc = slapi_search_internal_get_entry(area_sdn, NULL, &suffix, repl_get_plugin_identity(PLUGIN_MULTIMASTER_REPLICATION));
+ if (rc) {
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "get the suffix entry \"%s\".\n", slapi_sdn_get_dn(area_sdn));
+ goto done;
+ }
- /* Time to make sure it exists a keep alive subentry for that replica */
- replica = (Replica*) object_get_data(prp->replica_object);
- if (replica)
- {
- rid = replica_get_rid(replica);
- }
- replica_subentry_check(area_sdn, rid);
-
- slapi_search_internal_set_pb (pb, slapi_sdn_get_dn (area_sdn),
- LDAP_SCOPE_SUBTREE, "(|(objectclass=ldapsubentry)(objectclass=nstombstone)(nsuniqueid=*))", NULL, 0, ctrls, NULL,
- repl_get_plugin_identity (PLUGIN_MULTIMASTER_REPLICATION), 0);
-
- cb_data.prp = prp;
- cb_data.rc = 0;
- cb_data.num_entries = 0UL;
- cb_data.sleep_on_busy = 0UL;
- cb_data.last_busy = current_time ();
- cb_data.flowcontrol_detection = 0;
- cb_data.lock = PR_NewLock();
-
- /* This allows during perform_operation to check the callback data
- * especially to do flow contol on delta send msgid / recv msgid
- */
- conn_set_tot_update_cb(prp->conn, (void *) &cb_data);
+ cb_data.prp = prp;
+ cb_data.rc = 0;
+ cb_data.num_entries = 1UL;
+ cb_data.sleep_on_busy = 0UL;
+ cb_data.last_busy = current_time ();
+ cb_data.flowcontrol_detection = 0;
+ cb_data.lock = PR_NewLock();
+
+ /* This allows during perform_operation to check the callback data
+ * especially to do flow contol on delta send msgid / recv msgid
+ */
+ conn_set_tot_update_cb(prp->conn, (void *) &cb_data);
+
+ /* Send suffix first. */
+ rc = send_entry(suffix, (void *)&cb_data);
+ if (rc) {
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "send the suffix entry \"%s\" to the consumer.\n", slapi_sdn_get_dn(area_sdn));
+ goto done;
+ }
+
+ /* we need to provide managedsait control so that referral entries can
+ be replicated */
+ ctrls = (LDAPControl **)slapi_ch_calloc (3, sizeof (LDAPControl *));
+ ctrls[0] = create_managedsait_control ();
+ ctrls[1] = create_backend_control(area_sdn);
+
+ /* Time to make sure it exists a keep alive subentry for that replica */
+ if (replica)
+ {
+ rid = replica_get_rid(replica);
+ }
+ replica_subentry_check(area_sdn, rid);
+ /* Send the subtree of the suffix in the order of parentid index plus ldapsubentry and nstombstone. */
+ slapi_search_internal_set_pb(pb, slapi_sdn_get_dn (area_sdn),
+ LDAP_SCOPE_SUBTREE, "(parentid>=1)", NULL, 0, ctrls, NULL,
+ repl_get_plugin_identity (PLUGIN_MULTIMASTER_REPLICATION), OP_FLAG_BULK_IMPORT);
+ cb_data.num_entries = 0UL;
+ } else {
+ /* Original total update */
+ /* we need to provide managedsait control so that referral entries can
+ be replicated */
+ ctrls = (LDAPControl **)slapi_ch_calloc (3, sizeof (LDAPControl *));
+ ctrls[0] = create_managedsait_control ();
+ ctrls[1] = create_backend_control(area_sdn);
+
+ /* Time to make sure it exists a keep alive subentry for that replica */
+ replica = (Replica*) object_get_data(prp->replica_object);
+ if (replica)
+ {
+ rid = replica_get_rid(replica);
+ }
+ replica_subentry_check(area_sdn, rid);
+
+ slapi_search_internal_set_pb (pb, slapi_sdn_get_dn (area_sdn),
+ LDAP_SCOPE_SUBTREE, "(|(objectclass=ldapsubentry)(objectclass=nstombstone)(nsuniqueid=*))", NULL, 0, ctrls, NULL,
+ repl_get_plugin_identity (PLUGIN_MULTIMASTER_REPLICATION), 0);
+
+ cb_data.prp = prp;
+ cb_data.rc = 0;
+ cb_data.num_entries = 0UL;
+ cb_data.sleep_on_busy = 0UL;
+ cb_data.last_busy = current_time ();
+ cb_data.flowcontrol_detection = 0;
+ cb_data.lock = PR_NewLock();
+
+ /* This allows during perform_operation to check the callback data
+ * especially to do flow contol on delta send msgid / recv msgid
+ */
+ conn_set_tot_update_cb(prp->conn, (void *) &cb_data);
+ }
+
/* Before we get started on sending entries to the replica, we need to
* setup things for async propagation:
* 1. Create a thread that will read the LDAP results from the connection.
@@ -470,7 +547,7 @@ retry:
slapi_search_internal_callback_pb (pb, &cb_data /* callback data */,
get_result /* result callback */,
send_entry /* entry callback */,
- NULL /* referral callback*/);
+ NULL /* referral callback*/);
/*
* After completing the sending operation (or optionally failing), we need to clean up
diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h
index 9499292..2d77a8a 100644
--- a/ldap/servers/slapd/back-ldbm/back-ldbm.h
+++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h
@@ -132,7 +132,7 @@ typedef unsigned short u_int16_t;
#define BDB_BACKEND "libback-ldbm" /* This backend plugin */
#define BDB_NEWIDL "newidl" /* new idl format */
#define BDB_RDNFORMAT "rdn-format" /* Subtree rename enabled */
-#define BDB_RDNFORMAT_VERSION "2" /* rdn-format version (by default, 0) */
+#define BDB_RDNFORMAT_VERSION "3" /* rdn-format version (by default, 0) */
#define BDB_DNFORMAT "dn-4514" /* DN format RFC 4514 compliant */
#define BDB_DNFORMAT_VERSION "1" /* DN format version */
@@ -808,11 +808,11 @@ typedef struct _back_search_result_set
/* #define LDBM_ENTRYRDN_OID "2.16.840.1.113730.3.1.2097" */
#define LDBM_ANCESTORID_STR "ancestorid"
-#define LDBM_ENTRYDN_STR "entrydn"
+#define LDBM_ENTRYDN_STR SLAPI_ATTR_ENTRYDN
#define LDBM_ENTRYRDN_STR "entryrdn"
#define LDBM_NUMSUBORDINATES_STR "numsubordinates"
#define LDBM_TOMBSTONE_NUMSUBORDINATES_STR "tombstonenumsubordinates"
-#define LDBM_PARENTID_STR "parentid"
+#define LDBM_PARENTID_STR SLAPI_ATTR_PARENTID
/* Name of psuedo attribute used to track default indexes */
#define LDBM_PSEUDO_ATTR_DEFAULT ".default"
diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c
index 33506f4..9e74d9b 100644
--- a/ldap/servers/slapd/back-ldbm/dblayer.c
+++ b/ldap/servers/slapd/back-ldbm/dblayer.c
@@ -7485,6 +7485,11 @@ ldbm_back_get_info(Slapi_Backend *be, int cmd, void **info)
}
break;
}
+ case BACK_INFO_IS_ENTRYRDN:
+ {
+ *(int *)info = entryrdn_get_switch();
+ break;
+ }
default:
break;
}
diff --git a/ldap/servers/slapd/back-ldbm/filterindex.c b/ldap/servers/slapd/back-ldbm/filterindex.c
index 9c14de4..e87cf30 100644
--- a/ldap/servers/slapd/back-ldbm/filterindex.c
+++ b/ldap/servers/slapd/back-ldbm/filterindex.c
@@ -552,6 +552,8 @@ range_candidates(
struct berval *low = NULL, *high = NULL;
struct berval **lows = NULL, **highs = NULL;
back_txn txn = {NULL};
+ int operator = 0;
+ Operation *op = NULL;
LDAPDebug(LDAP_DEBUG_TRACE, "=> range_candidates attr=%s\n", type, 0, 0);
@@ -579,17 +581,24 @@ range_candidates(
high = attr_value_lowest(highs, slapi_berval_cmp);
}
+ /* Check if it is for bulk import. */
+ slapi_pblock_get(pb, SLAPI_OPERATION, &op);
+ if (entryrdn_get_switch() && operation_is_flag_set(op, OP_FLAG_INTERNAL) &&
+ operation_is_flag_set(op, OP_FLAG_BULK_IMPORT)) {
+ /* parentid is treated specially that is needed for the bulk import. (See #48755) */
+ operator = SLAPI_OP_RANGE_NO_IDL_SORT|SLAPI_OP_RANGE_NO_ALLIDS;
+ }
if (low == NULL) {
- idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY,
- SLAPI_OP_LESS_OR_EQUAL,
+ operator |= SLAPI_OP_LESS_OR_EQUAL;
+ idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, operator,
high, NULL, 0, &txn, err, allidslimit);
} else if (high == NULL) {
- idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY,
- SLAPI_OP_GREATER_OR_EQUAL,
+ operator |= SLAPI_OP_GREATER_OR_EQUAL;
+ idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, operator,
low, NULL, 0, &txn, err, allidslimit);
} else {
- idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY,
- SLAPI_OP_LESS_OR_EQUAL,
+ operator |= SLAPI_OP_LESS_OR_EQUAL;
+ idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, operator,
low, high, 1, &txn, err, allidslimit);
}
diff --git a/ldap/servers/slapd/back-ldbm/idl_new.c b/ldap/servers/slapd/back-ldbm/idl_new.c
index 25b3bfa..63df49f 100644
--- a/ldap/servers/slapd/back-ldbm/idl_new.c
+++ b/ldap/servers/slapd/back-ldbm/idl_new.c
@@ -350,17 +350,31 @@ error:
return idl;
}
+typedef struct _range_id_pair {
+ ID key;
+ ID id;
+} idl_range_id_pair;
/*
* Perform the range search in the idl layer instead of the index layer
* to improve the performance.
*/
+/*
+ * NOTE:
+ * In the total update (bulk import), an entry requires its ancestors already added.
+ * To guarantee it, the range search with parentid is used with setting the flag
+ * SLAPI_OP_RANGE_NO_IDL_SORT in operator.
+ *
+ * If the flag is set,
+ * 1. the IDList is not sorted by the ID.
+ * 2. holding to add an ID to the IDList unless the key is found in the IDList.
+ */
IDList *
idl_new_range_fetch(
- backend *be,
- DB* db,
- DBT *lowerkey,
+ backend *be,
+ DB* db,
+ DBT *lowerkey,
DBT *upperkey,
- DB_TXN *txn,
+ DB_TXN *txn,
struct attrinfo *ai,
int *flag_err,
int allidslimit,
@@ -380,7 +394,7 @@ idl_new_range_fetch(
size_t count = 0;
#ifdef DB_USE_BULK_FETCH
/* beware that a large buffer on the stack might cause a stack overflow on some platforms */
- char buffer[BULK_FETCH_BUFFER_SIZE];
+ char buffer[BULK_FETCH_BUFFER_SIZE];
void *ptr;
DBT dataret;
#endif
@@ -388,15 +402,21 @@ idl_new_range_fetch(
struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
time_t curtime;
void *saved_key = NULL;
+ int coreop = operator & SLAPI_OP_RANGE;
+ ID key;
+ ID suffix;
+ idl_range_id_pair *leftover = NULL;
+ size_t leftoverlen = 32;
+ int leftovercnt = 0;
if (NULL == flag_err) {
return NULL;
}
- *flag_err = 0;
if (NEW_IDL_NOOP == *flag_err) {
return NULL;
}
+
dblayer_txn_init(li, &s_txn);
if (txn) {
dblayer_read_txn_begin(be, txn, &s_txn);
@@ -460,7 +480,7 @@ idl_new_range_fetch(
#ifdef DB_USE_BULK_FETCH
while (cur_key.data &&
(upperkey && upperkey->data ?
- ((operator == SLAPI_OP_LESS) ?
+ ((coreop == SLAPI_OP_LESS) ?
DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) < 0 :
DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) <= 0) :
PR_TRUE /* e.g., (x > a) */)) {
@@ -496,6 +516,9 @@ idl_new_range_fetch(
goto error;
}
}
+ if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) {
+ key = (ID)strtol((char *)cur_key.data+1 , (char **)NULL, 10);
+ }
while (PR_TRUE) {
DB_MULTIPLE_NEXT(ptr, &data, dataret.data, dataret.size);
if (dataret.data == NULL) break;
@@ -524,7 +547,29 @@ idl_new_range_fetch(
/* note the last id read to check for dups */
lastid = id;
/* we got another ID, add it to our IDL */
- idl_rc = idl_append_extend(&idl, id);
+ if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) {
+ if (!idl) {
+ /* First time. Keep the suffix ID. */
+ suffix = key;
+ idl_rc = idl_append_extend(&idl, id);
+ } else if ((key == suffix) || idl_id_is_in_idlist(idl, key)) {
+ /* the parent is the suffix or already in idl. */
+ idl_rc = idl_append_extend(&idl, id);
+ } else {
+ /* Otherwise, keep the {key,id} in leftover array */
+ if (!leftover) {
+ leftover = (idl_range_id_pair *)slapi_ch_calloc(leftoverlen, sizeof(idl_range_id_pair));
+ } else if (leftovercnt == leftoverlen) {
+ leftover = (idl_range_id_pair *)slapi_ch_realloc((char *)leftover, 2 * leftoverlen * sizeof(idl_range_id_pair));
+ memset(leftover + leftovercnt, 0, leftoverlen);
+ leftoverlen *= 2;
+ }
+ leftover[leftovercnt].key = key;
+ leftover[leftovercnt++].id = id;
+ }
+ } else {
+ idl_rc = idl_append_extend(&idl, id);
+ }
if (idl_rc) {
LDAPDebug1Arg(LDAP_DEBUG_ANY,
"unable to extend id list (err=%d)\n", idl_rc);
@@ -581,7 +626,7 @@ idl_new_range_fetch(
}
#else
while (upperkey && upperkey->data ?
- ((operator == SLAPI_OP_LESS) ?
+ ((coreop == SLAPI_OP_LESS) ?
DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) < 0 :
DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) <= 0) :
PR_TRUE /* e.g., (x > a) */) {
@@ -698,9 +743,27 @@ error:
*flag_err = ret;
/* sort idl */
- if (idl && !ALLIDS(idl)) {
- qsort((void *)&idl->b_ids[0], idl->b_nids,
- (size_t)sizeof(ID), idl_sort_cmp);
+ if (idl && !ALLIDS(idl) && !(operator & SLAPI_OP_RANGE_NO_IDL_SORT)) {
+ qsort((void *)&idl->b_ids[0], idl->b_nids, (size_t)sizeof(ID), idl_sort_cmp);
+ }
+ if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) {
+ int i;
+ int left = leftovercnt;
+ while (left) {
+ for (i = 0; i < leftovercnt; i++) {
+ if (leftover[i].key && idl_id_is_in_idlist(idl, leftover[i].key)) {
+ idl_rc = idl_append_extend(&idl, leftover[i].id);
+ if (idl_rc) {
+ LDAPDebug1Arg(LDAP_DEBUG_ANY, "unable to extend id list (err=%d)\n", idl_rc);
+ idl_free(&idl);
+ return NULL;
+ }
+ leftover[i].key = 0;
+ left--;
+ }
+ }
+ }
+ slapi_ch_free((void **)&leftover);
}
return idl;
}
diff --git a/ldap/servers/slapd/back-ldbm/index.c b/ldap/servers/slapd/back-ldbm/index.c
index 00e78a7..81d6621 100644
--- a/ldap/servers/slapd/back-ldbm/index.c
+++ b/ldap/servers/slapd/back-ldbm/index.c
@@ -1232,6 +1232,7 @@ index_range_read_ext(
int timelimit = -1;
back_search_result_set *sr = NULL;
int isroot = 0;
+ int coreop = operator & SLAPI_OP_RANGE;
if (!pb) {
LDAPDebug(LDAP_DEBUG_ANY, "index_range_read: NULL pblock\n",
@@ -1278,7 +1279,7 @@ index_range_read_ext(
LDAPDebug1Arg(LDAP_DEBUG_TRACE, "index_range_read lookthrough_limit=%d\n",
lookthrough_limit);
- switch( operator ) {
+ switch( coreop ) {
case SLAPI_OP_LESS:
case SLAPI_OP_LESS_OR_EQUAL:
case SLAPI_OP_GREATER_OR_EQUAL:
@@ -1287,7 +1288,7 @@ index_range_read_ext(
default:
LDAPDebug( LDAP_DEBUG_ANY,
"<= index_range_read(%s,%s) NULL (operator %i)\n",
- type, prefix, operator );
+ type, prefix, coreop );
index_free_prefix(prefix);
return( NULL );
}
@@ -1343,7 +1344,7 @@ index_range_read_ext(
if (range != 1) { /* open range search */
char *tmpbuf = NULL;
/* this is a search with only one boundary value */
- switch( operator ) {
+ switch( coreop ) {
case SLAPI_OP_LESS:
case SLAPI_OP_LESS_OR_EQUAL:
lowerkey.dptr = slapi_ch_strdup(prefix);
@@ -1451,8 +1452,17 @@ index_range_read_ext(
cur_key.data = lowerkey.data;
cur_key.size = lowerkey.size;
lowerkey.data = NULL; /* Don't need this any more, since the memory will be freed from cur_key */
- if (operator == SLAPI_OP_GREATER) {
- *err = index_range_next_key(db,&cur_key,db_txn);
+ *err = 0;
+ if (coreop == SLAPI_OP_GREATER) {
+ *err = index_range_next_key(db, &cur_key, db_txn);
+ if (*err) {
+ LDAPDebug(LDAP_DEBUG_ANY, "<= index_range_read(%s,%s) op==GREATER, no next key: %i)\n",
+ type, prefix, *err );
+ goto error;
+ }
+ }
+ if (operator & SLAPI_OP_RANGE_NO_ALLIDS) {
+ *err = NEW_IDL_NO_ALLID;
}
if (idl_get_idl_new()) { /* new idl */
idl = idl_new_range_fetch(be, db, &cur_key, &upperkey, db_txn,
@@ -1462,7 +1472,7 @@ index_range_read_ext(
int retry_count = 0;
while (*err == 0 &&
(upperkey.data &&
- (operator == SLAPI_OP_LESS) ?
+ (coreop == SLAPI_OP_LESS) ?
DBTcmp(&cur_key, &upperkey, ai->ai_key_cmp_fn) < 0 :
DBTcmp(&cur_key, &upperkey, ai->ai_key_cmp_fn) <= 0)) {
/* exit the loop when we either run off the end of the table,
diff --git a/ldap/servers/slapd/back-ldbm/init.c b/ldap/servers/slapd/back-ldbm/init.c
index a531abb..04cc936 100644
--- a/ldap/servers/slapd/back-ldbm/init.c
+++ b/ldap/servers/slapd/back-ldbm/init.c
@@ -36,7 +36,7 @@ ldbm_back_add_schema( Slapi_PBlock *pb )
SLAPI_ATTR_FLAG_NOUSERMOD );
rc |= slapi_add_internal_attr_syntax( LDBM_PARENTID_STR,
- LDBM_PARENTID_OID, DIRSTRING_SYNTAX_OID, CASEIGNOREMATCH_NAME,
+ LDBM_PARENTID_OID, INTEGER_SYNTAX_OID, INTEGERMATCH_NAME,
SLAPI_ATTR_FLAG_SINGLE|SLAPI_ATTR_FLAG_NOUSERMOD );
rc |= slapi_add_internal_attr_syntax( "entryid",
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_search.c b/ldap/servers/slapd/back-ldbm/ldbm_search.c
index 8ed6b4d..535529c 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_search.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_search.c
@@ -1200,6 +1200,8 @@ subtree_candidates(
int isroot = 0;
struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private;
int allidslimit = compute_allids_limit(pb, li);
+ Operation *op = NULL;
+ PRBool is_bulk_import = PR_FALSE;
/* make (|(originalfilter)(objectclass=referral)) */
ftop= create_subtree_filter(filter, managedsait, &focref, &forr);
@@ -1216,6 +1218,12 @@ subtree_candidates(
has_tombstone_filter = (filter->f_flags & SLAPI_FILTER_TOMBSTONE);
slapi_pblock_get( pb, SLAPI_REQUESTOR_ISROOT, &isroot );
+ /* Check if it is for bulk import. */
+ slapi_pblock_get(pb, SLAPI_OPERATION, &op);
+ if (entryrdn_get_switch() && operation_is_flag_set(op, OP_FLAG_INTERNAL) &&
+ operation_is_flag_set(op, OP_FLAG_BULK_IMPORT)) {
+ is_bulk_import = PR_TRUE;
+ }
/*
* Apply the DN components if the candidate list is greater than
@@ -1237,13 +1245,13 @@ subtree_candidates(
candidates = idl_intersection(be, candidates, descendants);
idl_free(&tmp);
idl_free(&descendants);
- } else if (!has_tombstone_filter) {
+ } else if (!has_tombstone_filter && !is_bulk_import) {
*err = ldbm_ancestorid_read_ext(be, &txn, e->ep_id, &descendants, allidslimit);
idl_insert(&descendants, e->ep_id);
candidates = idl_intersection(be, candidates, descendants);
idl_free(&tmp);
idl_free(&descendants);
- } /* else == has_tombstone_filter: do nothing */
+ } /* else == has_tombstone_filter OR is_bulk_import: do nothing */
}
return( candidates );
@@ -1683,13 +1691,17 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
* >0 an ldap error code
*/
int filter_test = -1;
+ int is_bulk_import = operation_is_flag_set(op, OP_FLAG_BULK_IMPORT);
- if((slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_LDAPSUBENTRY)
- && !filter_flag_is_set(filter,SLAPI_FILTER_LDAPSUBENTRY)) ||
- (slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_FLAG_TOMBSTONE)
- && ((!isroot && !filter_flag_is_set(filter, SLAPI_FILTER_RUV)) ||
- !filter_flag_is_set(filter, SLAPI_FILTER_TOMBSTONE))))
- {
+ if (is_bulk_import) {
+ /* If it is from bulk import, no need to check. */
+ filter_test = 0;
+ slimit = -1; /* no sizelimit applied */
+ } else if ((slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_LDAPSUBENTRY) &&
+ !filter_flag_is_set(filter,SLAPI_FILTER_LDAPSUBENTRY)) ||
+ (slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_FLAG_TOMBSTONE) &&
+ ((!isroot && !filter_flag_is_set(filter, SLAPI_FILTER_RUV)) ||
+ !filter_flag_is_set(filter, SLAPI_FILTER_TOMBSTONE)))) {
/* If the entry is an LDAP subentry and filter don't filter subentries OR
* the entry is a TombStone and filter don't filter Tombstone
* don't return the entry. We make a special case to allow a non-root user
@@ -1736,7 +1748,8 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
/* ugaston - if filter failed due to subentries or tombstones (filter_test=-1),
* just forget about it, since we don't want to return anything at all. */
{
- if ( slapi_uniqueIDCompareString(target_uniqueid, e->ep_entry->e_uniqueid) ||
+ if ( is_bulk_import ||
+ slapi_uniqueIDCompareString(target_uniqueid, e->ep_entry->e_uniqueid) ||
slapi_sdn_scope_test_ext( backentry_get_sdn(e), basesdn, scope, e->ep_entry->e_flags ))
{
/* check size limit */
@@ -1781,7 +1794,7 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
}
rc = 0;
goto bail;
- }
+ }
else
{
CACHE_RETURN ( &inst->inst_cache, &(sr->sr_entry) );
diff --git a/ldap/servers/slapd/back-ldbm/misc.c b/ldap/servers/slapd/back-ldbm/misc.c
index fe3d01b..77c1e70 100644
--- a/ldap/servers/slapd/back-ldbm/misc.c
+++ b/ldap/servers/slapd/back-ldbm/misc.c
@@ -79,6 +79,7 @@ static const char *systemIndexes[] = {
SLAPI_ATTR_NSCP_ENTRYDN,
ATTR_NSDS5_REPLCONFLICT,
SLAPI_ATTR_ENTRYUSN,
+ SLAPI_ATTR_PARENTID,
NULL
};
diff --git a/ldap/servers/slapd/entry.c b/ldap/servers/slapd/entry.c
index 3124ff6..d38f970 100644
--- a/ldap/servers/slapd/entry.c
+++ b/ldap/servers/slapd/entry.c
@@ -3429,8 +3429,8 @@ slapi_entry_rename(Slapi_Entry *e, const char *newrdn, int deleteoldrdn, Slapi_D
/* We remove the parentid and entrydn since the backend will change these.
* We don't want to give the caller an inconsistent entry. */
- slapi_entry_attr_delete(e, "parentid");
- slapi_entry_attr_delete(e, "entrydn");
+ slapi_entry_attr_delete(e, SLAPI_ATTR_PARENTID);
+ slapi_entry_attr_delete(e, SLAPI_ATTR_ENTRYDN);
/* Build new DN. If newsuperior is set, just use "newrdn,newsuperior". If
* newsuperior is not set, need to add newrdn to old superior. */
diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h
index 6615868..4ce6a3d 100644
--- a/ldap/servers/slapd/slap.h
+++ b/ldap/servers/slapd/slap.h
@@ -529,12 +529,16 @@ typedef int (*SyntaxEnumFunc)(char **names, Slapi_PluginDesc *plugindesc,
/* OIDs for some commonly used matching rules */
#define DNMATCH_OID "2.5.13.1" /* distinguishedNameMatch */
#define CASEIGNOREMATCH_OID "2.5.13.2" /* caseIgnoreMatch */
+#define INTEGERMATCH_OID "2.5.13.14" /* integerMatch */
+#define INTEGERORDERINGMATCH_OID "2.5.13.15" /* integerOrderingMatch */
#define INTFIRSTCOMPMATCH_OID "2.5.13.29" /* integerFirstComponentMatch */
#define OIDFIRSTCOMPMATCH_OID "2.5.13.30" /* objectIdentifierFirstComponentMatch */
/* Names for some commonly used matching rules */
#define DNMATCH_NAME "distinguishedNameMatch"
#define CASEIGNOREMATCH_NAME "caseIgnoreMatch"
+#define INTEGERMATCH_NAME "integerMatch"
+#define INTEGERORDERINGMATCH_NAME "integerOrderingMatch"
#define INTFIRSTCOMPMATCH_NAME "integerFirstComponentMatch"
#define OIDFIRSTCOMPMATCH_NAME "objectIdentifierFirstComponentMatch"
diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h
index a193aad..b296060 100644
--- a/ldap/servers/slapd/slapi-plugin.h
+++ b/ldap/servers/slapd/slapi-plugin.h
@@ -483,6 +483,7 @@ NSPR_API(PRUint32) PR_fprintf(struct PRFileDesc* fd, const char *fmt, ...)
#define SLAPI_ATTR_ENTRYDN "entrydn"
#define SLAPI_ATTR_DN "dn"
#define SLAPI_ATTR_RDN "rdn"
+#define SLAPI_ATTR_PARENTID "parentid"
#define SLAPI_ATTR_UNIQUEID_LENGTH 10
#define SLAPI_ATTR_OBJECTCLASS_LENGTH 11
#define SLAPI_ATTR_VALUE_TOMBSTONE_LENGTH 11
@@ -494,6 +495,7 @@ NSPR_API(PRUint32) PR_fprintf(struct PRFileDesc* fd, const char *fmt, ...)
#define SLAPI_ATTR_ENTRYDN_LENGTH 7
#define SLAPI_ATTR_DN_LENGTH 2
#define SLAPI_ATTR_RDN_LENGTH 3
+#define SLAPI_ATTR_PARENTID_LENGTH 8
/* plugin shared config area */
#define SLAPI_PLUGIN_SHARED_CONFIG_AREA "nsslapd-pluginConfigArea"
@@ -6975,6 +6977,9 @@ typedef struct slapi_plugindesc {
#define SLAPI_OP_GREATER_OR_EQUAL 4
#define SLAPI_OP_GREATER 5
#define SLAPI_OP_SUBSTRING 6
+#define SLAPI_OP_RANGE 0xff
+#define SLAPI_OP_RANGE_NO_IDL_SORT 0x100
+#define SLAPI_OP_RANGE_NO_ALLIDS 0x200
/* Defined values of SLAPI_PLUGIN_MR_USAGE: */
#define SLAPI_PLUGIN_MR_USAGE_INDEX 0
@@ -7552,7 +7557,8 @@ enum
BACK_INFO_CRYPT_ENCRYPT_VALUE, /* Ctrl: clcrypt_encrypt_value */
BACK_INFO_CRYPT_DECRYPT_VALUE, /* Ctrl: clcrypt_decrypt_value */
BACK_INFO_DIRECTORY, /* Get the directory path */
- BACK_INFO_LOG_DIRECTORY /* Get the txn log directory */
+ BACK_INFO_LOG_DIRECTORY, /* Get the txn log directory */
+ BACK_INFO_IS_ENTRYRDN /* Get the flag for entryrdn */
};
struct _back_info_crypt_init {
diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h
index 63fe8cc..562ddea 100644
--- a/ldap/servers/slapd/slapi-private.h
+++ b/ldap/servers/slapd/slapi-private.h
@@ -416,6 +416,7 @@ char *slapi_filter_to_string_internal( const struct slapi_filter *f, char *buf,
#define OP_FLAG_REVERSE_CANDIDATE_ORDER 0x100000 /* reverse the search candidate list */
#define OP_FLAG_NEVER_CACHE 0x200000 /* never keep the entry in cache */
#define OP_FLAG_TOMBSTONE_FIXUP 0x400000 /* operation is tombstone fixup op */
+#define OP_FLAG_BULK_IMPORT 0x800000 /* operation is bulk import */
/* reverse search states */
#define REV_STARTED 1
7 years, 10 months
2 commits - dirsrvtests/tests ldap/admin ldap/ldif ldap/servers Makefile.am Makefile.in
by Noriko Hosoi
Makefile.am | 3
Makefile.in | 3
dirsrvtests/tests/tickets/ticket48755_test.py | 61 ++++-
ldap/admin/src/scripts/91reindex.pl.in | 103 +++++++++
ldap/admin/src/scripts/91subtreereindex.pl | 8
ldap/admin/src/scripts/setup-ds.res.in | 1
ldap/ldif/template-dse.ldif.in | 1
ldap/servers/plugins/replication/repl5_replica_config.c | 9
ldap/servers/plugins/replication/repl5_tot_protocol.c | 169 +++++++++++-----
ldap/servers/slapd/back-ldbm/back-ldbm.h | 6
ldap/servers/slapd/back-ldbm/dblayer.c | 5
ldap/servers/slapd/back-ldbm/filterindex.c | 21 +
ldap/servers/slapd/back-ldbm/idl_new.c | 87 +++++++-
ldap/servers/slapd/back-ldbm/index.c | 22 +-
ldap/servers/slapd/back-ldbm/init.c | 2
ldap/servers/slapd/back-ldbm/ldbm_search.c | 33 ++-
ldap/servers/slapd/back-ldbm/misc.c | 1
ldap/servers/slapd/entry.c | 4
ldap/servers/slapd/slap.h | 4
ldap/servers/slapd/slapi-plugin.h | 8
ldap/servers/slapd/slapi-private.h | 1
21 files changed, 445 insertions(+), 107 deletions(-)
New commits:
commit a20111b446a62c9a514ce23ca1a7eafabdfc904b
Author: Noriko Hosoi <nhosoi(a)redhat.com>
Date: Thu Jun 16 17:32:49 2016 -0700
Ticket #48755 - CI test: test case for ticket 48755
Description: moving an entry could make the online init fail
In addition to the existing test case, added password policy entries,
which are ldapsubentry.
diff --git a/dirsrvtests/tests/tickets/ticket48755_test.py b/dirsrvtests/tests/tickets/ticket48755_test.py
index e3b7b61..db4ae39 100644
--- a/dirsrvtests/tests/tickets/ticket48755_test.py
+++ b/dirsrvtests/tests/tickets/ticket48755_test.py
@@ -149,6 +149,44 @@ def add_user_entry(server, idx, myparent):
'cn': 'Test User%d' % idx,
'userpassword': 'password'})))
+def del_user_entry(server, idx, myparent):
+ name = 'tuser%d' % idx
+ dn = 'uid=%s,%s' % (name, myparent)
+ server.delete_s(dn)
+
+def add_ldapsubentry(server, myparent):
+ name = 'nsPwPolicyContainer'
+ container = 'cn=%s,%s' % (name, myparent)
+ server.add_s(Entry((container, {'objectclass': ['top', 'nsContainer'],
+ 'cn': '%s' % name})))
+
+ name = 'nsPwPolicyEntry'
+ pwpentry = 'cn=%s,%s' % (name, myparent)
+ pwpdn = 'cn="%s",%s' % (pwpentry, container)
+ server.add_s(Entry((pwpdn, {'objectclass': ['top', 'ldapsubentry', 'passwordpolicy'],
+ 'passwordStorageScheme': 'ssha',
+ 'passwordCheckSyntax': 'on',
+ 'passwordInHistory': '6',
+ 'passwordChange': 'on',
+ 'passwordMinAge': '0',
+ 'passwordExp': 'off',
+ 'passwordMustChange': 'off',
+ 'cn': '%s' % pwpentry})))
+
+ name = 'nsPwTemplateEntry'
+ tmplentry = 'cn=%s,%s' % (name, myparent)
+ tmpldn = 'cn="%s",%s' % (tmplentry, container)
+ server.add_s(Entry((tmpldn, {'objectclass': ['top', 'ldapsubentry', 'costemplate', 'extensibleObject'],
+ 'cosPriority': '1',
+ 'cn': '%s' % tmplentry})))
+
+ name = 'nsPwPolicy_CoS'
+ cos = 'cn=%s,%s' % (name, myparent)
+ server.add_s(Entry((cos, {'objectclass': ['top', 'ldapsubentry', 'cosPointerDefinition', 'cosSuperDefinition'],
+ 'costemplatedn': '%s' % tmpldn,
+ 'cosAttribute': 'pwdpolicysubentry default operational-default',
+ 'cn': '%s' % name})))
+
def test_ticket48755(topology):
log.info("Ticket 48755 - moving an entry could make the online init fail")
@@ -163,9 +201,14 @@ def test_ticket48755(topology):
parent0 = '%s,%s' % (ou0, DEFAULT_SUFFIX)
add_ou_entry(M1, idx, parent0)
+ add_ldapsubentry(M1, parent0)
+
parent00 = 'ou=OU%d,%s' % (idx, parent0)
for idx in range(0, 9):
add_user_entry(M1, idx, parent00)
+ if idx % 2 == 0:
+ log.info("Turning tuser%d into a tombstone entry" % idx)
+ del_user_entry(M1, idx, parent00)
log.info('%s => %s => %s => 10 USERS' % (DEFAULT_SUFFIX, parent0, parent00))
@@ -176,6 +219,8 @@ def test_ticket48755(topology):
parent1 = 'ou=OU%d,%s' % (idx, DEFAULT_SUFFIX)
add_ou_entry(M1, idx, parent1)
+ add_ldapsubentry(M1, parent1)
+
log.info("Moving %s to DIT_1" % parent00)
M1.rename_s(parent00, ou0, newsuperior=parent1, delold=1)
@@ -186,20 +231,14 @@ def test_ticket48755(topology):
parent001 = '%s,%s' % (ou0, parent01)
log.info("Moving USERS to %s" % parent0)
for idx in range(0, 9):
- name = 'tuser%d' % idx
- rdn = 'uid=%s' % name
- dn = 'uid=%s,%s' % (name, parent01)
- M1.rename_s(dn, rdn, newsuperior=parent001, delold=1)
+ if idx % 2 == 1:
+ name = 'tuser%d' % idx
+ rdn = 'uid=%s' % name
+ dn = 'uid=%s,%s' % (name, parent01)
+ M1.rename_s(dn, rdn, newsuperior=parent001, delold=1)
log.info('%s => %s => %s => %s => 10 USERS' % (DEFAULT_SUFFIX, parent1, parent01, parent001))
- log.info("Deleting 5 USERS to turn them into a tombstone entries")
- for idx in range(5, 9):
- name = 'tuser%d' % idx
- rdn = 'uid=%s' % name
- dn = 'uid=%s,%s' % (name, parent001)
- M1.delete_s(dn)
-
log.info("Run Consumer Initialization.")
global m1_m2_agmt
M1.startReplication_async(m1_m2_agmt)
commit f50e8ec950980af70d892cc45eac07684d6721a5
Author: Noriko Hosoi <nhosoi(a)redhat.com>
Date: Mon Jun 27 22:33:40 2016 -0700
Ticket #48755 - moving an entry could make the online init fail
Bug Description: Online init (aka Total update, bulk import) scans the
primary id2entry db in the order of ID. If Entry A is moved under a
new superior Entry B which was generated after Entry A, when Entry A
is sent to a consumer using online init, its parent entry does not
exist on the consumer and the online init fails.
Fix Description:
- Added a command BACK_INFO_IS_ENTRYRDN to slapi_back_get_info, which
returns the status of entryrdn switch maintained in the backend.
- If slapi_backend_get_info(BACK_INFO_IS_ENTRYRDN) returns true for
the replicated backend, repl5_tot_run searches the entry with the
filter:
(parentid>=1)
instead of:
(|(objectclass=ldapsubentry)(objectclass=nstombstone)(nsuniqueid=*))".
The filter MUST be this single range filter since the IDList created
for this bulk-import is not sorted by ID, but in the order of parentid.
- Added a new flag OP_FLAG_BULK_IMPORT to the operation. Setting the
flag implies to skip the internal operations such as eliminating
ldapsubentry from the search results or a filter check.
- In addition, idl_new_range_fetch is modified so that ...
* A range search for parentid ignores nsslapd-idlistscanlimit by
setting SLAPI_OP_RANGE_NO_ALLIDS as well as it skips sorting the
IDlist by ID by setting SLAPI_OP_RANGE_NO_IDL_SORT.
* In case SLAPI_OP_RANGE_NO_IDL_SORT is set, idl_new_range_fetch
checks whether the key (in this case parentid) is in the IDlist.
If it exists, the ID is appended. If it does not, the ID is in
the leftover list and appended when the parent ID is found in the
IDlist.
- Increased the version of rdn-format-# in DBVERSION to 3.
- Upgrade script 91reindex.pl.in is added which reindex the parentid
index file in the integer order if the version of rdn-format-# in
DBVERSION is less than 3.
- Notes: In case entryrdn is used (nsslapd-subtree-rename-switch: on),
all the type of entries including the tombstone entry and the ldap-
subentry are indexed with parentid.
- Another notes: This special treatment is enabled only when nsslapd-
subtree-rename-switch is on. If entrydn is used instead of entryrdn,
there is no possibility to establish the order that child comes first.
Plus, to replace nsslapd-subtree-rename-switch on with off, the main
db file id2entry.db needs to be recreated so that each entry stores
DN in it. It requires export then import, which corrects the order
of the entries and the order problem disappears.
https://fedorahosted.org/389/ticket/48755
Reviewed by mreynolds@redhat and wibrown(a)redhat.com (Thank you, Mark and William!!)
Also, thanks to lkrispen(a)redhat.com and tbordaz(a)redhat.com for their discussion!
diff --git a/Makefile.am b/Makefile.am
index a0ba4b6..ed3d462 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -841,7 +841,8 @@ update_DATA = ldap/admin/src/scripts/exampleupdate.pl \
ldap/admin/src/scripts/50AES-pbe-plugin.ldif\
ldap/admin/src/scripts/50updateconfig.ldif \
ldap/admin/src/scripts/52updateAESplugin.pl \
- ldap/admin/src/scripts/dnaplugindepends.ldif
+ ldap/admin/src/scripts/dnaplugindepends.ldif \
+ ldap/admin/src/scripts/91reindex.pl
update_SCRIPTS = ldap/admin/src/scripts/exampleupdate.sh
diff --git a/Makefile.in b/Makefile.in
index 0c0176b..8109469 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -2259,7 +2259,8 @@ update_DATA = ldap/admin/src/scripts/exampleupdate.pl \
ldap/admin/src/scripts/50AES-pbe-plugin.ldif\
ldap/admin/src/scripts/50updateconfig.ldif \
ldap/admin/src/scripts/52updateAESplugin.pl \
- ldap/admin/src/scripts/dnaplugindepends.ldif
+ ldap/admin/src/scripts/dnaplugindepends.ldif \
+ ldap/admin/src/scripts/91reindex.pl
update_SCRIPTS = ldap/admin/src/scripts/exampleupdate.sh
diff --git a/ldap/admin/src/scripts/91reindex.pl.in b/ldap/admin/src/scripts/91reindex.pl.in
new file mode 100644
index 0000000..c861f64
--- /dev/null
+++ b/ldap/admin/src/scripts/91reindex.pl.in
@@ -0,0 +1,103 @@
+use Mozilla::LDAP::Conn;
+use Mozilla::LDAP::Utils qw(normalizeDN);
+use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn);
+use DSUpdate qw(isOffline);
+
+sub runinst {
+ my ($inf, $inst, $dseldif, $conn) = @_;
+ my $rc, @errs;
+
+ # List of index to be reindexed
+ my @toreindex = qw(parentid);
+ # rdn-format value. See $rdn_format set below.
+ # If equal to or greater than this value, no need to reindex.
+ # If it needs to be unconditionally reindexed, set 0.
+ my @rdnconditions = (4)
+
+ my $config = $conn->search("cn=config", "base", "(objectclass=*)");
+ if (!$config) {
+ push @errs, ['error_finding_config_entry', 'cn=config',
+ $conn->getErrorString()];
+ return @errs;
+ }
+
+ ($rc, @errs) = isOffline($inf, $inst, $conn);
+ if (!$rc) {
+ return @errs;
+ }
+
+ my $reindex = "@sbindir@/db2index -Z $inst";
+ my @errs;
+ my $instconf = $conn->search("cn=ldbm database,cn=plugins,cn=config", "onelevel", "(objectclass=*)");
+ if (!$instconf) {
+ push @errs, ['error_finding_config_entry', 'cn=*,cn=ldbm database,cn=plugins,cn=config', $conn->getErrorString()];
+ return @errs;
+ }
+
+ my $dbconf = $conn->search("cn=config,cn=ldbm database,cn=plugins,cn=config", "base", "(objectclass=*)");
+ if (!$dbconf) {
+ push @errs, ['error_finding_config_entry',
+ 'cn=config,cn=ldbm database,cn=plugins,cn=config',
+ $conn->getErrorString()];
+ return @errs;
+ }
+
+ # Get the value of nsslapd-subtree-rename-switch.
+ my $switch = $dbconf->getValues('nsslapd-subtree-rename-switch');
+ if ("" eq $switch) {
+ return (); # subtree-rename-switch does not exist; do nothing.
+ } elsif ("off" eq $switch || "OFF" eq $switch) {
+ return (); # subtree-rename-switch is OFF; do nothing.
+ }
+
+ my $dbdir = $dbconf->getValues('nsslapd-directory');
+ my $dbversion0 = $dbdir . "/DBVERSION";
+ my $rdn_format = 0;
+ my $dbversionstr = "";
+ if (!open(DBVERSION, "$dbversion0")) {
+ push @errs, ['error_opening_file', $dbversion0, $!];
+ return @errs;
+ } else {
+ while (<DBVERSION>) {
+ if ($_ =~ /rdn-format/) {
+ $rdn_format = 1;
+ $dbversionstr = $_;
+ if ($_ =~ /rdn-format-1/) {
+ $rdn_format = 2;
+ } elsif ($_ =~ /rdn-format-2/) {
+ $rdn_format = 3;
+ } elsif ($_ =~ /rdn-format-3/) {
+ $rdn_format = 4;
+ } elsif ($_ =~ /rdn-format-4/) {
+ $rdn_format = 5;
+ } elsif ($_ =~ /rdn-format-5/) {
+ $rdn_format = 6;
+ } elsif ($_ =~ /rdn-format-/) {
+ # assume greater than -5
+ $rdn_format = 7;
+ }
+ }
+ }
+ close DBVERSION;
+ }
+
+ while ($instconf) {
+ my $backend= $instconf->getValues('cn');
+ if (($backend eq "config") || ($backend eq "monitor")) {
+ goto NEXT;
+ }
+
+ for (my $idx = 0; $ <= $#toreindex; $idx++) {
+ if (0 == $rdnconditions[$idx] || $rdnconditions[$idx] > $rdn_format) {
+ my $rc = system("$reindex -n $backend -t $idx");
+ if ($rc) {
+ push @errs, ["error_reindexng", $idx, $backend, $rc];
+ }
+ }
+ }
+NEXT:
+ $instconf = $conn->nextEntry();
+ }
+
+ return @errs;
+}
diff --git a/ldap/admin/src/scripts/91subtreereindex.pl b/ldap/admin/src/scripts/91subtreereindex.pl
index a031cc1..c4b40a3 100644
--- a/ldap/admin/src/scripts/91subtreereindex.pl
+++ b/ldap/admin/src/scripts/91subtreereindex.pl
@@ -51,14 +51,18 @@ sub runinst {
if ($_ =~ /rdn-format-1/) {
$is_rdn_format = 2;
}
- if ($_ =~ /rdn-format-2/) {
+ elsif ($_ =~ /rdn-format-2/) {
$is_rdn_format = 3;
}
+ elsif ($_ =~ /rdn-format-/) {
+ # assume greater than -2
+ $is_rdn_format = 4;
+ }
}
}
close DBVERSION;
- if (3 == $is_rdn_format) {
+ if (3 <= $is_rdn_format) {
return (); # DB already has the new rdn format.
}
diff --git a/ldap/admin/src/scripts/setup-ds.res.in b/ldap/admin/src/scripts/setup-ds.res.in
index fa37567..e46b858 100644
--- a/ldap/admin/src/scripts/setup-ds.res.in
+++ b/ldap/admin/src/scripts/setup-ds.res.in
@@ -209,3 +209,4 @@ error_opening_file = Opening file '%s' failed. Error: %s\n
error_format_error = '%s' has invalid format.\n
error_update_not_offline = Error: offline mode selected but the server [%s] is still running.\n
error_update_all = Failed to update all the Directory Server instances.\n
+error_reindexing = Failed to reindex '%s' in backend '%s'. Error: %s\n
diff --git a/ldap/ldif/template-dse.ldif.in b/ldap/ldif/template-dse.ldif.in
index f18c082..46b416b 100644
--- a/ldap/ldif/template-dse.ldif.in
+++ b/ldap/ldif/template-dse.ldif.in
@@ -930,6 +930,7 @@ objectclass: nsIndex
cn: parentid
nssystemindex: true
nsindextype: eq
+nsmatchingrule: integerOrderingMatch
dn: cn=seeAlso,cn=default indexes, cn=config,cn=ldbm database,cn=plugins,cn=config
objectclass: top
diff --git a/ldap/servers/plugins/replication/repl5_replica_config.c b/ldap/servers/plugins/replication/repl5_replica_config.c
index 7655ac4..59d3374 100644
--- a/ldap/servers/plugins/replication/repl5_replica_config.c
+++ b/ldap/servers/plugins/replication/repl5_replica_config.c
@@ -405,12 +405,12 @@ replica_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry*
{
if (apply_mods)
replica_set_precise_purging(r, 0);
- }
+ }
else if (strcasecmp (config_attr, type_replicaReleaseTimeout) == 0 )
{
if (apply_mods)
replica_set_release_timeout(r, 0);
- }
+ }
else
{
*returncode = LDAP_UNWILLING_TO_PERFORM;
@@ -572,8 +572,7 @@ replica_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry*
{
if (apply_mods)
{
- if (apply_mods && config_attr_value[0])
- {
+ if (config_attr_value[0]) {
PRUint64 on_off = 0;
if (strcasecmp(config_attr_value, "on") == 0){
@@ -592,7 +591,7 @@ replica_config_modify (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry*
break;
}
replica_set_precise_purging(r, on_off);
- } else if (apply_mods) {
+ } else {
replica_set_precise_purging(r, 0);
}
}
diff --git a/ldap/servers/plugins/replication/repl5_tot_protocol.c b/ldap/servers/plugins/replication/repl5_tot_protocol.c
index d0c4402..aac89bd 100644
--- a/ldap/servers/plugins/replication/repl5_tot_protocol.c
+++ b/ldap/servers/plugins/replication/repl5_tot_protocol.c
@@ -323,6 +323,10 @@ repl5_tot_run(Private_Repl_Protocol *prp)
int init_retry = 0;
Replica *replica;
ReplicaId rid = 0; /* Used to create the replica keep alive subentry */
+ Slapi_Entry *suffix = NULL;
+ char **instances = NULL;
+ Slapi_Backend *be = NULL;
+ int is_entryrdn = 0;
PR_ASSERT(NULL != prp);
@@ -354,21 +358,21 @@ retry:
*/
if (rc != ACQUIRE_SUCCESS)
{
- int optype, ldaprc, wait_retry;
- conn_get_error(prp->conn, &optype, &ldaprc);
- if (rc == ACQUIRE_TRANSIENT_ERROR && INIT_RETRY_MAX > init_retry++) {
- wait_retry = init_retry * INIT_RETRY_INT;
- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
- "acquire replica for total update, error: %d,"
+ int optype, ldaprc, wait_retry;
+ conn_get_error(prp->conn, &optype, &ldaprc);
+ if (rc == ACQUIRE_TRANSIENT_ERROR && INIT_RETRY_MAX > init_retry++) {
+ wait_retry = init_retry * INIT_RETRY_INT;
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "acquire replica for total update, error: %d,"
" retrying in %d seconds.\n",
- ldaprc, wait_retry);
- DS_Sleep(PR_SecondsToInterval(wait_retry));
- goto retry;
- } else {
- agmt_set_last_init_status(prp->agmt, ldaprc,
- prp->last_acquire_response_code, 0, NULL);
- goto done;
- }
+ ldaprc, wait_retry);
+ DS_Sleep(PR_SecondsToInterval(wait_retry));
+ goto retry;
+ } else {
+ agmt_set_last_init_status(prp->agmt, ldaprc,
+ prp->last_acquire_response_code, 0, NULL);
+ goto done;
+ }
}
else if (prp->terminate)
{
@@ -405,48 +409,121 @@ retry:
and that the order implies that perent entry is always ahead of the
child entry in the list. Otherwise, the consumer would not be
properly updated because bulk import at the moment skips orphand entries. */
- /* XXXggood above assumption may not be valid if orphaned entry moved???? */
+ /* XXXggood above assumption may not be valid if orphaned entry moved???? */
agmt_set_last_init_status(prp->agmt, 0, 0, 0, "Total update in progress");
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Beginning total update of replica "
- "\"%s\".\n", agmt_get_long_name(prp->agmt));
+ "\"%s\".\n", agmt_get_long_name(prp->agmt));
/* RMREPL - need to send schema here */
pb = slapi_pblock_new ();
- /* we need to provide managedsait control so that referral entries can
- be replicated */
- ctrls = (LDAPControl **)slapi_ch_calloc (3, sizeof (LDAPControl *));
- ctrls[0] = create_managedsait_control ();
- ctrls[1] = create_backend_control(area_sdn);
+ replica = (Replica*) object_get_data(prp->replica_object);
+ /*
+ * Get the info about the entryrdn vs. entrydn from the backend.
+ * If NOT is_entryrdn, its ancestor entries are always found prior to an entry.
+ */
+ rc = slapi_lookup_instance_name_by_suffix((char *)slapi_sdn_get_dn(area_sdn), NULL, &instances, 1);
+ if (rc || !instances) {
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "get the instance name for the suffix \"%s\".\n", slapi_sdn_get_dn(area_sdn));
+ goto done;
+ }
+ be = slapi_be_select_by_instance_name(instances[0]);
+ if (!be) {
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "get the instance for the suffix \"%s\".\n", slapi_sdn_get_dn(area_sdn));
+ goto done;
+ }
+ rc = slapi_back_get_info(be, BACK_INFO_IS_ENTRYRDN, (void **)&is_entryrdn);
+ if (is_entryrdn) {
+ /*
+ * Supporting entries out of order -- parent could have a larger id than its children.
+ * Entires are retireved sorted by parentid without the allid threshold.
+ */
+ /* Get suffix */
+ rc = slapi_search_internal_get_entry(area_sdn, NULL, &suffix, repl_get_plugin_identity(PLUGIN_MULTIMASTER_REPLICATION));
+ if (rc) {
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "get the suffix entry \"%s\".\n", slapi_sdn_get_dn(area_sdn));
+ goto done;
+ }
- /* Time to make sure it exists a keep alive subentry for that replica */
- replica = (Replica*) object_get_data(prp->replica_object);
- if (replica)
- {
- rid = replica_get_rid(replica);
- }
- replica_subentry_check(area_sdn, rid);
-
- slapi_search_internal_set_pb (pb, slapi_sdn_get_dn (area_sdn),
- LDAP_SCOPE_SUBTREE, "(|(objectclass=ldapsubentry)(objectclass=nstombstone)(nsuniqueid=*))", NULL, 0, ctrls, NULL,
- repl_get_plugin_identity (PLUGIN_MULTIMASTER_REPLICATION), 0);
-
- cb_data.prp = prp;
- cb_data.rc = 0;
- cb_data.num_entries = 0UL;
- cb_data.sleep_on_busy = 0UL;
- cb_data.last_busy = current_time ();
- cb_data.flowcontrol_detection = 0;
- cb_data.lock = PR_NewLock();
-
- /* This allows during perform_operation to check the callback data
- * especially to do flow contol on delta send msgid / recv msgid
- */
- conn_set_tot_update_cb(prp->conn, (void *) &cb_data);
+ cb_data.prp = prp;
+ cb_data.rc = 0;
+ cb_data.num_entries = 1UL;
+ cb_data.sleep_on_busy = 0UL;
+ cb_data.last_busy = current_time ();
+ cb_data.flowcontrol_detection = 0;
+ cb_data.lock = PR_NewLock();
+
+ /* This allows during perform_operation to check the callback data
+ * especially to do flow contol on delta send msgid / recv msgid
+ */
+ conn_set_tot_update_cb(prp->conn, (void *) &cb_data);
+
+ /* Send suffix first. */
+ rc = send_entry(suffix, (void *)&cb_data);
+ if (rc) {
+ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Warning: unable to "
+ "send the suffix entry \"%s\" to the consumer.\n", slapi_sdn_get_dn(area_sdn));
+ goto done;
+ }
+
+ /* we need to provide managedsait control so that referral entries can
+ be replicated */
+ ctrls = (LDAPControl **)slapi_ch_calloc (3, sizeof (LDAPControl *));
+ ctrls[0] = create_managedsait_control ();
+ ctrls[1] = create_backend_control(area_sdn);
+
+ /* Time to make sure it exists a keep alive subentry for that replica */
+ if (replica)
+ {
+ rid = replica_get_rid(replica);
+ }
+ replica_subentry_check(area_sdn, rid);
+ /* Send the subtree of the suffix in the order of parentid index plus ldapsubentry and nstombstone. */
+ slapi_search_internal_set_pb(pb, slapi_sdn_get_dn (area_sdn),
+ LDAP_SCOPE_SUBTREE, "(parentid>=1)", NULL, 0, ctrls, NULL,
+ repl_get_plugin_identity (PLUGIN_MULTIMASTER_REPLICATION), OP_FLAG_BULK_IMPORT);
+ cb_data.num_entries = 0UL;
+ } else {
+ /* Original total update */
+ /* we need to provide managedsait control so that referral entries can
+ be replicated */
+ ctrls = (LDAPControl **)slapi_ch_calloc (3, sizeof (LDAPControl *));
+ ctrls[0] = create_managedsait_control ();
+ ctrls[1] = create_backend_control(area_sdn);
+
+ /* Time to make sure it exists a keep alive subentry for that replica */
+ replica = (Replica*) object_get_data(prp->replica_object);
+ if (replica)
+ {
+ rid = replica_get_rid(replica);
+ }
+ replica_subentry_check(area_sdn, rid);
+
+ slapi_search_internal_set_pb (pb, slapi_sdn_get_dn (area_sdn),
+ LDAP_SCOPE_SUBTREE, "(|(objectclass=ldapsubentry)(objectclass=nstombstone)(nsuniqueid=*))", NULL, 0, ctrls, NULL,
+ repl_get_plugin_identity (PLUGIN_MULTIMASTER_REPLICATION), 0);
+
+ cb_data.prp = prp;
+ cb_data.rc = 0;
+ cb_data.num_entries = 0UL;
+ cb_data.sleep_on_busy = 0UL;
+ cb_data.last_busy = current_time ();
+ cb_data.flowcontrol_detection = 0;
+ cb_data.lock = PR_NewLock();
+
+ /* This allows during perform_operation to check the callback data
+ * especially to do flow contol on delta send msgid / recv msgid
+ */
+ conn_set_tot_update_cb(prp->conn, (void *) &cb_data);
+ }
+
/* Before we get started on sending entries to the replica, we need to
* setup things for async propagation:
* 1. Create a thread that will read the LDAP results from the connection.
@@ -470,7 +547,7 @@ retry:
slapi_search_internal_callback_pb (pb, &cb_data /* callback data */,
get_result /* result callback */,
send_entry /* entry callback */,
- NULL /* referral callback*/);
+ NULL /* referral callback*/);
/*
* After completing the sending operation (or optionally failing), we need to clean up
diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h
index 9499292..2d77a8a 100644
--- a/ldap/servers/slapd/back-ldbm/back-ldbm.h
+++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h
@@ -132,7 +132,7 @@ typedef unsigned short u_int16_t;
#define BDB_BACKEND "libback-ldbm" /* This backend plugin */
#define BDB_NEWIDL "newidl" /* new idl format */
#define BDB_RDNFORMAT "rdn-format" /* Subtree rename enabled */
-#define BDB_RDNFORMAT_VERSION "2" /* rdn-format version (by default, 0) */
+#define BDB_RDNFORMAT_VERSION "3" /* rdn-format version (by default, 0) */
#define BDB_DNFORMAT "dn-4514" /* DN format RFC 4514 compliant */
#define BDB_DNFORMAT_VERSION "1" /* DN format version */
@@ -808,11 +808,11 @@ typedef struct _back_search_result_set
/* #define LDBM_ENTRYRDN_OID "2.16.840.1.113730.3.1.2097" */
#define LDBM_ANCESTORID_STR "ancestorid"
-#define LDBM_ENTRYDN_STR "entrydn"
+#define LDBM_ENTRYDN_STR SLAPI_ATTR_ENTRYDN
#define LDBM_ENTRYRDN_STR "entryrdn"
#define LDBM_NUMSUBORDINATES_STR "numsubordinates"
#define LDBM_TOMBSTONE_NUMSUBORDINATES_STR "tombstonenumsubordinates"
-#define LDBM_PARENTID_STR "parentid"
+#define LDBM_PARENTID_STR SLAPI_ATTR_PARENTID
/* Name of psuedo attribute used to track default indexes */
#define LDBM_PSEUDO_ATTR_DEFAULT ".default"
diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c
index bc290cb..93d42be 100644
--- a/ldap/servers/slapd/back-ldbm/dblayer.c
+++ b/ldap/servers/slapd/back-ldbm/dblayer.c
@@ -7315,6 +7315,11 @@ ldbm_back_get_info(Slapi_Backend *be, int cmd, void **info)
}
break;
}
+ case BACK_INFO_IS_ENTRYRDN:
+ {
+ *(int *)info = entryrdn_get_switch();
+ break;
+ }
default:
break;
}
diff --git a/ldap/servers/slapd/back-ldbm/filterindex.c b/ldap/servers/slapd/back-ldbm/filterindex.c
index 9c14de4..e87cf30 100644
--- a/ldap/servers/slapd/back-ldbm/filterindex.c
+++ b/ldap/servers/slapd/back-ldbm/filterindex.c
@@ -552,6 +552,8 @@ range_candidates(
struct berval *low = NULL, *high = NULL;
struct berval **lows = NULL, **highs = NULL;
back_txn txn = {NULL};
+ int operator = 0;
+ Operation *op = NULL;
LDAPDebug(LDAP_DEBUG_TRACE, "=> range_candidates attr=%s\n", type, 0, 0);
@@ -579,17 +581,24 @@ range_candidates(
high = attr_value_lowest(highs, slapi_berval_cmp);
}
+ /* Check if it is for bulk import. */
+ slapi_pblock_get(pb, SLAPI_OPERATION, &op);
+ if (entryrdn_get_switch() && operation_is_flag_set(op, OP_FLAG_INTERNAL) &&
+ operation_is_flag_set(op, OP_FLAG_BULK_IMPORT)) {
+ /* parentid is treated specially that is needed for the bulk import. (See #48755) */
+ operator = SLAPI_OP_RANGE_NO_IDL_SORT|SLAPI_OP_RANGE_NO_ALLIDS;
+ }
if (low == NULL) {
- idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY,
- SLAPI_OP_LESS_OR_EQUAL,
+ operator |= SLAPI_OP_LESS_OR_EQUAL;
+ idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, operator,
high, NULL, 0, &txn, err, allidslimit);
} else if (high == NULL) {
- idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY,
- SLAPI_OP_GREATER_OR_EQUAL,
+ operator |= SLAPI_OP_GREATER_OR_EQUAL;
+ idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, operator,
low, NULL, 0, &txn, err, allidslimit);
} else {
- idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY,
- SLAPI_OP_LESS_OR_EQUAL,
+ operator |= SLAPI_OP_LESS_OR_EQUAL;
+ idl = index_range_read_ext(pb, be, type, (char*)indextype_EQUALITY, operator,
low, high, 1, &txn, err, allidslimit);
}
diff --git a/ldap/servers/slapd/back-ldbm/idl_new.c b/ldap/servers/slapd/back-ldbm/idl_new.c
index 25b3bfa..63df49f 100644
--- a/ldap/servers/slapd/back-ldbm/idl_new.c
+++ b/ldap/servers/slapd/back-ldbm/idl_new.c
@@ -350,17 +350,31 @@ error:
return idl;
}
+typedef struct _range_id_pair {
+ ID key;
+ ID id;
+} idl_range_id_pair;
/*
* Perform the range search in the idl layer instead of the index layer
* to improve the performance.
*/
+/*
+ * NOTE:
+ * In the total update (bulk import), an entry requires its ancestors already added.
+ * To guarantee it, the range search with parentid is used with setting the flag
+ * SLAPI_OP_RANGE_NO_IDL_SORT in operator.
+ *
+ * If the flag is set,
+ * 1. the IDList is not sorted by the ID.
+ * 2. holding to add an ID to the IDList unless the key is found in the IDList.
+ */
IDList *
idl_new_range_fetch(
- backend *be,
- DB* db,
- DBT *lowerkey,
+ backend *be,
+ DB* db,
+ DBT *lowerkey,
DBT *upperkey,
- DB_TXN *txn,
+ DB_TXN *txn,
struct attrinfo *ai,
int *flag_err,
int allidslimit,
@@ -380,7 +394,7 @@ idl_new_range_fetch(
size_t count = 0;
#ifdef DB_USE_BULK_FETCH
/* beware that a large buffer on the stack might cause a stack overflow on some platforms */
- char buffer[BULK_FETCH_BUFFER_SIZE];
+ char buffer[BULK_FETCH_BUFFER_SIZE];
void *ptr;
DBT dataret;
#endif
@@ -388,15 +402,21 @@ idl_new_range_fetch(
struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
time_t curtime;
void *saved_key = NULL;
+ int coreop = operator & SLAPI_OP_RANGE;
+ ID key;
+ ID suffix;
+ idl_range_id_pair *leftover = NULL;
+ size_t leftoverlen = 32;
+ int leftovercnt = 0;
if (NULL == flag_err) {
return NULL;
}
- *flag_err = 0;
if (NEW_IDL_NOOP == *flag_err) {
return NULL;
}
+
dblayer_txn_init(li, &s_txn);
if (txn) {
dblayer_read_txn_begin(be, txn, &s_txn);
@@ -460,7 +480,7 @@ idl_new_range_fetch(
#ifdef DB_USE_BULK_FETCH
while (cur_key.data &&
(upperkey && upperkey->data ?
- ((operator == SLAPI_OP_LESS) ?
+ ((coreop == SLAPI_OP_LESS) ?
DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) < 0 :
DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) <= 0) :
PR_TRUE /* e.g., (x > a) */)) {
@@ -496,6 +516,9 @@ idl_new_range_fetch(
goto error;
}
}
+ if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) {
+ key = (ID)strtol((char *)cur_key.data+1 , (char **)NULL, 10);
+ }
while (PR_TRUE) {
DB_MULTIPLE_NEXT(ptr, &data, dataret.data, dataret.size);
if (dataret.data == NULL) break;
@@ -524,7 +547,29 @@ idl_new_range_fetch(
/* note the last id read to check for dups */
lastid = id;
/* we got another ID, add it to our IDL */
- idl_rc = idl_append_extend(&idl, id);
+ if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) {
+ if (!idl) {
+ /* First time. Keep the suffix ID. */
+ suffix = key;
+ idl_rc = idl_append_extend(&idl, id);
+ } else if ((key == suffix) || idl_id_is_in_idlist(idl, key)) {
+ /* the parent is the suffix or already in idl. */
+ idl_rc = idl_append_extend(&idl, id);
+ } else {
+ /* Otherwise, keep the {key,id} in leftover array */
+ if (!leftover) {
+ leftover = (idl_range_id_pair *)slapi_ch_calloc(leftoverlen, sizeof(idl_range_id_pair));
+ } else if (leftovercnt == leftoverlen) {
+ leftover = (idl_range_id_pair *)slapi_ch_realloc((char *)leftover, 2 * leftoverlen * sizeof(idl_range_id_pair));
+ memset(leftover + leftovercnt, 0, leftoverlen);
+ leftoverlen *= 2;
+ }
+ leftover[leftovercnt].key = key;
+ leftover[leftovercnt++].id = id;
+ }
+ } else {
+ idl_rc = idl_append_extend(&idl, id);
+ }
if (idl_rc) {
LDAPDebug1Arg(LDAP_DEBUG_ANY,
"unable to extend id list (err=%d)\n", idl_rc);
@@ -581,7 +626,7 @@ idl_new_range_fetch(
}
#else
while (upperkey && upperkey->data ?
- ((operator == SLAPI_OP_LESS) ?
+ ((coreop == SLAPI_OP_LESS) ?
DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) < 0 :
DBTcmp(&cur_key, upperkey, ai->ai_key_cmp_fn) <= 0) :
PR_TRUE /* e.g., (x > a) */) {
@@ -698,9 +743,27 @@ error:
*flag_err = ret;
/* sort idl */
- if (idl && !ALLIDS(idl)) {
- qsort((void *)&idl->b_ids[0], idl->b_nids,
- (size_t)sizeof(ID), idl_sort_cmp);
+ if (idl && !ALLIDS(idl) && !(operator & SLAPI_OP_RANGE_NO_IDL_SORT)) {
+ qsort((void *)&idl->b_ids[0], idl->b_nids, (size_t)sizeof(ID), idl_sort_cmp);
+ }
+ if (operator & SLAPI_OP_RANGE_NO_IDL_SORT) {
+ int i;
+ int left = leftovercnt;
+ while (left) {
+ for (i = 0; i < leftovercnt; i++) {
+ if (leftover[i].key && idl_id_is_in_idlist(idl, leftover[i].key)) {
+ idl_rc = idl_append_extend(&idl, leftover[i].id);
+ if (idl_rc) {
+ LDAPDebug1Arg(LDAP_DEBUG_ANY, "unable to extend id list (err=%d)\n", idl_rc);
+ idl_free(&idl);
+ return NULL;
+ }
+ leftover[i].key = 0;
+ left--;
+ }
+ }
+ }
+ slapi_ch_free((void **)&leftover);
}
return idl;
}
diff --git a/ldap/servers/slapd/back-ldbm/index.c b/ldap/servers/slapd/back-ldbm/index.c
index 00e78a7..81d6621 100644
--- a/ldap/servers/slapd/back-ldbm/index.c
+++ b/ldap/servers/slapd/back-ldbm/index.c
@@ -1232,6 +1232,7 @@ index_range_read_ext(
int timelimit = -1;
back_search_result_set *sr = NULL;
int isroot = 0;
+ int coreop = operator & SLAPI_OP_RANGE;
if (!pb) {
LDAPDebug(LDAP_DEBUG_ANY, "index_range_read: NULL pblock\n",
@@ -1278,7 +1279,7 @@ index_range_read_ext(
LDAPDebug1Arg(LDAP_DEBUG_TRACE, "index_range_read lookthrough_limit=%d\n",
lookthrough_limit);
- switch( operator ) {
+ switch( coreop ) {
case SLAPI_OP_LESS:
case SLAPI_OP_LESS_OR_EQUAL:
case SLAPI_OP_GREATER_OR_EQUAL:
@@ -1287,7 +1288,7 @@ index_range_read_ext(
default:
LDAPDebug( LDAP_DEBUG_ANY,
"<= index_range_read(%s,%s) NULL (operator %i)\n",
- type, prefix, operator );
+ type, prefix, coreop );
index_free_prefix(prefix);
return( NULL );
}
@@ -1343,7 +1344,7 @@ index_range_read_ext(
if (range != 1) { /* open range search */
char *tmpbuf = NULL;
/* this is a search with only one boundary value */
- switch( operator ) {
+ switch( coreop ) {
case SLAPI_OP_LESS:
case SLAPI_OP_LESS_OR_EQUAL:
lowerkey.dptr = slapi_ch_strdup(prefix);
@@ -1451,8 +1452,17 @@ index_range_read_ext(
cur_key.data = lowerkey.data;
cur_key.size = lowerkey.size;
lowerkey.data = NULL; /* Don't need this any more, since the memory will be freed from cur_key */
- if (operator == SLAPI_OP_GREATER) {
- *err = index_range_next_key(db,&cur_key,db_txn);
+ *err = 0;
+ if (coreop == SLAPI_OP_GREATER) {
+ *err = index_range_next_key(db, &cur_key, db_txn);
+ if (*err) {
+ LDAPDebug(LDAP_DEBUG_ANY, "<= index_range_read(%s,%s) op==GREATER, no next key: %i)\n",
+ type, prefix, *err );
+ goto error;
+ }
+ }
+ if (operator & SLAPI_OP_RANGE_NO_ALLIDS) {
+ *err = NEW_IDL_NO_ALLID;
}
if (idl_get_idl_new()) { /* new idl */
idl = idl_new_range_fetch(be, db, &cur_key, &upperkey, db_txn,
@@ -1462,7 +1472,7 @@ index_range_read_ext(
int retry_count = 0;
while (*err == 0 &&
(upperkey.data &&
- (operator == SLAPI_OP_LESS) ?
+ (coreop == SLAPI_OP_LESS) ?
DBTcmp(&cur_key, &upperkey, ai->ai_key_cmp_fn) < 0 :
DBTcmp(&cur_key, &upperkey, ai->ai_key_cmp_fn) <= 0)) {
/* exit the loop when we either run off the end of the table,
diff --git a/ldap/servers/slapd/back-ldbm/init.c b/ldap/servers/slapd/back-ldbm/init.c
index a531abb..04cc936 100644
--- a/ldap/servers/slapd/back-ldbm/init.c
+++ b/ldap/servers/slapd/back-ldbm/init.c
@@ -36,7 +36,7 @@ ldbm_back_add_schema( Slapi_PBlock *pb )
SLAPI_ATTR_FLAG_NOUSERMOD );
rc |= slapi_add_internal_attr_syntax( LDBM_PARENTID_STR,
- LDBM_PARENTID_OID, DIRSTRING_SYNTAX_OID, CASEIGNOREMATCH_NAME,
+ LDBM_PARENTID_OID, INTEGER_SYNTAX_OID, INTEGERMATCH_NAME,
SLAPI_ATTR_FLAG_SINGLE|SLAPI_ATTR_FLAG_NOUSERMOD );
rc |= slapi_add_internal_attr_syntax( "entryid",
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_search.c b/ldap/servers/slapd/back-ldbm/ldbm_search.c
index 8ed6b4d..535529c 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_search.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_search.c
@@ -1200,6 +1200,8 @@ subtree_candidates(
int isroot = 0;
struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private;
int allidslimit = compute_allids_limit(pb, li);
+ Operation *op = NULL;
+ PRBool is_bulk_import = PR_FALSE;
/* make (|(originalfilter)(objectclass=referral)) */
ftop= create_subtree_filter(filter, managedsait, &focref, &forr);
@@ -1216,6 +1218,12 @@ subtree_candidates(
has_tombstone_filter = (filter->f_flags & SLAPI_FILTER_TOMBSTONE);
slapi_pblock_get( pb, SLAPI_REQUESTOR_ISROOT, &isroot );
+ /* Check if it is for bulk import. */
+ slapi_pblock_get(pb, SLAPI_OPERATION, &op);
+ if (entryrdn_get_switch() && operation_is_flag_set(op, OP_FLAG_INTERNAL) &&
+ operation_is_flag_set(op, OP_FLAG_BULK_IMPORT)) {
+ is_bulk_import = PR_TRUE;
+ }
/*
* Apply the DN components if the candidate list is greater than
@@ -1237,13 +1245,13 @@ subtree_candidates(
candidates = idl_intersection(be, candidates, descendants);
idl_free(&tmp);
idl_free(&descendants);
- } else if (!has_tombstone_filter) {
+ } else if (!has_tombstone_filter && !is_bulk_import) {
*err = ldbm_ancestorid_read_ext(be, &txn, e->ep_id, &descendants, allidslimit);
idl_insert(&descendants, e->ep_id);
candidates = idl_intersection(be, candidates, descendants);
idl_free(&tmp);
idl_free(&descendants);
- } /* else == has_tombstone_filter: do nothing */
+ } /* else == has_tombstone_filter OR is_bulk_import: do nothing */
}
return( candidates );
@@ -1683,13 +1691,17 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
* >0 an ldap error code
*/
int filter_test = -1;
+ int is_bulk_import = operation_is_flag_set(op, OP_FLAG_BULK_IMPORT);
- if((slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_LDAPSUBENTRY)
- && !filter_flag_is_set(filter,SLAPI_FILTER_LDAPSUBENTRY)) ||
- (slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_FLAG_TOMBSTONE)
- && ((!isroot && !filter_flag_is_set(filter, SLAPI_FILTER_RUV)) ||
- !filter_flag_is_set(filter, SLAPI_FILTER_TOMBSTONE))))
- {
+ if (is_bulk_import) {
+ /* If it is from bulk import, no need to check. */
+ filter_test = 0;
+ slimit = -1; /* no sizelimit applied */
+ } else if ((slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_LDAPSUBENTRY) &&
+ !filter_flag_is_set(filter,SLAPI_FILTER_LDAPSUBENTRY)) ||
+ (slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_FLAG_TOMBSTONE) &&
+ ((!isroot && !filter_flag_is_set(filter, SLAPI_FILTER_RUV)) ||
+ !filter_flag_is_set(filter, SLAPI_FILTER_TOMBSTONE)))) {
/* If the entry is an LDAP subentry and filter don't filter subentries OR
* the entry is a TombStone and filter don't filter Tombstone
* don't return the entry. We make a special case to allow a non-root user
@@ -1736,7 +1748,8 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
/* ugaston - if filter failed due to subentries or tombstones (filter_test=-1),
* just forget about it, since we don't want to return anything at all. */
{
- if ( slapi_uniqueIDCompareString(target_uniqueid, e->ep_entry->e_uniqueid) ||
+ if ( is_bulk_import ||
+ slapi_uniqueIDCompareString(target_uniqueid, e->ep_entry->e_uniqueid) ||
slapi_sdn_scope_test_ext( backentry_get_sdn(e), basesdn, scope, e->ep_entry->e_flags ))
{
/* check size limit */
@@ -1781,7 +1794,7 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
}
rc = 0;
goto bail;
- }
+ }
else
{
CACHE_RETURN ( &inst->inst_cache, &(sr->sr_entry) );
diff --git a/ldap/servers/slapd/back-ldbm/misc.c b/ldap/servers/slapd/back-ldbm/misc.c
index fe3d01b..77c1e70 100644
--- a/ldap/servers/slapd/back-ldbm/misc.c
+++ b/ldap/servers/slapd/back-ldbm/misc.c
@@ -79,6 +79,7 @@ static const char *systemIndexes[] = {
SLAPI_ATTR_NSCP_ENTRYDN,
ATTR_NSDS5_REPLCONFLICT,
SLAPI_ATTR_ENTRYUSN,
+ SLAPI_ATTR_PARENTID,
NULL
};
diff --git a/ldap/servers/slapd/entry.c b/ldap/servers/slapd/entry.c
index 3124ff6..d38f970 100644
--- a/ldap/servers/slapd/entry.c
+++ b/ldap/servers/slapd/entry.c
@@ -3429,8 +3429,8 @@ slapi_entry_rename(Slapi_Entry *e, const char *newrdn, int deleteoldrdn, Slapi_D
/* We remove the parentid and entrydn since the backend will change these.
* We don't want to give the caller an inconsistent entry. */
- slapi_entry_attr_delete(e, "parentid");
- slapi_entry_attr_delete(e, "entrydn");
+ slapi_entry_attr_delete(e, SLAPI_ATTR_PARENTID);
+ slapi_entry_attr_delete(e, SLAPI_ATTR_ENTRYDN);
/* Build new DN. If newsuperior is set, just use "newrdn,newsuperior". If
* newsuperior is not set, need to add newrdn to old superior. */
diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h
index 645cdd5..b74f9bd 100644
--- a/ldap/servers/slapd/slap.h
+++ b/ldap/servers/slapd/slap.h
@@ -536,12 +536,16 @@ typedef int (*SyntaxEnumFunc)(char **names, Slapi_PluginDesc *plugindesc,
/* OIDs for some commonly used matching rules */
#define DNMATCH_OID "2.5.13.1" /* distinguishedNameMatch */
#define CASEIGNOREMATCH_OID "2.5.13.2" /* caseIgnoreMatch */
+#define INTEGERMATCH_OID "2.5.13.14" /* integerMatch */
+#define INTEGERORDERINGMATCH_OID "2.5.13.15" /* integerOrderingMatch */
#define INTFIRSTCOMPMATCH_OID "2.5.13.29" /* integerFirstComponentMatch */
#define OIDFIRSTCOMPMATCH_OID "2.5.13.30" /* objectIdentifierFirstComponentMatch */
/* Names for some commonly used matching rules */
#define DNMATCH_NAME "distinguishedNameMatch"
#define CASEIGNOREMATCH_NAME "caseIgnoreMatch"
+#define INTEGERMATCH_NAME "integerMatch"
+#define INTEGERORDERINGMATCH_NAME "integerOrderingMatch"
#define INTFIRSTCOMPMATCH_NAME "integerFirstComponentMatch"
#define OIDFIRSTCOMPMATCH_NAME "objectIdentifierFirstComponentMatch"
diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h
index d3a6b25..a7e544a 100644
--- a/ldap/servers/slapd/slapi-plugin.h
+++ b/ldap/servers/slapd/slapi-plugin.h
@@ -483,6 +483,7 @@ NSPR_API(PRUint32) PR_fprintf(struct PRFileDesc* fd, const char *fmt, ...)
#define SLAPI_ATTR_ENTRYDN "entrydn"
#define SLAPI_ATTR_DN "dn"
#define SLAPI_ATTR_RDN "rdn"
+#define SLAPI_ATTR_PARENTID "parentid"
#define SLAPI_ATTR_UNIQUEID_LENGTH 10
#define SLAPI_ATTR_OBJECTCLASS_LENGTH 11
#define SLAPI_ATTR_VALUE_TOMBSTONE_LENGTH 11
@@ -494,6 +495,7 @@ NSPR_API(PRUint32) PR_fprintf(struct PRFileDesc* fd, const char *fmt, ...)
#define SLAPI_ATTR_ENTRYDN_LENGTH 7
#define SLAPI_ATTR_DN_LENGTH 2
#define SLAPI_ATTR_RDN_LENGTH 3
+#define SLAPI_ATTR_PARENTID_LENGTH 8
/* plugin shared config area */
#define SLAPI_PLUGIN_SHARED_CONFIG_AREA "nsslapd-pluginConfigArea"
@@ -7024,6 +7026,9 @@ typedef struct slapi_plugindesc {
#define SLAPI_OP_GREATER_OR_EQUAL 4
#define SLAPI_OP_GREATER 5
#define SLAPI_OP_SUBSTRING 6
+#define SLAPI_OP_RANGE 0xff
+#define SLAPI_OP_RANGE_NO_IDL_SORT 0x100
+#define SLAPI_OP_RANGE_NO_ALLIDS 0x200
/* Defined values of SLAPI_PLUGIN_MR_USAGE: */
#define SLAPI_PLUGIN_MR_USAGE_INDEX 0
@@ -7607,7 +7612,8 @@ enum
BACK_INFO_CRYPT_ENCRYPT_VALUE, /* Ctrl: clcrypt_encrypt_value */
BACK_INFO_CRYPT_DECRYPT_VALUE, /* Ctrl: clcrypt_decrypt_value */
BACK_INFO_DIRECTORY, /* Get the directory path */
- BACK_INFO_LOG_DIRECTORY /* Get the txn log directory */
+ BACK_INFO_LOG_DIRECTORY, /* Get the txn log directory */
+ BACK_INFO_IS_ENTRYRDN /* Get the flag for entryrdn */
};
struct _back_info_crypt_init {
diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h
index 7636559..52d1c4a 100644
--- a/ldap/servers/slapd/slapi-private.h
+++ b/ldap/servers/slapd/slapi-private.h
@@ -416,6 +416,7 @@ char *slapi_filter_to_string_internal( const struct slapi_filter *f, char *buf,
#define OP_FLAG_REVERSE_CANDIDATE_ORDER 0x100000 /* reverse the search candidate list */
#define OP_FLAG_NEVER_CACHE 0x200000 /* never keep the entry in cache */
#define OP_FLAG_TOMBSTONE_FIXUP 0x400000 /* operation is tombstone fixup op */
+#define OP_FLAG_BULK_IMPORT 0x800000 /* operation is bulk import */
/* reverse search states */
#define REV_STARTED 1
7 years, 10 months
ldap/servers
by Noriko Hosoi
ldap/servers/slapd/result.c | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
New commits:
commit 0157eb14e34e0acce7b5363741bd7c8d84cc9b32
Author: Noriko Hosoi <nhosoi(a)redhat.com>
Date: Tue Jun 28 13:31:35 2016 -0700
Ticket #48904 - syncrepl search returning error 329; plugin sending a bad error code
Description: Implemented based upon the analysis by lkrispen(a)redhat.com.
> the problem is in the new function send_ldap_intermediate() introduced for
> sync_repl, it calls log_result (....,rc, ) where rc is the return code of
> ber_put_seq(). It should call log_result( ...,1, ) if rc==LBER_ERROR and
> (...,0,..) else.
Instead of using 1 and 0, use ldap code LDAP_OPERATIONS_ERROR and LDAP_SUCCESS,
respectively.
https://fedorahosted.org/389/ticket/48904
Reviewed by wibrown(a)redhat.com (Thank you, William!!)
diff --git a/ldap/servers/slapd/result.c b/ldap/servers/slapd/result.c
index cb465bf..506da92 100644
--- a/ldap/servers/slapd/result.c
+++ b/ldap/servers/slapd/result.c
@@ -255,17 +255,19 @@ log_and_return:
* set o_status
*/
+ if (rc == LBER_ERROR) {
+ rc = LDAP_OPERATIONS_ERROR;
+ } else {
+ rc = LDAP_SUCCESS;
+ }
+
if ( logit && operation_is_flag_set( operation,
OP_FLAG_ACTION_LOG_ACCESS )) {
log_result( pb, operation, rc, tag, 0 );
}
LDAPDebug( LDAP_DEBUG_TRACE, "<= send_ldap_intermediate\n", 0, 0, 0 );
- if (rc == LBER_ERROR) {
- return(1);
- } else {
- return(0);
- }
+ return rc;
}
static int
7 years, 10 months
ldap/servers
by William Brown
ldap/servers/plugins/pwdstorage/pwd_init.c | 52 +++++++----------------------
ldap/servers/plugins/rever/rever.c | 6 +--
ldap/servers/slapd/pblock.c | 2 -
3 files changed, 16 insertions(+), 44 deletions(-)
New commits:
commit 2309c38a2c510b011327354c0600e945ce2e53a5
Author: William Brown <firstyear(a)redhat.com>
Date: Tue Jun 28 12:15:03 2016 +1000
Ticket 48902 - Strdup pwdstoragescheme name to prevent misbehaving plugins
Bug Description: Some plugins would set the pwdstorageschemename to a value
from their stack. This would cause ns-slapd to segfault on shutdown due to
attempting to free this value.
Fix Description: pblock now strdups the pwdstorageschemename, so that plugins
can behave however they want, and we always do the right thing.
https://fedorahosted.org/389/ticket/48902
Author: wibrown
Review by: nhosoi (Thanks!)
diff --git a/ldap/servers/plugins/pwdstorage/pwd_init.c b/ldap/servers/plugins/pwdstorage/pwd_init.c
index d441d38..5c14c95 100644
--- a/ldap/servers/plugins/pwdstorage/pwd_init.c
+++ b/ldap/servers/plugins/pwdstorage/pwd_init.c
@@ -50,7 +50,6 @@ int
sha_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> sha_pwd_storage_scheme_init\n" );
@@ -62,9 +61,8 @@ sha_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) sha1_pw_enc);
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) sha1_pw_cmp );
- name = slapi_ch_strdup("SHA");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "SHA" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= sha_pwd_storage_scheme_init %d\n\n", rc );
@@ -75,7 +73,6 @@ int
ssha_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> ssha_pwd_storage_scheme_init\n" );
@@ -87,9 +84,8 @@ ssha_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) salted_sha1_pw_enc );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) sha1_pw_cmp );
- name = slapi_ch_strdup("SSHA");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "SSHA" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= ssha_pwd_storage_scheme_init %d\n\n", rc );
return( rc );
@@ -99,7 +95,6 @@ int
sha256_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> sha256_pwd_storage_scheme_init\n" );
@@ -111,9 +106,8 @@ sha256_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) sha256_pw_enc);
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) sha256_pw_cmp );
- name = slapi_ch_strdup("SHA256");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "SHA256" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= sha256_pwd_storage_scheme_init %d\n\n", rc );
@@ -124,7 +118,6 @@ int
ssha256_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> ssha256_pwd_storage_scheme_init\n" );
@@ -136,9 +129,8 @@ ssha256_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) salted_sha256_pw_enc );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) sha256_pw_cmp );
- name = slapi_ch_strdup("SSHA256");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "SSHA256" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= ssha256_pwd_storage_scheme_init %d\n\n", rc );
return( rc );
@@ -148,7 +140,6 @@ int
sha384_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> sha384_pwd_storage_scheme_init\n" );
@@ -160,9 +151,8 @@ sha384_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) sha384_pw_enc);
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) sha384_pw_cmp );
- name = slapi_ch_strdup("SHA384");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "SHA384" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= sha384_pwd_storage_scheme_init %d\n\n", rc );
@@ -173,7 +163,6 @@ int
ssha384_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> ssha384_pwd_storage_scheme_init\n" );
@@ -185,9 +174,8 @@ ssha384_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) salted_sha384_pw_enc );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) sha384_pw_cmp );
- name = slapi_ch_strdup("SSHA384");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "SSHA384" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= ssha384_pwd_storage_scheme_init %d\n\n", rc );
return( rc );
@@ -197,7 +185,6 @@ int
sha512_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> sha512_pwd_storage_scheme_init\n" );
@@ -209,9 +196,8 @@ sha512_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) sha512_pw_enc);
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) sha512_pw_cmp );
- name = slapi_ch_strdup("SHA512");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "SHA512" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= sha512_pwd_storage_scheme_init %d\n\n", rc );
@@ -222,7 +208,6 @@ int
ssha512_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> ssha512_pwd_storage_scheme_init\n" );
@@ -234,9 +219,8 @@ ssha512_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) salted_sha512_pw_enc );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) sha512_pw_cmp );
- name = slapi_ch_strdup("SSHA512");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "SSHA512" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= ssha512_pwd_storage_scheme_init %d\n\n", rc );
return( rc );
@@ -246,7 +230,6 @@ int
crypt_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> crypt_pwd_storage_scheme_init\n" );
@@ -259,9 +242,8 @@ crypt_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) crypt_pw_enc );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) crypt_pw_cmp );
- name = slapi_ch_strdup("CRYPT");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "CRYPT" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= crypt_pwd_storage_scheme_init %d\n\n", rc );
return( rc );
@@ -271,7 +253,6 @@ int
clear_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> clear_pwd_storage_scheme_init\n" );
@@ -283,9 +264,8 @@ clear_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) clear_pw_enc );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) clear_pw_cmp );
- name = slapi_ch_strdup("CLEAR");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "CLEAR" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= clear_pwd_storage_scheme_init %d\n\n", rc );
return( rc );
@@ -295,7 +275,6 @@ int
ns_mta_md5_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> ns_mta_md5_pwd_storage_scheme_init\n" );
@@ -307,9 +286,8 @@ ns_mta_md5_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) NULL );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) ns_mta_md5_pw_cmp );
- name = slapi_ch_strdup("NS-MTA-MD5");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "NS-MTA-MD5" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= ns_mta_md5_pwd_storage_scheme_init %d\n\n", rc );
return( rc );
@@ -319,7 +297,6 @@ int
md5_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> md5_pwd_storage_scheme_init\n" );
@@ -331,9 +308,8 @@ md5_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) md5_pw_enc );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) md5_pw_cmp );
- name = slapi_ch_strdup("MD5");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "MD5" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= md5_pwd_storage_scheme_init %d\n\n", rc );
return( rc );
@@ -343,7 +319,6 @@ int
smd5_pwd_storage_scheme_init( Slapi_PBlock *pb )
{
int rc;
- char *name;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> smd5_pwd_storage_scheme_init\n" );
@@ -355,9 +330,8 @@ smd5_pwd_storage_scheme_init( Slapi_PBlock *pb )
(void *) smd5_pw_enc );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN,
(void *) smd5_pw_cmp );
- name = slapi_ch_strdup("SMD5");
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME,
- name );
+ "SMD5" );
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "<= smd5_pwd_storage_scheme_init %d\n\n", rc );
return( rc );
diff --git a/ldap/servers/plugins/rever/rever.c b/ldap/servers/plugins/rever/rever.c
index c49eeec..719cbde 100644
--- a/ldap/servers/plugins/rever/rever.c
+++ b/ldap/servers/plugins/rever/rever.c
@@ -68,7 +68,6 @@ aes_dec( char *pwd, char *alg )
int
aes_init( Slapi_PBlock *pb)
{
- char *name = slapi_ch_strdup(AES_REVER_SCHEME_NAME);
int rc;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> aes_init\n" );
@@ -78,7 +77,7 @@ aes_init( Slapi_PBlock *pb)
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_ENC_FN, (void *) aes_enc);
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN, (void *) aes_cmp );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_DEC_FN, (void *) aes_dec );
- rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME, name );
+ rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME, AES_REVER_SCHEME_NAME );
init_pbe_plugin();
@@ -130,7 +129,6 @@ des_dec( char *pwd )
int
des_init( Slapi_PBlock *pb )
{
- char *name = slapi_ch_strdup(DES_REVER_SCHEME_NAME);
int rc;
slapi_log_error( SLAPI_LOG_PLUGIN, plugin_name, "=> des_init\n" );
@@ -140,7 +138,7 @@ des_init( Slapi_PBlock *pb )
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_ENC_FN, (void *) des_enc);
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_CMP_FN, (void *) des_cmp );
rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_DEC_FN, (void *) des_dec );
- rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME, name );
+ rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME, DES_REVER_SCHEME_NAME );
init_pbe_plugin();
diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c
index 7205337..db5a9e7 100644
--- a/ldap/servers/slapd/pblock.c
+++ b/ldap/servers/slapd/pblock.c
@@ -3473,7 +3473,7 @@ slapi_pblock_set( Slapi_PBlock *pblock, int arg, void *value )
/* password storage scheme (kexcoff) */
case SLAPI_PLUGIN_PWD_STORAGE_SCHEME_NAME:
- pblock->pb_plugin->plg_pwdstorageschemename = (char *)value;
+ pblock->pb_plugin->plg_pwdstorageschemename = slapi_ch_strdup((char *)value);
break;
case SLAPI_PLUGIN_PWD_STORAGE_SCHEME_USER_PWD:
pblock->pb_pwd_storage_scheme_user_passwd = (char *)value;
7 years, 10 months
Branch '389-ds-base-1.2.11' - ldap/admin
by Mark Reynolds
ldap/admin/src/scripts/repl-monitor.pl.in | 24 ++++++------------------
1 file changed, 6 insertions(+), 18 deletions(-)
New commits:
commit 66aed16a9bf30264c52965fc6d6f038f96f70f10
Author: Mark Reynolds <mreynolds(a)redhat.com>
Date: Tue Jun 28 16:54:54 2016 -0400
Ticket 47538 - Fix repl-monitor color and lag times
Bug Description: Colors do not match the legend
Fix Description: The color hash table needed to be sorted before processing,
and there was a previous commit that was not properly
cherry picked that needed to be cleaned up
https://fedorahosted.org/389/ticket/47538
Reviewed by: nhosoi(Thanks!)
diff --git a/ldap/admin/src/scripts/repl-monitor.pl.in b/ldap/admin/src/scripts/repl-monitor.pl.in
index 94a6a92..de0efa5 100755
--- a/ldap/admin/src/scripts/repl-monitor.pl.in
+++ b/ldap/admin/src/scripts/repl-monitor.pl.in
@@ -811,7 +811,9 @@ sub print_consumers
if ($c_ridx >= 0) {
$myruv = $allruvs {"$c_ridx:$mid"};
- ($c_maxcsn, $c_lastmodified) = split ( /;/, $myruv );
+ ($c_maxcsn, $c_lastmodified) = split ( /;/, "$myruv" );
+ ($c_maxcsn_str, $lag, $markcolor) = &cacl_time_lag ($m_maxcsn, $c_maxcsn);
+ if(!$opt_s){ $c_maxcsn_str =~ s/ /\<br\>/; }
($c_sidx, $c_replicaroot, $c_replicatype) = split (/:/, $allreplicas[$c_ridx]);
$c_replicaroot = "same as master" if $m_replicaroot eq $c_replicaroot;
}
@@ -846,17 +848,6 @@ sub print_consumers
if ($first_entry) {
$first_entry = 0;
$c_ldapurl = &get_ldap_url ($c_sidx, $conntype);
- if ($c_ridx >= 0) {
- ($c_maxcsn_str, $lag, $markcolor, $supplier_maxcsn) =
- &cacl_time_lag ($_->{nsDS5ReplicaRoot},
- $_->{cn},
- $_->{nsds5ReplicaHost},
- $_->{nsds5ReplicaPort},
- $s_ridx,
- $m_maxcsn,
- $c_maxcsn);
- if(!$opt_s){ $c_maxcsn_str =~ s/ /\<br\>/; }
- }
if($opt_s){
print "Receiver: $c_ldapurl\nType: $c_replicatype\n";
print "Time Lag: $lag\n";
@@ -1141,7 +1132,7 @@ sub get_ldap_url
sub to_decimal_csn
{
my ($maxcsn) = @_;
- if (!$maxcsn || $maxcsn eq "" || $maxcsn eq "Unavailable") {
+ if (!$maxcsn || $maxcsn eq "") {
return "none";
}
@@ -1161,9 +1152,6 @@ sub to_string_csn
if (!$rawcsn || $rawcsn eq "") {
return "none";
}
- if ($rawcsn eq "Unavailable"){
- return $rawcsn;
- }
my ($tm, $seq, $masterid, $subseq) = split(/ /, $decimalcsn);
my ($sec, $min, $hour, $mday, $mon, $year) = localtime($tm);
$mon++;
@@ -1182,9 +1170,9 @@ sub get_color
$lag_minute /= 60;
my ($color) = $allcolors { $colorkeys[0] };
- foreach ( keys %allcolors) {
+ foreach ( sort keys %allcolors) {
if ($lag_minute >= $_){
- $color = $allcolors {$_};
+ $color = $allcolors {$_};
}
}
return $color;
7 years, 10 months
Branch '389-ds-base-1.3.3' - ldap/admin man/man1
by Mark Reynolds
ldap/admin/src/scripts/repl-monitor.pl.in | 52 ++++++++++++++++----------
man/man1/repl-monitor.1 | 58 +++++++++++++++++++++++++++---
2 files changed, 86 insertions(+), 24 deletions(-)
New commits:
commit ad8ba5186bd9892917734c99fe8b793dd96548b5
Author: Mark Reynolds <mreynolds(a)redhat.com>
Date: Tue Jun 28 15:41:42 2016 -0400
Ticket 47538 - Fix repl-monitor color and lag times
Bug Description: Colors do not match the legend, and when the supplier or
consumer max csns are not available, the lag time is displayed
incorrectly.
Fix Description: The color hash table needed to be sorted before processing,
and we were not properly detecting the "Unavailable" state of
a replica which lead to the odd lag times. Also added the
string "Unavailable" for uninitialized values which allowed
for a cleaner report when replicas are offline.
Updated the man page as well.
https://fedorahosted.org/389/ticket/47538
Reviewed by: nhosoi(Thanks!)
(cherry picked from commit 34c929353280b93eebe214f1c21f5b788aff1c39)
diff --git a/ldap/admin/src/scripts/repl-monitor.pl.in b/ldap/admin/src/scripts/repl-monitor.pl.in
index 5354a8b..f30967d 100755
--- a/ldap/admin/src/scripts/repl-monitor.pl.in
+++ b/ldap/admin/src/scripts/repl-monitor.pl.in
@@ -626,7 +626,7 @@ sub process_suppliers
# Skip replicas without agreements defined yet
next if (! grep {$_->{ridx} == $ridx} @allagreements);
$maxcsn = &print_master_header ($ridx, $mid);
- if ( "$maxcsn" ne "none" ) {
+ if ( "$maxcsn" ne "Unavailable" ) {
&print_consumer_header ();
&print_consumers ($ridx, $mid);
}
@@ -784,12 +784,12 @@ sub print_consumers
my ($c_maxcsn, $c_maxcsn_str, $c_lastmodified, $c_sidx, $lag, $markcolor);
my ($c_replicaroot, $c_replicatype);
my ($first_entry, $s_ldapurl, $c_ldapurl);
- my $supplier_maxcsn = "_";
+ my $supplier_maxcsn = "Unavailable";
my ($nrows);
my ($found);
undef @ouragreements;
- $c_lastmodified = "";
+ $c_lastmodified = "Unavailable";
# Collect all the consumer replicas for the current master replica
push (@consumers, $m_ridx);
@@ -820,18 +820,20 @@ sub print_consumers
if ($c_ridx >= 0) {
$myruv = $allruvs {"$c_ridx:$mid"};
- ($c_maxcsn, $c_lastmodified) = split ( /;/, $myruv );
- ($c_sidx, $c_replicaroot, $c_replicatype) = split (/:/, $allreplicas[$c_ridx]);
- $c_replicaroot = "same as master" if $m_replicaroot eq $c_replicaroot;
+ if ($myruv) {
+ ($c_maxcsn, $c_lastmodified) = split ( /;/, $myruv );
+ ($c_sidx, $c_replicaroot, $c_replicatype) = split (/:/, $allreplicas[$c_ridx]);
+ $c_replicaroot = "same as master" if $m_replicaroot eq $c_replicaroot;
+ }
}
else {
# $c_ridx is actually -$c_sidx when c is not available
$c_sidx = -$c_ridx;
- $c_maxcsn_str = "_";
+ $c_maxcsn_str = "Unavailable";
$lag = "n/a";
$markcolor = "red";
- $c_replicaroot = "_";
- $c_replicatype = "_";
+ $c_replicaroot = "Unavailable";
+ $c_replicatype = "Unavailable";
}
$nrows = 0;
@@ -962,7 +964,7 @@ sub get_supplier_maxcsn
{
my ($ridx, $s, $cn, $h, $p) = @_;
my $decimalcsn;
- my $csn = "";
+ my $csn = "Unavailable";
# normalize suffix
$s =~ s/ //;
$s =~ lc $s;
@@ -974,7 +976,7 @@ sub get_supplier_maxcsn
last;
}
}
- if($csn ne ""){
+ if($csn && $csn ne "Unavailable"){
$decimalcsn = &to_decimal_csn ($csn);
return "$csn:$decimalcsn";
}
@@ -996,11 +998,16 @@ sub cacl_time_lag
$supplier_csn_str = &to_string_csn ($s_maxcsn);
$csn_str = &to_string_csn ($c_maxcsn);
- if ($s_maxcsn && !$c_maxcsn) {
+ if ((!$s_maxcsn || $s_maxcsn eq "Unavailable") &&
+ (!$c_maxcsn || $c_maxcsn eq "Unavailable")) {
+ $lag_str = "?:??:??";
+ $markcolor = "white"; # Both unknown
+ }
+ elsif ($s_maxcsn && (!$c_maxcsn || $c_maxcsn eq "Unavailable")) {
$lag_str = "- ?:??:??";
$markcolor = &get_color (36000); # assume consumer has big latency
}
- elsif (!$s_maxcsn && $c_maxcsn) {
+ elsif ((!$s_maxcsn || $s_maxcsn eq "Unavailable") && $c_maxcsn) {
$lag_str = "+ ?:??:??";
$markcolor = &get_color (1); # consumer is ahead of supplier
}
@@ -1210,7 +1217,7 @@ sub to_decimal_csn
{
my ($maxcsn) = @_;
if (!$maxcsn || $maxcsn eq "" || $maxcsn eq "Unavailable") {
- return "none";
+ return "Unavailable";
}
my ($tm, $seq, $masterid, $subseq) = unpack("a8 a4 a4 a4", $maxcsn);
@@ -1225,9 +1232,13 @@ sub to_decimal_csn
sub to_string_csn
{
- my ($rawcsn, $decimalcsn) = split(/:/, "@_");
+ my $str = shift;
+ if (!defined($str)){
+ return "Unavailable";
+ }
+ my ($rawcsn, $decimalcsn) = split(/:/, "$str");
if (!$rawcsn || $rawcsn eq "") {
- return "none";
+ return "Unavailable";
}
if ($rawcsn eq "Unavailable"){
return $rawcsn;
@@ -1251,7 +1262,7 @@ sub get_color
$lag_minute /= 60;
my ($color) = $allcolors { $colorkeys[0] };
- foreach ( keys %allcolors) {
+ foreach ( sort keys %allcolors) {
if ($lag_minute >= $_){
$color = $allcolors {$_};
}
@@ -1330,14 +1341,15 @@ sub print_legend
if($opt_s){ return; }
print "\n<center><p><font class=page-subtitle color=#0099cc>Time Lag Legend:</font><p>\n";
print "<table cellpadding=6 cols=$nlegends width=40%>\n<tr>\n";
+ print "\n<td bgcolor=white><center>Unknown</center></td>\n";
my ($i, $j);
for ($i = 0; $i < $nlegends - 1; $i++) {
$j = $colorkeys[$i];
- print "\n<td bgcolor=$allcolors{$j}><center>within $colorkeys[$i+1] min</center></td>\n";
+ print "\n<td bgcolor=$allcolors{$j}><center>Within $colorkeys[$i+1] minutes</center></td>\n";
}
$j = $colorkeys[$i];
- print "\n<td bgcolor=$allcolors{$j}><center>over $colorkeys[$i] min</center></td>\n";
- print "\n<td bgcolor=red><center>server n/a</center></td>\n";
+ print "\n<td bgcolor=$allcolors{$j}><center>Over $colorkeys[$i] minutes</center></td>\n";
+ print "\n<td bgcolor=red><center>Server n/a</center></td>\n";
print "</table></center>\n";
}
diff --git a/man/man1/repl-monitor.1 b/man/man1/repl-monitor.1
index 37a5fa4..17b9c4b 100644
--- a/man/man1/repl-monitor.1
+++ b/man/man1/repl-monitor.1
@@ -2,7 +2,7 @@
.\" First parameter, NAME, should be all caps
.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
.\" other parameters are allowed: see man(7), man(1)
-.TH REPL-MONITOR 1 "May 18, 2008"
+.TH REPL-MONITOR 1 "Jun 28, 2016"
.\" Please adjust this date whenever revising the manpage.
.\"
.\" Some roff macros, for reference:
@@ -68,6 +68,56 @@ Prompt for passwords
Print plain text report
.br
+.SH CONFIGURATION FILE
+This section describes the various directives that can be used in the configuration file.
+.TP
+.B [connection]
+The connection details about a replica
+.br
+
+host:port:binddn:bindpwd:cert_file
+.br
+
+or,
+.br
+
+host:port=shadowport:binddn:bindpwd:cert_file
+.TP
+.B [alias]
+Define an alias for a server, this alias is used in the report in place of the
+hostname/port
+
+.br
+alias = host:port
+.TP
+.B [color]
+Set a color based on the replicaton lag time lowmark (in minutes)
+.br
+
+.R lowmark = color
+.SH EXAMPLE
+Example of a configuration file:
+
+[connection]
+.br
+localhost.localdomain:3891:cn=directory manager:password:*
+.br
+localhost2.localdomain:3892:cn=directory manager:password:*
+
+[alias]
+.br
+MY_SYSTEM1 = localhost.localdomain:3891
+.br
+MY_SYSTEM2 = localhost2.localdomain:3892
+
+[color]
+.br
+0 = #CCFFCC
+.br
+5 = #FFFFCC
+.br
+60 = #FFCCCC
+
.SH AUTHOR
repl-monitor was written by the 389 Project.
.SH "REPORTING BUGS"
@@ -75,14 +125,14 @@ Report bugs to https://fedorahosted.org/389/newticket.
.SH COPYRIGHT
Copyright \(co 2001 Sun Microsystems, Inc. Used by permission.
.br
-Copyright \(co 2008 Red Hat, Inc.
+Copyright \(co 2016 Red Hat, Inc.
.br
This manual page was written by Michele Baldessari <michele(a)pupazzo.org>,
for the Debian project (but may be used by others).
.br
-Manual page updated by Mark Reynolds <mreynolds(a)redhat.com> 10/11/13
+Manual page updated by Mark Reynolds <mreynolds(a)redhat.com> 6/28/2016
.br
This is free software. You may redistribute copies of it under the terms of
the Directory Server license found in the LICENSE file of this
software distribution. This license is essentially the GNU General Public
-License version 2 with an exception for plug\(hyin distribution.
+License version 3 with an exception for plug\(hyin distribution.
7 years, 10 months
Branch '389-ds-base-1.3.4' - ldap/admin man/man1
by Mark Reynolds
ldap/admin/src/scripts/repl-monitor.pl.in | 52 ++++++++++++++++----------
man/man1/repl-monitor.1 | 58 +++++++++++++++++++++++++++---
2 files changed, 86 insertions(+), 24 deletions(-)
New commits:
commit 0fe8f374db9df553bbd494952511f6546ab52155
Author: Mark Reynolds <mreynolds(a)redhat.com>
Date: Tue Jun 28 15:41:42 2016 -0400
Ticket 47538 - Fix repl-monitor color and lag times
Bug Description: Colors do not match the legend, and when the supplier or
consumer max csns are not available, the lag time is displayed
incorrectly.
Fix Description: The color hash table needed to be sorted before processing,
and we were not properly detecting the "Unavailable" state of
a replica which lead to the odd lag times. Also added the
string "Unavailable" for uninitialized values which allowed
for a cleaner report when replicas are offline.
Updated the man page as well.
https://fedorahosted.org/389/ticket/47538
Reviewed by: nhosoi(Thanks!)
(cherry picked from commit 34c929353280b93eebe214f1c21f5b788aff1c39)
diff --git a/ldap/admin/src/scripts/repl-monitor.pl.in b/ldap/admin/src/scripts/repl-monitor.pl.in
index eca3db0..88655ff 100755
--- a/ldap/admin/src/scripts/repl-monitor.pl.in
+++ b/ldap/admin/src/scripts/repl-monitor.pl.in
@@ -597,7 +597,7 @@ sub process_suppliers
# Skip replicas without agreements defined yet
next if (! grep {$_->{ridx} == $ridx} @allagreements);
$maxcsn = &print_master_header ($ridx, $mid);
- if ( "$maxcsn" ne "none" ) {
+ if ( "$maxcsn" ne "Unavailable" ) {
&print_consumer_header ();
&print_consumers ($ridx, $mid);
}
@@ -755,12 +755,12 @@ sub print_consumers
my ($c_maxcsn, $c_maxcsn_str, $c_lastmodified, $c_sidx, $lag, $markcolor);
my ($c_replicaroot, $c_replicatype);
my ($first_entry, $s_ldapurl, $c_ldapurl);
- my $supplier_maxcsn = "_";
+ my $supplier_maxcsn = "Unavailable";
my ($nrows);
my ($found);
undef @ouragreements;
- $c_lastmodified = "";
+ $c_lastmodified = "Unavailable";
# Collect all the consumer replicas for the current master replica
push (@consumers, $m_ridx);
@@ -791,18 +791,20 @@ sub print_consumers
if ($c_ridx >= 0) {
$myruv = $allruvs {"$c_ridx:$mid"};
- ($c_maxcsn, $c_lastmodified) = split ( /;/, $myruv );
- ($c_sidx, $c_replicaroot, $c_replicatype) = split (/:/, $allreplicas[$c_ridx]);
- $c_replicaroot = "same as master" if $m_replicaroot eq $c_replicaroot;
+ if ($myruv) {
+ ($c_maxcsn, $c_lastmodified) = split ( /;/, $myruv );
+ ($c_sidx, $c_replicaroot, $c_replicatype) = split (/:/, $allreplicas[$c_ridx]);
+ $c_replicaroot = "same as master" if $m_replicaroot eq $c_replicaroot;
+ }
}
else {
# $c_ridx is actually -$c_sidx when c is not available
$c_sidx = -$c_ridx;
- $c_maxcsn_str = "_";
+ $c_maxcsn_str = "Unavailable";
$lag = "n/a";
$markcolor = "red";
- $c_replicaroot = "_";
- $c_replicatype = "_";
+ $c_replicaroot = "Unavailable";
+ $c_replicatype = "Unavailable";
}
$nrows = 0;
@@ -933,7 +935,7 @@ sub get_supplier_maxcsn
{
my ($ridx, $s, $cn, $h, $p) = @_;
my $decimalcsn;
- my $csn = "";
+ my $csn = "Unavailable";
# normalize suffix
$s =~ s/ //;
$s =~ lc $s;
@@ -945,7 +947,7 @@ sub get_supplier_maxcsn
last;
}
}
- if($csn ne ""){
+ if($csn && $csn ne "Unavailable"){
$decimalcsn = &to_decimal_csn ($csn);
return "$csn:$decimalcsn";
}
@@ -967,11 +969,16 @@ sub cacl_time_lag
$supplier_csn_str = &to_string_csn ($s_maxcsn);
$csn_str = &to_string_csn ($c_maxcsn);
- if ($s_maxcsn && !$c_maxcsn) {
+ if ((!$s_maxcsn || $s_maxcsn eq "Unavailable") &&
+ (!$c_maxcsn || $c_maxcsn eq "Unavailable")) {
+ $lag_str = "?:??:??";
+ $markcolor = "white"; # Both unknown
+ }
+ elsif ($s_maxcsn && (!$c_maxcsn || $c_maxcsn eq "Unavailable")) {
$lag_str = "- ?:??:??";
$markcolor = &get_color (36000); # assume consumer has big latency
}
- elsif (!$s_maxcsn && $c_maxcsn) {
+ elsif ((!$s_maxcsn || $s_maxcsn eq "Unavailable") && $c_maxcsn) {
$lag_str = "+ ?:??:??";
$markcolor = &get_color (1); # consumer is ahead of supplier
}
@@ -1181,7 +1188,7 @@ sub to_decimal_csn
{
my ($maxcsn) = @_;
if (!$maxcsn || $maxcsn eq "" || $maxcsn eq "Unavailable") {
- return "none";
+ return "Unavailable";
}
my ($tm, $seq, $masterid, $subseq) = unpack("a8 a4 a4 a4", $maxcsn);
@@ -1196,9 +1203,13 @@ sub to_decimal_csn
sub to_string_csn
{
- my ($rawcsn, $decimalcsn) = split(/:/, "@_");
+ my $str = shift;
+ if (!defined($str)){
+ return "Unavailable";
+ }
+ my ($rawcsn, $decimalcsn) = split(/:/, "$str");
if (!$rawcsn || $rawcsn eq "") {
- return "none";
+ return "Unavailable";
}
if ($rawcsn eq "Unavailable"){
return $rawcsn;
@@ -1222,7 +1233,7 @@ sub get_color
$lag_minute /= 60;
my ($color) = $allcolors { $colorkeys[0] };
- foreach ( keys %allcolors) {
+ foreach ( sort keys %allcolors) {
if ($lag_minute >= $_){
$color = $allcolors {$_};
}
@@ -1301,14 +1312,15 @@ sub print_legend
if($opt_s){ return; }
print "\n<center><p><font class=page-subtitle color=#0099cc>Time Lag Legend:</font><p>\n";
print "<table cellpadding=6 cols=$nlegends width=40%>\n<tr>\n";
+ print "\n<td bgcolor=white><center>Unknown</center></td>\n";
my ($i, $j);
for ($i = 0; $i < $nlegends - 1; $i++) {
$j = $colorkeys[$i];
- print "\n<td bgcolor=$allcolors{$j}><center>within $colorkeys[$i+1] min</center></td>\n";
+ print "\n<td bgcolor=$allcolors{$j}><center>Within $colorkeys[$i+1] minutes</center></td>\n";
}
$j = $colorkeys[$i];
- print "\n<td bgcolor=$allcolors{$j}><center>over $colorkeys[$i] min</center></td>\n";
- print "\n<td bgcolor=red><center>server n/a</center></td>\n";
+ print "\n<td bgcolor=$allcolors{$j}><center>Over $colorkeys[$i] minutes</center></td>\n";
+ print "\n<td bgcolor=red><center>Server n/a</center></td>\n";
print "</table></center>\n";
}
diff --git a/man/man1/repl-monitor.1 b/man/man1/repl-monitor.1
index 37a5fa4..17b9c4b 100644
--- a/man/man1/repl-monitor.1
+++ b/man/man1/repl-monitor.1
@@ -2,7 +2,7 @@
.\" First parameter, NAME, should be all caps
.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
.\" other parameters are allowed: see man(7), man(1)
-.TH REPL-MONITOR 1 "May 18, 2008"
+.TH REPL-MONITOR 1 "Jun 28, 2016"
.\" Please adjust this date whenever revising the manpage.
.\"
.\" Some roff macros, for reference:
@@ -68,6 +68,56 @@ Prompt for passwords
Print plain text report
.br
+.SH CONFIGURATION FILE
+This section describes the various directives that can be used in the configuration file.
+.TP
+.B [connection]
+The connection details about a replica
+.br
+
+host:port:binddn:bindpwd:cert_file
+.br
+
+or,
+.br
+
+host:port=shadowport:binddn:bindpwd:cert_file
+.TP
+.B [alias]
+Define an alias for a server, this alias is used in the report in place of the
+hostname/port
+
+.br
+alias = host:port
+.TP
+.B [color]
+Set a color based on the replicaton lag time lowmark (in minutes)
+.br
+
+.R lowmark = color
+.SH EXAMPLE
+Example of a configuration file:
+
+[connection]
+.br
+localhost.localdomain:3891:cn=directory manager:password:*
+.br
+localhost2.localdomain:3892:cn=directory manager:password:*
+
+[alias]
+.br
+MY_SYSTEM1 = localhost.localdomain:3891
+.br
+MY_SYSTEM2 = localhost2.localdomain:3892
+
+[color]
+.br
+0 = #CCFFCC
+.br
+5 = #FFFFCC
+.br
+60 = #FFCCCC
+
.SH AUTHOR
repl-monitor was written by the 389 Project.
.SH "REPORTING BUGS"
@@ -75,14 +125,14 @@ Report bugs to https://fedorahosted.org/389/newticket.
.SH COPYRIGHT
Copyright \(co 2001 Sun Microsystems, Inc. Used by permission.
.br
-Copyright \(co 2008 Red Hat, Inc.
+Copyright \(co 2016 Red Hat, Inc.
.br
This manual page was written by Michele Baldessari <michele(a)pupazzo.org>,
for the Debian project (but may be used by others).
.br
-Manual page updated by Mark Reynolds <mreynolds(a)redhat.com> 10/11/13
+Manual page updated by Mark Reynolds <mreynolds(a)redhat.com> 6/28/2016
.br
This is free software. You may redistribute copies of it under the terms of
the Directory Server license found in the LICENSE file of this
software distribution. This license is essentially the GNU General Public
-License version 2 with an exception for plug\(hyin distribution.
+License version 3 with an exception for plug\(hyin distribution.
7 years, 10 months