dirsrvtests/tickets/ticket48252_test.py | 178 +++++++++++++++++++++++++++++++
ldap/servers/slapd/back-ldbm/ldif2ldbm.c | 66 +++++++----
2 files changed, 222 insertions(+), 22 deletions(-)
New commits:
commit 2231ee4673d52e92c6132daec1ee6e19e38f0c88
Author: Noriko Hosoi <nhosoi(a)redhat.com>
Date: Fri Aug 21 17:00:08 2015 -0700
Ticket #48252 - CI test: added test cases for ticket 48252
Description: db2index creates index entry from deleted records
(cherry picked from commit b011ae3aed0a6d24816708d34f3a5ae578c5cd63)
diff --git a/dirsrvtests/tickets/ticket48252_test.py
b/dirsrvtests/tickets/ticket48252_test.py
new file mode 100644
index 0000000..5970d70
--- /dev/null
+++ b/dirsrvtests/tickets/ticket48252_test.py
@@ -0,0 +1,178 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2015 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+#
+import os
+import sys
+import time
+import ldap
+import logging
+import pytest
+from lib389 import DirSrv, Entry, tools, tasks
+from lib389.tools import DirSrvTools
+from lib389._constants import *
+from lib389.properties import *
+from lib389.tasks import *
+
+log = logging.getLogger(__name__)
+
+installation_prefix = None
+
+# Assuming DEFAULT_SUFFIX is "dc=example,dc=com", otherwise it does not work...
:(
+USER_NUM = 10
+TEST_USER = "test_user"
+
+class TopologyStandalone(object):
+ def __init__(self, standalone):
+ standalone.open()
+ self.standalone = standalone
+
+
+(a)pytest.fixture(scope="module")
+def topology(request):
+ '''
+ This fixture is used to standalone topology for the 'module'.
+ '''
+ global installation_prefix
+
+ if installation_prefix:
+ args_instance[SER_DEPLOYED_DIR] = installation_prefix
+
+ standalone = DirSrv(verbose=False)
+
+ # Args for the standalone instance
+ args_instance[SER_HOST] = HOST_STANDALONE
+ args_instance[SER_PORT] = PORT_STANDALONE
+ args_instance[SER_SERVERID_PROP] = SERVERID_STANDALONE
+ args_standalone = args_instance.copy()
+ standalone.allocate(args_standalone)
+
+ # Get the status of the instance and restart it if it exists
+ instance_standalone = standalone.exists()
+
+ # Remove the instance
+ if instance_standalone:
+ standalone.delete()
+
+ # Create the instance
+ standalone.create()
+
+ # Used to retrieve configuration information (dbdir, confdir...)
+ standalone.open()
+
+ # clear the tmp directory
+ standalone.clearTmpDir(__file__)
+
+ # Here we have standalone instance up and running
+ return TopologyStandalone(standalone)
+
+def test_ticket48252_setup(topology):
+ """
+ Enable USN plug-in for enabling tombstones
+ Add test entries
+ """
+
+ log.info("Enable the USN plugin...")
+ try:
+ topology.standalone.plugins.enable(name=PLUGIN_USN)
+ except e:
+ log.error("Failed to enable USN Plugin: error " +
e.message['desc'])
+ assert False
+
+ log.info("Adding test entries...")
+ for id in range(USER_NUM):
+ name = "%s%d" % (TEST_USER, id)
+ topology.standalone.add_s(Entry(("cn=%s,%s" % (name, SUFFIX), {
+ 'objectclass': "top
person".split(),
+ 'sn': name,
+ 'cn': name})))
+def in_index_file(topology, id, index):
+ key = "%s%s" % (TEST_USER, id)
+ log.info(" dbscan - checking %s is in index file %s..." % (key, index))
+ dbscanOut = topology.standalone.dbscan(DEFAULT_BENAME, index)
+
+ if key in dbscanOut:
+ found = True
+ topology.standalone.log.info("Found key %s in dbscan output" % key)
+ else:
+ found = False
+ topology.standalone.log.info("Did not found key %s in dbscan output" %
key)
+
+ return found
+
+def test_ticket48252_run_0(topology):
+ """
+ Delete an entry cn=test_entry0
+ Check it is not in the 'cn' index file
+ """
+ log.info("Case 1 - Check deleted entry is not in the 'cn' index
file")
+ del_rdn = "cn=%s0" % TEST_USER
+ del_entry = "%s,%s" % (del_rdn, SUFFIX)
+ log.info(" Deleting a test entry %s..." % del_entry)
+ topology.standalone.delete_s(del_entry)
+
+ assert in_index_file(topology, 0, 'cn') == False
+
+ log.info(" db2index - reindexing %s ..." % 'cn')
+ assert topology.standalone.db2index(DEFAULT_BENAME, 'cn')
+
+ assert in_index_file(topology, 0, 'cn') == False
+ log.info(" entry %s is not in the cn index file after reindexed." %
del_entry)
+ log.info('Case 1 - PASSED')
+
+def test_ticket48252_run_1(topology):
+ """
+ Delete an entry cn=test_entry1
+ Check it is in the 'objectclass' index file as a tombstone entry
+ """
+ log.info("Case 2 - Check deleted entry is in the 'objectclass' index
file as a tombstone entry")
+ del_rdn = "cn=%s1" % TEST_USER
+ del_entry = "%s,%s" % (del_rdn, SUFFIX)
+ log.info(" Deleting a test entry %s..." % del_entry)
+ topology.standalone.delete_s(del_entry)
+
+ entry = topology.standalone.search_s(SUFFIX, ldap.SCOPE_SUBTREE,
'(&(objectclass=nstombstone)(%s))' % del_rdn)
+ assert len(entry) == 1
+ log.info(" entry %s is in the objectclass index file." % del_entry)
+
+ log.info(" db2index - reindexing %s ..." % 'objectclass')
+ assert topology.standalone.db2index(DEFAULT_BENAME, 'objectclass')
+
+ entry = topology.standalone.search_s(SUFFIX, ldap.SCOPE_SUBTREE,
'(&(objectclass=nstombstone)(%s))' % del_rdn)
+ assert len(entry) == 1
+ log.info(" entry %s is in the objectclass index file after reindexed." %
del_entry)
+ log.info('Case 2 - PASSED')
+
+def test_ticket48252_final(topology):
+ topology.standalone.delete()
+ log.info('Testing Ticket 48252 - PASSED.')
+
+def run_isolated():
+ '''
+ run_isolated is used to run these test cases independently of a test scheduler
(xunit, py.test..)
+ To run isolated without py.test, you need to
+ - edit this file and comment '(a)pytest.fixture' line before
'topology' function.
+ - set the installation prefix
+ - run this program
+ '''
+ global installation_prefix
+ installation_prefix = None
+
+ topo = topology(True)
+ log.info('Testing Ticket 48252 - db2index creates index entry from deleted
records')
+
+ test_ticket48252_setup(topo)
+
+ test_ticket48252_run_0(topo)
+ test_ticket48252_run_1(topo)
+
+ test_ticket48252_final(topo)
+
+
+if __name__ == '__main__':
+ run_isolated()
+
commit 86b96278871d8b08f7c6a75379935c2183f6668f
Author: Noriko Hosoi <nhosoi(a)redhat.com>
Date: Fri Aug 21 12:10:27 2015 -0700
Ticket #48252 - db2index creates index entry from deleted records
Bug Description: When an entry is deleted, its indexed attribute values
are also removed from each index file. But if the entry is turned to be
a tombstone entry, reindexing adds the removed attribute value back to
the index file.
Fix Description: In the reindexing function ldbm_back_ldbm2index, if the
entry is a tombstone entry, it skips reindexing operation unless the target
attribute is entryrdn or objectclass. If it is reindexing the objectclass,
it indexes just for the key "=nstombstone".
https://fedorahosted.org/389/ticket/48252
Reviewed by mreynolds(a)redhat.com (Thank you, Mark!!)
(cherry picked from commit ca4d38dff3bd820af2bf60c9bcc82fd64aac2556)
diff --git a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c
b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c
index a1b4a9f..3d07b2e 100644
--- a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c
+++ b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c
@@ -25,9 +25,12 @@
static char *sourcefile = "ldif2ldbm.c";
-#define DB2INDEX_ANCESTORID 0x1 /* index ancestorid */
-#define DB2INDEX_ENTRYRDN 0x2 /* index entryrdn */
-#define DB2LDIF_ENTRYRDN 0x4 /* export entryrdn */
+#define DB2INDEX_ANCESTORID 0x1 /* index ancestorid */
+#define DB2INDEX_ENTRYRDN 0x2 /* index entryrdn */
+#define DB2LDIF_ENTRYRDN 0x4 /* export entryrdn */
+#define DB2INDEX_OBJECTCLASS 0x10 /* for reindexing "objectclass:
nstombstone" */
+
+#define LDIF2LDBM_EXTBITS(x) ((x) & 0xf)
typedef struct _export_args {
struct backentry *ep;
@@ -1675,6 +1678,7 @@ ldbm_back_ldbm2index(Slapi_PBlock *pb)
struct vlvIndex *vlvip = NULL;
back_txn txn;
ID suffixid = NOID; /* holds the id of the suffix entry */
+ Slapi_Value **nstombstone_vals = NULL;
LDAPDebug( LDAP_DEBUG_TRACE, "=> ldbm_back_ldbm2index\n", 0, 0, 0 );
if ( g_get_shutdown() || c_get_shutdown() ) {
@@ -1795,7 +1799,7 @@ ldbm_back_ldbm2index(Slapi_PBlock *pb)
if (strcasecmp(attrs[i]+1, LDBM_ANCESTORID_STR) == 0) {
if (task) {
slapi_task_log_notice(task, "%s: Indexing %s",
- inst->inst_name, LDBM_ENTRYRDN_STR);
+ inst->inst_name, LDBM_ANCESTORID_STR);
}
LDAPDebug2Args(LDAP_DEBUG_ANY, "%s: Indexing %s\n",
inst->inst_name, LDBM_ANCESTORID_STR);
@@ -1848,6 +1852,9 @@ ldbm_back_ldbm2index(Slapi_PBlock *pb)
inst->inst_name, attrs[i] + 1);
}
} else {
+ if (strcasecmp(attrs[i]+1, SLAPI_ATTR_OBJECTCLASS) == 0) {
+ index_ext |= DB2INDEX_OBJECTCLASS;
+ }
charray_add(&indexAttrs, attrs[i]+1);
ai->ai_indexmask |= INDEX_OFFLINE;
if (task) {
@@ -1893,7 +1900,7 @@ ldbm_back_ldbm2index(Slapi_PBlock *pb)
* idl composed from the ancestorid list, instead of traversing the
* entire database.
*/
- if (!indexAttrs && !index_ext && pvlv) {
+ if (!indexAttrs && !LDIF2LDBM_EXTBITS(index_ext) && pvlv) {
int err;
char **suffix_list = NULL;
@@ -2152,11 +2159,13 @@ ldbm_back_ldbm2index(Slapi_PBlock *pb)
/*
* If this entry is a tombstone, update the 'nstombstonecsn' index
*/
- if(ep->ep_entry->e_flags & SLAPI_ENTRY_FLAG_TOMBSTONE){
- const CSN *tombstone_csn = NULL;
+ if (ep->ep_entry->e_flags & SLAPI_ENTRY_FLAG_TOMBSTONE) {
+ const CSN *tombstone_csn = entry_get_deletion_csn(ep->ep_entry);
char deletion_csn_str[CSN_STRSIZE];
- if((tombstone_csn = entry_get_deletion_csn(ep->ep_entry))){
+ nstombstone_vals = (Slapi_Value **) slapi_ch_calloc(2, sizeof(Slapi_Value
*));
+ *nstombstone_vals = slapi_value_new_string(SLAPI_ATTR_VALUE_TOMBSTONE);
+ if (tombstone_csn) {
if (!run_from_cmdline) {
rc = dblayer_txn_begin(be, NULL, &txn);
if (0 != rc) {
@@ -2215,18 +2224,31 @@ ldbm_back_ldbm2index(Slapi_PBlock *pb)
/*
* Update the attribute indexes
*/
- if (indexAttrs != NULL) {
+ if (indexAttrs) {
+ if (nstombstone_vals && !(index_ext &
(DB2INDEX_ENTRYRDN|DB2INDEX_OBJECTCLASS))) {
+ /* if it is a tombstone entry, just entryrdn or "objectclass:
nstombstone"
+ * need to be reindexed. the to-be-indexed list does not contain them.
*/
+ continue;
+ }
for (i = slapi_entry_first_attr(ep->ep_entry, &attr); i == 0;
i = slapi_entry_next_attr(ep->ep_entry, attr, &attr)) {
Slapi_Value **svals;
slapi_attr_get_type( attr, &type );
for ( j = 0; indexAttrs[j] != NULL; j++ ) {
+ int is_tombstone_obj = 0;
if ( g_get_shutdown() || c_get_shutdown() ) {
goto err_out;
}
- if (slapi_attr_type_cmp(indexAttrs[j], type,
- SLAPI_TYPE_CMP_SUBTYPE) == 0 ) {
+ if (slapi_attr_type_cmp(indexAttrs[j], type, SLAPI_TYPE_CMP_SUBTYPE)
== 0) {
+ if (nstombstone_vals) {
+ if (!slapi_attr_type_cmp(indexAttrs[j],
SLAPI_ATTR_OBJECTCLASS, SLAPI_TYPE_CMP_SUBTYPE)) {
+ is_tombstone_obj = 1; /* is tombstone && is
objectclass. need to index "nstombstone"*/
+ } else if (slapi_attr_type_cmp(indexAttrs[j],
LDBM_ENTRYRDN_STR, SLAPI_TYPE_CMP_SUBTYPE)) {
+ /* Entry is a tombstone && this index is not an
entryrdn. */
+ continue;
+ }
+ }
svals = attr_get_present_values(attr);
if (!run_from_cmdline) {
@@ -2250,10 +2272,12 @@ ldbm_back_ldbm2index(Slapi_PBlock *pb)
goto err_out;
}
}
- rc = index_addordel_values_sv(
- be, indexAttrs[j], svals,
- NULL, ep->ep_id, BE_INDEX_ADD, &txn);
- if (rc != 0) {
+ if (is_tombstone_obj) {
+ rc = index_addordel_values_sv(be, indexAttrs[j],
nstombstone_vals, NULL, ep->ep_id, BE_INDEX_ADD, &txn);
+ } else {
+ rc = index_addordel_values_sv(be, indexAttrs[j], svals, NULL,
ep->ep_id, BE_INDEX_ADD, &txn);
+ }
+ if (rc) {
LDAPDebug(LDAP_DEBUG_ANY,
"%s: ERROR: failed to update index
'%s'\n",
inst->inst_name, indexAttrs[j], 0);
@@ -2299,18 +2323,16 @@ ldbm_back_ldbm2index(Slapi_PBlock *pb)
}
/*
- * Update the Virtual List View indexes
+ * If it is NOT a tombstone entry, update the Virtual List View indexes.
*/
- for ( vlvidx = 0; vlvidx < numvlv; vlvidx++ ) {
+ for (vlvidx = 0; !nstombstone_vals && (vlvidx < numvlv); vlvidx++) {
char *ai = "Unknown index";
if ( g_get_shutdown() || c_get_shutdown() ) {
goto err_out;
}
- if(indexAttrs){
- if(indexAttrs[vlvidx]){
- ai = indexAttrs[vlvidx];
- }
+ if (indexAttrs && indexAttrs[vlvidx]) {
+ ai = indexAttrs[vlvidx];
}
if (!run_from_cmdline) {
rc = dblayer_txn_begin(be, NULL, &txn);
@@ -2364,7 +2386,7 @@ ldbm_back_ldbm2index(Slapi_PBlock *pb)
/*
* Update the ancestorid and entryrdn index
*/
- if (!entryrdn_get_noancestorid() && index_ext & DB2INDEX_ANCESTORID)
{
+ if (!entryrdn_get_noancestorid() && (index_ext &
DB2INDEX_ANCESTORID)) {
rc = ldbm_ancestorid_index_entry(be, ep, BE_INDEX_ADD, NULL);
if (rc != 0) {
LDAPDebug(LDAP_DEBUG_ANY,
--
389 commits mailing list
389-commits@%(host_name)s
http://lists.fedoraproject.org/postorius/389-commits@lists.fedoraproject.org