Author: nhosoi
Update of /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv2672/ldap/servers/slapd/back-ldbm
Modified Files:
back-ldbm.h index.c ldbm_attr.c ldbm_index_config.c
Log Message:
Resolves: #447353
Summary: RFE: search optimization and single character substring searches
Description: extended the substring key to have 3 types:
* begin (e.g., *^a)
* middle (e.g., *abc)
* end (e.g., *xy$)
* Usage: turn an index object to extensibleobject and set an integer value as
follows:
* dn: cn=sn, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config
* objectClass: extensibleObject
* nsSubStrBegin: 2
* nsSubStrMiddle: 3
* nsSubStrEnd: 2
* [...]
Index: back-ldbm.h
===================================================================
RCS file: /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm/back-ldbm.h,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -r1.13 -r1.14
--- back-ldbm.h 3 Apr 2008 16:52:47 -0000 1.13
+++ back-ldbm.h 15 Jul 2008 16:49:43 -0000 1.14
@@ -346,6 +346,34 @@
typedef struct idl_private idl_private;
typedef struct attrcrypt_private attrcrypt_private;
+/*
+ * Special attribute for an index entry
+ * Usage: turn an index object to extensibleobject and
+ * set an integer value for each
+ * dn: cn=sn, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config
+ * objectClass: extensibleObject
+ * nsSubStrBegin: 2
+ * nsSubStrMiddle: 3
+ * nsSubStrEnd: 2
+ * [...]
+ *
+ * By default, the minimum key length triplets of substring index is 2, 3, 2.
+ * The length is changed by setting this nsSubStrLen value.
+ *
+ * Note: If any of the key length value is modified, the index file needs
+ * to be regenerated. Otherwise, the index file is going to have mixed
+ * key length.
+ * To change the key length,
+ * 1) stop the server, 2) run db2index -t <attr>, 3) start the server.
+ */
+#define INDEX_ATTR_SUBSTRBEGIN "nsSubStrBegin"
+#define INDEX_ATTR_SUBSTRMIDDLE "nsSubStrMiddle"
+#define INDEX_ATTR_SUBSTREND "nsSubStrEnd"
+
+#define INDEX_SUBSTRBEGIN 0
+#define INDEX_SUBSTRMIDDLE 1
+#define INDEX_SUBSTREND 2
+
/* for the cache of attribute information (which are indexed, etc.) */
struct attrinfo {
@@ -382,6 +410,10 @@
specify an ORDERING matching rule, or the index
configuration must define an ORDERING matching rule.
*/
+ int *ai_substr_lens; /* if the attribute nsSubStrXxx is specivied in
+ * an index instance (dse.ldif), the substr key
+ * len value(s) are stored here. If not specified,
+ * the default length triplet is 2, 3, 2.
};
#define MAXDBCACHE 20
Index: index.c
===================================================================
RCS file: /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm/index.c,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -r1.14 -r1.15
--- index.c 20 Jun 2008 15:10:00 -0000 1.14
+++ index.c 15 Jul 2008 16:49:43 -0000 1.15
@@ -1830,8 +1830,13 @@
Slapi_Value **esubvals = NULL;
Slapi_Value **substresult = NULL;
Slapi_Value **origvals = NULL;
- slapi_call_syntax_values2keys_sv( ai->ai_plugin, vals, &ivals,
- LDAP_FILTER_SUBSTRINGS );
+ Slapi_PBlock pipb;
+
+ /* prepare pblock to pass ai_substr_lens */
+ pblock_init( &pipb );
+ slapi_pblock_set( &pipb, SLAPI_SYNTAX_SUBSTRLENS, ai->ai_substr_lens );
+ slapi_call_syntax_values2keys_sv_pb( ai->ai_plugin, vals, &ivals,
+ LDAP_FILTER_SUBSTRINGS, &pipb );
origvals = ivals;
/* delete only: if the attribute has multiple values,
@@ -1840,8 +1845,8 @@
* then get rid of them from the being deleted values
*/
if ( evals != NULL ) {
- slapi_call_syntax_values2keys_sv( ai->ai_plugin, evals, &esubvals,
- LDAP_FILTER_SUBSTRINGS );
+ slapi_call_syntax_values2keys_sv_pb( ai->ai_plugin, evals,
+ &esubvals, LDAP_FILTER_SUBSTRINGS, &pipb );
substresult = valuearray_minus_valuearray( ai->ai_plugin, ivals, esubvals
);
ivals = substresult;
valuearray_free( &esubvals );
Index: ldbm_attr.c
===================================================================
RCS file: /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm/ldbm_attr.c,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -r1.9 -r1.10
--- ldbm_attr.c 18 Oct 2007 00:08:34 -0000 1.9
+++ ldbm_attr.c 15 Jul 2008 16:49:43 -0000 1.10
@@ -50,14 +50,6 @@
attrinfo_new()
{
struct attrinfo *p= (struct attrinfo *)slapi_ch_calloc(1, sizeof(struct attrinfo));
- p->ai_type= 0;
- p->ai_indexmask= 0;
- p->ai_plugin= NULL;
- p->ai_index_rules= NULL;
- p->ai_dblayer= NULL;
- p->ai_dblayer_count = 0;
- p->ai_idl= NULL;
- p->ai_key_cmp_fn = NULL;
return p;
}
@@ -156,6 +148,25 @@
}
void
+_set_attr_substrlen(int index, char *str, int **substrlens)
+{
+ char *p = NULL;
+ /* nsSubStrXxx=<VAL> is passed;
+ set it to the attribute's plugin */
+ p = strchr(str, '=');
+ if (NULL != p) {
+ long sublen = strtol(++p, (char **)NULL, 10);
+ if (sublen > 0) { /* 0 is not acceptable */
+ if (NULL == *substrlens) {
+ *substrlens = (int *)slapi_ch_calloc(1,
+ sizeof(int) * INDEX_SUBSTRLEN);
+ }
+ (*substrlens)[index] = sublen;
+ }
+ }
+}
+
+void
attr_index_config(
backend *be,
char *fname,
@@ -172,6 +183,8 @@
char **index_rules = NULL;
struct attrinfo *a;
int return_value = -1;
+ char *p;
+ int *substrlens = NULL;
attrs = str2charray( argv[0], "," );
if ( argc > 1 ) {
@@ -189,34 +202,29 @@
attrsyntax_oid = slapi_ch_strdup(plugin_syntax2oid(a->ai_plugin));
if ( argc == 1 ) {
a->ai_indexmask = (INDEX_PRESENCE | INDEX_EQUALITY |
- INDEX_APPROX | INDEX_SUB);
+ INDEX_APPROX | INDEX_SUB);
} else {
a->ai_indexmask = 0;
for ( j = 0; indexes[j] != NULL; j++ ) {
- if ( strncasecmp( indexes[j], "pres", 4 )
- == 0 ) {
+ if ( strncasecmp( indexes[j], "pres", 4 ) == 0 ) {
a->ai_indexmask |= INDEX_PRESENCE;
- } else if ( strncasecmp( indexes[j], "eq", 2 )
- == 0 ) {
+ } else if ( strncasecmp( indexes[j], "eq", 2 ) == 0 ) {
a->ai_indexmask |= INDEX_EQUALITY;
- } else if ( strncasecmp( indexes[j], "approx",
- 6 ) == 0 ) {
+ } else if ( strncasecmp( indexes[j], "approx", 6 ) == 0 ) {
a->ai_indexmask |= INDEX_APPROX;
- } else if ( strncasecmp( indexes[j], "sub", 3 )
- == 0 ) {
+ } else if ( strncasecmp( indexes[j], "sub", 3 ) == 0 ) {
a->ai_indexmask |= INDEX_SUB;
- } else if ( strncasecmp( indexes[j], "none", 4 )
- == 0 ) {
+ } else if ( strncasecmp( indexes[j], "none", 4 ) == 0 ) {
if ( a->ai_indexmask != 0 ) {
LDAPDebug(LDAP_DEBUG_ANY,
"%s: line %d: index type \"none\" cannot be combined with other
types\n",
- fname, lineno, 0);
+ fname, lineno, 0);
}
a->ai_indexmask = INDEX_OFFLINE; /* note that the index isn't available */
} else {
LDAPDebug(LDAP_DEBUG_ANY,
"%s: line %d: unknown index type \"%s\" (ignored)\n",
- fname, lineno, indexes[j]);
+ fname, lineno, indexes[j]);
LDAPDebug(LDAP_DEBUG_ANY,
"valid index types are \"pres\", \"eq\",
\"approx\", or \"sub\"\n",
0, 0, 0);
@@ -227,53 +235,66 @@
j = 0;
if (index_rules != NULL) for (; index_rules[j] != NULL; ++j);
if (j > 0) { /* there are some candidates */
- char** official_rules = (char**)
- slapi_ch_malloc ((j + 1) * sizeof (char*));
- size_t k = 0;
- for (j = 0; index_rules[j] != NULL; ++j) {
- /* Check that index_rules[j] is an official OID */
- char* officialOID = NULL;
- IFP mrINDEX = NULL;
- Slapi_PBlock* pb = slapi_pblock_new();
- if (!slapi_pblock_set (pb, SLAPI_PLUGIN_MR_OID, index_rules[j]) &&
- !slapi_pblock_set (pb, SLAPI_PLUGIN_MR_TYPE, a->ai_type) &&
- !slapi_mr_indexer_create (pb) &&
- !slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX) &&
- mrINDEX != NULL &&
- !slapi_pblock_get (pb, SLAPI_PLUGIN_MR_OID, &officialOID) &&
- officialOID != NULL) {
- if (!strcasecmp (index_rules[j], officialOID)) {
- official_rules[k++] = slapi_ch_strdup (officialOID);
- } else {
- char* preamble = slapi_ch_smprintf("%s: line %d", fname, lineno);
- LDAPDebug (LDAP_DEBUG_ANY, "%s: use \"%s\" instead of
\"%s\" (ignored)\n",
- preamble, officialOID, index_rules[j] );
- slapi_ch_free((void**)&preamble);
- }
- } else if (!slapi_matchingrule_is_ordering(index_rules[j], attrsyntax_oid)) {
- LDAPDebug (LDAP_DEBUG_ANY, "%s: line %d: "
- "unknown or invalid matching rule \"%s\" in index
configuration (ignored)\n",
- fname, lineno, index_rules[j] );
- } else { /* assume builtin and use compare fn provided by syntax plugin */
- need_compare_fn = 1;
+ char** official_rules =
+ (char**)slapi_ch_malloc ((j + 1) * sizeof (char*));
+ size_t k = 0;
+ for (j = 0; index_rules[j] != NULL; ++j) {
+ /* Check that index_rules[j] is an official OID */
+ char* officialOID = NULL;
+ IFP mrINDEX = NULL;
+ Slapi_PBlock* pb = slapi_pblock_new();
+ if (!slapi_pblock_set (pb, SLAPI_PLUGIN_MR_OID, index_rules[j]) &&
+ !slapi_pblock_set (pb, SLAPI_PLUGIN_MR_TYPE, a->ai_type) &&
+ !slapi_mr_indexer_create (pb) &&
+ !slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX) &&
+ mrINDEX != NULL &&
+ !slapi_pblock_get (pb, SLAPI_PLUGIN_MR_OID, &officialOID) &&
+ officialOID != NULL) {
+ if (!strcasecmp (index_rules[j], officialOID)) {
+ official_rules[k++] = slapi_ch_strdup (officialOID);
+ } else {
+ char* preamble = slapi_ch_smprintf("%s: line %d", fname, lineno);
+ LDAPDebug (LDAP_DEBUG_ANY, "%s: use \"%s\" instead of
\"%s\" (ignored)\n",
+ preamble, officialOID, index_rules[j] );
+ slapi_ch_free((void**)&preamble);
+ }
+ } else if (p =
+ strstr(index_rules[j], INDEX_ATTR_SUBSTRBEGIN)) {
+ _set_attr_substrlen(INDEX_SUBSTRBEGIN, index_rules[j],
+ &substrlens);
+ } else if (p =
+ strstr(index_rules[j], INDEX_ATTR_SUBSTRMIDDLE)) {
+ _set_attr_substrlen(INDEX_SUBSTRMIDDLE, index_rules[j],
+ &substrlens);
+ } else if (p =
+ strstr(index_rules[j], INDEX_ATTR_SUBSTREND)) {
+ _set_attr_substrlen(INDEX_SUBSTREND, index_rules[j],
+ &substrlens);
+ } else if (!slapi_matchingrule_is_ordering(index_rules[j], attrsyntax_oid)) {
+ LDAPDebug (LDAP_DEBUG_ANY, "%s: line %d: "
+ "unknown or invalid matching rule \"%s\" in index configuration
(ignored)\n",
+ fname, lineno, index_rules[j] );
+ } else { /* assume builtin and use compare fn provided by syntax plugin */
+ need_compare_fn = 1;
+ }
+ {/* It would improve speed to save the indexer, for future use.
+ But, for simplicity, we destroy it now: */
+ IFP mrDESTROY = NULL;
+ if (!slapi_pblock_get (pb, SLAPI_PLUGIN_DESTROY_FN, &mrDESTROY) &&
+ mrDESTROY != NULL) {
+ mrDESTROY (pb);
+ }
+ }
+ slapi_pblock_destroy (pb);
}
- {/* It would improve speed to save the indexer, for future use.
- But, for simplicity, we destroy it now: */
- IFP mrDESTROY = NULL;
- if (!slapi_pblock_get (pb, SLAPI_PLUGIN_DESTROY_FN, &mrDESTROY) &&
- mrDESTROY != NULL) {
- mrDESTROY (pb);
- }
+ official_rules[k] = NULL;
+ a->ai_substr_lens = substrlens;
+ if (k > 0) {
+ a->ai_index_rules = official_rules;
+ a->ai_indexmask |= INDEX_RULES;
+ } else {
+ slapi_ch_free((void**)&official_rules);
}
- slapi_pblock_destroy (pb);
- }
- official_rules[k] = NULL;
- if (k > 0) {
- a->ai_index_rules = official_rules;
- a->ai_indexmask |= INDEX_RULES;
- } else {
- slapi_ch_free((void**)&official_rules);
- }
}
}
@@ -283,7 +304,7 @@
if (0 != return_value) {
/* fatal error, exit */
LDAPDebug(LDAP_DEBUG_ANY,"%s: line %d:Fatal Error: Failed to initialize attribute
structure\n",
- fname, lineno, 0);
+ fname, lineno, 0);
exit( 1 );
}
@@ -300,24 +321,24 @@
if (need_compare_fn) {
int rc = plugin_call_syntax_get_compare_fn( a->ai_plugin, &a->ai_key_cmp_fn
);
if (rc != LDAP_SUCCESS) {
- LDAPDebug(LDAP_DEBUG_ANY,
- "The attribute [%s] does not have a valid ORDERING matching rule\n",
- a->ai_type, 0, 0);
+ LDAPDebug(LDAP_DEBUG_ANY,
+ "The attribute [%s] does not have a valid ORDERING matching rule\n",
+ a->ai_type, 0, 0);
a->ai_key_cmp_fn = NULL;
}
}
if ( avl_insert( &inst->inst_attrs, a, ainfo_cmp, ainfo_dup ) != 0 ) {
/* duplicate - existing version updated */
- attrinfo_delete(&a);
+ attrinfo_delete(&a);
}
}
charray_free( attrs );
if ( indexes != NULL ) {
- charray_free( indexes );
+ charray_free( indexes );
}
if ( index_rules != NULL ) {
- charray_free( index_rules );
+ charray_free( index_rules );
}
}
Index: ldbm_index_config.c
===================================================================
RCS file: /cvs/dirsec/ldapserver/ldap/servers/slapd/back-ldbm/ldbm_index_config.c,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -r1.7 -r1.8
--- ldbm_index_config.c 10 Nov 2006 23:45:39 -0000 1.7
+++ ldbm_index_config.c 15 Jul 2008 16:49:43 -0000 1.8
@@ -122,7 +122,7 @@
/* used by the two callbacks below, to parse an index entry into something
* awkward that we can pass to attr_index_config().
*/
-#define MAX_TMPBUF 256
+#define MAX_TMPBUF 1024
#define ZCAT_SAFE(_buf, _x1, _x2) do { \
if (strlen(_buf) + strlen(_x1) + strlen(_x2) + 2 < MAX_TMPBUF) { \
strcat(_buf, _x1); \
@@ -170,18 +170,71 @@
arglist[argc++] = slapi_ch_strdup(tmpBuf);
}
+ tmpBuf[0] = 0;
/* Get the list of matching rules from the entry. */
if (0 == slapi_entry_attr_find(e, "nsMatchingRule", &attr)) {
for (i = slapi_attr_first_value(attr, &sval); i != -1;
i = slapi_attr_next_value(attr, i, &sval)) {
attrValue = slapi_value_get_berval(sval);
if (0 == i) {
- tmpBuf[0] = 0;
ZCAT_SAFE(tmpBuf, "", attrValue->bv_val);
} else {
ZCAT_SAFE(tmpBuf, ",", attrValue->bv_val);
}
}
+ }
+
+ /* Get the substr begin length. note: pick the first value. */
+ if (0 == slapi_entry_attr_find(e, INDEX_ATTR_SUBSTRBEGIN, &attr)) {
+ i = slapi_attr_first_value(attr, &sval);
+ if (-1 != i) {
+ attrValue = slapi_value_get_berval(sval);
+ if (0 == tmpBuf[0]) {
+ PR_snprintf(tmpBuf, MAX_TMPBUF, "%s=%s",
+ INDEX_ATTR_SUBSTRBEGIN, attrValue->bv_val);
+ } else {
+ int tmpbuflen = strlen(tmpBuf);
+ char *p = tmpBuf + tmpbuflen;
+ PR_snprintf(p, MAX_TMPBUF - tmpbuflen, ",%s=%s",
+ INDEX_ATTR_SUBSTRBEGIN, attrValue->bv_val);
+ }
+ }
+ }
+
+ /* Get the substr middle length. note: pick the first value. */
+ if (0 == slapi_entry_attr_find(e, INDEX_ATTR_SUBSTRMIDDLE, &attr)) {
+ i = slapi_attr_first_value(attr, &sval);
+ if (-1 != i) {
+ attrValue = slapi_value_get_berval(sval);
+ if (0 == tmpBuf[0]) {
+ PR_snprintf(tmpBuf, MAX_TMPBUF, "%s=%s",
+ INDEX_ATTR_SUBSTRMIDDLE, attrValue->bv_val);
+ } else {
+ int tmpbuflen = strlen(tmpBuf);
+ char *p = tmpBuf + tmpbuflen;
+ PR_snprintf(p, MAX_TMPBUF - tmpbuflen, ",%s=%s",
+ INDEX_ATTR_SUBSTRMIDDLE, attrValue->bv_val);
+ }
+ }
+ }
+
+ /* Get the substr end length. note: pick the first value. */
+ if (0 == slapi_entry_attr_find(e, INDEX_ATTR_SUBSTREND, &attr)) {
+ i = slapi_attr_first_value(attr, &sval);
+ if (-1 != i) {
+ attrValue = slapi_value_get_berval(sval);
+ if (0 == tmpBuf[0]) {
+ PR_snprintf(tmpBuf, MAX_TMPBUF, "%s=%s",
+ INDEX_ATTR_SUBSTREND, attrValue->bv_val);
+ } else {
+ int tmpbuflen = strlen(tmpBuf);
+ char *p = tmpBuf + tmpbuflen;
+ PR_snprintf(p, MAX_TMPBUF - tmpbuflen, ",%s=%s",
+ INDEX_ATTR_SUBSTREND, attrValue->bv_val);
+ }
+ }
+ }
+ if (0 != tmpBuf[0]) {
arglist[argc++] = slapi_ch_strdup(tmpBuf);
}